From 74379e5459a0928fca42ec28b991ff61ab530ca6 Mon Sep 17 00:00:00 2001 From: Brian Buchalter Date: Tue, 10 May 2022 11:03:01 -0600 Subject: [PATCH 001/536] Add failing test --- test/fixtures/if.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/fixtures/if.rb b/test/fixtures/if.rb index cabea4c3..607af05d 100644 --- a/test/fixtures/if.rb +++ b/test/fixtures/if.rb @@ -35,3 +35,9 @@ % if foo {} end +% +if not a + b +else + c +end From 2a3d9db5db34ff29adb7843ec0363ed89f516c9b Mon Sep 17 00:00:00 2001 From: Brian Buchalter Date: Tue, 10 May 2022 11:38:47 -0600 Subject: [PATCH 002/536] Disallow turning a conditional into a ternary if there's a Not node without parentheses in the predicate --- lib/syntax_tree/node.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index d153ef78..30fd8cf1 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -5192,7 +5192,7 @@ def call(q, node) else # Otherwise, we're going to check the conditional for certain cases. case node - in predicate: Assign | Command | CommandCall | MAssign | OpAssign + in predicate: Assign | Command | CommandCall | MAssign | Not | OpAssign false in { statements: { body: [truthy] }, From c5ac4b3cd876ed38acae704fae551616b9239643 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 10 May 2022 14:33:09 -0400 Subject: [PATCH 003/536] Fix nested hash patterns from accidentally adding a `then` to their output. --- CHANGELOG.md | 7 ++++++- Gemfile.lock | 2 +- lib/syntax_tree/node.rb | 7 +++++-- lib/syntax_tree/parser.rb | 10 ++++++++-- lib/syntax_tree/version.rb | 2 +- test/fixtures/hshptn.rb | 5 +++++ 6 files changed, 26 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 26e538c3..eeb9231f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +## [2.4.1] - 2022-05-10 + +- Fix nested hash patterns from accidentally adding a `then` to their output. + ## [2.4.0] - 2022-05-07 ### Added @@ -209,7 +213,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - 🎉 Initial release! 🎉 -[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.4.0...HEAD +[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.4.1...HEAD +[2.4.1]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.4.0...v2.4.1 [2.4.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.3.1...v2.4.0 [2.3.1]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.3.0...v2.3.1 [2.3.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.2.0...v2.3.0 diff --git a/Gemfile.lock b/Gemfile.lock index 8357fd92..b4eebdd4 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - syntax_tree (2.4.0) + syntax_tree (2.4.1) GEM remote: https://rubygems.org/ diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index d153ef78..fdb40631 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -5064,13 +5064,16 @@ def format(q) parts = keywords.map { |(key, value)| KeywordFormatter.new(key, value) } parts << KeywordRestFormatter.new(keyword_rest) if keyword_rest + nested = PATTERNS.include?(q.parent.class) contents = -> do q.group { q.seplist(parts) { |part| q.format(part, stackable: false) } } # If there isn't a constant, and there's a blank keyword_rest, then we # have an plain ** that needs to have a `then` after it in order to # parse correctly on the next parse. - q.text(" then") if !constant && keyword_rest && keyword_rest.value.nil? + if !constant && keyword_rest && keyword_rest.value.nil? && !nested + q.text(" then") + end end # If there is a constant, we're going to format to have the constant name @@ -5097,7 +5100,7 @@ def format(q) # If there's only one pair, then we'll just print the contents provided # we're not inside another pattern. - if !PATTERNS.include?(q.parent.class) && parts.size == 1 + if !nested && parts.size == 1 contents.call return end diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index 75d3c322..f5ffe47d 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -1671,9 +1671,15 @@ def on_heredoc_end(value) # (nil | VarField) keyword_rest # ) -> HshPtn def on_hshptn(constant, keywords, keyword_rest) - # Create an artificial VarField if we find an extra ** on the end - if !keyword_rest && (token = find_token(Op, "**", consume: false)) + if keyword_rest + # We're doing this to delete the token from the list so that it doesn't + # confuse future patterns by thinking they have an extra ** on the end. + find_token(Op, "**") + elsif (token = find_token(Op, "**", consume: false)) tokens.delete(token) + + # Create an artificial VarField if we find an extra ** on the end. This + # means the formatting will be a little more consistent. keyword_rest = VarField.new(value: nil, location: token.location) end diff --git a/lib/syntax_tree/version.rb b/lib/syntax_tree/version.rb index 894ff1b7..fbecb604 100644 --- a/lib/syntax_tree/version.rb +++ b/lib/syntax_tree/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SyntaxTree - VERSION = "2.4.0" + VERSION = "2.4.1" end diff --git a/test/fixtures/hshptn.rb b/test/fixtures/hshptn.rb index 2935f9c1..7a35b4d0 100644 --- a/test/fixtures/hshptn.rb +++ b/test/fixtures/hshptn.rb @@ -66,3 +66,8 @@ case foo in **nil end +% +case foo +in bar, { baz:, **nil } +in qux: +end From eb4721355d1d4afae92d64680f8d3865838e7004 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 10 May 2022 14:40:59 -0400 Subject: [PATCH 004/536] Update CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eeb9231f..fe3030e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [2.4.1] - 2022-05-10 -- Fix nested hash patterns from accidentally adding a `then` to their output. +- [#73](https://github.com/ruby-syntax-tree/syntax_tree/pull/73) - Fix nested hash patterns from accidentally adding a `then` to their output. ## [2.4.0] - 2022-05-07 From b39b1bf44142cfdb95743b4e2798c9d9cb3f177b Mon Sep 17 00:00:00 2001 From: Wender Freese Date: Tue, 10 May 2022 15:58:37 -0300 Subject: [PATCH 005/536] Add Rake test to run check and format commands --- lib/syntax_tree/rake/task.rb | 50 ++++++++++++++++++++++++++++++++++++ test/task_test.rb | 27 +++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 lib/syntax_tree/rake/task.rb create mode 100644 test/task_test.rb diff --git a/lib/syntax_tree/rake/task.rb b/lib/syntax_tree/rake/task.rb new file mode 100644 index 00000000..12279e0f --- /dev/null +++ b/lib/syntax_tree/rake/task.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +require "rake" +require "rake/tasklib" + +module SyntaxTree + module Rake + # A Rake task that runs check and format on a set of source files. + # + # Example: + # + # require 'syntax_tree/rake/task' + # + # SyntaxTree::Rake::Task.new do |t| + # t.source_files = '{app,config,lib}/**/*.rb' + # end + # + # This will create task that can be run with: + # + # rake syntax_tree:check_and_format + class Task < ::Rake::TaskLib + # Glob pattern to match source files. + # Defaults to 'lib/**/*.rb'. + attr_accessor :source_files + + def initialize + @source_files = "lib/**/*.rb" + + yield self if block_given? + define_task + end + + private + + def define_task + desc "Runs syntax_tree over source files" + task(:check_and_format) { run_task } + end + + def run_task + %w[check format].each do |command| + SyntaxTree::CLI.run([command, source_files].compact) + end + + # TODO: figure this out + # exit($?.exitstatus) if $?&.exited? + end + end + end +end diff --git a/test/task_test.rb b/test/task_test.rb new file mode 100644 index 00000000..c1e00b8b --- /dev/null +++ b/test/task_test.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +require_relative "test_helper" +require "syntax_tree/rake/task" + +module SyntaxTree + class TaskTest < Minitest::Test + Invoke = Struct.new(:args) + + def test_task + source_files = "{app,config,lib}/**/*.rb" + + SyntaxTree::Rake::Task.new do |t| + t.source_files = source_files + end + + invoke = [] + SyntaxTree::CLI.stub(:run, ->(args) { invoke << Invoke.new(args) }) do + ::Rake::Task["check_and_format"].invoke + end + + assert_equal( + [["check", source_files], ["format", source_files]], invoke.map(&:args) + ) + end + end +end From 695206e970df0e0ac8c3f563e180bf43a439084f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 May 2022 17:30:50 +0000 Subject: [PATCH 006/536] Bump rubocop from 1.29.0 to 1.29.1 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.29.0 to 1.29.1. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.29.0...v1.29.1) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index b4eebdd4..fe27163b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -14,9 +14,9 @@ GEM ast (~> 2.4.1) rainbow (3.1.1) rake (13.0.6) - regexp_parser (2.3.1) + regexp_parser (2.4.0) rexml (3.2.5) - rubocop (1.29.0) + rubocop (1.29.1) parallel (~> 1.10) parser (>= 3.1.0.0) rainbow (>= 2.2.2, < 4.0) From a69e958bbd00c8fadc38792dbab661ab24714a47 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 13 May 2022 11:17:36 -0400 Subject: [PATCH 007/536] Correct the pattern for checking if a dynamic symbol can be converted into a label as a hash key. --- CHANGELOG.md | 4 ++++ lib/syntax_tree/node.rb | 2 +- test/fixtures/assoc.rb | 2 ++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe3030e9..07da8abb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +### Changed + +- Correct the pattern for checking if a dynamic symbol can be converted into a label as a hash key. + ## [2.4.1] - 2022-05-10 - [#73](https://github.com/ruby-syntax-tree/syntax_tree/pull/73) - Fix nested hash patterns from accidentally adding a `then` to their output. diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index fdb40631..e8df1ea7 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -1388,7 +1388,7 @@ def format(q) module HashKeyFormatter # Formats the keys of a hash literal using labels. class Labels - LABEL = /^[@$_A-Za-z]([_A-Za-z0-9]*)?([!_=?A-Za-z0-9])?$/ + LABEL = /\A[A-Za-z_](\w*[\w!?])?\z/ def format_key(q, key) case key diff --git a/test/fixtures/assoc.rb b/test/fixtures/assoc.rb index cd3e5ed1..0fc60e6f 100644 --- a/test/fixtures/assoc.rb +++ b/test/fixtures/assoc.rb @@ -46,3 +46,5 @@ { foo: "bar" } % { "foo #{bar}": "baz" } +% +{ "foo=": "baz" } From f27590c43e61cdd2977c01d56601c3ba33ff00fa Mon Sep 17 00:00:00 2001 From: Wender Freese Date: Fri, 13 May 2022 16:05:25 -0300 Subject: [PATCH 008/536] Split "check_and_format" task into two different files "check" and "write" --- .../rake/{task.rb => check_task.rb} | 27 +++++----- lib/syntax_tree/rake/write_task.rb | 53 +++++++++++++++++++ test/check_task_test.rb | 23 ++++++++ test/task_test.rb | 27 ---------- test/write_task_test.rb | 23 ++++++++ 5 files changed, 114 insertions(+), 39 deletions(-) rename lib/syntax_tree/rake/{task.rb => check_task.rb} (54%) create mode 100644 lib/syntax_tree/rake/write_task.rb create mode 100644 test/check_task_test.rb delete mode 100644 test/task_test.rb create mode 100644 test/write_task_test.rb diff --git a/lib/syntax_tree/rake/task.rb b/lib/syntax_tree/rake/check_task.rb similarity index 54% rename from lib/syntax_tree/rake/task.rb rename to lib/syntax_tree/rake/check_task.rb index 12279e0f..0c0dc860 100644 --- a/lib/syntax_tree/rake/task.rb +++ b/lib/syntax_tree/rake/check_task.rb @@ -5,25 +5,31 @@ module SyntaxTree module Rake - # A Rake task that runs check and format on a set of source files. + # A Rake task that runs check on a set of source files. # # Example: # - # require 'syntax_tree/rake/task' + # require 'syntax_tree/rake/check_task' # - # SyntaxTree::Rake::Task.new do |t| + # SyntaxTree::Rake::CheckTask.new do |t| # t.source_files = '{app,config,lib}/**/*.rb' # end # # This will create task that can be run with: # - # rake syntax_tree:check_and_format - class Task < ::Rake::TaskLib + # rake stree_check + # + class CheckTask < ::Rake::TaskLib + # Name of the task. + # Defaults to :stree_check. + attr_accessor :name + # Glob pattern to match source files. # Defaults to 'lib/**/*.rb'. attr_accessor :source_files - def initialize + def initialize(name = :stree_check) + @name = name @source_files = "lib/**/*.rb" yield self if block_given? @@ -33,16 +39,13 @@ def initialize private def define_task - desc "Runs syntax_tree over source files" - task(:check_and_format) { run_task } + desc "Runs `stree check` over source files" + task(name) { run_task } end def run_task - %w[check format].each do |command| - SyntaxTree::CLI.run([command, source_files].compact) - end + SyntaxTree::CLI.run(["check", source_files].compact) - # TODO: figure this out # exit($?.exitstatus) if $?&.exited? end end diff --git a/lib/syntax_tree/rake/write_task.rb b/lib/syntax_tree/rake/write_task.rb new file mode 100644 index 00000000..08b6018c --- /dev/null +++ b/lib/syntax_tree/rake/write_task.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +require "rake" +require "rake/tasklib" + +module SyntaxTree + module Rake + # A Rake task that runs format on a set of source files. + # + # Example: + # + # require 'syntax_tree/rake/write_task' + # + # SyntaxTree::Rake::WriteTask.new do |t| + # t.source_files = '{app,config,lib}/**/*.rb' + # end + # + # This will create task that can be run with: + # + # rake stree_write + # + class WriteTask < ::Rake::TaskLib + # Name of the task. + # Defaults to :stree_write. + attr_accessor :name + + # Glob pattern to match source files. + # Defaults to 'lib/**/*.rb'. + attr_accessor :source_files + + def initialize(name = :stree_write) + @name = name + @source_files = "lib/**/*.rb" + + yield self if block_given? + define_task + end + + private + + def define_task + desc "Runs `stree write` over source files" + task(name) { run_task } + end + + def run_task + SyntaxTree::CLI.run(["write", source_files].compact) + + # exit($?.exitstatus) if $?&.exited? + end + end + end +end diff --git a/test/check_task_test.rb b/test/check_task_test.rb new file mode 100644 index 00000000..33333241 --- /dev/null +++ b/test/check_task_test.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +require_relative "test_helper" +require "syntax_tree/rake/check_task" + +module SyntaxTree + class CheckTaskTest < Minitest::Test + Invoke = Struct.new(:args) + + def test_task + source_files = "{app,config,lib}/**/*.rb" + + SyntaxTree::Rake::CheckTask.new { |t| t.source_files = source_files } + + invoke = nil + SyntaxTree::CLI.stub(:run, ->(args) { invoke = Invoke.new(args) }) do + ::Rake::Task["stree_check"].invoke + end + + assert_equal(["check", source_files], invoke.args) + end + end +end diff --git a/test/task_test.rb b/test/task_test.rb deleted file mode 100644 index c1e00b8b..00000000 --- a/test/task_test.rb +++ /dev/null @@ -1,27 +0,0 @@ -# frozen_string_literal: true - -require_relative "test_helper" -require "syntax_tree/rake/task" - -module SyntaxTree - class TaskTest < Minitest::Test - Invoke = Struct.new(:args) - - def test_task - source_files = "{app,config,lib}/**/*.rb" - - SyntaxTree::Rake::Task.new do |t| - t.source_files = source_files - end - - invoke = [] - SyntaxTree::CLI.stub(:run, ->(args) { invoke << Invoke.new(args) }) do - ::Rake::Task["check_and_format"].invoke - end - - assert_equal( - [["check", source_files], ["format", source_files]], invoke.map(&:args) - ) - end - end -end diff --git a/test/write_task_test.rb b/test/write_task_test.rb new file mode 100644 index 00000000..deb5acfd --- /dev/null +++ b/test/write_task_test.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +require_relative "test_helper" +require "syntax_tree/rake/write_task" + +module SyntaxTree + class WriteTaskTest < Minitest::Test + Invoke = Struct.new(:args) + + def test_task + source_files = "{app,config,lib}/**/*.rb" + + SyntaxTree::Rake::WriteTask.new { |t| t.source_files = source_files } + + invoke = nil + SyntaxTree::CLI.stub(:run, ->(args) { invoke = Invoke.new(args) }) do + ::Rake::Task["stree_write"].invoke + end + + assert_equal(["write", source_files], invoke.args) + end + end +end From 9601839439fb26c6d48b066430807f84cc3324b7 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 13 May 2022 21:22:23 -0400 Subject: [PATCH 009/536] Use prettier_print --- Gemfile | 2 - Gemfile.lock | 2 + lib/syntax_tree.rb | 26 +- lib/syntax_tree/formatter.rb | 2 +- lib/syntax_tree/node.rb | 100 +- lib/syntax_tree/prettyprint.rb | 1159 ---------------------- lib/syntax_tree/visitor/match_visitor.rb | 4 +- syntax_tree.gemspec | 3 + test/test_helper.rb | 2 - 9 files changed, 18 insertions(+), 1282 deletions(-) delete mode 100644 lib/syntax_tree/prettyprint.rb diff --git a/Gemfile b/Gemfile index 73418542..be173b20 100644 --- a/Gemfile +++ b/Gemfile @@ -3,5 +3,3 @@ source "https://rubygems.org" gemspec - -gem "rubocop" diff --git a/Gemfile.lock b/Gemfile.lock index fe27163b..b41fb897 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -2,6 +2,7 @@ PATH remote: . specs: syntax_tree (2.4.1) + prettier_print GEM remote: https://rubygems.org/ @@ -12,6 +13,7 @@ GEM parallel (1.22.1) parser (3.1.2.0) ast (~> 2.4.1) + prettier_print (0.1.0) rainbow (3.1.1) rake (13.0.6) regexp_parser (2.4.0) diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index c5e2d913..4d4c320e 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -2,7 +2,7 @@ require "json" require "pp" -require "prettyprint" +require "prettier_print" require "ripper" require "stringio" @@ -16,30 +16,6 @@ require_relative "syntax_tree/visitor/match_visitor" require_relative "syntax_tree/visitor/pretty_print_visitor" -# If PrettyPrint::Align isn't defined, then we haven't gotten the updated -# version of prettyprint. In that case we'll define our own. This is going to -# overwrite a bunch of methods, so silencing them as well. -unless PrettyPrint.const_defined?(:Align) - verbose = $VERBOSE - $VERBOSE = nil - - begin - require_relative "syntax_tree/prettyprint" - ensure - $VERBOSE = verbose - end -end - -# When PP is running, it expects that everything that interacts with it is going -# to flow through PP.pp, since that's the main entry into the module from the -# perspective of its uses in core Ruby. In doing so, it calls guard_inspect_key -# at the top of the PP.pp method, which establishes some thread-local hashes to -# check for cycles in the pretty printed tree. This means that if you want to -# manually call pp on some object _before_ you have established these hashes, -# you're going to break everything. So this call ensures that those hashes have -# been set up before anything uses pp manually. -PP.new(+"", 0).guard_inspect_key {} - # Syntax Tree is a suite of tools built on top of the internal CRuby parser. It # provides the ability to generate a syntax tree from source, as well as the # tools necessary to inspect and manipulate that syntax tree. It can be used to diff --git a/lib/syntax_tree/formatter.rb b/lib/syntax_tree/formatter.rb index 9959421a..88974be4 100644 --- a/lib/syntax_tree/formatter.rb +++ b/lib/syntax_tree/formatter.rb @@ -3,7 +3,7 @@ module SyntaxTree # A slightly enhanced PP that knows how to format recursively including # comments. - class Formatter < PP + class Formatter < PrettierPrint COMMENT_PRIORITY = 1 HEREDOC_PRIORITY = 2 diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index e8df1ea7..ef9aa8b5 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -123,7 +123,7 @@ def to_json(*opts) end def construct_keys - PP.format(+"") { |q| Visitor::MatchVisitor.new(q).visit(self) } + PrettierPrint.format(+"") { |q| Visitor::MatchVisitor.new(q).visit(self) } end end @@ -1666,52 +1666,6 @@ def format(q) end end - # This module will remove any breakables from the list of contents so that no - # newlines are present in the output. - module RemoveBreaks - class << self - def call(doc) - marker = Object.new - stack = [doc] - - while stack.any? - doc = stack.pop - - if doc == marker - stack.pop - next - end - - stack += [doc, marker] - - case doc - when PrettyPrint::Align, PrettyPrint::Indent, PrettyPrint::Group - doc.contents.map! { |child| remove_breaks(child) } - stack += doc.contents.reverse - when PrettyPrint::IfBreak - doc.flat_contents.map! { |child| remove_breaks(child) } - stack += doc.flat_contents.reverse - end - end - end - - private - - def remove_breaks(doc) - case doc - when PrettyPrint::Breakable - text = PrettyPrint::Text.new - text.add(object: doc.force? ? "; " : doc.separator, width: doc.width) - text - when PrettyPrint::IfBreak - PrettyPrint::Align.new(indent: 0, contents: doc.flat_contents) - else - doc - end - end - end - end - # BlockVar represents the parameters being declared for a block. Effectively # this node is everything contained within the pipes. This includes all of the # various parameter types, as well as block-local variable declarations. @@ -1752,8 +1706,7 @@ def deconstruct_keys(_keys) def format(q) q.group(0, "|", "|") do - doc = q.format(params) - RemoveBreaks.call(doc) + q.remove_breaks(q.format(params)) if locals.any? q.text("; ") @@ -3096,31 +3049,6 @@ def format(q) private - # This is a somewhat naive method that is attempting to sum up the width of - # the doc nodes that make up the given doc node. This is used to align - # content. - def doc_width(parent) - queue = [parent] - width = 0 - - until queue.empty? - doc = queue.shift - - case doc - when PrettyPrint::Text - width += doc.width - when PrettyPrint::Indent, PrettyPrint::Align, PrettyPrint::Group - queue = doc.contents + queue - when PrettyPrint::IfBreak - queue = doc.break_contents + queue - when PrettyPrint::Breakable - width = 0 - end - end - - width - end - def argument_alignment(q, doc) # Very special handling case for rspec matchers. In general with rspec # matchers you expect to see something like: @@ -3138,7 +3066,7 @@ def argument_alignment(q, doc) if %w[to not_to to_not].include?(message.value) 0 else - width = doc_width(doc) + 1 + width = q.last_position(doc) + 1 width > (q.maxwidth / 2) ? 0 : width end end @@ -4891,17 +4819,9 @@ def deconstruct_keys(_keys) end def format(q) - # This is a very specific behavior that should probably be included in the - # prettyprint module. It's when you want to force a newline, but don't - # want to force the break parent. - breakable = -> do - q.target << PrettyPrint::Breakable.new( - " ", - 1, - indent: false, - force: true - ) - end + # This is a very specific behavior where you want to force a newline, but + # don't want to force the break parent. + breakable = -> { q.breakable(indent: false, force: :skip_break_parent) } q.group do q.format(beginning) @@ -5325,9 +5245,8 @@ def format_ternary(q) # force it into the output but we _don't_ want to explicitly # break the parent. If a break-parent shows up in the tree, then # it's going to force it all the way up to the tree, which is - # going to negate the ternary. Maybe this should be an option in - # prettyprint? As in force: :no_break_parent or something. - q.target << PrettyPrint::Breakable.new(" ", 1, force: true) + # going to negate the ternary. + q.breakable(force: :skip_break_parent) q.format(node.consequent.statements) end end @@ -8314,8 +8233,7 @@ def format(q) # same line in the source, then we're going to leave them in place and # assume that's the way the developer wanted this expression # represented. - doc = q.group(0, '#{', "}") { q.format(statements) } - RemoveBreaks.call(doc) + q.remove_breaks(q.group(0, '#{', "}") { q.format(statements) }) else q.group do q.text('#{') diff --git a/lib/syntax_tree/prettyprint.rb b/lib/syntax_tree/prettyprint.rb deleted file mode 100644 index 7fe64a56..00000000 --- a/lib/syntax_tree/prettyprint.rb +++ /dev/null @@ -1,1159 +0,0 @@ -# frozen_string_literal: true -# -# This class implements a pretty printing algorithm. It finds line breaks and -# nice indentations for grouped structure. -# -# By default, the class assumes that primitive elements are strings and each -# byte in the strings is a single column in width. But it can be used for other -# situations by giving suitable arguments for some methods: -# -# * newline object and space generation block for PrettyPrint.new -# * optional width argument for PrettyPrint#text -# * PrettyPrint#breakable -# -# There are several candidate uses: -# * text formatting using proportional fonts -# * multibyte characters which has columns different to number of bytes -# * non-string formatting -# -# == Usage -# -# To use this module, you will need to generate a tree of print nodes that -# represent indentation and newline behavior before it gets sent to the printer. -# Each node has different semantics, depending on the desired output. -# -# The most basic node is a Text node. This represents plain text content that -# cannot be broken up even if it doesn't fit on one line. You would create one -# of those with the text method, as in: -# -# PrettyPrint.format { |q| q.text('my content') } -# -# No matter what the desired output width is, the output for the snippet above -# will always be the same. -# -# If you want to allow the printer to break up the content on the space -# character when there isn't enough width for the full string on the same line, -# you can use the Breakable and Group nodes. For example: -# -# PrettyPrint.format do |q| -# q.group do -# q.text('my') -# q.breakable -# q.text('content') -# end -# end -# -# Now, if everything fits on one line (depending on the maximum width specified) -# then it will be the same output as the first example. If, however, there is -# not enough room on the line, then you will get two lines of output, one for -# the first string and one for the second. -# -# There are other nodes for the print tree as well, described in the -# documentation below. They control alignment, indentation, conditional -# formatting, and more. -# -# == Bugs -# * Box based formatting? -# -# Report any bugs at http://bugs.ruby-lang.org -# -# == References -# Christian Lindig, Strictly Pretty, March 2000, -# https://lindig.github.io/papers/strictly-pretty-2000.pdf -# -# Philip Wadler, A prettier printer, March 1998, -# https://homepages.inf.ed.ac.uk/wadler/papers/prettier/prettier.pdf -# -# == Author -# Tanaka Akira -# -class PrettyPrint - # A node in the print tree that represents aligning nested nodes to a certain - # prefix width or string. - class Align - attr_reader :indent, :contents - - def initialize(indent:, contents: []) - @indent = indent - @contents = contents - end - - def pretty_print(q) - q.group(2, "align#{indent}([", "])") do - q.seplist(contents) { |content| q.pp(content) } - end - end - end - - # A node in the print tree that represents a place in the buffer that the - # content can be broken onto multiple lines. - class Breakable - attr_reader :separator, :width - - def initialize( - separator = " ", - width = separator.length, - force: false, - indent: true - ) - @separator = separator - @width = width - @force = force - @indent = indent - end - - def force? - @force - end - - def indent? - @indent - end - - def pretty_print(q) - q.text("breakable") - - attributes = [ - ("force=true" if force?), - ("indent=false" unless indent?) - ].compact - - if attributes.any? - q.text("(") - q.seplist(attributes, -> { q.text(", ") }) do |attribute| - q.text(attribute) - end - q.text(")") - end - end - end - - # A node in the print tree that forces the surrounding group to print out in - # the "break" mode as opposed to the "flat" mode. Useful for when you need to - # force a newline into a group. - class BreakParent - def pretty_print(q) - q.text("break-parent") - end - end - - # A node in the print tree that represents a group of items which the printer - # should try to fit onto one line. This is the basic command to tell the - # printer when to break. Groups are usually nested, and the printer will try - # to fit everything on one line, but if it doesn't fit it will break the - # outermost group first and try again. It will continue breaking groups until - # everything fits (or there are no more groups to break). - class Group - attr_reader :depth, :contents - - def initialize(depth, contents: []) - @depth = depth - @contents = contents - @break = false - end - - def break - @break = true - end - - def break? - @break - end - - def pretty_print(q) - q.group(2, break? ? "breakGroup([" : "group([", "])") do - q.seplist(contents) { |content| q.pp(content) } - end - end - end - - # A node in the print tree that represents printing one thing if the - # surrounding group node is broken and another thing if the surrounding group - # node is flat. - class IfBreak - attr_reader :break_contents, :flat_contents - - def initialize(break_contents: [], flat_contents: []) - @break_contents = break_contents - @flat_contents = flat_contents - end - - def pretty_print(q) - q.group(2, "if-break(", ")") do - q.breakable("") - q.group(2, "[", "],") do - q.seplist(break_contents) { |content| q.pp(content) } - end - q.breakable - q.group(2, "[", "]") do - q.seplist(flat_contents) { |content| q.pp(content) } - end - end - end - end - - # A node in the print tree that is a variant of the Align node that indents - # its contents by one level. - class Indent - attr_reader :contents - - def initialize(contents: []) - @contents = contents - end - - def pretty_print(q) - q.group(2, "indent([", "])") do - q.seplist(contents) { |content| q.pp(content) } - end - end - end - - # A node in the print tree that has its own special buffer for implementing - # content that should flush before any newline. - # - # Useful for implementating trailing content, as it's not always practical to - # constantly check where the line ends to avoid accidentally printing some - # content after a line suffix node. - class LineSuffix - DEFAULT_PRIORITY = 1 - - attr_reader :priority, :contents - - def initialize(priority: DEFAULT_PRIORITY, contents: []) - @priority = priority - @contents = contents - end - - def pretty_print(q) - q.group(2, "line-suffix([", "])") do - q.seplist(contents) { |content| q.pp(content) } - end - end - end - - # A node in the print tree that represents plain content that cannot be broken - # up (by default this assumes strings, but it can really be anything). - class Text - attr_reader :objects, :width - - def initialize - @objects = [] - @width = 0 - end - - def add(object: "", width: object.length) - @objects << object - @width += width - end - - def pretty_print(q) - q.group(2, "text([", "])") do - q.seplist(objects) { |object| q.pp(object) } - end - end - end - - # A node in the print tree that represents trimming all of the indentation of - # the current line, in the rare case that you need to ignore the indentation - # that you've already created. This node should be placed after a Breakable. - class Trim - def pretty_print(q) - q.text("trim") - end - end - - # When building up the contents in the output buffer, it's convenient to be - # able to trim trailing whitespace before newlines. If the output object is a - # string or array or strings, then we can do this with some gsub calls. If - # not, then this effectively just wraps the output object and forwards on - # calls to <<. - module Buffer - # This is the default output buffer that provides a base implementation of - # trim! that does nothing. It's effectively a wrapper around whatever output - # object was given to the format command. - class DefaultBuffer - attr_reader :output - - def initialize(output = []) - @output = output - end - - def <<(object) - @output << object - end - - def trim! - 0 - end - end - - # This is an output buffer that wraps a string output object. It provides a - # trim! method that trims off trailing whitespace from the string using - # gsub!. - class StringBuffer < DefaultBuffer - def initialize(output = "".dup) - super(output) - end - - def trim! - length = output.length - output.gsub!(/[\t ]*\z/, "") - length - output.length - end - end - - # This is an output buffer that wraps an array output object. It provides a - # trim! method that trims off trailing whitespace from the last element in - # the array if it's an unfrozen string using the same method as the - # StringBuffer. - class ArrayBuffer < DefaultBuffer - def initialize(output = []) - super(output) - end - - def trim! - return 0 if output.empty? - - trimmed = 0 - - while output.any? && output.last.is_a?(String) && - output.last.match?(/\A[\t ]*\z/) - trimmed += output.pop.length - end - - if output.any? && output.last.is_a?(String) && !output.last.frozen? - length = output.last.length - output.last.gsub!(/[\t ]*\z/, "") - trimmed += length - output.last.length - end - - trimmed - end - end - - # This is a switch for building the correct output buffer wrapper class for - # the given output object. - def self.for(output) - case output - when String - StringBuffer.new(output) - when Array - ArrayBuffer.new(output) - else - DefaultBuffer.new(output) - end - end - end - - # PrettyPrint::SingleLine is used by PrettyPrint.singleline_format - # - # It is passed to be similar to a PrettyPrint object itself, by responding to - # all of the same print tree node builder methods, as well as the #flush - # method. - # - # The significant difference here is that there are no line breaks in the - # output. If an IfBreak node is used, only the flat contents are printed. - # LineSuffix nodes are printed at the end of the buffer when #flush is called. - class SingleLine - # The output object. It stores rendered text and should respond to <<. - attr_reader :output - - # The current array of contents that the print tree builder methods should - # append to. - attr_reader :target - - # A buffer output that wraps any calls to line_suffix that will be flushed - # at the end of printing. - attr_reader :line_suffixes - - # Create a PrettyPrint::SingleLine object - # - # Arguments: - # * +output+ - String (or similar) to store rendered text. Needs to respond - # to '<<'. - # * +maxwidth+ - Argument position expected to be here for compatibility. - # This argument is a noop. - # * +newline+ - Argument position expected to be here for compatibility. - # This argument is a noop. - def initialize(output, _maxwidth = nil, _newline = nil) - @output = Buffer.for(output) - @target = @output - @line_suffixes = Buffer::ArrayBuffer.new - end - - # Flushes the line suffixes onto the output buffer. - def flush - line_suffixes.output.each { |doc| output << doc } - end - - # -------------------------------------------------------------------------- - # Markers node builders - # -------------------------------------------------------------------------- - - # Appends +separator+ to the text to be output. By default +separator+ is - # ' ' - # - # The +width+, +indent+, and +force+ arguments are here for compatibility. - # They are all noop arguments. - def breakable( - separator = " ", - _width = separator.length, - indent: nil, - force: nil - ) - target << separator - end - - # Here for compatibility, does nothing. - def break_parent - end - - # Appends +separator+ to the output buffer. +width+ is a noop here for - # compatibility. - def fill_breakable(separator = " ", _width = separator.length) - target << separator - end - - # Immediately trims the output buffer. - def trim - target.trim! - end - - # -------------------------------------------------------------------------- - # Container node builders - # -------------------------------------------------------------------------- - - # Opens a block for grouping objects to be pretty printed. - # - # Arguments: - # * +indent+ - noop argument. Present for compatibility. - # * +open_obj+ - text appended before the &block. Default is '' - # * +close_obj+ - text appended after the &block. Default is '' - # * +open_width+ - noop argument. Present for compatibility. - # * +close_width+ - noop argument. Present for compatibility. - def group( - _indent = nil, - open_object = "", - close_object = "", - _open_width = nil, - _close_width = nil - ) - target << open_object - yield - target << close_object - end - - # A class that wraps the ability to call #if_flat. The contents of the - # #if_flat block are executed immediately, so effectively this class and the - # #if_break method that triggers it are unnecessary, but they're here to - # maintain compatibility. - class IfBreakBuilder - def if_flat - yield - end - end - - # Effectively unnecessary, but here for compatibility. - def if_break - IfBreakBuilder.new - end - - # Also effectively unnecessary, but here for compatibility. - def if_flat - end - - # A noop that immediately yields. - def indent - yield - end - - # Changes the target output buffer to the line suffix output buffer which - # will get flushed at the end of printing. - def line_suffix - previous_target, @target = @target, line_suffixes - yield - @target = previous_target - end - - # Takes +indent+ arg, but does nothing with it. - # - # Yields to a block. - def nest(_indent) - yield - end - - # Add +object+ to the text to be output. - # - # +width+ argument is here for compatibility. It is a noop argument. - def text(object = "", _width = nil) - target << object - end - end - - # This object represents the current level of indentation within the printer. - # It has the ability to generate new levels of indentation through the #align - # and #indent methods. - class IndentLevel - IndentPart = Object.new - DedentPart = Object.new - - StringAlignPart = Struct.new(:n) - NumberAlignPart = Struct.new(:n) - - attr_reader :genspace, :value, :length, :queue, :root - - def initialize( - genspace:, - value: genspace.call(0), - length: 0, - queue: [], - root: nil - ) - @genspace = genspace - @value = value - @length = length - @queue = queue - @root = root - end - - # This can accept a whole lot of different kinds of objects, due to the - # nature of the flexibility of the Align node. - def align(n) - case n - when NilClass - self - when String - indent(StringAlignPart.new(n)) - else - indent(n < 0 ? DedentPart : NumberAlignPart.new(n)) - end - end - - def indent(part = IndentPart) - next_value = genspace.call(0) - next_length = 0 - next_queue = (part == DedentPart ? queue[0...-1] : [*queue, part]) - - last_spaces = 0 - - add_spaces = ->(count) do - next_value << genspace.call(count) - next_length += count - end - - flush_spaces = -> do - add_spaces[last_spaces] if last_spaces > 0 - last_spaces = 0 - end - - next_queue.each do |next_part| - case next_part - when IndentPart - flush_spaces.call - add_spaces.call(2) - when StringAlignPart - flush_spaces.call - next_value += next_part.n - next_length += next_part.n.length - when NumberAlignPart - last_spaces += next_part.n - end - end - - flush_spaces.call - - IndentLevel.new( - genspace: genspace, - value: next_value, - length: next_length, - queue: next_queue, - root: root - ) - end - end - - # When printing, you can optionally specify the value that should be used - # whenever a group needs to be broken onto multiple lines. In this case the - # default is \n. - DEFAULT_NEWLINE = "\n" - - # When generating spaces after a newline for indentation, by default we - # generate one space per character needed for indentation. You can change this - # behavior (for instance to use tabs) by passing a different genspace - # procedure. - DEFAULT_GENSPACE = ->(n) { " " * n } - - # There are two modes in printing, break and flat. When we're in break mode, - # any lines will use their newline, any if-breaks will use their break - # contents, etc. - MODE_BREAK = 1 - - # This is another print mode much like MODE_BREAK. When we're in flat mode, we - # attempt to print everything on one line until we either hit a broken group, - # a forced line, or the maximum width. - MODE_FLAT = 2 - - # This is a convenience method which is same as follows: - # - # begin - # q = PrettyPrint.new(output, maxwidth, newline, &genspace) - # ... - # q.flush - # output - # end - # - def self.format( - output = "".dup, - maxwidth = 80, - newline = DEFAULT_NEWLINE, - genspace = DEFAULT_GENSPACE - ) - q = new(output, maxwidth, newline, &genspace) - yield q - q.flush - output - end - - # This is similar to PrettyPrint::format but the result has no breaks. - # - # +maxwidth+, +newline+ and +genspace+ are ignored. - # - # The invocation of +breakable+ in the block doesn't break a line and is - # treated as just an invocation of +text+. - # - def self.singleline_format( - output = "".dup, - _maxwidth = nil, - _newline = nil, - _genspace = nil - ) - q = SingleLine.new(output) - yield q - output - end - - # The output object. It represents the final destination of the contents of - # the print tree. It should respond to <<. - # - # This defaults to "".dup - attr_reader :output - - # This is an output buffer that wraps the output object and provides - # additional functionality depending on its type. - # - # This defaults to Buffer::StringBuffer.new("".dup) - attr_reader :buffer - - # The maximum width of a line, before it is separated in to a newline - # - # This defaults to 80, and should be an Integer - attr_reader :maxwidth - - # The value that is appended to +output+ to add a new line. - # - # This defaults to "\n", and should be String - attr_reader :newline - - # An object that responds to call that takes one argument, of an Integer, and - # returns the corresponding number of spaces. - # - # By default this is: ->(n) { ' ' * n } - attr_reader :genspace - - # The stack of groups that are being printed. - attr_reader :groups - - # The current array of contents that calls to methods that generate print tree - # nodes will append to. - attr_reader :target - - # Creates a buffer for pretty printing. - # - # +output+ is an output target. If it is not specified, '' is assumed. It - # should have a << method which accepts the first argument +obj+ of - # PrettyPrint#text, the first argument +separator+ of PrettyPrint#breakable, - # the first argument +newline+ of PrettyPrint.new, and the result of a given - # block for PrettyPrint.new. - # - # +maxwidth+ specifies maximum line length. If it is not specified, 80 is - # assumed. However actual outputs may overflow +maxwidth+ if long - # non-breakable texts are provided. - # - # +newline+ is used for line breaks. "\n" is used if it is not specified. - # - # The block is used to generate spaces. ->(n) { ' ' * n } is used if it is not - # given. - def initialize( - output = "".dup, - maxwidth = 80, - newline = DEFAULT_NEWLINE, - &genspace - ) - @output = output - @buffer = Buffer.for(output) - @maxwidth = maxwidth - @newline = newline - @genspace = genspace || DEFAULT_GENSPACE - reset - end - - # Returns the group most recently added to the stack. - # - # Contrived example: - # out = "" - # => "" - # q = PrettyPrint.new(out) - # => # - # q.group { - # q.text q.current_group.inspect - # q.text q.newline - # q.group(q.current_group.depth + 1) { - # q.text q.current_group.inspect - # q.text q.newline - # q.group(q.current_group.depth + 1) { - # q.text q.current_group.inspect - # q.text q.newline - # q.group(q.current_group.depth + 1) { - # q.text q.current_group.inspect - # q.text q.newline - # } - # } - # } - # } - # => 284 - # puts out - # # - # # - # # - # # - def current_group - groups.last - end - - # Flushes all of the generated print tree onto the output buffer, then clears - # the generated tree from memory. - def flush - # First, get the root group, since we placed one at the top to begin with. - doc = groups.first - - # This represents how far along the current line we are. It gets reset - # back to 0 when we encounter a newline. - position = 0 - - # This is our command stack. A command consists of a triplet of an - # indentation level, the mode (break or flat), and a doc node. - commands = [[IndentLevel.new(genspace: genspace), MODE_BREAK, doc]] - - # This is a small optimization boolean. It keeps track of whether or not - # when we hit a group node we should check if it fits on the same line. - should_remeasure = false - - # This is a separate command stack that includes the same kind of triplets - # as the commands variable. It is used to keep track of things that should - # go at the end of printed lines once the other doc nodes are accounted for. - # Typically this is used to implement comments. - line_suffixes = [] - - # This is a special sort used to order the line suffixes by both the - # priority set on the line suffix and the index it was in the original - # array. - line_suffix_sort = ->(line_suffix) do - [-line_suffix.last, -line_suffixes.index(line_suffix)] - end - - # This is a linear stack instead of a mutually recursive call defined on - # the individual doc nodes for efficiency. - while (indent, mode, doc = commands.pop) - case doc - when Text - doc.objects.each { |object| buffer << object } - position += doc.width - when Array - doc.reverse_each { |part| commands << [indent, mode, part] } - when Indent - commands << [indent.indent, mode, doc.contents] - when Align - commands << [indent.align(doc.indent), mode, doc.contents] - when Trim - position -= buffer.trim! - when Group - if mode == MODE_FLAT && !should_remeasure - commands << [ - indent, - doc.break? ? MODE_BREAK : MODE_FLAT, - doc.contents - ] - else - should_remeasure = false - next_cmd = [indent, MODE_FLAT, doc.contents] - commands << if !doc.break? && - fits?(next_cmd, commands, maxwidth - position) - next_cmd - else - [indent, MODE_BREAK, doc.contents] - end - end - when IfBreak - if mode == MODE_BREAK && doc.break_contents.any? - commands << [indent, mode, doc.break_contents] - elsif mode == MODE_FLAT && doc.flat_contents.any? - commands << [indent, mode, doc.flat_contents] - end - when LineSuffix - line_suffixes << [indent, mode, doc.contents, doc.priority] - when Breakable - if mode == MODE_FLAT - if doc.force? - # This line was forced into the output even if we were in flat mode, - # so we need to tell the next group that no matter what, it needs to - # remeasure because the previous measurement didn't accurately - # capture the entire expression (this is necessary for nested - # groups). - should_remeasure = true - else - buffer << doc.separator - position += doc.width - next - end - end - - # If there are any commands in the line suffix buffer, then we're going - # to flush them now, as we are about to add a newline. - if line_suffixes.any? - commands << [indent, mode, doc] - commands += line_suffixes.sort_by(&line_suffix_sort) - line_suffixes = [] - next - end - - if !doc.indent? - buffer << newline - - if indent.root - buffer << indent.root.value - position = indent.root.length - else - position = 0 - end - else - position -= buffer.trim! - buffer << newline - buffer << indent.value - position = indent.length - end - when BreakParent - # do nothing - else - # Special case where the user has defined some way to get an extra doc - # node that we don't explicitly support into the list. In this case - # we're going to assume it's 0-width and just append it to the output - # buffer. - # - # This is useful behavior for putting marker nodes into the list so that - # you can know how things are getting mapped before they get printed. - buffer << doc - end - - if commands.empty? && line_suffixes.any? - commands += line_suffixes.sort_by(&line_suffix_sort) - line_suffixes = [] - end - end - - # Reset the group stack and target array so that this pretty printer object - # can continue to be used before calling flush again if desired. - reset - end - - # ---------------------------------------------------------------------------- - # Markers node builders - # ---------------------------------------------------------------------------- - - # This says "you can break a line here if necessary", and a +width+\-column - # text +separator+ is inserted if a line is not broken at the point. - # - # If +separator+ is not specified, ' ' is used. - # - # If +width+ is not specified, +separator.length+ is used. You will have to - # specify this when +separator+ is a multibyte character, for example. - # - # By default, if the surrounding group is broken and a newline is inserted, - # the printer will indent the subsequent line up to the current level of - # indentation. You can disable this behavior with the +indent+ argument if - # that's not desired (rare). - # - # By default, when you insert a Breakable into the print tree, it only breaks - # the surrounding group when the group's contents cannot fit onto the - # remaining space of the current line. You can force it to break the - # surrounding group instead if you always want the newline with the +force+ - # argument. - def breakable( - separator = " ", - width = separator.length, - indent: true, - force: false - ) - doc = Breakable.new(separator, width, indent: indent, force: force) - - target << doc - break_parent if force - - doc - end - - # This inserts a BreakParent node into the print tree which forces the - # surrounding and all parent group nodes to break. - def break_parent - doc = BreakParent.new - target << doc - - groups.reverse_each do |group| - break if group.break? - group.break - end - - doc - end - - # This is similar to #breakable except the decision to break or not is - # determined individually. - # - # Two #fill_breakable under a group may cause 4 results: - # (break,break), (break,non-break), (non-break,break), (non-break,non-break). - # This is different to #breakable because two #breakable under a group - # may cause 2 results: (break,break), (non-break,non-break). - # - # The text +separator+ is inserted if a line is not broken at this point. - # - # If +separator+ is not specified, ' ' is used. - # - # If +width+ is not specified, +separator.length+ is used. You will have to - # specify this when +separator+ is a multibyte character, for example. - def fill_breakable(separator = " ", width = separator.length) - group { breakable(separator, width) } - end - - # This inserts a Trim node into the print tree which, when printed, will clear - # all whitespace at the end of the output buffer. This is useful for the rare - # case where you need to delete printed indentation and force the next node - # to start at the beginning of the line. - def trim - doc = Trim.new - target << doc - - doc - end - - # ---------------------------------------------------------------------------- - # Container node builders - # ---------------------------------------------------------------------------- - - # Groups line break hints added in the block. The line break hints are all to - # be used or not. - # - # If +indent+ is specified, the method call is regarded as nested by - # nest(indent) { ... }. - # - # If +open_object+ is specified, text(open_object, open_width) is - # called before grouping. If +close_object+ is specified, - # text(close_object, close_width) is called after grouping. - def group( - indent = 0, - open_object = "", - close_object = "", - open_width = open_object.length, - close_width = close_object.length - ) - text(open_object, open_width) if open_object != "" - - doc = Group.new(groups.last.depth + 1) - groups << doc - target << doc - - with_target(doc.contents) do - if indent != 0 - nest(indent) { yield } - else - yield - end - end - - groups.pop - text(close_object, close_width) if close_object != "" - - doc - end - - # A small DSL-like object used for specifying the alternative contents to be - # printed if the surrounding group doesn't break for an IfBreak node. - class IfBreakBuilder - attr_reader :builder, :if_break - - def initialize(builder, if_break) - @builder = builder - @if_break = if_break - end - - def if_flat(&block) - builder.with_target(if_break.flat_contents, &block) - end - end - - # Inserts an IfBreak node with the contents of the block being added to its - # list of nodes that should be printed if the surrounding node breaks. If it - # doesn't, then you can specify the contents to be printed with the #if_flat - # method used on the return object from this method. For example, - # - # q.if_break { q.text('do') }.if_flat { q.text('{') } - # - # In the example above, if the surrounding group is broken it will print 'do' - # and if it is not it will print '{'. - def if_break - doc = IfBreak.new - target << doc - - with_target(doc.break_contents) { yield } - IfBreakBuilder.new(self, doc) - end - - # This is similar to if_break in that it also inserts an IfBreak node into the - # print tree, however it's starting from the flat contents, and cannot be used - # to build the break contents. - def if_flat - doc = IfBreak.new - target << doc - - with_target(doc.flat_contents) { yield } - end - - # Very similar to the #nest method, this indents the nested content by one - # level by inserting an Indent node into the print tree. The contents of the - # node are determined by the block. - def indent - doc = Indent.new - target << doc - - with_target(doc.contents) { yield } - doc - end - - # Inserts a LineSuffix node into the print tree. The contents of the node are - # determined by the block. - def line_suffix(priority: LineSuffix::DEFAULT_PRIORITY) - doc = LineSuffix.new(priority: priority) - target << doc - - with_target(doc.contents) { yield } - doc - end - - # Increases left margin after newline with +indent+ for line breaks added in - # the block. - def nest(indent) - doc = Align.new(indent: indent) - target << doc - - with_target(doc.contents) { yield } - doc - end - - # This adds +object+ as a text of +width+ columns in width. - # - # If +width+ is not specified, object.length is used. - def text(object = "", width = object.length) - doc = target.last - - unless doc.is_a?(Text) - doc = Text.new - target << doc - end - - doc.add(object: object, width: width) - doc - end - - # ---------------------------------------------------------------------------- - # Internal APIs - # ---------------------------------------------------------------------------- - - # A convenience method used by a lot of the print tree node builders that - # temporarily changes the target that the builders will append to. - def with_target(target) - previous_target, @target = @target, target - yield - @target = previous_target - end - - private - - # This method returns a boolean as to whether or not the remaining commands - # fit onto the remaining space on the current line. If we finish printing - # all of the commands or if we hit a newline, then we return true. Otherwise - # if we continue printing past the remaining space, we return false. - def fits?(next_command, rest_commands, remaining) - # This is the index in the remaining commands that we've handled so far. - # We reverse through the commands and add them to the stack if we've run - # out of nodes to handle. - rest_index = rest_commands.length - - # This is our stack of commands, very similar to the commands list in the - # print method. - commands = [next_command] - - # This is our output buffer, really only necessary to keep track of - # because we could encounter a Trim doc node that would actually add - # remaining space. - fit_buffer = buffer.class.new - - while remaining >= 0 - if commands.empty? - return true if rest_index == 0 - - rest_index -= 1 - commands << rest_commands[rest_index] - next - end - - indent, mode, doc = commands.pop - - case doc - when Text - doc.objects.each { |object| fit_buffer << object } - remaining -= doc.width - when Array - doc.reverse_each { |part| commands << [indent, mode, part] } - when Indent - commands << [indent.indent, mode, doc.contents] - when Align - commands << [indent.align(doc.indent), mode, doc.contents] - when Trim - remaining += fit_buffer.trim! - when Group - commands << [indent, doc.break? ? MODE_BREAK : mode, doc.contents] - when IfBreak - if mode == MODE_BREAK && doc.break_contents.any? - commands << [indent, mode, doc.break_contents] - elsif mode == MODE_FLAT && doc.flat_contents.any? - commands << [indent, mode, doc.flat_contents] - end - when Breakable - if mode == MODE_FLAT && !doc.force? - fit_buffer << doc.separator - remaining -= doc.width - next - end - - return true - end - end - - false - end - - # Resets the group stack and target array so that this pretty printer object - # can continue to be used before calling flush again if desired. - def reset - @groups = [Group.new(0)] - @target = @groups.last.contents - end -end diff --git a/lib/syntax_tree/visitor/match_visitor.rb b/lib/syntax_tree/visitor/match_visitor.rb index 205f2b90..e0bdaf08 100644 --- a/lib/syntax_tree/visitor/match_visitor.rb +++ b/lib/syntax_tree/visitor/match_visitor.rb @@ -22,7 +22,7 @@ def visit(node) # entire value into the output buffer. q.text(node.inspect) else - q.pp(node) + node.pretty_print(q) end end @@ -114,7 +114,7 @@ def text(name, value) q.nest(0) do q.text(name) q.text(": ") - q.pp(value) + value.pretty_print(q) end end end diff --git a/syntax_tree.gemspec b/syntax_tree.gemspec index 06a7ed78..820a61a0 100644 --- a/syntax_tree.gemspec +++ b/syntax_tree.gemspec @@ -25,8 +25,11 @@ Gem::Specification.new do |spec| spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } spec.require_paths = %w[lib] + spec.add_dependency "prettier_print" + spec.add_development_dependency "bundler" spec.add_development_dependency "minitest" spec.add_development_dependency "rake" + spec.add_development_dependency "rubocop" spec.add_development_dependency "simplecov" end diff --git a/test/test_helper.rb b/test/test_helper.rb index ce75aeb2..bb3ea67f 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -2,8 +2,6 @@ require "simplecov" SimpleCov.start do - add_filter("prettyprint.rb") - unless ENV["CI"] add_filter("accept_methods_test.rb") add_filter("idempotency_test.rb") From 0ff50302a7cbcca4fd0967088515da460b851103 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 13 May 2022 21:31:52 -0400 Subject: [PATCH 010/536] Support maxwidth on format --- CHANGELOG.md | 4 ++++ lib/syntax_tree.rb | 4 ++-- test/syntax_tree_test.rb | 4 ++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07da8abb..81c1d0fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +### Added + +- Support an optional `maxwidth` second argument to `SyntaxTree.format`. + ### Changed - Correct the pattern for checking if a dynamic symbol can be converted into a label as a hash key. diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index 4d4c320e..faefd4df 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -40,8 +40,8 @@ def self.parse(source) end # Parses the given source and returns the formatted source. - def self.format(source) - formatter = Formatter.new(source, []) + def self.format(source, maxwidth = 80) + formatter = Formatter.new(source, [], maxwidth) parse(source).format(formatter) formatter.flush diff --git a/test/syntax_tree_test.rb b/test/syntax_tree_test.rb index 3d5ae90e..05242d94 100644 --- a/test/syntax_tree_test.rb +++ b/test/syntax_tree_test.rb @@ -29,6 +29,10 @@ def test_parse_error assert_raises(Parser::ParseError) { SyntaxTree.parse("<>") } end + def test_maxwidth_format + assert_equal("foo +\n bar\n", SyntaxTree.format("foo + bar", 5)) + end + def test_read source = SyntaxTree.read(__FILE__) assert_equal(Encoding.default_external, source.encoding) From aa219efdb41eb9f2508bd8beb0bb4b6182b6c1da Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 13 May 2022 21:42:07 -0400 Subject: [PATCH 011/536] Disallow conditionals with `not` without parentheses in the predicate from turning into a ternary. --- CHANGELOG.md | 1 + lib/syntax_tree/node.rb | 4 +++- test/fixtures/if.rb | 8 ++++++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 81c1d0fc..ed3100f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ### Changed - Correct the pattern for checking if a dynamic symbol can be converted into a label as a hash key. +- Disallow conditionals with `not` without parentheses in the predicate from turning into a ternary. ## [2.4.1] - 2022-05-10 diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 2f0d9419..7667378d 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -5115,7 +5115,9 @@ def call(q, node) else # Otherwise, we're going to check the conditional for certain cases. case node - in predicate: Assign | Command | CommandCall | MAssign | Not | OpAssign + in predicate: Assign | Command | CommandCall | MAssign | OpAssign + false + in predicate: Not[parentheses: false] false in { statements: { body: [truthy] }, diff --git a/test/fixtures/if.rb b/test/fixtures/if.rb index 607af05d..9045e5bf 100644 --- a/test/fixtures/if.rb +++ b/test/fixtures/if.rb @@ -41,3 +41,11 @@ else c end +% +if not(a) + b +else + c +end +- +not(a) ? b : c From 785511212d2407b9750c09ca8b5c43dc0c7d51aa Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 13 May 2022 21:51:40 -0400 Subject: [PATCH 012/536] Bump to 2.5.0 --- CHANGELOG.md | 11 +++++++---- Gemfile.lock | 2 +- lib/syntax_tree/version.rb | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed3100f2..b6c8781d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,14 +6,16 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +## [2.5.0] - 2022-05-13 + ### Added -- Support an optional `maxwidth` second argument to `SyntaxTree.format`. +- [#79](https://github.com/ruby-syntax-tree/syntax_tree/pull/79) - Support an optional `maxwidth` second argument to `SyntaxTree.format`. ### Changed -- Correct the pattern for checking if a dynamic symbol can be converted into a label as a hash key. -- Disallow conditionals with `not` without parentheses in the predicate from turning into a ternary. +- [#77](https://github.com/ruby-syntax-tree/syntax_tree/pull/77) - Correct the pattern for checking if a dynamic symbol can be converted into a label as a hash key. +- [#72](https://github.com/ruby-syntax-tree/syntax_tree/pull/72) - Disallow conditionals with `not` without parentheses in the predicate from turning into a ternary. ## [2.4.1] - 2022-05-10 @@ -222,7 +224,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - 🎉 Initial release! 🎉 -[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.4.1...HEAD +[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.5.0...HEAD +[2.5.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.4.1...v2.5.0 [2.4.1]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.4.0...v2.4.1 [2.4.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.3.1...v2.4.0 [2.3.1]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.3.0...v2.3.1 diff --git a/Gemfile.lock b/Gemfile.lock index b41fb897..220985eb 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - syntax_tree (2.4.1) + syntax_tree (2.5.0) prettier_print GEM diff --git a/lib/syntax_tree/version.rb b/lib/syntax_tree/version.rb index fbecb604..d12b4964 100644 --- a/lib/syntax_tree/version.rb +++ b/lib/syntax_tree/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SyntaxTree - VERSION = "2.4.1" + VERSION = "2.5.0" end From dd1a0fe60955ae9e021c5863dfe5b8c9418f0e2c Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 13 May 2022 21:55:42 -0400 Subject: [PATCH 013/536] Document maxwidth on format --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8955a310..b1523080 100644 --- a/README.md +++ b/README.md @@ -223,7 +223,7 @@ This function takes an input string containing Ruby code and returns the syntax ### SyntaxTree.format(source) -This function takes an input string containing Ruby code, parses it into its underlying syntax tree, and formats it back out to a string. +This function takes an input string containing Ruby code, parses it into its underlying syntax tree, and formats it back out to a string. You can optionally pass a second argument to this method as well that is the maximum width to print. It defaults to `80`. ## Nodes From 66f708bd7f196c97a5a6889ace263b731a4779cc Mon Sep 17 00:00:00 2001 From: Wender Freese Date: Mon, 16 May 2022 09:44:04 -0300 Subject: [PATCH 014/536] Remove explicit call for exit() --- lib/syntax_tree/rake/check_task.rb | 2 -- lib/syntax_tree/rake/write_task.rb | 2 -- 2 files changed, 4 deletions(-) diff --git a/lib/syntax_tree/rake/check_task.rb b/lib/syntax_tree/rake/check_task.rb index 0c0dc860..5fc4ce56 100644 --- a/lib/syntax_tree/rake/check_task.rb +++ b/lib/syntax_tree/rake/check_task.rb @@ -45,8 +45,6 @@ def define_task def run_task SyntaxTree::CLI.run(["check", source_files].compact) - - # exit($?.exitstatus) if $?&.exited? end end end diff --git a/lib/syntax_tree/rake/write_task.rb b/lib/syntax_tree/rake/write_task.rb index 08b6018c..6143d6b9 100644 --- a/lib/syntax_tree/rake/write_task.rb +++ b/lib/syntax_tree/rake/write_task.rb @@ -45,8 +45,6 @@ def define_task def run_task SyntaxTree::CLI.run(["write", source_files].compact) - - # exit($?.exitstatus) if $?&.exited? end end end From cff7953adf0fad6125073f440459acf0e671c55b Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 16 May 2022 10:12:17 -0400 Subject: [PATCH 015/536] Document rake task --- .github/workflows/main.yml | 2 +- CHANGELOG.md | 4 +++ README.md | 41 ++++++++++++++++++++++++++++++ Rakefile | 25 ++++-------------- lib/syntax_tree/rake/check_task.rb | 23 ++++++++++++++--- lib/syntax_tree/rake/write_task.rb | 23 ++++++++++++++--- lib/syntax_tree/rake_tasks.rb | 4 +++ test/check_task_test.rb | 23 ----------------- test/rake_test.rb | 36 ++++++++++++++++++++++++++ test/write_task_test.rb | 23 ----------------- 10 files changed, 129 insertions(+), 75 deletions(-) create mode 100644 lib/syntax_tree/rake_tasks.rb delete mode 100644 test/check_task_test.rb create mode 100644 test/rake_test.rb delete mode 100644 test/write_task_test.rb diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7f5ac15c..ed3c51fd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -38,7 +38,7 @@ jobs: ruby-version: '3.1' - name: Check run: | - bundle exec rake check + bundle exec rake stree:check bundle exec rubocop automerge: diff --git a/CHANGELOG.md b/CHANGELOG.md index b6c8781d..479863f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +### Added + +- [#74](https://github.com/ruby-syntax-tree/syntax_tree/pull/74) - Add Rake test to run check and format commands. + ## [2.5.0] - 2022-05-13 ### Added diff --git a/README.md b/README.md index b1523080..2657852d 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,7 @@ It is built with only standard library dependencies. It additionally ships with - [syntaxTree/visualizing](#syntaxtreevisualizing) - [Plugins](#plugins) - [Integration](#integration) + - [Rake](#rake) - [RuboCop](#rubocop) - [VSCode](#vscode) - [Contributing](#contributing) @@ -436,6 +437,46 @@ Below are listed all of the "official" language plugins hosted under the same Gi Syntax Tree's goal is to seemlessly integrate into your workflow. To this end, it provides a couple of additional tools beyond the CLI and the Ruby library. +### Rake + +Syntax Tree ships with the ability to define [rake](https://github.com/ruby/rake) tasks that will trigger runs of the CLI. To define them in your application, add the following configuration to your `Rakefile`: + +```ruby +require "syntax_tree/rake_tasks" +SyntaxTree::Rake::CheckTask.new +SyntaxTree::Rake::WriteTask.new +``` + +These calls will define `rake stree:check` and `rake stree:write` (equivalent to calling `stree check` and `stree write` respectively). You can configure them by either passing arguments to the `new` method or by using a block. + +#### `name` + +If you'd like to change the default name of the rake task, you can pass that as the first optioon, as in: + +```ruby +SyntaxTree::Rake::WriteTask.new(:format) +``` + +#### `source_files` + +If you wanted to configure Syntax Tree to check or write different files than the default (`lib/**/*.rb`), you can set the `source_files` field, as in: + +```ruby +SyntaxTree::Rake::WriteTask.new do |t| + t.source_files = FileList[%w[Gemfile Rakefile lib/**/*.rb test/**/*.rb]] +end +``` + +#### `plugins` + +If you're running Syntax Tree with plugins (either your own or the pre-built ones), you can pass that to the `plugins` field, as in: + +```ruby +SyntaxTree::Rake::WriteTask.new do |t| + t.plugins = ["plugin/single_quotes"] +end +``` + ### RuboCop RuboCop and Syntax Tree serve different purposes, but there is overlap with some of RuboCop's functionality. Syntax Tree provides a RuboCop configuration file to disable rules that are redundant with Syntax Tree. To use this configuration file, add the following snippet to the top of your project's `.rubocop.yml`: diff --git a/Rakefile b/Rakefile index 4b3de39a..6ba17fe9 100644 --- a/Rakefile +++ b/Rakefile @@ -2,6 +2,7 @@ require "bundler/gem_tasks" require "rake/testtask" +require "syntax_tree/rake_tasks" Rake::TestTask.new(:test) do |t| t.libs << "test" @@ -11,24 +12,8 @@ end task default: :test -FILEPATHS = %w[ - Gemfile - Rakefile - syntax_tree.gemspec - lib/**/*.rb - test/*.rb -].freeze +SOURCE_FILES = + FileList[%w[Gemfile Rakefile syntax_tree.gemspec lib/**/*.rb test/*.rb]] -task :syntax_tree do - $:.unshift File.expand_path("lib", __dir__) - require "syntax_tree" - require "syntax_tree/cli" -end - -task check: :syntax_tree do - exit SyntaxTree::CLI.run(["check"] + FILEPATHS) -end - -task format: :syntax_tree do - exit SyntaxTree::CLI.run(["write"] + FILEPATHS) -end +SyntaxTree::Rake::CheckTask.new { |t| t.source_files = SOURCE_FILES } +SyntaxTree::Rake::WriteTask.new { |t| t.source_files = SOURCE_FILES } diff --git a/lib/syntax_tree/rake/check_task.rb b/lib/syntax_tree/rake/check_task.rb index 5fc4ce56..354cd172 100644 --- a/lib/syntax_tree/rake/check_task.rb +++ b/lib/syntax_tree/rake/check_task.rb @@ -3,6 +3,9 @@ require "rake" require "rake/tasklib" +require "syntax_tree" +require "syntax_tree/cli" + module SyntaxTree module Rake # A Rake task that runs check on a set of source files. @@ -21,16 +24,25 @@ module Rake # class CheckTask < ::Rake::TaskLib # Name of the task. - # Defaults to :stree_check. + # Defaults to :"stree:check". attr_accessor :name # Glob pattern to match source files. # Defaults to 'lib/**/*.rb'. attr_accessor :source_files - def initialize(name = :stree_check) + # The set of plugins to require. + # Defaults to []. + attr_accessor :plugins + + def initialize( + name = :"stree:check", + source_files = ::Rake::FileList["lib/**/*.rb"], + plugins = [] + ) @name = name - @source_files = "lib/**/*.rb" + @source_files = source_files + @plugins = plugins yield self if block_given? define_task @@ -44,7 +56,10 @@ def define_task end def run_task - SyntaxTree::CLI.run(["check", source_files].compact) + arguments = ["check"] + arguments << "--plugins=#{plugins.join(",")}" if plugins.any? + + SyntaxTree::CLI.run(arguments + Array(source_files)) end end end diff --git a/lib/syntax_tree/rake/write_task.rb b/lib/syntax_tree/rake/write_task.rb index 6143d6b9..5a957480 100644 --- a/lib/syntax_tree/rake/write_task.rb +++ b/lib/syntax_tree/rake/write_task.rb @@ -3,6 +3,9 @@ require "rake" require "rake/tasklib" +require "syntax_tree" +require "syntax_tree/cli" + module SyntaxTree module Rake # A Rake task that runs format on a set of source files. @@ -21,16 +24,25 @@ module Rake # class WriteTask < ::Rake::TaskLib # Name of the task. - # Defaults to :stree_write. + # Defaults to :"stree:write". attr_accessor :name # Glob pattern to match source files. # Defaults to 'lib/**/*.rb'. attr_accessor :source_files - def initialize(name = :stree_write) + # The set of plugins to require. + # Defaults to []. + attr_accessor :plugins + + def initialize( + name = :"stree:write", + source_files = ::Rake::FileList["lib/**/*.rb"], + plugins = [] + ) @name = name - @source_files = "lib/**/*.rb" + @source_files = source_files + @plugins = plugins yield self if block_given? define_task @@ -44,7 +56,10 @@ def define_task end def run_task - SyntaxTree::CLI.run(["write", source_files].compact) + arguments = ["write"] + arguments << "--plugins=#{plugins.join(",")}" if plugins.any? + + SyntaxTree::CLI.run(arguments + Array(source_files)) end end end diff --git a/lib/syntax_tree/rake_tasks.rb b/lib/syntax_tree/rake_tasks.rb new file mode 100644 index 00000000..b53743e5 --- /dev/null +++ b/lib/syntax_tree/rake_tasks.rb @@ -0,0 +1,4 @@ +# frozen_string_literal: true + +require_relative "rake/check_task" +require_relative "rake/write_task" diff --git a/test/check_task_test.rb b/test/check_task_test.rb deleted file mode 100644 index 33333241..00000000 --- a/test/check_task_test.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -require_relative "test_helper" -require "syntax_tree/rake/check_task" - -module SyntaxTree - class CheckTaskTest < Minitest::Test - Invoke = Struct.new(:args) - - def test_task - source_files = "{app,config,lib}/**/*.rb" - - SyntaxTree::Rake::CheckTask.new { |t| t.source_files = source_files } - - invoke = nil - SyntaxTree::CLI.stub(:run, ->(args) { invoke = Invoke.new(args) }) do - ::Rake::Task["stree_check"].invoke - end - - assert_equal(["check", source_files], invoke.args) - end - end -end diff --git a/test/rake_test.rb b/test/rake_test.rb new file mode 100644 index 00000000..57364859 --- /dev/null +++ b/test/rake_test.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +require_relative "test_helper" +require "syntax_tree/rake_tasks" + +module SyntaxTree + module Rake + class CheckTaskTest < Minitest::Test + Invoke = Struct.new(:args) + + def test_check_task + source_files = "{app,config,lib}/**/*.rb" + CheckTask.new { |t| t.source_files = source_files } + + invoke = nil + SyntaxTree::CLI.stub(:run, ->(args) { invoke = Invoke.new(args) }) do + ::Rake::Task["stree:check"].invoke + end + + assert_equal(["check", source_files], invoke.args) + end + + def test_write_task + source_files = "{app,config,lib}/**/*.rb" + WriteTask.new { |t| t.source_files = source_files } + + invoke = nil + SyntaxTree::CLI.stub(:run, ->(args) { invoke = Invoke.new(args) }) do + ::Rake::Task["stree:write"].invoke + end + + assert_equal(["write", source_files], invoke.args) + end + end + end +end diff --git a/test/write_task_test.rb b/test/write_task_test.rb deleted file mode 100644 index deb5acfd..00000000 --- a/test/write_task_test.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -require_relative "test_helper" -require "syntax_tree/rake/write_task" - -module SyntaxTree - class WriteTaskTest < Minitest::Test - Invoke = Struct.new(:args) - - def test_task - source_files = "{app,config,lib}/**/*.rb" - - SyntaxTree::Rake::WriteTask.new { |t| t.source_files = source_files } - - invoke = nil - SyntaxTree::CLI.stub(:run, ->(args) { invoke = Invoke.new(args) }) do - ::Rake::Task["stree_write"].invoke - end - - assert_equal(["write", source_files], invoke.args) - end - end -end From 744e5eb4ed17b8b40dd419ecc08e963a7b4366ba Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 16 May 2022 10:13:45 -0400 Subject: [PATCH 016/536] Fix up documentation typos --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2657852d..3fed780c 100644 --- a/README.md +++ b/README.md @@ -447,11 +447,11 @@ SyntaxTree::Rake::CheckTask.new SyntaxTree::Rake::WriteTask.new ``` -These calls will define `rake stree:check` and `rake stree:write` (equivalent to calling `stree check` and `stree write` respectively). You can configure them by either passing arguments to the `new` method or by using a block. +These calls will define `rake stree:check` and `rake stree:write` (equivalent to calling `stree check` and `stree write` with the CLI respectively). You can configure them by either passing arguments to the `new` method or by using a block. #### `name` -If you'd like to change the default name of the rake task, you can pass that as the first optioon, as in: +If you'd like to change the default name of the rake task, you can pass that as the first argument, as in: ```ruby SyntaxTree::Rake::WriteTask.new(:format) From 8b8cecc345ca4393e6d5d46955ad9869fc3a6ea2 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 16 May 2022 11:17:42 -0400 Subject: [PATCH 017/536] Support trailing commas --- README.md | 7 +- lib/syntax_tree/formatter.rb | 8 +- lib/syntax_tree/formatter/trailing_comma.rb | 13 +++ lib/syntax_tree/node.rb | 23 +++++ lib/syntax_tree/plugin/trailing_comma.rb | 4 + test/formatter/single_quotes_test.rb | 72 +++++++-------- test/formatter/trailing_comma_test.rb | 97 +++++++++++++++++++++ 7 files changed, 187 insertions(+), 37 deletions(-) create mode 100644 lib/syntax_tree/formatter/trailing_comma.rb create mode 100644 lib/syntax_tree/plugin/trailing_comma.rb create mode 100644 test/formatter/trailing_comma_test.rb diff --git a/README.md b/README.md index 3fed780c..81dfdd71 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,8 @@ It is built with only standard library dependencies. It additionally ships with - [textDocument/inlayHints](#textdocumentinlayhints) - [syntaxTree/visualizing](#syntaxtreevisualizing) - [Plugins](#plugins) + - [Configuration](#configuration) + - [Languages](#languages) - [Integration](#integration) - [Rake](#rake) - [RuboCop](#rubocop) @@ -409,9 +411,12 @@ You can register additional configuration and additional languages that can flow ### Configuration -To register additional configuration, define a file somewhere in your load path named `syntax_tree/my_plugin` directory. Then when invoking the CLI, you will pass `--plugins=my_plugin`. That will get required. In this way, you can modify Syntax Tree however you would like. Some plugins ship with Syntax Tree itself. They are: +To register additional configuration, define a file somewhere in your load path named `syntax_tree/my_plugin`. Then when invoking the CLI, you will pass `--plugins=my_plugin`. To require multiple, separate them by a comma. In this way, you can modify Syntax Tree however you would like. Some plugins ship with Syntax Tree itself. They are: * `plugin/single_quotes` - This will change all of your string literals to use single quotes instead of the default double quotes. +* `plugin/trailing_comma` - This will put trailing commas into multiline array literals, hash literals, and method calls that can support trailing commas. + +If you're using Syntax Tree as a library, you should require those files directly. ### Languages diff --git a/lib/syntax_tree/formatter.rb b/lib/syntax_tree/formatter.rb index 88974be4..5d362129 100644 --- a/lib/syntax_tree/formatter.rb +++ b/lib/syntax_tree/formatter.rb @@ -7,7 +7,12 @@ class Formatter < PrettierPrint COMMENT_PRIORITY = 1 HEREDOC_PRIORITY = 2 - attr_reader :source, :stack, :quote + attr_reader :source, :stack + + # These options are overridden in plugins to we need to make sure they are + # available here. + attr_reader :quote, :trailing_comma + alias trailing_comma? trailing_comma def initialize(source, ...) super(...) @@ -15,6 +20,7 @@ def initialize(source, ...) @source = source @stack = [] @quote = "\"" + @trailing_comma = false end def self.format(source, node) diff --git a/lib/syntax_tree/formatter/trailing_comma.rb b/lib/syntax_tree/formatter/trailing_comma.rb new file mode 100644 index 00000000..63fe2e9a --- /dev/null +++ b/lib/syntax_tree/formatter/trailing_comma.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module SyntaxTree + class Formatter + # This module overrides the trailing_comma? method on the formatter to + # return true. + module TrailingComma + def trailing_comma? + true + end + end + end +end diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 7667378d..a96b9794 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -597,10 +597,30 @@ def format(q) q.indent do q.breakable("") q.format(arguments) + q.if_break { q.text(",") } if q.trailing_comma? && trailing_comma? end q.breakable("") end end + + private + + def trailing_comma? + case arguments + in Args[parts: [*, ArgBlock]] + # If the last argument is a block, then we can't put a trailing comma + # after it without resulting in a syntax error. + false + in Args[parts: [Command | CommandCall]] + # If the only argument is a command or command call, then a trailing + # comma would be parsed as part of that expression instead of on this + # one, so we don't want to add a trailing comma. + false + else + # Otherwise, we should be okay to add a trailing comma. + true + end + end end # Args represents a list of arguments being passed to a method call or array @@ -859,6 +879,7 @@ def format(q) end q.seplist(contents.parts, separator) { |part| q.format(part) } + q.if_break { q.text(",") } if q.trailing_comma? end q.breakable("") end @@ -954,6 +975,7 @@ def format(q) q.indent do q.breakable("") q.format(contents) + q.if_break { q.text(",") } if q.trailing_comma? end end @@ -4751,6 +4773,7 @@ def format_contents(q) q.indent do q.breakable q.seplist(assocs) { |assoc| q.format(assoc) } + q.if_break { q.text(",") } if q.trailing_comma? end q.breakable end diff --git a/lib/syntax_tree/plugin/trailing_comma.rb b/lib/syntax_tree/plugin/trailing_comma.rb new file mode 100644 index 00000000..eaa8cb6a --- /dev/null +++ b/lib/syntax_tree/plugin/trailing_comma.rb @@ -0,0 +1,4 @@ +# frozen_string_literal: true + +require "syntax_tree/formatter/trailing_comma" +SyntaxTree::Formatter.prepend(SyntaxTree::Formatter::TrailingComma) diff --git a/test/formatter/single_quotes_test.rb b/test/formatter/single_quotes_test.rb index 8bf82cb8..ac5103a1 100644 --- a/test/formatter/single_quotes_test.rb +++ b/test/formatter/single_quotes_test.rb @@ -5,41 +5,43 @@ module SyntaxTree class Formatter - class TestFormatter < Formatter - prepend Formatter::SingleQuotes - end - - def test_empty_string_literal - assert_format("''\n", "\"\"") - end - - def test_string_literal - assert_format("'string'\n", "\"string\"") - end - - def test_string_literal_with_interpolation - assert_format("\"\#{foo}\"\n") - end - - def test_dyna_symbol - assert_format(":'symbol'\n", ":\"symbol\"") - end - - def test_label - assert_format( - "{ foo => foo, :'bar' => bar }\n", - "{ foo => foo, \"bar\": bar }" - ) - end - - private - - def assert_format(expected, source = expected) - formatter = TestFormatter.new(source, []) - SyntaxTree.parse(source).format(formatter) - - formatter.flush - assert_equal(expected, formatter.output.join) + class SingleQuotesTest < Minitest::Test + class TestFormatter < Formatter + prepend Formatter::SingleQuotes + end + + def test_empty_string_literal + assert_format("''\n", "\"\"") + end + + def test_string_literal + assert_format("'string'\n", "\"string\"") + end + + def test_string_literal_with_interpolation + assert_format("\"\#{foo}\"\n") + end + + def test_dyna_symbol + assert_format(":'symbol'\n", ":\"symbol\"") + end + + def test_label + assert_format( + "{ foo => foo, :'bar' => bar }\n", + "{ foo => foo, \"bar\": bar }" + ) + end + + private + + def assert_format(expected, source = expected) + formatter = TestFormatter.new(source, []) + SyntaxTree.parse(source).format(formatter) + + formatter.flush + assert_equal(expected, formatter.output.join) + end end end end diff --git a/test/formatter/trailing_comma_test.rb b/test/formatter/trailing_comma_test.rb new file mode 100644 index 00000000..f6585772 --- /dev/null +++ b/test/formatter/trailing_comma_test.rb @@ -0,0 +1,97 @@ +# frozen_string_literal: true + +require_relative "../test_helper" +require "syntax_tree/formatter/trailing_comma" + +module SyntaxTree + class Formatter + class TrailingCommaTest < Minitest::Test + class TestFormatter < Formatter + prepend Formatter::TrailingComma + end + + def test_arg_paren_flat + assert_format("foo(a)\n") + end + + def test_arg_paren_break + assert_format(<<~EXPECTED, <<~SOURCE) + foo( + #{"a" * 80}, + ) + EXPECTED + foo(#{"a" * 80}) + SOURCE + end + + def test_arg_paren_block + assert_format(<<~EXPECTED, <<~SOURCE) + foo( + &#{"a" * 80} + ) + EXPECTED + foo(&#{"a" * 80}) + SOURCE + end + + def test_arg_paren_command + assert_format(<<~EXPECTED, <<~SOURCE) + foo( + bar #{"a" * 80} + ) + EXPECTED + foo(bar #{"a" * 80}) + SOURCE + end + + def test_arg_paren_command_call + assert_format(<<~EXPECTED, <<~SOURCE) + foo( + bar.baz #{"a" * 80} + ) + EXPECTED + foo(bar.baz #{"a" * 80}) + SOURCE + end + + def test_array_literal_flat + assert_format("[a]\n") + end + + def test_array_literal_break + assert_format(<<~EXPECTED, <<~SOURCE) + [ + #{"a" * 80}, + ] + EXPECTED + [#{"a" * 80}] + SOURCE + end + + def test_hash_literal_flat + assert_format("{ a: a }\n") + end + + def test_hash_literal_break + assert_format(<<~EXPECTED, <<~SOURCE) + { + a: + #{"a" * 80}, + } + EXPECTED + { a: #{"a" * 80} } + SOURCE + end + + private + + def assert_format(expected, source = expected) + formatter = TestFormatter.new(source, []) + SyntaxTree.parse(source).format(formatter) + + formatter.flush + assert_equal(expected, formatter.output.join) + end + end + end +end From 5ee6fae73c30042b7be6a19f6763c2a138942349 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 16 May 2022 13:51:54 -0400 Subject: [PATCH 018/536] Handle lambda-local variables --- lib/syntax_tree/node.rb | 96 +++++++++++++++++-- lib/syntax_tree/parser.rb | 113 +++++++++++++++++++++++ lib/syntax_tree/visitor.rb | 3 + lib/syntax_tree/visitor/field_visitor.rb | 8 ++ test/fixtures/lambda.rb | 26 ++++++ 5 files changed, 236 insertions(+), 10 deletions(-) diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index a96b9794..5663fac3 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -5900,7 +5900,7 @@ def deconstruct_keys(_keys) # ->(value) { value * 2 } # class Lambda < Node - # [Params | Paren] the parameter declaration for this lambda + # [LambdaVar | Paren] the parameter declaration for this lambda attr_reader :params # [BodyStmt | Statements] the expressions to be executed in this lambda @@ -5955,24 +5955,100 @@ def format(q) node.is_a?(Command) || node.is_a?(CommandCall) end - q.text(force_parens ? "{" : "do") - q.indent do + if force_parens + q.text("{") + + unless statements.empty? + q.indent do + q.breakable + q.format(statements) + end + q.breakable + end + + q.text("}") + else + q.text("do") + + unless statements.empty? + q.indent do + q.breakable + q.format(statements) + end + end + q.breakable - q.format(statements) + q.text("end") end - - q.breakable - q.text(force_parens ? "}" : "end") end .if_flat do - q.text("{ ") - q.format(statements) - q.text(" }") + q.text("{") + + unless statements.empty? + q.text(" ") + q.format(statements) + q.text(" ") + end + + q.text("}") end end end end + # LambdaVar represents the parameters being declared for a lambda. Effectively + # this node is everything contained within the parentheses. This includes all + # of the various parameter types, as well as block-local variable + # declarations. + # + # -> (positional, optional = value, keyword:, █ local) do + # end + # + class LambdaVar < Node + # [Params] the parameters being declared with the block + attr_reader :params + + # [Array[ Ident ]] the list of block-local variable declarations + attr_reader :locals + + # [Array[ Comment | EmbDoc ]] the comments attached to this node + attr_reader :comments + + def initialize(params:, locals:, location:, comments: []) + @params = params + @locals = locals + @location = location + @comments = comments + end + + def accept(visitor) + visitor.visit_lambda_var(self) + end + + def child_nodes + [params, *locals] + end + + alias deconstruct child_nodes + + def deconstruct_keys(_keys) + { params: params, locals: locals, location: location, comments: comments } + end + + def empty? + params.empty? && locals.empty? + end + + def format(q) + q.format(params) + + if locals.any? + q.text("; ") + q.seplist(locals, -> { q.text(", ") }) { |local| q.format(local) } + end + end + end + # LBrace represents the use of a left brace, i.e., {. class LBrace < Node # [String] the left brace diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index f5ffe47d..2de295f3 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -1940,6 +1940,41 @@ def on_lambda(params, statements) token.location.start_char > beginning.location.start_char end + # We need to do some special mapping here. Since ripper doesn't support + # capturing lambda var until 3.2, we need to normalize all of that here. + params = + case params + in Paren[contents: Params] + # In this case we've gotten to the <3.2 parentheses wrapping a set of + # parameters case. Here we need to manually scan for lambda locals. + range = (params.location.start_char + 1)...params.location.end_char + locals = lambda_locals(source[range]) + + location = params.contents.location + location = location.to(locals.last.location) if locals.any? + + Paren.new( + lparen: params.lparen, + contents: + LambdaVar.new( + params: params.contents, + locals: locals, + location: location + ), + location: params.location, + comments: params.comments + ) + in Params + # In this case we've gotten to the <3.2 plain set of parameters. In + # this case there cannot be lambda locals, so we will wrap the + # parameters into a lambda var that has no locals. + LambdaVar.new(params: params, locals: [], location: params.location) + in LambdaVar + # In this case we've gotten to 3.2+ lambda var. In this case we don't + # need to do anything and can just the value as given. + params + end + if braces opening = find_token(TLamBeg) closing = find_token(RBrace) @@ -1962,6 +1997,84 @@ def on_lambda(params, statements) ) end + # :call-seq: + # on_lambda_var: (Params params, Array[ Ident ] locals) -> LambdaVar + def on_lambda_var(params, locals) + location = params.location + location = location.to(locals.last.location) if locals.any? + + LambdaVar.new(params: params, locals: locals || [], location: location) + end + + # Ripper doesn't support capturing lambda local variables until 3.2. To + # mitigate this, we have to parse that code for ourselves. We use the range + # from the parentheses to find where we _should_ be looking. Then we check + # if the resulting tokens match a pattern that we determine means that the + # declaration has block-local variables. Once it does, we parse those out + # and convert them into Ident nodes. + def lambda_locals(source) + tokens = Ripper.lex(source) + + # First, check that we have a semi-colon. If we do, then we can start to + # parse the tokens _after_ the semicolon. + index = tokens.rindex { |token| token[1] == :on_semicolon } + return [] unless index + + # Next, map over the tokens and convert them into Ident nodes. Bail out + # midway through if we encounter a token we didn't expect. Basically we're + # making our own mini-parser here. To do that we'll walk through a small + # state machine: + # + # ┌────────┐ ┌────────┐ ┌─────────┐ + # │ │ │ │ │┌───────┐│ + # ──> │ item │ ─── ident ──> │ next │ ─── rparen ──> ││ final ││ + # │ │ <── comma ─── │ │ │└───────┘│ + # └────────┘ └────────┘ └─────────┘ + # │ ^ │ ^ + # └──┘ └──┘ + # ignored_nl, sp nl, sp + # + state = :item + transitions = { + item: { + on_ignored_nl: :item, + on_sp: :item, + on_ident: :next + }, + next: { + on_nl: :next, + on_sp: :next, + on_comma: :item, + on_rparen: :final + }, + final: { + } + } + + tokens[(index + 1)..].each_with_object([]) do |token, locals| + (lineno, column), type, value, = token + + # Make the state transition for the parser. This is going to raise a + # KeyError if we don't have a transition for the current state and type. + # But that shouldn't actually be possible because ripper would have + # found a syntax error by then. + state = transitions[state].fetch(type) + + # If we hit an identifier, then add it to our list. + next if type != :on_ident + + location = + Location.token( + line: lineno, + char: line_counts[lineno - 1][column], + column: column, + size: value.size + ) + + locals << Ident.new(value: value, location: location) + end + end + # :call-seq: # on_lbrace: (String value) -> LBrace def on_lbrace(value) diff --git a/lib/syntax_tree/visitor.rb b/lib/syntax_tree/visitor.rb index 57794ddb..fa1173eb 100644 --- a/lib/syntax_tree/visitor.rb +++ b/lib/syntax_tree/visitor.rb @@ -301,6 +301,9 @@ def visit_child_nodes(node) # Visit a Lambda node. alias visit_lambda visit_child_nodes + # Visit a LambdaVar node. + alias visit_lambda_var visit_child_nodes + # Visit a LBrace node. alias visit_lbrace visit_child_nodes diff --git a/lib/syntax_tree/visitor/field_visitor.rb b/lib/syntax_tree/visitor/field_visitor.rb index 631084e8..4527e0d3 100644 --- a/lib/syntax_tree/visitor/field_visitor.rb +++ b/lib/syntax_tree/visitor/field_visitor.rb @@ -586,6 +586,14 @@ def visit_lambda(node) end end + def visit_lambda_var(node) + node(node, "lambda_var") do + field("params", node.params) + list("locals", node.locals) if node.locals.any? + comments(node) + end + end + def visit_lbrace(node) visit_token(node, "lbrace") end diff --git a/test/fixtures/lambda.rb b/test/fixtures/lambda.rb index 043ceb5a..50e406b1 100644 --- a/test/fixtures/lambda.rb +++ b/test/fixtures/lambda.rb @@ -1,4 +1,6 @@ % +-> {} +% -> { foo } % ->(foo, bar) { baz } @@ -40,3 +42,27 @@ -> { -> foo do bar end.baz }.qux - -> { ->(foo) { bar }.baz }.qux +% +->(;a) {} +- +->(; a) {} +% +->(; a) {} +% +->(; a,b) {} +- +->(; a, b) {} +% +->(; a, b) {} +% +->(; +a +) {} +- +->(; a) {} +% +->(; a , +b +) {} +- +->(; a, b) {} From a487b9a71f3091a0e53c0de35f9fd353ffa02eca Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 16 May 2022 16:30:24 -0400 Subject: [PATCH 019/536] Trailing operators on command calls --- lib/syntax_tree/node.rb | 16 ++++++++++++++-- test/fixtures/command_call.rb | 4 ++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 5663fac3..0a1fc394 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -3052,8 +3052,20 @@ def format(q) doc = q.nest(0) do q.format(receiver) - q.format(CallOperatorFormatter.new(operator), stackable: false) - q.format(message) + + # If there are leading comments on the message then we know we have + # a newline in the source that is forcing these things apart. In + # this case we will have to use a trailing operator. + if message.comments.any?(&:leading?) + q.format(CallOperatorFormatter.new(operator), stackable: false) + q.indent do + q.breakable("") + q.format(message) + end + else + q.format(CallOperatorFormatter.new(operator), stackable: false) + q.format(message) + end end case arguments diff --git a/test/fixtures/command_call.rb b/test/fixtures/command_call.rb index 5060ffa4..fb0d084a 100644 --- a/test/fixtures/command_call.rb +++ b/test/fixtures/command_call.rb @@ -28,3 +28,7 @@ % foo.bar baz do end +% +foo. + # comment + bar baz From 0c5728f718870208ee80a318ac8d3aced63a166c Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 16 May 2022 19:34:21 -0400 Subject: [PATCH 020/536] Update CHANGELOG --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 479863f3..65ca8eab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ### Added - [#74](https://github.com/ruby-syntax-tree/syntax_tree/pull/74) - Add Rake test to run check and format commands. +- [#83](https://github.com/ruby-syntax-tree/syntax_tree/pull/83) - Add a trailing commas plugin. +- [#84](https://github.com/ruby-syntax-tree/syntax_tree/pull/84) - Handle lambda block-local variables. + +### Changed + +- [#85](https://github.com/ruby-syntax-tree/syntax_tree/pull/85) - Better handle trailing operators on command calls. ## [2.5.0] - 2022-05-13 From 0fd7a5fcf45a4372aac23424c751c6b9e5984548 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 16 May 2022 19:38:42 -0400 Subject: [PATCH 021/536] Ensure lambda block-local variables are handled properly --- lib/syntax_tree/parser.rb | 9 ++++----- test/fixtures/lambda.rb | 10 ++++++++++ 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index 2de295f3..6bff0838 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -2054,11 +2054,10 @@ def lambda_locals(source) tokens[(index + 1)..].each_with_object([]) do |token, locals| (lineno, column), type, value, = token - # Make the state transition for the parser. This is going to raise a - # KeyError if we don't have a transition for the current state and type. - # But that shouldn't actually be possible because ripper would have - # found a syntax error by then. - state = transitions[state].fetch(type) + # Make the state transition for the parser. If there isn't a transition + # from the current state to a new state for this type, then we're in a + # pattern that isn't actually locals. In that case we can return []. + state = transitions[state].fetch(type) { return [] } # If we hit an identifier, then add it to our list. next if type != :on_ident diff --git a/test/fixtures/lambda.rb b/test/fixtures/lambda.rb index 50e406b1..d0cc6f9b 100644 --- a/test/fixtures/lambda.rb +++ b/test/fixtures/lambda.rb @@ -66,3 +66,13 @@ ) {} - ->(; a, b) {} +% +->(a = (b; c)) {} +- +->( + a = ( + b + c + ) +) do +end From 3ee5d0cb94c3ec471740ade57de42ff424172423 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 16 May 2022 19:39:33 -0400 Subject: [PATCH 022/536] Bump to v2.6.0 --- CHANGELOG.md | 5 ++++- Gemfile.lock | 2 +- lib/syntax_tree/version.rb | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 65ca8eab..3c57136c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +## [2.6.0] - 2022-05-16 + ### Added - [#74](https://github.com/ruby-syntax-tree/syntax_tree/pull/74) - Add Rake test to run check and format commands. @@ -234,7 +236,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - 🎉 Initial release! 🎉 -[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.5.0...HEAD +[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.6.0...HEAD +[2.6.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.5.0...v2.6.0 [2.5.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.4.1...v2.5.0 [2.4.1]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.4.0...v2.4.1 [2.4.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.3.1...v2.4.0 diff --git a/Gemfile.lock b/Gemfile.lock index 220985eb..b642d5dc 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - syntax_tree (2.5.0) + syntax_tree (2.6.0) prettier_print GEM diff --git a/lib/syntax_tree/version.rb b/lib/syntax_tree/version.rb index d12b4964..afa6cc12 100644 --- a/lib/syntax_tree/version.rb +++ b/lib/syntax_tree/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SyntaxTree - VERSION = "2.5.0" + VERSION = "2.6.0" end From 13d4faf77269978204e0a3d871d03e89e2a79d40 Mon Sep 17 00:00:00 2001 From: Boris Petrov Date: Tue, 17 May 2022 17:01:25 +0300 Subject: [PATCH 023/536] Fix converting to the other type of quotes when that would lead to escaping in the string --- lib/syntax_tree/node.rb | 10 +++++----- test/fixtures/string_literal.rb | 6 +++++- test/formatter/single_quotes_test.rb | 4 ++++ 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 0a1fc394..5e1a353a 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -3864,9 +3864,9 @@ module Quotes # whichever quote the user chose. (If they chose single quotes, then double # quoting would activate the escape sequence, and if they chose double # quotes, then single quotes would deactivate it.) - def self.locked?(node) + def self.locked?(node, quote) node.parts.any? do |part| - !part.is_a?(TStringContent) || part.value.match?(/\\|#[@${]/) + !part.is_a?(TStringContent) || part.value.match?(/\\|#[@${]|#{quote}/) end end @@ -3981,12 +3981,12 @@ def quotes(q) if matched [quote, matching] - elsif Quotes.locked?(self) + elsif Quotes.locked?(self, q.quote) ["#{":" unless hash_key}'", "'"] else ["#{":" unless hash_key}#{q.quote}", q.quote] end - elsif Quotes.locked?(self) + elsif Quotes.locked?(self, q.quote) if quote.start_with?(":") [hash_key ? quote[1..] : quote, quote[1..]] else @@ -8404,7 +8404,7 @@ def format(q) end opening_quote, closing_quote = - if !Quotes.locked?(self) + if !Quotes.locked?(self, q.quote) [q.quote, q.quote] elsif quote.start_with?("%") [quote, Quotes.matching(quote[/%[qQ]?(.)/, 1])] diff --git a/test/fixtures/string_literal.rb b/test/fixtures/string_literal.rb index ebe56a40..d8ee0cdb 100644 --- a/test/fixtures/string_literal.rb +++ b/test/fixtures/string_literal.rb @@ -41,4 +41,8 @@ % '"foo"' - -"\"foo\"" +'"foo"' +% +"'foo'" +- +"'foo'" diff --git a/test/formatter/single_quotes_test.rb b/test/formatter/single_quotes_test.rb index ac5103a1..78f9ae3d 100644 --- a/test/formatter/single_quotes_test.rb +++ b/test/formatter/single_quotes_test.rb @@ -26,6 +26,10 @@ def test_dyna_symbol assert_format(":'symbol'\n", ":\"symbol\"") end + def test_single_quote_in_string + assert_format("\"str'ing\"\n") + end + def test_label assert_format( "{ foo => foo, :'bar' => bar }\n", From 933f685c8751db8eff83e786b211a447663a5536 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 17 May 2022 15:45:48 -0400 Subject: [PATCH 024/536] Provide a BasicVisitor --- README.md | 15 +++++ lib/syntax_tree.rb | 2 + lib/syntax_tree/basic_visitor.rb | 74 ++++++++++++++++++++++++ lib/syntax_tree/visitor.rb | 67 +-------------------- lib/syntax_tree/visitor/field_visitor.rb | 4 +- 5 files changed, 93 insertions(+), 69 deletions(-) create mode 100644 lib/syntax_tree/basic_visitor.rb diff --git a/README.md b/README.md index 81dfdd71..e3e995cf 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ It is built with only standard library dependencies. It additionally ships with - [construct_keys](#construct_keys) - [Visitor](#visitor) - [visit_method](#visit_method) + - [BasicVisitor](#basicvisitor) - [Language server](#language-server) - [textDocument/formatting](#textdocumentformatting) - [textDocument/inlayHints](#textdocumentinlayhints) @@ -373,6 +374,20 @@ Did you mean? visit_binary from bin/console:8:in `
' ``` +### BasicVisitor + +When you're defining your own visitor, by default it will walk down the tree even if you don't define `visit_*` methods. This is to ensure you can define a subset of the necessary methods in order to only interact with the nodes you're interested in. If you'd like to change this default to instead raise an error if you visit a node you haven't explicitly handled, you can instead inherit from `BasicVisitor`. + +```ruby +class MyVisitor < SyntaxTree::BasicVisitor + def visit_int(node) + # ... + end +end +``` + +The visitor defined above will error out unless it's only visiting a `SyntaxTree::Int` node. This is useful in a couple of ways, e.g., if you're trying to define a visitor to handle the whole tree but it's currently a work-in-progress. + ## Language server Syntax Tree additionally ships with a language server conforming to the [language server protocol](https://microsoft.github.io/language-server-protocol/). It can be invoked through the CLI by running: diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index faefd4df..60979d04 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -10,6 +10,8 @@ require_relative "syntax_tree/node" require_relative "syntax_tree/parser" require_relative "syntax_tree/version" + +require_relative "syntax_tree/basic_visitor" require_relative "syntax_tree/visitor" require_relative "syntax_tree/visitor/field_visitor" require_relative "syntax_tree/visitor/json_visitor" diff --git a/lib/syntax_tree/basic_visitor.rb b/lib/syntax_tree/basic_visitor.rb new file mode 100644 index 00000000..1ad6a80f --- /dev/null +++ b/lib/syntax_tree/basic_visitor.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +module SyntaxTree + # BasicVisitor is the parent class of the Visitor class that provides the + # ability to walk down the tree. It does not define any handlers, so you + # should extend this class if you want your visitor to raise an error if you + # attempt to visit a node that you don't handle. + class BasicVisitor + # This is raised when you use the Visitor.visit_method method and it fails. + # It is correctable to through DidYouMean. + class VisitMethodError < StandardError + attr_reader :visit_method + + def initialize(visit_method) + @visit_method = visit_method + super("Invalid visit method: #{visit_method}") + end + end + + # This class is used by DidYouMean to offer corrections to invalid visit + # method names. + class VisitMethodChecker + attr_reader :visit_method + + def initialize(error) + @visit_method = error.visit_method + end + + def corrections + @corrections ||= + DidYouMean::SpellChecker.new( + dictionary: Visitor.visit_methods + ).correct(visit_method) + end + + DidYouMean.correct_error(VisitMethodError, self) + end + + class << self + # This method is here to help folks write visitors. + # + # It's not always easy to ensure you're writing the correct method name in + # the visitor since it's perfectly valid to define methods that don't + # override these parent methods. + # + # If you use this method, you can ensure you're writing the correct method + # name. It will raise an error if the visit method you're defining isn't + # actually a method on the parent visitor. + def visit_method(method_name) + return if visit_methods.include?(method_name) + + raise VisitMethodError, method_name + end + + # This is the list of all of the valid visit methods. + def visit_methods + @visit_methods ||= + Visitor.instance_methods.grep(/^visit_(?!child_nodes)/) + end + end + + def visit(node) + node&.accept(self) + end + + def visit_all(nodes) + nodes.map { |node| visit(node) } + end + + def visit_child_nodes(node) + visit_all(node.child_nodes) + end + end +end diff --git a/lib/syntax_tree/visitor.rb b/lib/syntax_tree/visitor.rb index fa1173eb..348a05a2 100644 --- a/lib/syntax_tree/visitor.rb +++ b/lib/syntax_tree/visitor.rb @@ -4,72 +4,7 @@ module SyntaxTree # Visitor is a parent class that provides the ability to walk down the tree # and handle a subset of nodes. By defining your own subclass, you can # explicitly handle a node type by defining a visit_* method. - class Visitor - # This is raised when you use the Visitor.visit_method method and it fails. - # It is correctable to through DidYouMean. - class VisitMethodError < StandardError - attr_reader :visit_method - - def initialize(visit_method) - @visit_method = visit_method - super("Invalid visit method: #{visit_method}") - end - end - - # This class is used by DidYouMean to offer corrections to invalid visit - # method names. - class VisitMethodChecker - attr_reader :visit_method - - def initialize(error) - @visit_method = error.visit_method - end - - def corrections - @corrections ||= - DidYouMean::SpellChecker.new( - dictionary: Visitor.visit_methods - ).correct(visit_method) - end - - DidYouMean.correct_error(VisitMethodError, self) - end - - class << self - # This method is here to help folks write visitors. - # - # It's not always easy to ensure you're writing the correct method name in - # the visitor since it's perfectly valid to define methods that don't - # override these parent methods. - # - # If you use this method, you can ensure you're writing the correct method - # name. It will raise an error if the visit method you're defining isn't - # actually a method on the parent visitor. - def visit_method(method_name) - return if visit_methods.include?(method_name) - - raise VisitMethodError, method_name - end - - # This is the list of all of the valid visit methods. - def visit_methods - @visit_methods ||= - Visitor.instance_methods.grep(/^visit_(?!child_nodes)/) - end - end - - def visit(node) - node&.accept(self) - end - - def visit_all(nodes) - nodes.map { |node| visit(node) } - end - - def visit_child_nodes(node) - visit_all(node.child_nodes) - end - + class Visitor < BasicVisitor # Visit an ARef node. alias visit_aref visit_child_nodes diff --git a/lib/syntax_tree/visitor/field_visitor.rb b/lib/syntax_tree/visitor/field_visitor.rb index 4527e0d3..1cc74f3d 100644 --- a/lib/syntax_tree/visitor/field_visitor.rb +++ b/lib/syntax_tree/visitor/field_visitor.rb @@ -49,9 +49,7 @@ class Visitor # of circumstances, like when visiting the list of optional parameters # defined on a method. # - class FieldVisitor < Visitor - attr_reader :q - + class FieldVisitor < BasicVisitor def visit_aref(node) node(node, "aref") do field("collection", node.collection) From b8ec43eb46d0c28b740f46dbc8924a1452c81a79 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 19 May 2022 17:30:18 -0400 Subject: [PATCH 025/536] Better formatting for array patterns in rassign --- lib/syntax_tree/node.rb | 53 +++++++++++++++++++++++++++++----------- test/fixtures/rassign.rb | 8 ++++++ 2 files changed, 47 insertions(+), 14 deletions(-) diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 0a1fc394..6c2617cc 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -1131,7 +1131,11 @@ def format(q) q.group do q.format(constant) q.text("[") - q.seplist(parts) { |part| q.format(part) } + q.indent do + q.breakable("") + q.seplist(parts) { |part| q.format(part) } + end + q.breakable("") q.text("]") end @@ -1141,7 +1145,11 @@ def format(q) parent = q.parent if parts.length == 1 || PATTERNS.include?(parent.class) q.text("[") - q.seplist(parts) { |part| q.format(part) } + q.indent do + q.breakable("") + q.seplist(parts) { |part| q.format(part) } + end + q.breakable("") q.text("]") elsif parts.empty? q.text("[]") @@ -2777,10 +2785,17 @@ def format(q) q.format(value) q.text(" ") q.format(operator) - q.group do - q.indent do - q.breakable - q.format(pattern) + + case pattern + in AryPtn | FndPtn | HshPtn + q.text(" ") + q.format(pattern) + else + q.group do + q.indent do + q.breakable + q.format(pattern) + end end end end @@ -4573,16 +4588,26 @@ def deconstruct_keys(_keys) def format(q) q.format(constant) if constant - q.group(0, "[", "]") do - q.text("*") - q.format(left) - q.comma_breakable - q.seplist(values) { |value| q.format(value) } - q.comma_breakable + q.group do + q.text("[") - q.text("*") - q.format(right) + q.indent do + q.breakable("") + + q.text("*") + q.format(left) + q.comma_breakable + + q.seplist(values) { |value| q.format(value) } + q.comma_breakable + + q.text("*") + q.format(right) + end + + q.breakable("") + q.text("]") end end end diff --git a/test/fixtures/rassign.rb b/test/fixtures/rassign.rb index 882ce890..ce749550 100644 --- a/test/fixtures/rassign.rb +++ b/test/fixtures/rassign.rb @@ -12,3 +12,11 @@ - foooooooooooooooooooooooooooooooooooooo => barrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr +% +foo => [ + ConstantConstantConstant, + ConstantConstantConstant, + ConstantConstantConstant, + ConstantConstantConstant, + ConstantConstantConstant +] From 07bd320f7ebea0ca3d06b23852e9277e20de7d20 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 19 May 2022 20:21:56 -0400 Subject: [PATCH 026/536] Bump to v2.7.0 --- CHANGELOG.md | 13 ++++++++++++- Gemfile.lock | 2 +- lib/syntax_tree/version.rb | 2 +- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c57136c..1d80cfc3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,16 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +## [2.7.0] - 2022-05-19 + +### Added + +- [#88](https://github.com/ruby-syntax-tree/syntax_tree/pull/88) - Provide a `SyntaxTree::BasicVisitor` that has no visit methods implemented. + +### Changed + +- [#90](https://github.com/ruby-syntax-tree/syntax_tree/pull/90) - Provide better formatting for `SyntaxTree::AryPtn` when its nested inside a `SyntaxTree::RAssign`. + ## [2.6.0] - 2022-05-16 ### Added @@ -236,7 +246,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - 🎉 Initial release! 🎉 -[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.6.0...HEAD +[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.7.0...HEAD +[2.7.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.6.0...v2.7.0 [2.6.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.5.0...v2.6.0 [2.5.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.4.1...v2.5.0 [2.4.1]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.4.0...v2.4.1 diff --git a/Gemfile.lock b/Gemfile.lock index b642d5dc..3f892652 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - syntax_tree (2.6.0) + syntax_tree (2.7.0) prettier_print GEM diff --git a/lib/syntax_tree/version.rb b/lib/syntax_tree/version.rb index afa6cc12..851d9565 100644 --- a/lib/syntax_tree/version.rb +++ b/lib/syntax_tree/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SyntaxTree - VERSION = "2.6.0" + VERSION = "2.7.0" end From eb264abb152e559cf021ee93d5e3ab9bc60caa0a Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 24 May 2022 12:06:55 -0400 Subject: [PATCH 027/536] Always use [] with aryptn --- lib/syntax_tree/node.rb | 32 ++++++---------------------- test/fixtures/aryptn.rb | 47 ++++++++++++++++++++++++++++++++++++++--- test/fixtures/hshptn.rb | 5 +++++ test/fixtures/in.rb | 6 ++++-- 4 files changed, 60 insertions(+), 30 deletions(-) diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 6c2617cc..1e8afa4c 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -1123,38 +1123,20 @@ def deconstruct_keys(_keys) end def format(q) - parts = [*requireds] - parts << RestFormatter.new(rest) if rest - parts += posts - - if constant - q.group do - q.format(constant) - q.text("[") - q.indent do - q.breakable("") - q.seplist(parts) { |part| q.format(part) } - end - q.breakable("") - q.text("]") - end - - return - end - - parent = q.parent - if parts.length == 1 || PATTERNS.include?(parent.class) + q.group do + q.format(constant) if constant q.text("[") q.indent do q.breakable("") + + parts = [*requireds] + parts << RestFormatter.new(rest) if rest + parts += posts + q.seplist(parts) { |part| q.format(part) } end q.breakable("") q.text("]") - elsif parts.empty? - q.text("[]") - else - q.group { q.seplist(parts) { |part| q.format(part) } } end end end diff --git a/test/fixtures/aryptn.rb b/test/fixtures/aryptn.rb index c5562305..eddd8e3f 100644 --- a/test/fixtures/aryptn.rb +++ b/test/fixtures/aryptn.rb @@ -6,51 +6,92 @@ case foo in _, _ end +- +case foo +in [_, _] +end % case foo in bar, baz end +- +case foo +in [bar, baz] +end % case foo in [bar] end % case foo -in [bar, baz] +in [bar] +in [baz] end -- +% case foo -in bar, baz +in [bar, baz] end % case foo in bar, *baz end +- +case foo +in [bar, *baz] +end % case foo in *bar, baz end +- +case foo +in [*bar, baz] +end % case foo in bar, *, baz end +- +case foo +in [bar, *, baz] +end % case foo in *, bar, baz end +- +case foo +in [*, bar, baz] +end % case foo in Constant[bar] end % case foo +in Constant(bar) +end +- +case foo +in Constant[bar] +end +% +case foo in Constant[bar, baz] end % case foo in bar, [baz, _] => qux end +- +case foo +in [bar, [baz, _] => qux] +end % case foo in bar, baz if bar == baz end +- +case foo +in [bar, baz] if bar == baz +end diff --git a/test/fixtures/hshptn.rb b/test/fixtures/hshptn.rb index 7a35b4d0..f8733170 100644 --- a/test/fixtures/hshptn.rb +++ b/test/fixtures/hshptn.rb @@ -71,3 +71,8 @@ in bar, { baz:, **nil } in qux: end +- +case foo +in [bar, { baz:, **nil }] +in qux: +end diff --git a/test/fixtures/in.rb b/test/fixtures/in.rb index 1e1b2282..59102505 100644 --- a/test/fixtures/in.rb +++ b/test/fixtures/in.rb @@ -14,8 +14,10 @@ end - case foo -in fooooooooooooooooooooooooooooooooooooo, - barrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr +in [ + fooooooooooooooooooooooooooooooooooooo, + barrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr + ] baz end % From f1e1c10f18a24e492e55437edcc044f0aed380ed Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 24 May 2022 14:01:19 -0400 Subject: [PATCH 028/536] Tests for the language server --- lib/syntax_tree/language_server.rb | 4 +- .../language_server/inlay_hints.rb | 9 + lib/syntax_tree/node.rb | 32 ++- test/language_server_test.rb | 201 ++++++++++++++++++ 4 files changed, 233 insertions(+), 13 deletions(-) create mode 100644 test/language_server_test.rb diff --git a/lib/syntax_tree/language_server.rb b/lib/syntax_tree/language_server.rb index 1e305cca..b73fe0f7 100644 --- a/lib/syntax_tree/language_server.rb +++ b/lib/syntax_tree/language_server.rb @@ -70,9 +70,7 @@ def run id:, params: { textDocument: { uri: } } } - output = [] - PP.pp(SyntaxTree.parse(store[uri]), output) - write(id: id, result: output.join) + write(id: id, result: PP.pp(SyntaxTree.parse(store[uri]), +"")) in method: %r{\$/.+} # ignored else diff --git a/lib/syntax_tree/language_server/inlay_hints.rb b/lib/syntax_tree/language_server/inlay_hints.rb index 69fc5ce4..aba8c4c6 100644 --- a/lib/syntax_tree/language_server/inlay_hints.rb +++ b/lib/syntax_tree/language_server/inlay_hints.rb @@ -38,6 +38,7 @@ def visit(node) # def visit_assign(node) parentheses(node.location) if stack[-2].is_a?(Params) + super end # Adds parentheses around binary expressions to make it clear which @@ -57,6 +58,8 @@ def visit_binary(node) parentheses(node.location) else end + + super end # Adds parentheses around ternary operators contained within certain @@ -73,6 +76,8 @@ def visit_if_op(node) if stack[-2] in Assign | Binary | IfOp | OpAssign parentheses(node.location) end + + super end # Adds the implicitly rescued StandardError into a bare rescue clause. For @@ -92,6 +97,8 @@ def visit_rescue(node) if node.exception.nil? after[node.location.start_char + "rescue".length] << " StandardError" end + + super end # Adds parentheses around unary statements using the - operator that are @@ -107,6 +114,8 @@ def visit_unary(node) if stack[-2].is_a?(Binary) && (node.operator == "-") parentheses(node.location) end + + super end def self.find(program) diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 92947735..4f12b144 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -2129,11 +2129,13 @@ def format(q) # # break # - in [Paren[ - contents: { - body: [ArrayLiteral[contents: { parts: [_, _, *] }] => array] - } - ]] + in [ + Paren[ + contents: { + body: [ArrayLiteral[contents: { parts: [_, _, *] }] => array] + } + ] + ] # Here we have a single argument that is a set of parentheses wrapping # an array literal that has at least 2 elements. We're going to print # the contents of the array directly. This would be like if we had: @@ -2146,7 +2148,9 @@ def format(q) # q.text(" ") format_array_contents(q, array) - in [Paren[contents: { body: [ArrayLiteral => statement] }]] + in [ + Paren[contents: { body: [ArrayLiteral => statement] }] + ] # Here we have a single argument that is a set of parentheses wrapping # an array literal that has 0 or 1 elements. We're going to skip the # parentheses but print the array itself. This would be like if we @@ -2174,7 +2178,9 @@ def format(q) # q.text(" ") q.format(statement) - in [Paren => part] + in [ + Paren => part + ] # Here we have a single argument that is a set of parentheses. We're # going to print the parentheses themselves as if they were the set of # arguments. This would be like if we had: @@ -2182,7 +2188,9 @@ def format(q) # break(foo.bar) # q.format(part) - in [ArrayLiteral[contents: { parts: [_, _, *] }] => array] + in [ + ArrayLiteral[contents: { parts: [_, _, *] }] => array + ] # Here there is a single argument that is an array literal with at # least two elements. We skip directly into the array literal's # elements in order to print the contents. This would be like if we @@ -2196,7 +2204,9 @@ def format(q) # q.text(" ") format_array_contents(q, array) - in [ArrayLiteral => part] + in [ + ArrayLiteral => part + ] # Here there is a single argument that is an array literal with 0 or 1 # elements. In this case we're going to print the array as it is # because skipping the brackets would change the remaining. This would @@ -2207,7 +2217,9 @@ def format(q) # q.text(" ") q.format(part) - in [_] + in [ + _ + ] # Here there is a single argument that hasn't matched one of our # previous cases. We're going to print the argument as it is. This # would be like if we had: diff --git a/test/language_server_test.rb b/test/language_server_test.rb new file mode 100644 index 00000000..ba33a05b --- /dev/null +++ b/test/language_server_test.rb @@ -0,0 +1,201 @@ +# frozen_string_literal: true + +require_relative "test_helper" +require "syntax_tree/language_server" + +module SyntaxTree + class LanguageServerTest < Minitest::Test + class Initialize < Struct.new(:id) + def to_hash + { method: "initialize", id: id } + end + end + + class Shutdown + def to_hash + { method: "shutdown" } + end + end + + class TextDocumentDidOpen < Struct.new(:uri, :text) + def to_hash + { + method: "textDocument/didOpen", + params: { + textDocument: { + uri: uri, + text: text + } + } + } + end + end + + class TextDocumentDidChange < Struct.new(:uri, :text) + def to_hash + { + method: "textDocument/didChange", + params: { + textDocument: { + uri: uri + }, + contentChanges: [{ text: text }] + } + } + end + end + + class TextDocumentDidClose < Struct.new(:uri) + def to_hash + { + method: "textDocument/didClose", + params: { + textDocument: { + uri: uri + } + } + } + end + end + + class TextDocumentFormatting < Struct.new(:id, :uri) + def to_hash + { + method: "textDocument/formatting", + id: id, + params: { + textDocument: { + uri: uri + } + } + } + end + end + + class TextDocumentInlayHints < Struct.new(:id, :uri) + def to_hash + { + method: "textDocument/inlayHints", + id: id, + params: { + textDocument: { + uri: uri + } + } + } + end + end + + class SyntaxTreeVisualizing < Struct.new(:id, :uri) + def to_hash + { + method: "syntaxTree/visualizing", + id: id, + params: { + textDocument: { + uri: uri + } + } + } + end + end + + def test_formatting + messages = [ + Initialize.new(1), + TextDocumentDidOpen.new("file:///path/to/file.rb", "class Foo; end"), + TextDocumentDidChange.new("file:///path/to/file.rb", "class Bar; end"), + TextDocumentFormatting.new(2, "file:///path/to/file.rb"), + TextDocumentDidClose.new("file:///path/to/file.rb"), + Shutdown.new + ] + + case run_server(messages) + in { id: 1, result: { capabilities: Hash } }, + { id: 2, result: [{ newText: new_text }] } + assert_equal("class Bar\nend\n", new_text) + end + end + + def test_inlay_hints + messages = [ + Initialize.new(1), + TextDocumentDidOpen.new("file:///path/to/file.rb", <<~RUBY), + begin + 1 + 2 * 3 + rescue + end + RUBY + TextDocumentInlayHints.new(2, "file:///path/to/file.rb"), + Shutdown.new + ] + + case run_server(messages) + in { id: 1, result: { capabilities: Hash } }, + { id: 2, result: { before:, after: } } + assert_equal(1, before.length) + assert_equal(2, after.length) + end + end + + def test_visualizing + messages = [ + Initialize.new(1), + TextDocumentDidOpen.new("file:///path/to/file.rb", "1 + 2"), + SyntaxTreeVisualizing.new(2, "file:///path/to/file.rb"), + Shutdown.new + ] + + case run_server(messages) + in { id: 1, result: { capabilities: Hash } }, { id: 2, result: } + assert_equal( + "(program (statements ((binary (int \"1\") + (int \"2\")))))\n", + result + ) + end + end + + def test_reading_file + Tempfile.open(%w[test- .rb]) do |file| + file.write("class Foo; end") + file.rewind + + messages = [ + Initialize.new(1), + TextDocumentFormatting.new(2, "file://#{file.path}"), + Shutdown.new + ] + + case run_server(messages) + in { id: 1, result: { capabilities: Hash } }, + { id: 2, result: [{ newText: new_text }] } + assert_equal("class Foo\nend\n", new_text) + end + end + end + + private + + def write(content) + request = content.to_hash.merge(jsonrpc: "2.0").to_json + "Content-Length: #{request.bytesize}\r\n\r\n#{request}" + end + + def read(content) + [].tap do |messages| + while (headers = content.gets("\r\n\r\n")) + source = content.read(headers[/Content-Length: (\d+)/i, 1].to_i) + messages << JSON.parse(source, symbolize_names: true) + end + end + end + + def run_server(messages) + input = StringIO.new(messages.map { |message| write(message) }.join) + output = StringIO.new + + LanguageServer.new(input: input, output: output).run + read(output.tap(&:rewind)) + end + end +end From bc1c0cc149d706f9aed4913b61026dcd7b3bdc76 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 24 May 2022 14:15:10 -0400 Subject: [PATCH 029/536] Tests for inlay hints --- lib/syntax_tree/language_server.rb | 4 -- test/language_server/inlay_hints_test.rb | 57 ++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 4 deletions(-) create mode 100644 test/language_server/inlay_hints_test.rb diff --git a/lib/syntax_tree/language_server.rb b/lib/syntax_tree/language_server.rb index b73fe0f7..6587ae08 100644 --- a/lib/syntax_tree/language_server.rb +++ b/lib/syntax_tree/language_server.rb @@ -107,10 +107,6 @@ def format(source) } end - def log(message) - write(method: "window/logMessage", params: { type: 4, message: message }) - end - def inlay_hints(source) inlay_hints = InlayHints.find(SyntaxTree.parse(source)) serialize = ->(position, text) { { position: position, text: text } } diff --git a/test/language_server/inlay_hints_test.rb b/test/language_server/inlay_hints_test.rb new file mode 100644 index 00000000..f652f6d8 --- /dev/null +++ b/test/language_server/inlay_hints_test.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +require_relative "../test_helper" +require "syntax_tree/language_server" + +module SyntaxTree + class LanguageServer + class InlayHintsTest < Minitest::Test + def test_assignments_in_parameters + hints = find("def foo(a = b = c); end") + + assert_equal(1, hints.before.length) + assert_equal(1, hints.after.length) + end + + def test_operators_in_binaries + hints = find("1 + 2 * 3") + + assert_equal(1, hints.before.length) + assert_equal(1, hints.after.length) + end + + def test_binaries_in_assignments + hints = find("a = 1 + 2") + + assert_equal(1, hints.before.length) + assert_equal(1, hints.after.length) + end + + def test_nested_ternaries + hints = find("a ? b : c ? d : e") + + assert_equal(1, hints.before.length) + assert_equal(1, hints.after.length) + end + + def test_bare_rescue + hints = find("begin; rescue; end") + + assert_equal(1, hints.after.length) + end + + def test_unary_in_binary + hints = find("-a + b") + + assert_equal(1, hints.before.length) + assert_equal(1, hints.after.length) + end + + private + + def find(source) + InlayHints.find(SyntaxTree.parse(source)) + end + end + end +end From 43bd2e938db702fb099a99081d92236561dd05d0 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 24 May 2022 14:36:20 -0400 Subject: [PATCH 030/536] Test plugin requiring --- test/cli_test.rb | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/test/cli_test.rb b/test/cli_test.rb index ade1485c..cf0d3265 100644 --- a/test/cli_test.rb +++ b/test/cli_test.rb @@ -142,11 +142,26 @@ def test_generic_error end end + def test_plugins + Dir.mktmpdir do |directory| + Dir.mkdir(File.join(directory, "syntax_tree")) + $:.unshift(directory) + + File.write( + File.join(directory, "syntax_tree", "plugin.rb"), + "puts 'Hello, world!'" + ) + result = run_cli("format", "--plugins=plugin") + + assert_equal("Hello, world!\ntest\n", result.stdio) + end + end + private Result = Struct.new(:status, :stdio, :stderr, keyword_init: true) - def run_cli(command, file: nil) + def run_cli(command, *args, file: nil) if file.nil? file = Tempfile.new(%w[test- .rb]) file.puts("test") @@ -156,7 +171,7 @@ def run_cli(command, file: nil) status = nil stdio, stderr = - capture_io { status = SyntaxTree::CLI.run([command, file.path]) } + capture_io { status = SyntaxTree::CLI.run([command, *args, file.path]) } Result.new(status: status, stdio: stdio, stderr: stderr) ensure From 595b2f445f2d960752d163b9702e84738915ef83 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 24 May 2022 14:51:55 -0400 Subject: [PATCH 031/536] Test bad requests to language server --- lib/syntax_tree/language_server.rb | 2 +- test/language_server_test.rb | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/syntax_tree/language_server.rb b/lib/syntax_tree/language_server.rb index 6587ae08..3853ee18 100644 --- a/lib/syntax_tree/language_server.rb +++ b/lib/syntax_tree/language_server.rb @@ -74,7 +74,7 @@ def run in method: %r{\$/.+} # ignored else - raise "Unhandled: #{request}" + raise ArgumentError, "Unhandled: #{request}" end end end diff --git a/test/language_server_test.rb b/test/language_server_test.rb index ba33a05b..acbb07f8 100644 --- a/test/language_server_test.rb +++ b/test/language_server_test.rb @@ -174,6 +174,12 @@ def test_reading_file end end + def test_bogus_request + assert_raises(ArgumentError) do + run_server([{ method: "textDocument/bogus" }]) + end + end + private def write(content) From 1735dd675e69eda653c1d32faef8827e555c94a5 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 24 May 2022 15:01:31 -0400 Subject: [PATCH 032/536] Test spawning language server --- test/cli_test.rb | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/cli_test.rb b/test/cli_test.rb index cf0d3265..7f2bcd26 100644 --- a/test/cli_test.rb +++ b/test/cli_test.rb @@ -157,6 +157,21 @@ def test_plugins end end + def test_language_server + prev_stdin = $stdin + prev_stdout = $stdout + + request = { method: "shutdown" }.merge(jsonrpc: "2.0").to_json + $stdin = + StringIO.new("Content-Length: #{request.bytesize}\r\n\r\n#{request}") + $stdout = StringIO.new + + assert_equal(0, SyntaxTree::CLI.run(["lsp"])) + ensure + $stdin = prev_stdin + $stdout = prev_stdout + end + private Result = Struct.new(:status, :stdio, :stderr, keyword_init: true) From e68f2256e056c397c00808c6fa1e5b21188661f3 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 24 May 2022 16:19:57 -0400 Subject: [PATCH 033/536] More test cases, more tests --- lib/syntax_tree/node.rb | 12 ++++++----- lib/syntax_tree/parser.rb | 18 +++++----------- test/fixtures/aryptn.rb | 16 ++++++++++++++ test/fixtures/call.rb | 39 +++++++++++++++++++++++++++++++++++ test/fixtures/command_call.rb | 2 ++ test/fixtures/do_block.rb | 12 +++++++++++ test/fixtures/hshptn.rb | 8 +++++++ test/fixtures/if.rb | 10 +++++++++ test/fixtures/ifop.rb | 6 ++++++ test/location_test.rb | 30 +++++++++++++++++++++++++++ test/node_test.rb | 14 +++++++++++++ test/parser_test.rb | 18 ++++++++++++++++ 12 files changed, 167 insertions(+), 18 deletions(-) create mode 100644 test/location_test.rb diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 4f12b144..8451545a 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -5484,12 +5484,14 @@ def format_flat(q) q.format(predicate) q.text(" ?") - q.breakable - q.format(truthy) - q.text(" :") + q.indent do + q.breakable + q.format(truthy) + q.text(" :") - q.breakable - q.format(falsy) + q.breakable + q.format(falsy) + end end end diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index 6bff0838..fdffbeb9 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -548,13 +548,6 @@ def on_aryptn(constant, requireds, rest, posts) parts[0].location.to(parts[-1].location) end - # If there's the optional then keyword, then we'll delete that and use it - # as the end bounds of the location. - if (token = find_token(Kw, "then", consume: false)) - tokens.delete(token) - location = location.to(token.location) - end - # If there is a plain *, then we're going to fix up the location of it # here because it currently doesn't have anything to use for its precise # location. If we hit a comma, then we've gone too far. @@ -1698,12 +1691,6 @@ def on_hshptn(constant, keywords, keyword_rest) end end - # Delete the optional then keyword - if (token = find_token(Kw, "then", consume: false)) - parts << token - tokens.delete(token) - end - HshPtn.new( constant: constant, keywords: keywords || [], @@ -3013,6 +3000,11 @@ def on_stmts_new # (StringEmbExpr | StringDVar | TStringContent) part # ) -> StringContent def on_string_add(string, part) + # Due to some eccentricities in how ripper works, you need this here in + # case you have a syntax error with an embedded expression that doesn't + # finish, as in: "#{" + return string if part.is_a?(String) + location = string.parts.any? ? string.location.to(part.location) : part.location diff --git a/test/fixtures/aryptn.rb b/test/fixtures/aryptn.rb index eddd8e3f..64d5d9d0 100644 --- a/test/fixtures/aryptn.rb +++ b/test/fixtures/aryptn.rb @@ -4,6 +4,22 @@ end % case foo +in [] then +end +- +case foo +in [] +end +% +case foo +in * then +end +- +case foo +in [*] +end +% +case foo in _, _ end - diff --git a/test/fixtures/call.rb b/test/fixtures/call.rb index f3333276..c41ee4ac 100644 --- a/test/fixtures/call.rb +++ b/test/fixtures/call.rb @@ -1,6 +1,8 @@ % foo.bar % +foo.bar(baz) +% foo.() % foo::() @@ -21,3 +23,40 @@ .barrrrrrrrrrrrrrrrrrr {} .bazzzzzzzzzzzzzzzzzzzzzzzzzz .quxxxxxxxxx +% +foo. # comment + bar +% +foo + .bar + .baz # comment + .qux + .quux +% +foo + .bar + .baz. + # comment + qux + .quux +% +{ a: 1, b: 2 }.fooooooooooooooooo.barrrrrrrrrrrrrrrrrrr.bazzzzzzzzzzzz.quxxxxxxxxxxxx +- +{ a: 1, b: 2 }.fooooooooooooooooo + .barrrrrrrrrrrrrrrrrrr + .bazzzzzzzzzzzz + .quxxxxxxxxxxxx +% +fooooooooooooooooo.barrrrrrrrrrrrrrrrrrr.bazzzzzzzzzzzz.quxxxxxxxxxxxx.each { block } +- +fooooooooooooooooo.barrrrrrrrrrrrrrrrrrr.bazzzzzzzzzzzz.quxxxxxxxxxxxx.each do + block +end +% +foo.bar.baz.each do + block1 + block2 +end +% +a b do +end.c d diff --git a/test/fixtures/command_call.rb b/test/fixtures/command_call.rb index fb0d084a..4a0f60f0 100644 --- a/test/fixtures/command_call.rb +++ b/test/fixtures/command_call.rb @@ -32,3 +32,5 @@ foo. # comment bar baz +% +foo.bar baz ? qux : qaz diff --git a/test/fixtures/do_block.rb b/test/fixtures/do_block.rb index 016f27b2..8ea4f75f 100644 --- a/test/fixtures/do_block.rb +++ b/test/fixtures/do_block.rb @@ -14,3 +14,15 @@ foo :bar do baz end +% +sig do + override.params(contacts: Contact::ActiveRecord_Relation).returns( + Customer::ActiveRecord_Relation + ) +end +- +sig do + override + .params(contacts: Contact::ActiveRecord_Relation) + .returns(Customer::ActiveRecord_Relation) +end diff --git a/test/fixtures/hshptn.rb b/test/fixtures/hshptn.rb index f8733170..505336b8 100644 --- a/test/fixtures/hshptn.rb +++ b/test/fixtures/hshptn.rb @@ -64,6 +64,14 @@ end % case foo +in {} then +end +- +case foo +in {} +end +% +case foo in **nil end % diff --git a/test/fixtures/if.rb b/test/fixtures/if.rb index 9045e5bf..e5e88103 100644 --- a/test/fixtures/if.rb +++ b/test/fixtures/if.rb @@ -49,3 +49,13 @@ end - not(a) ? b : c +% +(if foo then bar else baz end) +- +( + if foo + bar + else + baz + end +) diff --git a/test/fixtures/ifop.rb b/test/fixtures/ifop.rb index 541e667e..e56eb987 100644 --- a/test/fixtures/ifop.rb +++ b/test/fixtures/ifop.rb @@ -10,3 +10,9 @@ end % foo bar ? 1 : 2 +% +foooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo ? break : baz +- +foooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo ? + break : + baz diff --git a/test/location_test.rb b/test/location_test.rb new file mode 100644 index 00000000..35e5a6df --- /dev/null +++ b/test/location_test.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +require_relative "test_helper" + +module SyntaxTree + class LocationTest < Minitest::Test + def test_lines + location = Location.fixed(line: 1, char: 0, column: 0) + location = location.to(Location.fixed(line: 3, char: 3, column: 3)) + + assert_equal(1..3, location.lines) + end + + def test_deconstruct + location = Location.fixed(line: 1, char: 0, column: 0) + + case location + in [1, 0, 0, *] + end + end + + def test_deconstruct_keys + location = Location.fixed(line: 1, char: 0, column: 0) + + case location + in { start_line: 1 } + end + end + end +end diff --git a/test/node_test.rb b/test/node_test.rb index 6bde39bc..ffd00fa5 100644 --- a/test/node_test.rb +++ b/test/node_test.rb @@ -1032,6 +1032,20 @@ def test_multibyte_column_positions assert_node(Command, source, at: at) end + def test_root_class_raises_not_implemented_errors + { + accept: [nil], + child_nodes: [], + deconstruct: [], + deconstruct_keys: [[]], + format: [nil] + }.each do |method, arguments| + assert_raises(NotImplementedError) do + Node.new.public_send(method, *arguments) + end + end + end + private def location(lines: 1..1, chars: 0..0, columns: 0..0) diff --git a/test/parser_test.rb b/test/parser_test.rb index 8aadbfc2..59ea199a 100644 --- a/test/parser_test.rb +++ b/test/parser_test.rb @@ -30,5 +30,23 @@ def test_parses_ripper_methods # Finally, assert that we have no remaining events. assert_empty(events) end + + def test_errors_on_missing_token_with_location + assert_raises(Parser::ParseError) do + SyntaxTree.parse("\"foo") + end + end + + def test_errors_on_missing_token_without_location + assert_raises(Parser::ParseError) do + SyntaxTree.parse(":\"foo") + end + end + + def test_handles_strings_with_non_terminated_embedded_expressions + assert_raises(Parser::ParseError) do + SyntaxTree.parse('"#{"') + end + end end end From f23454d0cc577dcba9402596d0b286267c598913 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 24 May 2022 16:36:33 -0400 Subject: [PATCH 034/536] Remodel the plugins so that the options are available --- lib/syntax_tree/formatter.rb | 22 ++++- lib/syntax_tree/formatter/single_quotes.rb | 13 --- lib/syntax_tree/formatter/trailing_comma.rb | 13 --- lib/syntax_tree/plugin/single_quotes.rb | 3 +- lib/syntax_tree/plugin/trailing_comma.rb | 3 +- test/formatter/single_quotes_test.rb | 51 ----------- test/formatter/trailing_comma_test.rb | 97 --------------------- test/plugin/single_quotes_test.rb | 46 ++++++++++ test/plugin/trailing_comma_test.rb | 92 +++++++++++++++++++ test/test_helper.rb | 22 ++++- 10 files changed, 179 insertions(+), 183 deletions(-) delete mode 100644 lib/syntax_tree/formatter/single_quotes.rb delete mode 100644 lib/syntax_tree/formatter/trailing_comma.rb delete mode 100644 test/formatter/single_quotes_test.rb delete mode 100644 test/formatter/trailing_comma_test.rb create mode 100644 test/plugin/single_quotes_test.rb create mode 100644 test/plugin/trailing_comma_test.rb diff --git a/lib/syntax_tree/formatter.rb b/lib/syntax_tree/formatter.rb index 5d362129..c088b302 100644 --- a/lib/syntax_tree/formatter.rb +++ b/lib/syntax_tree/formatter.rb @@ -4,6 +4,18 @@ module SyntaxTree # A slightly enhanced PP that knows how to format recursively including # comments. class Formatter < PrettierPrint + # We want to minimize as much as possible the number of options that are + # available in syntax tree. For the most part, if users want non-default + # formatting, they should override the format methods on the specific nodes + # themselves. However, because of some history with prettier and the fact + # that folks have become entrenched in their ways, we decided to provide a + # small amount of configurability. + # + # Note that we're keeping this in a global-ish hash instead of just + # overriding methods on classes so that other plugins can reference this if + # necessary. For example, the RBS plugin references the quote style. + OPTIONS = { quote: "\"", trailing_comma: false } + COMMENT_PRIORITY = 1 HEREDOC_PRIORITY = 2 @@ -14,13 +26,15 @@ class Formatter < PrettierPrint attr_reader :quote, :trailing_comma alias trailing_comma? trailing_comma - def initialize(source, ...) - super(...) + def initialize(source, *args, quote: OPTIONS[:quote], trailing_comma: OPTIONS[:trailing_comma]) + super(*args) @source = source @stack = [] - @quote = "\"" - @trailing_comma = false + + # Memoizing these values per formatter to make access faster. + @quote = quote + @trailing_comma = trailing_comma end def self.format(source, node) diff --git a/lib/syntax_tree/formatter/single_quotes.rb b/lib/syntax_tree/formatter/single_quotes.rb deleted file mode 100644 index 4d1f41b3..00000000 --- a/lib/syntax_tree/formatter/single_quotes.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -module SyntaxTree - class Formatter - # This module overrides the quote method on the formatter to use single - # quotes for everything instead of double quotes. - module SingleQuotes - def quote - "'" - end - end - end -end diff --git a/lib/syntax_tree/formatter/trailing_comma.rb b/lib/syntax_tree/formatter/trailing_comma.rb deleted file mode 100644 index 63fe2e9a..00000000 --- a/lib/syntax_tree/formatter/trailing_comma.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -module SyntaxTree - class Formatter - # This module overrides the trailing_comma? method on the formatter to - # return true. - module TrailingComma - def trailing_comma? - true - end - end - end -end diff --git a/lib/syntax_tree/plugin/single_quotes.rb b/lib/syntax_tree/plugin/single_quotes.rb index d8034084..c6e829e0 100644 --- a/lib/syntax_tree/plugin/single_quotes.rb +++ b/lib/syntax_tree/plugin/single_quotes.rb @@ -1,4 +1,3 @@ # frozen_string_literal: true -require "syntax_tree/formatter/single_quotes" -SyntaxTree::Formatter.prepend(SyntaxTree::Formatter::SingleQuotes) +SyntaxTree::Formatter::OPTIONS[:quote] = "'" diff --git a/lib/syntax_tree/plugin/trailing_comma.rb b/lib/syntax_tree/plugin/trailing_comma.rb index eaa8cb6a..878703c3 100644 --- a/lib/syntax_tree/plugin/trailing_comma.rb +++ b/lib/syntax_tree/plugin/trailing_comma.rb @@ -1,4 +1,3 @@ # frozen_string_literal: true -require "syntax_tree/formatter/trailing_comma" -SyntaxTree::Formatter.prepend(SyntaxTree::Formatter::TrailingComma) +SyntaxTree::Formatter::OPTIONS[:trailing_comma] = true diff --git a/test/formatter/single_quotes_test.rb b/test/formatter/single_quotes_test.rb deleted file mode 100644 index 78f9ae3d..00000000 --- a/test/formatter/single_quotes_test.rb +++ /dev/null @@ -1,51 +0,0 @@ -# frozen_string_literal: true - -require_relative "../test_helper" -require "syntax_tree/formatter/single_quotes" - -module SyntaxTree - class Formatter - class SingleQuotesTest < Minitest::Test - class TestFormatter < Formatter - prepend Formatter::SingleQuotes - end - - def test_empty_string_literal - assert_format("''\n", "\"\"") - end - - def test_string_literal - assert_format("'string'\n", "\"string\"") - end - - def test_string_literal_with_interpolation - assert_format("\"\#{foo}\"\n") - end - - def test_dyna_symbol - assert_format(":'symbol'\n", ":\"symbol\"") - end - - def test_single_quote_in_string - assert_format("\"str'ing\"\n") - end - - def test_label - assert_format( - "{ foo => foo, :'bar' => bar }\n", - "{ foo => foo, \"bar\": bar }" - ) - end - - private - - def assert_format(expected, source = expected) - formatter = TestFormatter.new(source, []) - SyntaxTree.parse(source).format(formatter) - - formatter.flush - assert_equal(expected, formatter.output.join) - end - end - end -end diff --git a/test/formatter/trailing_comma_test.rb b/test/formatter/trailing_comma_test.rb deleted file mode 100644 index f6585772..00000000 --- a/test/formatter/trailing_comma_test.rb +++ /dev/null @@ -1,97 +0,0 @@ -# frozen_string_literal: true - -require_relative "../test_helper" -require "syntax_tree/formatter/trailing_comma" - -module SyntaxTree - class Formatter - class TrailingCommaTest < Minitest::Test - class TestFormatter < Formatter - prepend Formatter::TrailingComma - end - - def test_arg_paren_flat - assert_format("foo(a)\n") - end - - def test_arg_paren_break - assert_format(<<~EXPECTED, <<~SOURCE) - foo( - #{"a" * 80}, - ) - EXPECTED - foo(#{"a" * 80}) - SOURCE - end - - def test_arg_paren_block - assert_format(<<~EXPECTED, <<~SOURCE) - foo( - &#{"a" * 80} - ) - EXPECTED - foo(&#{"a" * 80}) - SOURCE - end - - def test_arg_paren_command - assert_format(<<~EXPECTED, <<~SOURCE) - foo( - bar #{"a" * 80} - ) - EXPECTED - foo(bar #{"a" * 80}) - SOURCE - end - - def test_arg_paren_command_call - assert_format(<<~EXPECTED, <<~SOURCE) - foo( - bar.baz #{"a" * 80} - ) - EXPECTED - foo(bar.baz #{"a" * 80}) - SOURCE - end - - def test_array_literal_flat - assert_format("[a]\n") - end - - def test_array_literal_break - assert_format(<<~EXPECTED, <<~SOURCE) - [ - #{"a" * 80}, - ] - EXPECTED - [#{"a" * 80}] - SOURCE - end - - def test_hash_literal_flat - assert_format("{ a: a }\n") - end - - def test_hash_literal_break - assert_format(<<~EXPECTED, <<~SOURCE) - { - a: - #{"a" * 80}, - } - EXPECTED - { a: #{"a" * 80} } - SOURCE - end - - private - - def assert_format(expected, source = expected) - formatter = TestFormatter.new(source, []) - SyntaxTree.parse(source).format(formatter) - - formatter.flush - assert_equal(expected, formatter.output.join) - end - end - end -end diff --git a/test/plugin/single_quotes_test.rb b/test/plugin/single_quotes_test.rb new file mode 100644 index 00000000..719f33c1 --- /dev/null +++ b/test/plugin/single_quotes_test.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +require_relative "../test_helper" + +module SyntaxTree + class SingleQuotesTest < Minitest::Test + OPTIONS = Plugin.options("syntax_tree/plugin/single_quotes") + + def test_empty_string_literal + assert_format("''\n", "\"\"") + end + + def test_string_literal + assert_format("'string'\n", "\"string\"") + end + + def test_string_literal_with_interpolation + assert_format("\"\#{foo}\"\n") + end + + def test_dyna_symbol + assert_format(":'symbol'\n", ":\"symbol\"") + end + + def test_single_quote_in_string + assert_format("\"str'ing\"\n") + end + + def test_label + assert_format( + "{ foo => foo, :'bar' => bar }\n", + "{ foo => foo, \"bar\": bar }" + ) + end + + private + + def assert_format(expected, source = expected) + formatter = Formatter.new(source, [], **OPTIONS) + SyntaxTree.parse(source).format(formatter) + + formatter.flush + assert_equal(expected, formatter.output.join) + end + end +end diff --git a/test/plugin/trailing_comma_test.rb b/test/plugin/trailing_comma_test.rb new file mode 100644 index 00000000..ba9ad846 --- /dev/null +++ b/test/plugin/trailing_comma_test.rb @@ -0,0 +1,92 @@ +# frozen_string_literal: true + +require_relative "../test_helper" + +module SyntaxTree + class TrailingCommaTest < Minitest::Test + OPTIONS = Plugin.options("syntax_tree/plugin/trailing_comma") + + def test_arg_paren_flat + assert_format("foo(a)\n") + end + + def test_arg_paren_break + assert_format(<<~EXPECTED, <<~SOURCE) + foo( + #{"a" * 80}, + ) + EXPECTED + foo(#{"a" * 80}) + SOURCE + end + + def test_arg_paren_block + assert_format(<<~EXPECTED, <<~SOURCE) + foo( + &#{"a" * 80} + ) + EXPECTED + foo(&#{"a" * 80}) + SOURCE + end + + def test_arg_paren_command + assert_format(<<~EXPECTED, <<~SOURCE) + foo( + bar #{"a" * 80} + ) + EXPECTED + foo(bar #{"a" * 80}) + SOURCE + end + + def test_arg_paren_command_call + assert_format(<<~EXPECTED, <<~SOURCE) + foo( + bar.baz #{"a" * 80} + ) + EXPECTED + foo(bar.baz #{"a" * 80}) + SOURCE + end + + def test_array_literal_flat + assert_format("[a]\n") + end + + def test_array_literal_break + assert_format(<<~EXPECTED, <<~SOURCE) + [ + #{"a" * 80}, + ] + EXPECTED + [#{"a" * 80}] + SOURCE + end + + def test_hash_literal_flat + assert_format("{ a: a }\n") + end + + def test_hash_literal_break + assert_format(<<~EXPECTED, <<~SOURCE) + { + a: + #{"a" * 80}, + } + EXPECTED + { a: #{"a" * 80} } + SOURCE + end + + private + + def assert_format(expected, source = expected) + formatter = Formatter.new(source, [], **OPTIONS) + SyntaxTree.parse(source).format(formatter) + + formatter.flush + assert_equal(expected, formatter.output.join) + end + end +end diff --git a/test/test_helper.rb b/test/test_helper.rb index bb3ea67f..895fbc82 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -77,10 +77,30 @@ def assert_syntax_tree(node) end RUBY end + + Minitest::Test.include(self) end end -Minitest::Test.include(SyntaxTree::Assertions) +module SyntaxTree + module Plugin + # A couple of plugins modify the options hash on the formatter. They're + # modeled as files that should be required so that it's simple for the CLI + # and the library to use the same code path. In this case we're going to + # require the file for the plugin but ensure it doesn't make any lasting + # changes. + def self.options(path) + previous_options = SyntaxTree::Formatter::OPTIONS.dup + + begin + require path + SyntaxTree::Formatter::OPTIONS.dup + ensure + SyntaxTree::Formatter::OPTIONS.merge!(previous_options) + end + end + end +end # There are a bunch of fixtures defined in test/fixtures. They exercise every # possible combination of syntax that leads to variations in the types of nodes. From ac41ba75a79ee63b62700fc31ee9dcabb8890ec6 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 24 May 2022 16:44:14 -0400 Subject: [PATCH 035/536] Reformat with syntax tree --- lib/syntax_tree/formatter.rb | 7 ++++++- lib/syntax_tree/node.rb | 20 +++++--------------- test/language_server_test.rb | 20 +++++++++++++------- test/location_test.rb | 2 +- test/parser_test.rb | 12 +++--------- 5 files changed, 28 insertions(+), 33 deletions(-) diff --git a/lib/syntax_tree/formatter.rb b/lib/syntax_tree/formatter.rb index c088b302..56de6a4a 100644 --- a/lib/syntax_tree/formatter.rb +++ b/lib/syntax_tree/formatter.rb @@ -26,7 +26,12 @@ class Formatter < PrettierPrint attr_reader :quote, :trailing_comma alias trailing_comma? trailing_comma - def initialize(source, *args, quote: OPTIONS[:quote], trailing_comma: OPTIONS[:trailing_comma]) + def initialize( + source, + *args, + quote: OPTIONS[:quote], + trailing_comma: OPTIONS[:trailing_comma] + ) super(*args) @source = source diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 8451545a..85956c8c 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -2148,9 +2148,7 @@ def format(q) # q.text(" ") format_array_contents(q, array) - in [ - Paren[contents: { body: [ArrayLiteral => statement] }] - ] + in [Paren[contents: { body: [ArrayLiteral => statement] }]] # Here we have a single argument that is a set of parentheses wrapping # an array literal that has 0 or 1 elements. We're going to skip the # parentheses but print the array itself. This would be like if we @@ -2178,9 +2176,7 @@ def format(q) # q.text(" ") q.format(statement) - in [ - Paren => part - ] + in [Paren => part] # Here we have a single argument that is a set of parentheses. We're # going to print the parentheses themselves as if they were the set of # arguments. This would be like if we had: @@ -2188,9 +2184,7 @@ def format(q) # break(foo.bar) # q.format(part) - in [ - ArrayLiteral[contents: { parts: [_, _, *] }] => array - ] + in [ArrayLiteral[contents: { parts: [_, _, *] }] => array] # Here there is a single argument that is an array literal with at # least two elements. We skip directly into the array literal's # elements in order to print the contents. This would be like if we @@ -2204,9 +2198,7 @@ def format(q) # q.text(" ") format_array_contents(q, array) - in [ - ArrayLiteral => part - ] + in [ArrayLiteral => part] # Here there is a single argument that is an array literal with 0 or 1 # elements. In this case we're going to print the array as it is # because skipping the brackets would change the remaining. This would @@ -2217,9 +2209,7 @@ def format(q) # q.text(" ") q.format(part) - in [ - _ - ] + in [_] # Here there is a single argument that hasn't matched one of our # previous cases. We're going to print the argument as it is. This # would be like if we had: diff --git a/test/language_server_test.rb b/test/language_server_test.rb index acbb07f8..f8a61003 100644 --- a/test/language_server_test.rb +++ b/test/language_server_test.rb @@ -111,8 +111,10 @@ def test_formatting ] case run_server(messages) - in { id: 1, result: { capabilities: Hash } }, - { id: 2, result: [{ newText: new_text }] } + in [ + { id: 1, result: { capabilities: Hash } }, + { id: 2, result: [{ newText: new_text }] } + ] assert_equal("class Bar\nend\n", new_text) end end @@ -131,8 +133,10 @@ def test_inlay_hints ] case run_server(messages) - in { id: 1, result: { capabilities: Hash } }, - { id: 2, result: { before:, after: } } + in [ + { id: 1, result: { capabilities: Hash } }, + { id: 2, result: { before:, after: } } + ] assert_equal(1, before.length) assert_equal(2, after.length) end @@ -147,7 +151,7 @@ def test_visualizing ] case run_server(messages) - in { id: 1, result: { capabilities: Hash } }, { id: 2, result: } + in [{ id: 1, result: { capabilities: Hash } }, { id: 2, result: }] assert_equal( "(program (statements ((binary (int \"1\") + (int \"2\")))))\n", result @@ -167,8 +171,10 @@ def test_reading_file ] case run_server(messages) - in { id: 1, result: { capabilities: Hash } }, - { id: 2, result: [{ newText: new_text }] } + in [ + { id: 1, result: { capabilities: Hash } }, + { id: 2, result: [{ newText: new_text }] } + ] assert_equal("class Foo\nend\n", new_text) end end diff --git a/test/location_test.rb b/test/location_test.rb index 35e5a6df..7fa6fbd2 100644 --- a/test/location_test.rb +++ b/test/location_test.rb @@ -23,7 +23,7 @@ def test_deconstruct_keys location = Location.fixed(line: 1, char: 0, column: 0) case location - in { start_line: 1 } + in start_line: 1 end end end diff --git a/test/parser_test.rb b/test/parser_test.rb index 59ea199a..b36c1a5f 100644 --- a/test/parser_test.rb +++ b/test/parser_test.rb @@ -32,21 +32,15 @@ def test_parses_ripper_methods end def test_errors_on_missing_token_with_location - assert_raises(Parser::ParseError) do - SyntaxTree.parse("\"foo") - end + assert_raises(Parser::ParseError) { SyntaxTree.parse("\"foo") } end def test_errors_on_missing_token_without_location - assert_raises(Parser::ParseError) do - SyntaxTree.parse(":\"foo") - end + assert_raises(Parser::ParseError) { SyntaxTree.parse(":\"foo") } end def test_handles_strings_with_non_terminated_embedded_expressions - assert_raises(Parser::ParseError) do - SyntaxTree.parse('"#{"') - end + assert_raises(Parser::ParseError) { SyntaxTree.parse('"#{"') } end end end From b22cb5f3824ddd3d2d5864949463c34c4cd8a60f Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 24 May 2022 22:47:13 -0400 Subject: [PATCH 036/536] Fix up Ruby 2.7 pattern matching usage --- .rubocop.yml | 3 +++ lib/syntax_tree/language_server/inlay_hints.rb | 4 +++- test/location_test.rb | 6 ++++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index 8c1bc99e..8cf5f209 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -78,3 +78,6 @@ Style/PerlBackrefs: Style/SpecialGlobalVars: Enabled: false + +Style/StructInheritance: + Enabled: false diff --git a/lib/syntax_tree/language_server/inlay_hints.rb b/lib/syntax_tree/language_server/inlay_hints.rb index aba8c4c6..089355a7 100644 --- a/lib/syntax_tree/language_server/inlay_hints.rb +++ b/lib/syntax_tree/language_server/inlay_hints.rb @@ -73,8 +73,10 @@ def visit_binary(node) # a ? b : ₍c ? d : e₎ # def visit_if_op(node) - if stack[-2] in Assign | Binary | IfOp | OpAssign + case stack[-2] + in Assign | Binary | IfOp | OpAssign parentheses(node.location) + else end super diff --git a/test/location_test.rb b/test/location_test.rb index 7fa6fbd2..2a697281 100644 --- a/test/location_test.rb +++ b/test/location_test.rb @@ -15,7 +15,8 @@ def test_deconstruct location = Location.fixed(line: 1, char: 0, column: 0) case location - in [1, 0, 0, *] + in [start_line, 0, 0, *] + assert_equal(1, start_line) end end @@ -23,7 +24,8 @@ def test_deconstruct_keys location = Location.fixed(line: 1, char: 0, column: 0) case location - in start_line: 1 + in start_line: + assert_equal(1, start_line) end end end From 8d77b80d174d266e9769f51f6edee83762af52a1 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 25 May 2022 09:38:44 -0400 Subject: [PATCH 037/536] Bump to v2.7.1 --- CHANGELOG.md | 16 +++++++++++++++- Gemfile.lock | 2 +- lib/syntax_tree/version.rb | 2 +- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d80cfc3..aa928c0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,19 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +## [2.7.1] - 2022-05-25 + +### Added + +- [#92](https://github.com/ruby-syntax-tree/syntax_tree/pull/92) - (Internal) Drastically increase test coverage, including many more tests for the language server and the CLI. + +### Changed + +- [#87](https://github.com/ruby-syntax-tree/syntax_tree/pull/87) - Don't convert quotes on strings if it would result in more escapes. +- [#91](https://github.com/ruby-syntax-tree/syntax_tree/pull/91) - Always use `[]` with array patterns. There are just too many edge cases where you have to use them anyway. This simplifies the look and makes it more consistent. +- [#92](https://github.com/ruby-syntax-tree/syntax_tree/pull/92) - Remodel the currently shipped plugins such that they're modifying an options hash instead of overriding methods. This should make it easier for other plugins to reference the already loaded plugins, e.g., the RBS plugin referencing the quotes. +- [#92](https://github.com/ruby-syntax-tree/syntax_tree/pull/92) - Fix up the language server inlay hints to continue walking the tree once a pattern is found. This should increase useability. + ## [2.7.0] - 2022-05-19 ### Added @@ -246,7 +259,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - 🎉 Initial release! 🎉 -[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.7.0...HEAD +[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.7.1...HEAD +[2.7.1]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.7.0...v2.7.1 [2.7.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.6.0...v2.7.0 [2.6.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.5.0...v2.6.0 [2.5.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.4.1...v2.5.0 diff --git a/Gemfile.lock b/Gemfile.lock index 3f892652..d707afd5 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - syntax_tree (2.7.0) + syntax_tree (2.7.1) prettier_print GEM diff --git a/lib/syntax_tree/version.rb b/lib/syntax_tree/version.rb index 851d9565..7754cf7a 100644 --- a/lib/syntax_tree/version.rb +++ b/lib/syntax_tree/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SyntaxTree - VERSION = "2.7.0" + VERSION = "2.7.1" end From 2f69135484bb6a1a06693ef2320c01a2f0ff1734 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 May 2022 17:30:02 +0000 Subject: [PATCH 038/536] Bump rubocop from 1.29.1 to 1.30.0 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.29.1 to 1.30.0. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.29.1...v1.30.0) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index d707afd5..f8898163 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -18,16 +18,16 @@ GEM rake (13.0.6) regexp_parser (2.4.0) rexml (3.2.5) - rubocop (1.29.1) + rubocop (1.30.0) parallel (~> 1.10) parser (>= 3.1.0.0) rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 1.8, < 3.0) rexml (>= 3.2.5, < 4.0) - rubocop-ast (>= 1.17.0, < 2.0) + rubocop-ast (>= 1.18.0, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 1.4.0, < 3.0) - rubocop-ast (1.17.0) + rubocop-ast (1.18.0) parser (>= 3.1.1.0) ruby-progressbar (1.11.0) simplecov (0.21.2) From 009c570c97a6e923551f382ade1d057b46817c15 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 1 Jun 2022 10:54:29 -0400 Subject: [PATCH 039/536] Parallel CLI execution --- lib/syntax_tree.rb | 1 + lib/syntax_tree/cli.rb | 172 ++++++++++++++++++++++++++++------------- 2 files changed, 120 insertions(+), 53 deletions(-) diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index 60979d04..1dbd3ac8 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +require "etc" require "json" require "pp" require "prettier_print" diff --git a/lib/syntax_tree/cli.rb b/lib/syntax_tree/cli.rb index 64848ca6..e5bf8cf0 100644 --- a/lib/syntax_tree/cli.rb +++ b/lib/syntax_tree/cli.rb @@ -34,9 +34,41 @@ def self.yellow(value) end end + # An item of work that corresponds to a file to be processed. + class FileItem + attr_reader :filepath + + def initialize(filepath) + @filepath = filepath + end + + def handler + HANDLERS[File.extname(filepath)] + end + + def source + handler.read(filepath) + end + end + + # An item of work that corresponds to the stdin content. + class STDINItem + def handler + HANDLERS[".rb"] + end + + def filepath + :stdin + end + + def source + $stdin.read + end + end + # The parent action class for the CLI that implements the basics. class Action - def run(handler, filepath, source) + def run(item) end def success @@ -48,8 +80,8 @@ def failure # An action of the CLI that prints out the AST for the given source. class AST < Action - def run(handler, _filepath, source) - pp handler.parse(source) + def run(item) + pp item.handler.parse(item.source) end end @@ -59,10 +91,11 @@ class Check < Action class UnformattedError < StandardError end - def run(handler, filepath, source) - raise UnformattedError if source != handler.format(source) + def run(item) + source = item.source + raise UnformattedError if source != item.handler.format(source) rescue StandardError - warn("[#{Color.yellow("warn")}] #{filepath}") + warn("[#{Color.yellow("warn")}] #{item.filepath}") raise end @@ -81,9 +114,11 @@ class Debug < Action class NonIdempotentFormatError < StandardError end - def run(handler, filepath, source) - warning = "[#{Color.yellow("warn")}] #{filepath}" - formatted = handler.format(source) + def run(item) + handler = item.handler + + warning = "[#{Color.yellow("warn")}] #{item.filepath}" + formatted = handler.format(item.source) raise NonIdempotentFormatError if formatted != handler.format(formatted) rescue StandardError @@ -102,25 +137,27 @@ def failure # An action of the CLI that prints out the doc tree IR for the given source. class Doc < Action - def run(handler, _filepath, source) + def run(item) + source = item.source + formatter = Formatter.new(source, []) - handler.parse(source).format(formatter) + item.handler.parse(source).format(formatter) pp formatter.groups.first end end # An action of the CLI that formats the input source and prints it out. class Format < Action - def run(handler, _filepath, source) - puts handler.format(source) + def run(item) + puts item.handler.format(item.source) end end # An action of the CLI that converts the source into its equivalent JSON # representation. class Json < Action - def run(handler, _filepath, source) - object = Visitor::JSONVisitor.new.visit(handler.parse(source)) + def run(item) + object = Visitor::JSONVisitor.new.visit(item.handler.parse(item.source)) puts JSON.pretty_generate(object) end end @@ -128,27 +165,28 @@ def run(handler, _filepath, source) # An action of the CLI that outputs a pattern-matching Ruby expression that # would match the input given. class Match < Action - def run(handler, _filepath, source) - puts handler.parse(source).construct_keys + def run(item) + puts item.handler.parse(item.source).construct_keys end end # An action of the CLI that formats the input source and writes the # formatted output back to the file. class Write < Action - def run(handler, filepath, source) - print filepath + def run(item) + filepath = item.filepath start = Time.now - formatted = handler.format(source) + source = item.source + formatted = item.handler.format(source) File.write(filepath, formatted) if filepath != :stdin color = source == formatted ? Color.gray(filepath) : filepath delta = ((Time.now - start) * 1000).round - puts "\r#{color} #{delta}ms" + puts "#{color} #{delta}ms" rescue StandardError - puts "\r#{filepath}" + puts filepath raise end end @@ -258,24 +296,41 @@ def run(argv) plugins.split(",").each { |plugin| require "syntax_tree/#{plugin}" } end - # Track whether or not there are any errors from any of the files that - # we take action on so that we can properly clean up and exit. - errored = false - - each_file(arguments) do |handler, filepath, source| - action.run(handler, filepath, source) - rescue Parser::ParseError => error - warn("Error: #{error.message}") - highlight_error(error, source) - errored = true - rescue Check::UnformattedError, Debug::NonIdempotentFormatError - errored = true - rescue StandardError => error - warn(error.message) - warn(error.backtrace) - errored = true + # We're going to build up a queue of items to process. + queue = Queue.new + + # If we're reading from stdin, then we'll just add the stdin object to + # the queue. Otherwise, we'll add each of the filepaths to the queue. + if $stdin.tty? || arguments.any? + arguments.each do |pattern| + Dir + .glob(pattern) + .each do |filepath| + queue << FileItem.new(filepath) if File.file?(filepath) + end + end + else + queue << STDINItem.new end + # At the end, we're going to return whether or not this worker ever + # encountered an error. + errored = + with_workers(queue) do |item| + action.run(item) + false + rescue Parser::ParseError => error + warn("Error: #{error.message}") + highlight_error(error, item.source) + true + rescue Check::UnformattedError, Debug::NonIdempotentFormatError + true + rescue StandardError => error + warn(error.message) + warn(error.backtrace) + true + end + if errored action.failure 1 @@ -287,22 +342,33 @@ def run(argv) private - def each_file(arguments) - if $stdin.tty? || arguments.any? - arguments.each do |pattern| - Dir - .glob(pattern) - .each do |filepath| - next unless File.file?(filepath) - - handler = HANDLERS[File.extname(filepath)] - source = handler.read(filepath) - yield handler, filepath, source - end + def with_workers(queue) + # If the queue is just 1 item, then we're not going to bother going + # through the whole ceremony of parallelizing the work. + return yield queue.shift if queue.size == 1 + + workers = + Etc.nprocessors.times.map do + Thread.new do + # Propagate errors in the worker threads up to the parent thread. + Thread.current.abort_on_exception = true + + # Track whether or not there are any errors from any of the files + # that we take action on so that we can properly clean up and + # exit. + errored = false + + # While there is still work left to do, shift off the queue and + # process the item. + (errored ||= yield queue.shift) until queue.empty? + + # At the end, we're going to return whether or not this worker + # ever encountered an error. + errored + end end - else - yield HANDLERS[".rb"], :stdin, $stdin.read - end + + workers.inject(false) { |accum, thread| accum || thread.value } end # Highlights a snippet from a source and parse error. From 531c6f9f9de787495dd29fd2ab8527c7ef0b0bb4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jun 2022 17:38:36 +0000 Subject: [PATCH 040/536] Bump rubocop from 1.30.0 to 1.30.1 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.30.0 to 1.30.1. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.30.0...v1.30.1) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index f8898163..b1398e59 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -16,9 +16,9 @@ GEM prettier_print (0.1.0) rainbow (3.1.1) rake (13.0.6) - regexp_parser (2.4.0) + regexp_parser (2.5.0) rexml (3.2.5) - rubocop (1.30.0) + rubocop (1.30.1) parallel (~> 1.10) parser (>= 3.1.0.0) rainbow (>= 2.2.2, < 4.0) From 640db96e0d19f634bc6b237c57a6de57b787b901 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 10 Jun 2022 11:14:57 -0400 Subject: [PATCH 041/536] Fix up did_you_mean nastiness --- lib/syntax_tree/basic_visitor.rb | 6 +++++- test/visitor_test.rb | 15 ++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/lib/syntax_tree/basic_visitor.rb b/lib/syntax_tree/basic_visitor.rb index 1ad6a80f..9e6a84c1 100644 --- a/lib/syntax_tree/basic_visitor.rb +++ b/lib/syntax_tree/basic_visitor.rb @@ -33,7 +33,11 @@ def corrections ).correct(visit_method) end - DidYouMean.correct_error(VisitMethodError, self) + # In some setups with Ruby you can turn off DidYouMean, so we're going to + # respect that setting here. + if defined?(DidYouMean) && DidYouMean.method_defined?(:correct_error) + DidYouMean.correct_error(VisitMethodError, self) + end end class << self diff --git a/test/visitor_test.rb b/test/visitor_test.rb index 5e4f134d..27bad364 100644 --- a/test/visitor_test.rb +++ b/test/visitor_test.rb @@ -40,9 +40,18 @@ def initialize end end - def test_visit_method_correction - error = assert_raises { Visitor.visit_method(:visit_binar) } - assert_match(/visit_binary/, error.message) + if defined?(DidYouMean) && DidYouMean.method_defined?(:correct_error) + def test_visit_method_correction + error = assert_raises { Visitor.visit_method(:visit_binar) } + message = + if Exception.method_defined?(:detailed_message) + error.detailed_message + else + error.message + end + + assert_match(/visit_binary/, message) + end end end end From 3c98f3665fb213621400393ad926e97e54a80bff Mon Sep 17 00:00:00 2001 From: Maple Ong Date: Fri, 10 Jun 2022 11:20:53 -0400 Subject: [PATCH 042/536] Add new node, HeredocEnd --- lib/syntax_tree/node.rb | 45 ++++++++++++++++++++++-- lib/syntax_tree/parser.rb | 12 ++++++- lib/syntax_tree/visitor.rb | 3 ++ lib/syntax_tree/visitor/field_visitor.rb | 4 +++ 4 files changed, 60 insertions(+), 4 deletions(-) diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 85956c8c..58335b00 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -4813,7 +4813,7 @@ class Heredoc < Node # [HeredocBeg] the opening of the heredoc attr_reader :beginning - # [String] the ending of the heredoc + # [HeredocEnd] the ending of the heredoc attr_reader :ending # [Integer] how far to dedent the heredoc @@ -4847,7 +4847,7 @@ def accept(visitor) end def child_nodes - [beginning, *parts] + [beginning, *parts, ending] end alias deconstruct child_nodes @@ -4883,7 +4883,7 @@ def format(q) end end - q.text(ending) + q.format(ending) end end end @@ -4929,6 +4929,45 @@ def format(q) end end + # HeredocEnd represents the closing declaration of a heredoc. + # + # <<~DOC + # contents + # DOC + # + # In the example above the HeredocEnd node represents the closing DOC. + class HeredocEnd < Node + # [String] the closing declaration of the heredoc + attr_reader :value + + # [Array[ Comment | EmbDoc ]] the comments attached to this node + attr_reader :comments + + def initialize(value:, location:, comments: []) + @value = value + @location = location + @comments = comments + end + + def accept(visitor) + visitor.visit_heredoc_end(self) + end + + def child_nodes + [] + end + + alias deconstruct child_nodes + + def deconstruct_keys(_keys) + { value: value, location: location, comments: comments } + end + + def format(q) + q.text(value) + end + end + # HshPtn represents matching against a hash pattern using the Ruby 2.7+ # pattern matching syntax. # diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index fdffbeb9..0f8332b1 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -1640,9 +1640,19 @@ def on_heredoc_dedent(string, width) def on_heredoc_end(value) heredoc = @heredocs[-1] + location = + Location.token( + line: lineno, + char: char_pos, + column: current_column, + size: value.size + 1 + ) + + heredoc_end = HeredocEnd.new(value: value.chomp, location: location) + @heredocs[-1] = Heredoc.new( beginning: heredoc.beginning, - ending: value.chomp, + ending: heredoc_end, dedent: heredoc.dedent, parts: heredoc.parts, location: diff --git a/lib/syntax_tree/visitor.rb b/lib/syntax_tree/visitor.rb index 348a05a2..e3b52077 100644 --- a/lib/syntax_tree/visitor.rb +++ b/lib/syntax_tree/visitor.rb @@ -194,6 +194,9 @@ class Visitor < BasicVisitor # Visit a HeredocBeg node. alias visit_heredoc_beg visit_child_nodes + # Visit a HeredocEnd node. + alias visit_heredoc_end visit_child_nodes + # Visit a HshPtn node. alias visit_hshptn visit_child_nodes diff --git a/lib/syntax_tree/visitor/field_visitor.rb b/lib/syntax_tree/visitor/field_visitor.rb index 1cc74f3d..6c5c6139 100644 --- a/lib/syntax_tree/visitor/field_visitor.rb +++ b/lib/syntax_tree/visitor/field_visitor.rb @@ -497,6 +497,10 @@ def visit_heredoc_beg(node) visit_token(node, "heredoc_beg") end + def visit_heredoc_end(node) + visit_token(node, "heredoc_end") + end + def visit_hshptn(node) node(node, "hshptn") do field("constant", node.constant) if node.constant From 0276c4458ef907d765eb05a7aaef4fa0a9448615 Mon Sep 17 00:00:00 2001 From: Maple Ong Date: Fri, 10 Jun 2022 11:21:00 -0400 Subject: [PATCH 043/536] Add test for HeredocEnd --- test/node_test.rb | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/node_test.rb b/test/node_test.rb index ffd00fa5..30776f9d 100644 --- a/test/node_test.rb +++ b/test/node_test.rb @@ -546,6 +546,17 @@ def test_heredoc_beg assert_node(HeredocBeg, source, at: at, &:beginning) end + def test_heredoc_end + source = <<~SOURCE + <<~HEREDOC + contents + HEREDOC + SOURCE + + at = location(lines: 3..3, chars: 22..31, columns: 0..9) + assert_node(HeredocEnd, source, at: at, &:ending) + end + def test_hshptn source = <<~SOURCE case value From b9b8d2ffdfaf38fc3ae3ebb5cea063ebfb8f308a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 15 Jun 2022 17:28:54 +0000 Subject: [PATCH 044/536] Bump minitest from 5.15.0 to 5.16.0 Bumps [minitest](https://github.com/seattlerb/minitest) from 5.15.0 to 5.16.0. - [Release notes](https://github.com/seattlerb/minitest/releases) - [Changelog](https://github.com/minitest/minitest/blob/master/History.rdoc) - [Commits](https://github.com/seattlerb/minitest/compare/v5.15.0...v5.16.0) --- updated-dependencies: - dependency-name: minitest dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index b1398e59..d610a836 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -9,7 +9,7 @@ GEM specs: ast (2.4.2) docile (1.4.0) - minitest (5.15.0) + minitest (5.16.0) parallel (1.22.1) parser (3.1.2.0) ast (~> 2.4.1) From ac16838709dedf8a82dc8546ffb4cd445893ce25 Mon Sep 17 00:00:00 2001 From: Tony Spataro Date: Sun, 19 Jun 2022 15:06:51 -0700 Subject: [PATCH 045/536] Respond to LSP Shutdown command to prevent VS Code deadlocks --- lib/syntax_tree/language_server.rb | 3 ++- test/language_server_test.rb | 36 +++++++++++++++++++++--------- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/lib/syntax_tree/language_server.rb b/lib/syntax_tree/language_server.rb index 3853ee18..31c91f9c 100644 --- a/lib/syntax_tree/language_server.rb +++ b/lib/syntax_tree/language_server.rb @@ -36,8 +36,9 @@ def run write(id: id, result: { capabilities: capabilities }) in method: "initialized" # ignored - in method: "shutdown" + in method: "shutdown" # tolerate missing ID to be a good citizen store.clear + write(id: request[:id], result: {}) return in { method: "textDocument/didChange", diff --git a/test/language_server_test.rb b/test/language_server_test.rb index f8a61003..519bada3 100644 --- a/test/language_server_test.rb +++ b/test/language_server_test.rb @@ -11,9 +11,9 @@ def to_hash end end - class Shutdown + class Shutdown < Struct.new(:id) def to_hash - { method: "shutdown" } + { method: "shutdown", id: id } end end @@ -107,13 +107,14 @@ def test_formatting TextDocumentDidChange.new("file:///path/to/file.rb", "class Bar; end"), TextDocumentFormatting.new(2, "file:///path/to/file.rb"), TextDocumentDidClose.new("file:///path/to/file.rb"), - Shutdown.new + Shutdown.new(3) ] case run_server(messages) in [ { id: 1, result: { capabilities: Hash } }, - { id: 2, result: [{ newText: new_text }] } + { id: 2, result: [{ newText: new_text }] }, + { id: 3, result: {} } ] assert_equal("class Bar\nend\n", new_text) end @@ -129,13 +130,14 @@ def test_inlay_hints end RUBY TextDocumentInlayHints.new(2, "file:///path/to/file.rb"), - Shutdown.new + Shutdown.new(3) ] case run_server(messages) in [ { id: 1, result: { capabilities: Hash } }, - { id: 2, result: { before:, after: } } + { id: 2, result: { before:, after: } }, + { id: 3, result: {} } ] assert_equal(1, before.length) assert_equal(2, after.length) @@ -147,11 +149,15 @@ def test_visualizing Initialize.new(1), TextDocumentDidOpen.new("file:///path/to/file.rb", "1 + 2"), SyntaxTreeVisualizing.new(2, "file:///path/to/file.rb"), - Shutdown.new + Shutdown.new(3) ] case run_server(messages) - in [{ id: 1, result: { capabilities: Hash } }, { id: 2, result: }] + in [ + { id: 1, result: { capabilities: Hash } }, + { id: 2, result: }, + { id: 3, result: {} } + ] assert_equal( "(program (statements ((binary (int \"1\") + (int \"2\")))))\n", result @@ -167,13 +173,14 @@ def test_reading_file messages = [ Initialize.new(1), TextDocumentFormatting.new(2, "file://#{file.path}"), - Shutdown.new + Shutdown.new(3) ] case run_server(messages) in [ { id: 1, result: { capabilities: Hash } }, - { id: 2, result: [{ newText: new_text }] } + { id: 2, result: [{ newText: new_text }] }, + { id: 3, result: {} } ] assert_equal("class Foo\nend\n", new_text) end @@ -186,6 +193,15 @@ def test_bogus_request end end + def test_clean_shutdown + messages = [Initialize.new(1), Shutdown.new(2)] + + case run_server(messages) + in [{ id: 1, result: { capabilities: Hash } }, { id: 2, result: {} }] + assert_equal(true, true) + end + end + private def write(content) From 480f095b658059ac880d3c8f0e91b4facd7f718d Mon Sep 17 00:00:00 2001 From: Tony Spataro Date: Sat, 18 Jun 2022 16:23:37 -0700 Subject: [PATCH 046/536] Allow CLI flags for lsp --- lib/syntax_tree/cli.rb | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/lib/syntax_tree/cli.rb b/lib/syntax_tree/cli.rb index e5bf8cf0..a7c6a684 100644 --- a/lib/syntax_tree/cli.rb +++ b/lib/syntax_tree/cli.rb @@ -218,7 +218,7 @@ def run(item) #{Color.bold("stree help")} Display this help message - #{Color.bold("stree lsp")} + #{Color.bold("stree lsp [OPTIONS]")} Run syntax tree in language server mode #{Color.bold("stree version")} @@ -239,6 +239,20 @@ class << self def run(argv) name, *arguments = argv + # If there are any plugins specified on the command line, then load them + # by requiring them here. We do this by transforming something like + # + # stree format --plugins=haml template.haml + # + # into + # + # require "syntax_tree/haml" + # + if arguments.first&.start_with?("--plugins=") + plugins = arguments.shift[/^--plugins=(.*)$/, 1] + plugins.split(",").each { |plugin| require "syntax_tree/#{plugin}" } + end + case name when "help" puts HELP @@ -282,20 +296,6 @@ def run(argv) return 1 end - # If there are any plugins specified on the command line, then load them - # by requiring them here. We do this by transforming something like - # - # stree format --plugins=haml template.haml - # - # into - # - # require "syntax_tree/haml" - # - if arguments.first&.start_with?("--plugins=") - plugins = arguments.shift[/^--plugins=(.*)$/, 1] - plugins.split(",").each { |plugin| require "syntax_tree/#{plugin}" } - end - # We're going to build up a queue of items to process. queue = Queue.new From 653b7260759c9ff9600c36004b1baa43e7712ad2 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 21 Jun 2022 13:16:32 -0400 Subject: [PATCH 047/536] Bump to version 2.8.0 --- CHANGELOG.md | 16 +++++++++++++++- Gemfile.lock | 2 +- lib/syntax_tree/version.rb | 2 +- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa928c0c..95d1f92c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,19 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +## [2.8.0] - 2022-06-21 + +### Added + +- [#95](https://github.com/ruby-syntax-tree/syntax_tree/pull/95) - The `HeredocEnd` node has been added which effectively results in the ability to determine the location of the ending of a heredoc from source. +- [#99](https://github.com/ruby-syntax-tree/syntax_tree/pull/99) - The LSP now allows you to pass the same configuration options as the other CLI commands which allows formatting to be modified in the VSCode extension. +- [#100](https://github.com/ruby-syntax-tree/syntax_tree/pull/100) - The LSP now explicitly responds to the shutdown request so that VSCode never deadlocks. + +### Changed + +- [#96](https://github.com/ruby-syntax-tree/syntax_tree/pull/96) - The CLI now runs in parallel by default. There is a worker created for each processor on the running machine (as determined by `Etc.nprocessors`). +- [#97](https://github.com/ruby-syntax-tree/syntax_tree/pull/97) - Syntax Tree now handles the case where `DidYouMean` is not available for whatever reason, as well as handles the newer `detailed_message` API for errors. + ## [2.7.1] - 2022-05-25 ### Added @@ -259,7 +272,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - 🎉 Initial release! 🎉 -[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.7.1...HEAD +[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.8.0...HEAD +[2.8.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.7.1...v2.8.0 [2.7.1]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.7.0...v2.7.1 [2.7.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.6.0...v2.7.0 [2.6.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.5.0...v2.6.0 diff --git a/Gemfile.lock b/Gemfile.lock index d610a836..92ad8d38 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - syntax_tree (2.7.1) + syntax_tree (2.8.0) prettier_print GEM diff --git a/lib/syntax_tree/version.rb b/lib/syntax_tree/version.rb index 7754cf7a..881c65aa 100644 --- a/lib/syntax_tree/version.rb +++ b/lib/syntax_tree/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SyntaxTree - VERSION = "2.7.1" + VERSION = "2.8.0" end From 9b1e8e1daca6568c04f0ee7b2181c226dad3bc33 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Jun 2022 17:52:56 +0000 Subject: [PATCH 048/536] Bump minitest from 5.16.0 to 5.16.1 Bumps [minitest](https://github.com/seattlerb/minitest) from 5.16.0 to 5.16.1. - [Release notes](https://github.com/seattlerb/minitest/releases) - [Changelog](https://github.com/minitest/minitest/blob/master/History.rdoc) - [Commits](https://github.com/seattlerb/minitest/compare/v5.16.0...v5.16.1) --- updated-dependencies: - dependency-name: minitest dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 92ad8d38..116a6547 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -9,7 +9,7 @@ GEM specs: ast (2.4.2) docile (1.4.0) - minitest (5.16.0) + minitest (5.16.1) parallel (1.22.1) parser (3.1.2.0) ast (~> 2.4.1) From a9ad4b45781a706f48ce4a473307973b0464669a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jun 2022 17:48:46 +0000 Subject: [PATCH 049/536] Bump rubocop from 1.30.1 to 1.31.0 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.30.1 to 1.31.0. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.30.1...v1.31.0) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 116a6547..a5f6f44a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -18,7 +18,7 @@ GEM rake (13.0.6) regexp_parser (2.5.0) rexml (3.2.5) - rubocop (1.30.1) + rubocop (1.31.0) parallel (~> 1.10) parser (>= 3.1.0.0) rainbow (>= 2.2.2, < 4.0) @@ -36,7 +36,7 @@ GEM simplecov_json_formatter (~> 0.1) simplecov-html (0.12.3) simplecov_json_formatter (0.1.4) - unicode-display_width (2.1.0) + unicode-display_width (2.2.0) PLATFORMS arm64-darwin-21 From 2e85a4934e8193c7210c5088225d30dd7f164ffd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 29 Jun 2022 17:37:13 +0000 Subject: [PATCH 050/536] Bump rubocop from 1.31.0 to 1.31.1 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.31.0 to 1.31.1. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.31.0...v1.31.1) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index a5f6f44a..b1b89bfd 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -9,6 +9,7 @@ GEM specs: ast (2.4.2) docile (1.4.0) + json (2.6.2) minitest (5.16.1) parallel (1.22.1) parser (3.1.2.0) @@ -18,7 +19,8 @@ GEM rake (13.0.6) regexp_parser (2.5.0) rexml (3.2.5) - rubocop (1.31.0) + rubocop (1.31.1) + json (~> 2.3) parallel (~> 1.10) parser (>= 3.1.0.0) rainbow (>= 2.2.2, < 4.0) From ba2a6e65a546e30169abefca6a407e3d15794813 Mon Sep 17 00:00:00 2001 From: Tony Spataro Date: Sun, 3 Jul 2022 15:51:36 -0700 Subject: [PATCH 051/536] Add inlay hint support per LSP specification. Closes #103. --- lib/syntax_tree/language_server.rb | 27 +++++++++++++---- .../language_server/inlay_hints.rb | 30 ++++++++++++++++--- test/language_server/inlay_hints_test.rb | 6 ++++ 3 files changed, 53 insertions(+), 10 deletions(-) diff --git a/lib/syntax_tree/language_server.rb b/lib/syntax_tree/language_server.rb index 31c91f9c..1f06b48e 100644 --- a/lib/syntax_tree/language_server.rb +++ b/lib/syntax_tree/language_server.rb @@ -61,11 +61,19 @@ def run } write(id: id, result: [format(store[uri])]) in { + # official RPC in LSP spec 3.17 + method: "textDocument/inlayHint", + id:, + params: { textDocument: { uri: } } + } + write(id: id, result: inlay_hints(store[uri], false)) + in { + # proprietary RPC (deprecated) between this gem and vscode-syntax-tree method: "textDocument/inlayHints", id:, params: { textDocument: { uri: } } } - write(id: id, result: inlay_hints(store[uri])) + write(id: id, result: inlay_hints(store[uri], true)) in { method: "syntaxTree/visualizing", id:, @@ -85,6 +93,9 @@ def run def capabilities { documentFormattingProvider: true, + inlayHintProvider: { + resolveProvider: false + }, textDocumentSync: { change: 1, openClose: true @@ -108,14 +119,18 @@ def format(source) } end - def inlay_hints(source) + def inlay_hints(source, proprietary) inlay_hints = InlayHints.find(SyntaxTree.parse(source)) serialize = ->(position, text) { { position: position, text: text } } - { - before: inlay_hints.before.map(&serialize), - after: inlay_hints.after.map(&serialize) - } + if proprietary + { + before: inlay_hints.before.map(&serialize), + after: inlay_hints.after.map(&serialize) + } + else + inlay_hints.all + end rescue Parser::ParseError # If there is a parse error, then we're not going to return any inlay # hints for this source. diff --git a/lib/syntax_tree/language_server/inlay_hints.rb b/lib/syntax_tree/language_server/inlay_hints.rb index 089355a7..4be8a765 100644 --- a/lib/syntax_tree/language_server/inlay_hints.rb +++ b/lib/syntax_tree/language_server/inlay_hints.rb @@ -2,18 +2,19 @@ module SyntaxTree class LanguageServer - # This class provides inlay hints for the language server. It is loosely - # designed around the LSP spec, but existed before the spec was finalized so - # is a little different for now. + # This class provides inlay hints for the language server. It existed + # before the spec was finalized so, so it provides two result formats: + # aligned with the spec (`#all`) and proprietary (`#before` and `#after`). # # For more information, see the spec here: # https://github.com/microsoft/language-server-protocol/issues/956. # class InlayHints < Visitor - attr_reader :stack, :before, :after + attr_reader :stack, :all, :before, :after def initialize @stack = [] + @all = [] @before = Hash.new { |hash, key| hash[key] = +"" } @after = Hash.new { |hash, key| hash[key] = +"" } end @@ -98,6 +99,13 @@ def visit_if_op(node) def visit_rescue(node) if node.exception.nil? after[node.location.start_char + "rescue".length] << " StandardError" + all << { + position: { + line: node.location.start_line - 1, + character: node.location.start_column + "rescue".length + }, + label: " StandardError" + } end super @@ -129,6 +137,20 @@ def self.find(program) private def parentheses(location) + all << { + position: { + line: location.start_line - 1, + character: location.start_column + }, + label: "₍" + } + all << { + position: { + line: location.end_line - 1, + character: location.end_column + }, + label: "₎" + } before[location.start_char] << "₍" after[location.end_char] << "₎" end diff --git a/test/language_server/inlay_hints_test.rb b/test/language_server/inlay_hints_test.rb index f652f6d8..35db365a 100644 --- a/test/language_server/inlay_hints_test.rb +++ b/test/language_server/inlay_hints_test.rb @@ -11,6 +11,7 @@ def test_assignments_in_parameters assert_equal(1, hints.before.length) assert_equal(1, hints.after.length) + assert_equal(2, hints.all.length) end def test_operators_in_binaries @@ -18,6 +19,7 @@ def test_operators_in_binaries assert_equal(1, hints.before.length) assert_equal(1, hints.after.length) + assert_equal(2, hints.all.length) end def test_binaries_in_assignments @@ -25,6 +27,7 @@ def test_binaries_in_assignments assert_equal(1, hints.before.length) assert_equal(1, hints.after.length) + assert_equal(2, hints.all.length) end def test_nested_ternaries @@ -32,12 +35,14 @@ def test_nested_ternaries assert_equal(1, hints.before.length) assert_equal(1, hints.after.length) + assert_equal(2, hints.all.length) end def test_bare_rescue hints = find("begin; rescue; end") assert_equal(1, hints.after.length) + assert_equal(1, hints.all.length) end def test_unary_in_binary @@ -45,6 +50,7 @@ def test_unary_in_binary assert_equal(1, hints.before.length) assert_equal(1, hints.after.length) + assert_equal(2, hints.all.length) end private From 0d59abea754f4dcd9919cf1fa5b49b4f5c4fd63e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Jul 2022 17:26:55 +0000 Subject: [PATCH 052/536] Bump minitest from 5.16.1 to 5.16.2 Bumps [minitest](https://github.com/seattlerb/minitest) from 5.16.1 to 5.16.2. - [Release notes](https://github.com/seattlerb/minitest/releases) - [Changelog](https://github.com/minitest/minitest/blob/master/History.rdoc) - [Commits](https://github.com/seattlerb/minitest/compare/v5.16.1...v5.16.2) --- updated-dependencies: - dependency-name: minitest dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index b1b89bfd..7ad92e0b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -10,7 +10,7 @@ GEM ast (2.4.2) docile (1.4.0) json (2.6.2) - minitest (5.16.1) + minitest (5.16.2) parallel (1.22.1) parser (3.1.2.0) ast (~> 2.4.1) From 201d27dce2f0d3149d5a1b7b31448b62cc02d8d0 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 4 Jul 2022 19:39:55 -0400 Subject: [PATCH 053/536] Bump to v2.9.0 --- CHANGELOG.md | 7 ++++++- Gemfile.lock | 2 +- lib/syntax_tree/version.rb | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95d1f92c..2c5e8bbd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +## [2.9.0] - 2022-07-04 + +- [#106](https://github.com/ruby-syntax-tree/syntax_tree/pull/106) - Add inlay hint support to match the LSP specification. + ## [2.8.0] - 2022-06-21 ### Added @@ -272,7 +276,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - 🎉 Initial release! 🎉 -[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.8.0...HEAD +[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.1.0...HEAD +[2.1.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.8.0...v2.1.0 [2.8.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.7.1...v2.8.0 [2.7.1]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.7.0...v2.7.1 [2.7.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.6.0...v2.7.0 diff --git a/Gemfile.lock b/Gemfile.lock index 7ad92e0b..effdb9aa 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - syntax_tree (2.8.0) + syntax_tree (2.9.0) prettier_print GEM diff --git a/lib/syntax_tree/version.rb b/lib/syntax_tree/version.rb index 881c65aa..5622a4da 100644 --- a/lib/syntax_tree/version.rb +++ b/lib/syntax_tree/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SyntaxTree - VERSION = "2.8.0" + VERSION = "2.9.0" end From 5ee15c7a6eb2bea96409f74fbb3c3b4ec76dbcd5 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 4 Jul 2022 20:01:17 -0400 Subject: [PATCH 054/536] Remove old inlay hints code --- README.md | 6 +- lib/syntax_tree/language_server.rb | 26 ++----- .../language_server/inlay_hints.rb | 76 ++++++++++--------- lib/syntax_tree/parser.rb | 12 +-- test/language_server/inlay_hints_test.rb | 42 +++------- test/language_server_test.rb | 13 ++-- 6 files changed, 71 insertions(+), 104 deletions(-) diff --git a/README.md b/README.md index e3e995cf..4c472e37 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ It is built with only standard library dependencies. It additionally ships with - [BasicVisitor](#basicvisitor) - [Language server](#language-server) - [textDocument/formatting](#textdocumentformatting) - - [textDocument/inlayHints](#textdocumentinlayhints) + - [textDocument/inlayHint](#textdocumentinlayhint) - [syntaxTree/visualizing](#syntaxtreevisualizing) - [Plugins](#plugins) - [Configuration](#configuration) @@ -402,7 +402,7 @@ By default, the language server is relatively minimal, mostly meant to provide a As mentioned above, the language server responds to formatting requests with the formatted document. It typically responds on the order of tens of milliseconds, so it should be fast enough for any IDE. -### textDocument/inlayHints +### textDocument/inlayHint The language server also responds to the relatively new inlay hints request. This request allows the language server to define additional information that should exist in the source code as helpful hints to the developer. In our case we use it to display things like implicit parentheses. For example, if you had the following code: @@ -410,7 +410,7 @@ The language server also responds to the relatively new inlay hints request. Thi 1 + 2 * 3 ``` -Implicity, the `2 * 3` is going to be executed first because the `*` operator has higher precedence than the `+` operator. However, to ease mental overhead, our language server includes small parentheses to make this explicit, as in: +Implicity, the `2 * 3` is going to be executed first because the `*` operator has higher precedence than the `+` operator. To ease mental overhead, our language server includes small parentheses to make this explicit, as in: ```ruby 1 + ₍2 * 3₎ diff --git a/lib/syntax_tree/language_server.rb b/lib/syntax_tree/language_server.rb index 1f06b48e..e42c77fd 100644 --- a/lib/syntax_tree/language_server.rb +++ b/lib/syntax_tree/language_server.rb @@ -66,14 +66,7 @@ def run id:, params: { textDocument: { uri: } } } - write(id: id, result: inlay_hints(store[uri], false)) - in { - # proprietary RPC (deprecated) between this gem and vscode-syntax-tree - method: "textDocument/inlayHints", - id:, - params: { textDocument: { uri: } } - } - write(id: id, result: inlay_hints(store[uri], true)) + write(id: id, result: inlay_hints(store[uri])) in { method: "syntaxTree/visualizing", id:, @@ -119,21 +112,14 @@ def format(source) } end - def inlay_hints(source, proprietary) - inlay_hints = InlayHints.find(SyntaxTree.parse(source)) - serialize = ->(position, text) { { position: position, text: text } } - - if proprietary - { - before: inlay_hints.before.map(&serialize), - after: inlay_hints.after.map(&serialize) - } - else - inlay_hints.all - end + def inlay_hints(source) + visitor = InlayHints.new + SyntaxTree.parse(source).accept(visitor) + visitor.hints rescue Parser::ParseError # If there is a parse error, then we're not going to return any inlay # hints for this source. + [] end def write(value) diff --git a/lib/syntax_tree/language_server/inlay_hints.rb b/lib/syntax_tree/language_server/inlay_hints.rb index 4be8a765..12c10230 100644 --- a/lib/syntax_tree/language_server/inlay_hints.rb +++ b/lib/syntax_tree/language_server/inlay_hints.rb @@ -2,21 +2,37 @@ module SyntaxTree class LanguageServer - # This class provides inlay hints for the language server. It existed - # before the spec was finalized so, so it provides two result formats: - # aligned with the spec (`#all`) and proprietary (`#before` and `#after`). - # - # For more information, see the spec here: + # This class provides inlay hints for the language server. For more + # information, see the spec here: # https://github.com/microsoft/language-server-protocol/issues/956. - # class InlayHints < Visitor - attr_reader :stack, :all, :before, :after + # This represents a hint that is going to be displayed in the editor. + class Hint + attr_reader :line, :character, :label + + def initialize(line:, character:, label:) + @line = line + @character = character + @label = label + end + + # This is the shape that the LSP expects. + def to_json(*opts) + { + position: { + line: line, + character: character + }, + label: label + }.to_json(*opts) + end + end + + attr_reader :stack, :hints def initialize @stack = [] - @all = [] - @before = Hash.new { |hash, key| hash[key] = +"" } - @after = Hash.new { |hash, key| hash[key] = +"" } + @hints = [] end def visit(node) @@ -98,14 +114,11 @@ def visit_if_op(node) # def visit_rescue(node) if node.exception.nil? - after[node.location.start_char + "rescue".length] << " StandardError" - all << { - position: { - line: node.location.start_line - 1, - character: node.location.start_column + "rescue".length - }, + hints << Hint.new( + line: node.location.start_line - 1, + character: node.location.start_column + "rescue".length, label: " StandardError" - } + ) end super @@ -128,31 +141,20 @@ def visit_unary(node) super end - def self.find(program) - visitor = new - visitor.visit(program) - visitor - end - private def parentheses(location) - all << { - position: { - line: location.start_line - 1, - character: location.start_column - }, + hints << Hint.new( + line: location.start_line - 1, + character: location.start_column, label: "₍" - } - all << { - position: { - line: location.end_line - 1, - character: location.end_column - }, + ) + + hints << Hint.new( + line: location.end_line - 1, + character: location.end_column, label: "₎" - } - before[location.start_char] << "₍" - after[location.end_char] << "₎" + ) end end end diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index 0f8332b1..6e6e4b1c 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -1641,12 +1641,12 @@ def on_heredoc_end(value) heredoc = @heredocs[-1] location = - Location.token( - line: lineno, - char: char_pos, - column: current_column, - size: value.size + 1 - ) + Location.token( + line: lineno, + char: char_pos, + column: current_column, + size: value.size + 1 + ) heredoc_end = HeredocEnd.new(value: value.chomp, location: location) diff --git a/test/language_server/inlay_hints_test.rb b/test/language_server/inlay_hints_test.rb index 35db365a..d3741894 100644 --- a/test/language_server/inlay_hints_test.rb +++ b/test/language_server/inlay_hints_test.rb @@ -7,56 +7,36 @@ module SyntaxTree class LanguageServer class InlayHintsTest < Minitest::Test def test_assignments_in_parameters - hints = find("def foo(a = b = c); end") - - assert_equal(1, hints.before.length) - assert_equal(1, hints.after.length) - assert_equal(2, hints.all.length) + assert_hints(2, "def foo(a = b = c); end") end def test_operators_in_binaries - hints = find("1 + 2 * 3") - - assert_equal(1, hints.before.length) - assert_equal(1, hints.after.length) - assert_equal(2, hints.all.length) + assert_hints(2, "1 + 2 * 3") end def test_binaries_in_assignments - hints = find("a = 1 + 2") - - assert_equal(1, hints.before.length) - assert_equal(1, hints.after.length) - assert_equal(2, hints.all.length) + assert_hints(2, "a = 1 + 2") end def test_nested_ternaries - hints = find("a ? b : c ? d : e") - - assert_equal(1, hints.before.length) - assert_equal(1, hints.after.length) - assert_equal(2, hints.all.length) + assert_hints(2, "a ? b : c ? d : e") end def test_bare_rescue - hints = find("begin; rescue; end") - - assert_equal(1, hints.after.length) - assert_equal(1, hints.all.length) + assert_hints(1, "begin; rescue; end") end def test_unary_in_binary - hints = find("-a + b") - - assert_equal(1, hints.before.length) - assert_equal(1, hints.after.length) - assert_equal(2, hints.all.length) + assert_hints(2, "-a + b") end private - def find(source) - InlayHints.find(SyntaxTree.parse(source)) + def assert_hints(expected, source) + visitor = InlayHints.new + SyntaxTree.parse(source).accept(visitor) + + assert_equal(expected, visitor.hints.length) end end end diff --git a/test/language_server_test.rb b/test/language_server_test.rb index 519bada3..0ea8b955 100644 --- a/test/language_server_test.rb +++ b/test/language_server_test.rb @@ -72,10 +72,10 @@ def to_hash end end - class TextDocumentInlayHints < Struct.new(:id, :uri) + class TextDocumentInlayHint < Struct.new(:id, :uri) def to_hash { - method: "textDocument/inlayHints", + method: "textDocument/inlayHint", id: id, params: { textDocument: { @@ -120,7 +120,7 @@ def test_formatting end end - def test_inlay_hints + def test_inlay_hint messages = [ Initialize.new(1), TextDocumentDidOpen.new("file:///path/to/file.rb", <<~RUBY), @@ -129,18 +129,17 @@ def test_inlay_hints rescue end RUBY - TextDocumentInlayHints.new(2, "file:///path/to/file.rb"), + TextDocumentInlayHint.new(2, "file:///path/to/file.rb"), Shutdown.new(3) ] case run_server(messages) in [ { id: 1, result: { capabilities: Hash } }, - { id: 2, result: { before:, after: } }, + { id: 2, result: hints }, { id: 3, result: {} } ] - assert_equal(1, before.length) - assert_equal(2, after.length) + assert_equal(3, hints.length) end end From f8839b45bbe7cfed4a018b6dd5b26af32d15fe87 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 4 Jul 2022 20:54:56 -0400 Subject: [PATCH 055/536] Fix for #102, handle files not existing --- lib/syntax_tree/formatter.rb | 4 ++- lib/syntax_tree/language_server.rb | 51 +++++++++++------------------- test/language_server_test.rb | 17 ++++++++++ 3 files changed, 38 insertions(+), 34 deletions(-) diff --git a/lib/syntax_tree/formatter.rb b/lib/syntax_tree/formatter.rb index 56de6a4a..6efad8d8 100644 --- a/lib/syntax_tree/formatter.rb +++ b/lib/syntax_tree/formatter.rb @@ -68,7 +68,9 @@ def format(node, stackable: true) # going to just print out the node as it was seen in the source. doc = if leading.last&.ignore? - text(source[node.location.start_char...node.location.end_char]) + range = source[node.location.start_char...node.location.end_char] + separator = -> { breakable(indent: false, force: true) } + seplist(range.split(/\r?\n/, -1), separator) { |line| text(line) } else node.format(self) end diff --git a/lib/syntax_tree/language_server.rb b/lib/syntax_tree/language_server.rb index e42c77fd..2eb8228b 100644 --- a/lib/syntax_tree/language_server.rb +++ b/lib/syntax_tree/language_server.rb @@ -20,66 +20,51 @@ def initialize(input: $stdin, output: $stdout) @output = output.binmode end + # rubocop:disable Layout/LineLength def run store = Hash.new do |hash, uri| - hash[uri] = File.binread(CGI.unescape(URI.parse(uri).path)) + filepath = CGI.unescape(URI.parse(uri).path) + File.exist?(filepath) ? (hash[uri] = File.read(filepath)) : nil end while (headers = input.gets("\r\n\r\n")) source = input.read(headers[/Content-Length: (\d+)/i, 1].to_i) request = JSON.parse(source, symbolize_names: true) + # stree-ignore case request in { method: "initialize", id: } store.clear write(id: id, result: { capabilities: capabilities }) - in method: "initialized" + in { method: "initialized" } # ignored - in method: "shutdown" # tolerate missing ID to be a good citizen + in { method: "shutdown" } # tolerate missing ID to be a good citizen store.clear write(id: request[:id], result: {}) return - in { - method: "textDocument/didChange", - params: { textDocument: { uri: }, contentChanges: [{ text: }, *] } - } + in { method: "textDocument/didChange", params: { textDocument: { uri: }, contentChanges: [{ text: }, *] } } store[uri] = text - in { - method: "textDocument/didOpen", - params: { textDocument: { uri:, text: } } - } + in { method: "textDocument/didOpen", params: { textDocument: { uri:, text: } } } store[uri] = text - in { - method: "textDocument/didClose", params: { textDocument: { uri: } } - } + in { method: "textDocument/didClose", params: { textDocument: { uri: } } } store.delete(uri) - in { - method: "textDocument/formatting", - id:, - params: { textDocument: { uri: } } - } - write(id: id, result: [format(store[uri])]) - in { - # official RPC in LSP spec 3.17 - method: "textDocument/inlayHint", - id:, - params: { textDocument: { uri: } } - } - write(id: id, result: inlay_hints(store[uri])) - in { - method: "syntaxTree/visualizing", - id:, - params: { textDocument: { uri: } } - } + in { method: "textDocument/formatting", id:, params: { textDocument: { uri: } } } + contents = store[uri] + write(id: id, result: contents ? [format(store[uri])] : nil) + in { method: "textDocument/inlayHint", id:, params: { textDocument: { uri: } } } + contents = store[uri] + write(id: id, result: contents ? inlay_hints(store[uri]) : nil) + in { method: "syntaxTree/visualizing", id:, params: { textDocument: { uri: } } } write(id: id, result: PP.pp(SyntaxTree.parse(store[uri]), +"")) - in method: %r{\$/.+} + in { method: %r{\$/.+} } # ignored else raise ArgumentError, "Unhandled: #{request}" end end end + # rubocop:enable Layout/LineLength private diff --git a/test/language_server_test.rb b/test/language_server_test.rb index 0ea8b955..fc26054d 100644 --- a/test/language_server_test.rb +++ b/test/language_server_test.rb @@ -201,6 +201,23 @@ def test_clean_shutdown end end + def test_file_that_does_not_exist + messages = [ + Initialize.new(1), + TextDocumentFormatting.new(2, "file:///path/to/file.rb"), + Shutdown.new(3) + ] + + case run_server(messages) + in [ + { id: 1, result: { capabilities: Hash } }, + { id: 2, result: nil }, + { id: 3, result: {} } + ] + assert_equal(true, true) + end + end + private def write(content) From 8a3fc6efaa2c203554988c7078e03d5f68d56420 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 4 Jul 2022 21:03:34 -0400 Subject: [PATCH 056/536] Update CHANGELOG links --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c5e8bbd..daf48cb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -276,8 +276,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - 🎉 Initial release! 🎉 -[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.1.0...HEAD -[2.1.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.8.0...v2.1.0 +[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.9.0...HEAD +[2.9.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.8.0...v2.9.0 [2.8.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.7.1...v2.8.0 [2.7.1]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.7.0...v2.7.1 [2.7.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.6.0...v2.7.0 From a16ffae4ad62c5bf9aa3adbdec43873bdb8c2add Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 4 Jul 2022 21:09:58 -0400 Subject: [PATCH 057/536] Bump to version 3.0.0 --- CHANGELOG.md | 12 ++++++++++++ Gemfile.lock | 2 +- lib/syntax_tree/version.rb | 2 +- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index daf48cb1..d5f9ba82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,20 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +## [3.0.0] - 2022-07-04 + +### Changed + +- [#102](https://github.com/ruby-syntax-tree/syntax_tree/issues/102) - Handle requests to the language server for files that do not yet exist on disk. + +### Removed + +- [#108](https://github.com/ruby-syntax-tree/syntax_tree/pull/108) - Remove old inlay hints code. + ## [2.9.0] - 2022-07-04 +### Added + - [#106](https://github.com/ruby-syntax-tree/syntax_tree/pull/106) - Add inlay hint support to match the LSP specification. ## [2.8.0] - 2022-06-21 diff --git a/Gemfile.lock b/Gemfile.lock index effdb9aa..62415795 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - syntax_tree (2.9.0) + syntax_tree (3.0.0) prettier_print GEM diff --git a/lib/syntax_tree/version.rb b/lib/syntax_tree/version.rb index 5622a4da..d3f929e6 100644 --- a/lib/syntax_tree/version.rb +++ b/lib/syntax_tree/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SyntaxTree - VERSION = "2.9.0" + VERSION = "3.0.0" end From e9ff7d6f63f16e418c9674fe6ef2914190cf0b0a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 7 Jul 2022 17:42:25 +0000 Subject: [PATCH 058/536] Bump rubocop from 1.31.1 to 1.31.2 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.31.1 to 1.31.2. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.31.1...v1.31.2) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 62415795..80ff09cc 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -19,7 +19,7 @@ GEM rake (13.0.6) regexp_parser (2.5.0) rexml (3.2.5) - rubocop (1.31.1) + rubocop (1.31.2) json (~> 2.3) parallel (~> 1.10) parser (>= 3.1.0.0) From bfb4a8424a3d73a70f56aff1cc147c777593ec96 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 15 Jul 2022 11:55:29 -0400 Subject: [PATCH 059/536] Fix parallel CLI execution When running the CLI in parallel, you can't use `||` on outputs because that will potentially not evaluate the RHS of the expression. Instead, we'll use `|` to combine outputs. --- lib/syntax_tree/cli.rb | 50 +++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/lib/syntax_tree/cli.rb b/lib/syntax_tree/cli.rb index a7c6a684..c8e42831 100644 --- a/lib/syntax_tree/cli.rb +++ b/lib/syntax_tree/cli.rb @@ -315,23 +315,7 @@ def run(argv) # At the end, we're going to return whether or not this worker ever # encountered an error. - errored = - with_workers(queue) do |item| - action.run(item) - false - rescue Parser::ParseError => error - warn("Error: #{error.message}") - highlight_error(error, item.source) - true - rescue Check::UnformattedError, Debug::NonIdempotentFormatError - true - rescue StandardError => error - warn(error.message) - warn(error.backtrace) - true - end - - if errored + if process_queue(queue, action) action.failure 1 else @@ -342,13 +326,11 @@ def run(argv) private - def with_workers(queue) - # If the queue is just 1 item, then we're not going to bother going - # through the whole ceremony of parallelizing the work. - return yield queue.shift if queue.size == 1 - + # Processes each item in the queue with the given action. Returns whether + # or not any errors were encountered. + def process_queue(queue, action) workers = - Etc.nprocessors.times.map do + [Etc.nprocessors, queue.size].min.times.map do Thread.new do # Propagate errors in the worker threads up to the parent thread. Thread.current.abort_on_exception = true @@ -360,7 +342,25 @@ def with_workers(queue) # While there is still work left to do, shift off the queue and # process the item. - (errored ||= yield queue.shift) until queue.empty? + until queue.empty? + item = queue.shift + errored |= + begin + action.run(item) + false + rescue Parser::ParseError => error + warn("Error: #{error.message}") + highlight_error(error, item.source) + true + rescue Check::UnformattedError, + Debug::NonIdempotentFormatError + true + rescue StandardError => error + warn(error.message) + warn(error.backtrace) + true + end + end # At the end, we're going to return whether or not this worker # ever encountered an error. @@ -368,7 +368,7 @@ def with_workers(queue) end end - workers.inject(false) { |accum, thread| accum || thread.value } + workers.map(&:value).inject(:|) end # Highlights a snippet from a source and parse error. From b5024c3733181f4d037cf4abeb2d11784e9b7dc2 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 15 Jul 2022 12:03:32 -0400 Subject: [PATCH 060/536] Bump to v3.0.1 --- CHANGELOG.md | 10 +++++++++- Gemfile.lock | 2 +- lib/syntax_tree/version.rb | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d5f9ba82..ee7261ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +## [3.0.1] - 2022-07-15 + +### Changed + +- [#112](https://github.com/ruby-syntax-tree/syntax_tree/pull/112) - Fix parallel CLI execution by not short-circuiting with the `||` operator. + ## [3.0.0] - 2022-07-04 ### Changed @@ -288,7 +294,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - 🎉 Initial release! 🎉 -[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.9.0...HEAD +[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.0.1...HEAD +[3.0.1]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.0.0...v3.0.1 +[3.0.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.9.0...v3.0.0 [2.9.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.8.0...v2.9.0 [2.8.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.7.1...v2.8.0 [2.7.1]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.7.0...v2.7.1 diff --git a/Gemfile.lock b/Gemfile.lock index 80ff09cc..ac7403ba 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - syntax_tree (3.0.0) + syntax_tree (3.0.1) prettier_print GEM diff --git a/lib/syntax_tree/version.rb b/lib/syntax_tree/version.rb index d3f929e6..3a740b84 100644 --- a/lib/syntax_tree/version.rb +++ b/lib/syntax_tree/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SyntaxTree - VERSION = "3.0.0" + VERSION = "3.0.1" end From 483bc026ec0266406b98c4b4a6674842943de931 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 19 Jul 2022 11:16:13 -0400 Subject: [PATCH 061/536] Allow specifying --print-width in the CLI --- README.md | 18 +++++++ lib/syntax_tree.rb | 7 ++- lib/syntax_tree/cli.rb | 107 ++++++++++++++++++++++++++++------------- test/cli_test.rb | 8 +++ 4 files changed, 106 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 4c472e37..9c33fd42 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,12 @@ If there are files with unformatted code, you will receive: The listed files did not match the expected format. ``` +To change the print width that you are checking against, specify the `--print-width` option, as in: + +```sh +stree check --print-width=100 path/to/file.rb +``` + ### format This command will output the formatted version of each of the listed files. Importantly, it will not write that content back to the source files. It is meant to display the formatted version only. @@ -132,6 +138,12 @@ For a file that contains `1 + 1`, you will receive: 1 + 1 ``` +To change the print width that you are formatting with, specify the `--print-width` option, as in: + +```sh +stree format --print-width=100 path/to/file.rb +``` + ### json This command will output a JSON representation of the syntax tree that is functionally equivalent to the input. This is mostly used in contexts where you need to access the tree from JavaScript or serialize it over a network. @@ -213,6 +225,12 @@ This will list every file that is being formatted. It will output light gray if path/to/file.rb 0ms ``` +To change the print width that you are writing with, specify the `--print-width` option, as in: + +```sh +stree write --print-width=100 path/to/file.rb +``` + ## Library Syntax Tree can be used as a library to access the syntax tree underlying Ruby source code. diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index 1dbd3ac8..5772b821 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -29,6 +29,11 @@ module SyntaxTree HANDLERS = {} HANDLERS.default = SyntaxTree + # This is the default print width when formatting. It can be overridden in the + # CLI by passing the --print-width option or here in the API by passing the + # optional second argument to ::format. + DEFAULT_PRINT_WIDTH = 80 + # This is a hook provided so that plugins can register themselves as the # handler for a particular file type. def self.register_handler(extension, handler) @@ -43,7 +48,7 @@ def self.parse(source) end # Parses the given source and returns the formatted source. - def self.format(source, maxwidth = 80) + def self.format(source, maxwidth = DEFAULT_PRINT_WIDTH) formatter = Formatter.new(source, [], maxwidth) parse(source).format(formatter) diff --git a/lib/syntax_tree/cli.rb b/lib/syntax_tree/cli.rb index c8e42831..2484a5b7 100644 --- a/lib/syntax_tree/cli.rb +++ b/lib/syntax_tree/cli.rb @@ -91,9 +91,17 @@ class Check < Action class UnformattedError < StandardError end + attr_reader :print_width + + def initialize(print_width:) + @print_width = print_width + end + def run(item) source = item.source - raise UnformattedError if source != item.handler.format(source) + if source != item.handler.format(source, print_width) + raise UnformattedError + end rescue StandardError warn("[#{Color.yellow("warn")}] #{item.filepath}") raise @@ -114,13 +122,21 @@ class Debug < Action class NonIdempotentFormatError < StandardError end + attr_reader :print_width + + def initialize(print_width:) + @print_width = print_width + end + def run(item) handler = item.handler warning = "[#{Color.yellow("warn")}] #{item.filepath}" - formatted = handler.format(item.source) + formatted = handler.format(item.source, print_width) - raise NonIdempotentFormatError if formatted != handler.format(formatted) + if formatted != handler.format(formatted, print_width) + raise NonIdempotentFormatError + end rescue StandardError warn(warning) raise @@ -148,8 +164,14 @@ def run(item) # An action of the CLI that formats the input source and prints it out. class Format < Action + attr_reader :print_width + + def initialize(print_width:) + @print_width = print_width + end + def run(item) - puts item.handler.format(item.source) + puts item.handler.format(item.source, print_width) end end @@ -173,12 +195,18 @@ def run(item) # An action of the CLI that formats the input source and writes the # formatted output back to the file. class Write < Action + attr_reader :print_width + + def initialize(print_width:) + @print_width = print_width + end + def run(item) filepath = item.filepath start = Time.now source = item.source - formatted = item.handler.format(source) + formatted = item.handler.format(source, print_width) File.write(filepath, formatted) if filepath != :stdin color = source == formatted ? Color.gray(filepath) : filepath @@ -194,43 +222,44 @@ def run(item) # The help message displayed if the input arguments are not correctly # ordered or formatted. HELP = <<~HELP - #{Color.bold("stree ast [OPTIONS] [FILE]")} + #{Color.bold("stree ast [--plugins=...] [--print-width=NUMBER] FILE")} Print out the AST corresponding to the given files - #{Color.bold("stree check [OPTIONS] [FILE]")} + #{Color.bold("stree check [--plugins=...] [--print-width=NUMBER] FILE")} Check that the given files are formatted as syntax tree would format them - #{Color.bold("stree debug [OPTIONS] [FILE]")} + #{Color.bold("stree debug [--plugins=...] [--print-width=NUMBER] FILE")} Check that the given files can be formatted idempotently - #{Color.bold("stree doc [OPTIONS] [FILE]")} + #{Color.bold("stree doc [--plugins=...] FILE")} Print out the doc tree that would be used to format the given files - #{Color.bold("stree format [OPTIONS] [FILE]")} + #{Color.bold("stree format [--plugins=...] [--print-width=NUMBER] FILE")} Print out the formatted version of the given files - #{Color.bold("stree json [OPTIONS] [FILE]")} + #{Color.bold("stree json [--plugins=...] FILE")} Print out the JSON representation of the given files - #{Color.bold("stree match [OPTIONS] [FILE]")} + #{Color.bold("stree match [--plugins=...] FILE")} Print out a pattern-matching Ruby expression that would match the given files #{Color.bold("stree help")} Display this help message - #{Color.bold("stree lsp [OPTIONS]")} + #{Color.bold("stree lsp [--plugins=...]")} Run syntax tree in language server mode #{Color.bold("stree version")} Output the current version of syntax tree - #{Color.bold("stree write [OPTIONS] [FILE]")} + #{Color.bold("stree write [--plugins=...] [--print-width=NUMBER] FILE")} Read, format, and write back the source of the given files - [OPTIONS] - --plugins=... A comma-separated list of plugins to load. + + --print-width=NUMBER + The maximum line width to use when formatting. HELP class << self @@ -238,19 +267,31 @@ class << self # passed to the invocation. def run(argv) name, *arguments = argv - - # If there are any plugins specified on the command line, then load them - # by requiring them here. We do this by transforming something like - # - # stree format --plugins=haml template.haml - # - # into - # - # require "syntax_tree/haml" - # - if arguments.first&.start_with?("--plugins=") - plugins = arguments.shift[/^--plugins=(.*)$/, 1] - plugins.split(",").each { |plugin| require "syntax_tree/#{plugin}" } + print_width = DEFAULT_PRINT_WIDTH + + while arguments.first&.start_with?("--") + case (argument = arguments.shift) + when /^--plugins=(.+)$/ + # If there are any plugins specified on the command line, then load + # them by requiring them here. We do this by transforming something + # like + # + # stree format --plugins=haml template.haml + # + # into + # + # require "syntax_tree/haml" + # + $1.split(",").each { |plugin| require "syntax_tree/#{plugin}" } + when /^--print-width=(\d+)$/ + # If there is a print width specified on the command line, then + # parse that out here and use it when formatting. + print_width = Integer($1) + else + warn("Unknown CLI option: #{argument}") + warn(HELP) + return 1 + end end case name @@ -271,9 +312,9 @@ def run(argv) when "a", "ast" AST.new when "c", "check" - Check.new + Check.new(print_width: print_width) when "debug" - Debug.new + Debug.new(print_width: print_width) when "doc" Doc.new when "j", "json" @@ -281,9 +322,9 @@ def run(argv) when "m", "match" Match.new when "f", "format" - Format.new + Format.new(print_width: print_width) when "w", "write" - Write.new + Write.new(print_width: print_width) else warn(HELP) return 1 diff --git a/test/cli_test.rb b/test/cli_test.rb index 7f2bcd26..31e4b7e2 100644 --- a/test/cli_test.rb +++ b/test/cli_test.rb @@ -52,6 +52,14 @@ def test_check_unformatted assert_includes(result.stderr, "expected") end + def test_check_print_width + file = Tempfile.new(%w[test- .rb]) + file.write("#{"a" * 40} + #{"b" * 40}\n") + + result = run_cli("check", "--print-width=100", file: file) + assert_includes(result.stdio, "match") + end + def test_debug result = run_cli("debug") assert_includes(result.stdio, "idempotently") From 068f7a6e4deac3db8e4fd20e523288038d07d902 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 19 Jul 2022 11:26:04 -0400 Subject: [PATCH 062/536] Bump to v3.1.0 --- CHANGELOG.md | 9 ++++++++- Gemfile.lock | 2 +- lib/syntax_tree/version.rb | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ee7261ea..5376027a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +## [3.1.0] - 2022-07-19 + +### Added + +- [#115](https://github.com/ruby-syntax-tree/syntax_tree/pull/115) - Support the `--print-width` option in the CLI for the actions that support it. + ## [3.0.1] - 2022-07-15 ### Changed @@ -294,7 +300,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - 🎉 Initial release! 🎉 -[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.0.1...HEAD +[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.1.0...HEAD +[3.1.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.0.1...v3.1.0 [3.0.1]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.0.0...v3.0.1 [3.0.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.9.0...v3.0.0 [2.9.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.8.0...v2.9.0 diff --git a/Gemfile.lock b/Gemfile.lock index ac7403ba..00b88281 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - syntax_tree (3.0.1) + syntax_tree (3.1.0) prettier_print GEM diff --git a/lib/syntax_tree/version.rb b/lib/syntax_tree/version.rb index 3a740b84..d4c0d91d 100644 --- a/lib/syntax_tree/version.rb +++ b/lib/syntax_tree/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SyntaxTree - VERSION = "3.0.1" + VERSION = "3.1.0" end From 8291fcb2f798c35a1debee1a9f682385f0354e4c Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 19 Jul 2022 12:22:32 -0400 Subject: [PATCH 063/536] Pass `--print-width` to LSP --- lib/syntax_tree/cli.rb | 4 ++-- lib/syntax_tree/language_server.rb | 11 ++++++++--- test/language_server_test.rb | 28 ++++++++++++++++++++++++++-- 3 files changed, 36 insertions(+), 7 deletions(-) diff --git a/lib/syntax_tree/cli.rb b/lib/syntax_tree/cli.rb index 2484a5b7..cad4fc35 100644 --- a/lib/syntax_tree/cli.rb +++ b/lib/syntax_tree/cli.rb @@ -246,7 +246,7 @@ def run(item) #{Color.bold("stree help")} Display this help message - #{Color.bold("stree lsp [--plugins=...]")} + #{Color.bold("stree lsp [--plugins=...] [--print-width=NUMBER]")} Run syntax tree in language server mode #{Color.bold("stree version")} @@ -300,7 +300,7 @@ def run(argv) return 0 when "lsp" require "syntax_tree/language_server" - LanguageServer.new.run + LanguageServer.new(print_width: print_width).run return 0 when "version" puts SyntaxTree::VERSION diff --git a/lib/syntax_tree/language_server.rb b/lib/syntax_tree/language_server.rb index 2eb8228b..41b80af1 100644 --- a/lib/syntax_tree/language_server.rb +++ b/lib/syntax_tree/language_server.rb @@ -13,11 +13,16 @@ module SyntaxTree # stree lsp # class LanguageServer - attr_reader :input, :output + attr_reader :input, :output, :print_width - def initialize(input: $stdin, output: $stdout) + def initialize( + input: $stdin, + output: $stdout, + print_width: DEFAULT_PRINT_WIDTH + ) @input = input.binmode @output = output.binmode + @print_width = print_width end # rubocop:disable Layout/LineLength @@ -93,7 +98,7 @@ def format(source) character: 0 } }, - newText: SyntaxTree.format(source) + newText: SyntaxTree.format(source, print_width) } end diff --git a/test/language_server_test.rb b/test/language_server_test.rb index fc26054d..31062e87 100644 --- a/test/language_server_test.rb +++ b/test/language_server_test.rb @@ -120,6 +120,26 @@ def test_formatting end end + def test_formatting_print_width + contents = "#{"a" * 40} + #{"b" * 40}\n" + messages = [ + Initialize.new(1), + TextDocumentDidOpen.new("file:///path/to/file.rb", contents), + TextDocumentFormatting.new(2, "file:///path/to/file.rb"), + TextDocumentDidClose.new("file:///path/to/file.rb"), + Shutdown.new(3) + ] + + case run_server(messages, print_width: 100) + in [ + { id: 1, result: { capabilities: Hash } }, + { id: 2, result: [{ newText: new_text }] }, + { id: 3, result: {} } + ] + assert_equal(contents, new_text) + end + end + def test_inlay_hint messages = [ Initialize.new(1), @@ -234,11 +254,15 @@ def read(content) end end - def run_server(messages) + def run_server(messages, print_width: DEFAULT_PRINT_WIDTH) input = StringIO.new(messages.map { |message| write(message) }.join) output = StringIO.new - LanguageServer.new(input: input, output: output).run + LanguageServer.new( + input: input, + output: output, + print_width: print_width + ).run read(output.tap(&:rewind)) end end From ef53c06c0fae82244cc6a79174a55845de6d651a Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 19 Jul 2022 12:42:12 -0400 Subject: [PATCH 064/536] Bump to v3.2.0 --- CHANGELOG.md | 9 ++++++++- Gemfile.lock | 2 +- lib/syntax_tree/version.rb | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5376027a..b219009b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +## [3.2.0] - 2022-07-19 + +### Added + +- [#116](https://github.com/ruby-syntax-tree/syntax_tree/pull/116) - Pass the `--print-width` option in the CLI to the language server. + ## [3.1.0] - 2022-07-19 ### Added @@ -300,7 +306,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - 🎉 Initial release! 🎉 -[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.1.0...HEAD +[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.2.0...HEAD +[3.2.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.1.0...v3.2.0 [3.1.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.0.1...v3.1.0 [3.0.1]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.0.0...v3.0.1 [3.0.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.9.0...v3.0.0 diff --git a/Gemfile.lock b/Gemfile.lock index 00b88281..ab79db80 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - syntax_tree (3.1.0) + syntax_tree (3.2.0) prettier_print GEM diff --git a/lib/syntax_tree/version.rb b/lib/syntax_tree/version.rb index d4c0d91d..92f18514 100644 --- a/lib/syntax_tree/version.rb +++ b/lib/syntax_tree/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SyntaxTree - VERSION = "3.1.0" + VERSION = "3.2.0" end From d3d883dd7ef9ab9c75ba7f085c3a05ec079d4164 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Jul 2022 17:27:36 +0000 Subject: [PATCH 065/536] Bump rubocop from 1.31.2 to 1.32.0 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.31.2 to 1.32.0. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.31.2...v1.32.0) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index ab79db80..937e7f3a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -19,17 +19,17 @@ GEM rake (13.0.6) regexp_parser (2.5.0) rexml (3.2.5) - rubocop (1.31.2) + rubocop (1.32.0) json (~> 2.3) parallel (~> 1.10) parser (>= 3.1.0.0) rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 1.8, < 3.0) rexml (>= 3.2.5, < 4.0) - rubocop-ast (>= 1.18.0, < 2.0) + rubocop-ast (>= 1.19.1, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 1.4.0, < 3.0) - rubocop-ast (1.18.0) + rubocop-ast (1.19.1) parser (>= 3.1.1.0) ruby-progressbar (1.11.0) simplecov (0.21.2) From 46d2fbf4fefb2542d3f7f3ae73c1b301e95e8696 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 22 Jul 2022 13:09:32 -0400 Subject: [PATCH 066/536] Properly handle conditionals in assignment --- lib/syntax_tree/node.rb | 2 +- test/fixtures/if.rb | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 58335b00..b633a1c9 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -5173,7 +5173,7 @@ def self.call(parent) while (node = queue.shift) return true if [Assign, MAssign, OpAssign].include?(node.class) - queue += node.child_nodes + queue += node.child_nodes.compact end false diff --git a/test/fixtures/if.rb b/test/fixtures/if.rb index e5e88103..1963d301 100644 --- a/test/fixtures/if.rb +++ b/test/fixtures/if.rb @@ -59,3 +59,7 @@ baz end ) +% +if (x = x + 1).to_i + x +end From 3f5b736b9c81587382192385421358dc5209ea62 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 22 Jul 2022 13:19:34 -0400 Subject: [PATCH 067/536] Bump to v3.2.1 --- CHANGELOG.md | 9 ++++++++- Gemfile.lock | 2 +- lib/syntax_tree/version.rb | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b219009b..6634f331 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +## [3.2.1] - 2022-07-22 + +### Changed + +- [#119](https://github.com/ruby-syntax-tree/syntax_tree/pull/119) - If there are conditionals in the assignment we cannot convert it to the modifier form. There was a bug where it would stop checking for assignment nodes if there were any optional child nodes. + ## [3.2.0] - 2022-07-19 ### Added @@ -306,7 +312,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - 🎉 Initial release! 🎉 -[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.2.0...HEAD +[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.2.1...HEAD +[3.2.1]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.2.0...v3.2.1 [3.2.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.1.0...v3.2.0 [3.1.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.0.1...v3.1.0 [3.0.1]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.0.0...v3.0.1 diff --git a/Gemfile.lock b/Gemfile.lock index 937e7f3a..0fbbbf84 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - syntax_tree (3.2.0) + syntax_tree (3.2.1) prettier_print GEM diff --git a/lib/syntax_tree/version.rb b/lib/syntax_tree/version.rb index 92f18514..f920098f 100644 --- a/lib/syntax_tree/version.rb +++ b/lib/syntax_tree/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SyntaxTree - VERSION = "3.2.0" + VERSION = "3.2.1" end From 35cf38838b88a072dcd1362669e24cf05aeced6f Mon Sep 17 00:00:00 2001 From: Tony Spataro Date: Fri, 22 Jul 2022 12:20:19 -0700 Subject: [PATCH 068/536] Deal better with multi-statement blocks inside parenthesized method calls. Fixes #120. --- lib/syntax_tree/node.rb | 2 +- test/fixtures/command_call.rb | 40 +++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index b633a1c9..d7b6d6cf 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -1978,7 +1978,7 @@ def unchangeable_bounds?(q) # If we hit a statements, then we're safe to use whatever since we # know for certain we're going to get split over multiple lines # anyway. - break false if parent.is_a?(Statements) + break false if parent.is_a?(Statements) || parent.is_a?(ArgParen) [Command, CommandCall].include?(parent.class) end diff --git a/test/fixtures/command_call.rb b/test/fixtures/command_call.rb index 4a0f60f0..7c055e8d 100644 --- a/test/fixtures/command_call.rb +++ b/test/fixtures/command_call.rb @@ -34,3 +34,43 @@ bar baz % foo.bar baz ? qux : qaz +% +expect foo, bar.map { |i| { quux: bazzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz } } +- +expect foo, + bar.map { |i| + { + quux: + bazzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz + } + } +% +expect(foo, bar.map { |i| {quux: bazzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz} }) +- +expect( + foo, + bar.map do |i| + { + quux: + bazzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz + } + end +) +% +expect(foo.map { |i| { bar: i.bazzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz } } ).to match(baz.map { |i| { bar: i.bazzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz } }) +- +expect( + foo.map do |i| + { + bar: + i.bazzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz + } + end +).to match( + baz.map do |i| + { + bar: + i.bazzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz + } + end +) From b5758af911279847db62ff632f624f6b927af478 Mon Sep 17 00:00:00 2001 From: Mo Lawson Date: Tue, 26 Jul 2022 13:13:07 -0500 Subject: [PATCH 069/536] Add print_width config option for rake tasks --- README.md | 10 ++++++++++ lib/syntax_tree/rake/check_task.rb | 9 ++++++++- lib/syntax_tree/rake/write_task.rb | 9 ++++++++- 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9c33fd42..3a3803ec 100644 --- a/README.md +++ b/README.md @@ -505,6 +505,16 @@ SyntaxTree::Rake::WriteTask.new do |t| end ``` +#### `print_width` + +If you want to use a different print width from the default (80), you can pass that to the `print_width` field, as in: + +```ruby +SyntaxTree::Rake::WriteTask.new do |t| + t.print_width = 100 +end +``` + #### `plugins` If you're running Syntax Tree with plugins (either your own or the pre-built ones), you can pass that to the `plugins` field, as in: diff --git a/lib/syntax_tree/rake/check_task.rb b/lib/syntax_tree/rake/check_task.rb index 354cd172..edaa1000 100644 --- a/lib/syntax_tree/rake/check_task.rb +++ b/lib/syntax_tree/rake/check_task.rb @@ -35,14 +35,20 @@ class CheckTask < ::Rake::TaskLib # Defaults to []. attr_accessor :plugins + # Max line length. + # Defaults to 80. + attr_accessor :print_width + def initialize( name = :"stree:check", source_files = ::Rake::FileList["lib/**/*.rb"], - plugins = [] + plugins = [], + print_width = DEFAULT_PRINT_WIDTH ) @name = name @source_files = source_files @plugins = plugins + @print_width = print_width yield self if block_given? define_task @@ -58,6 +64,7 @@ def define_task def run_task arguments = ["check"] arguments << "--plugins=#{plugins.join(",")}" if plugins.any? + arguments << "--print-width=#{print_width}" if print_width != DEFAULT_PRINT_WIDTH SyntaxTree::CLI.run(arguments + Array(source_files)) end diff --git a/lib/syntax_tree/rake/write_task.rb b/lib/syntax_tree/rake/write_task.rb index 5a957480..83c0e77a 100644 --- a/lib/syntax_tree/rake/write_task.rb +++ b/lib/syntax_tree/rake/write_task.rb @@ -35,14 +35,20 @@ class WriteTask < ::Rake::TaskLib # Defaults to []. attr_accessor :plugins + # Max line length. + # Defaults to 80. + attr_accessor :print_width + def initialize( name = :"stree:write", source_files = ::Rake::FileList["lib/**/*.rb"], - plugins = [] + plugins = [], + print_width = DEFAULT_PRINT_WIDTH ) @name = name @source_files = source_files @plugins = plugins + @print_width = print_width yield self if block_given? define_task @@ -58,6 +64,7 @@ def define_task def run_task arguments = ["write"] arguments << "--plugins=#{plugins.join(",")}" if plugins.any? + arguments << "--print-width=#{print_width}" if print_width != DEFAULT_PRINT_WIDTH SyntaxTree::CLI.run(arguments + Array(source_files)) end From f1ed1e94c240df7d669b357a2911b338a74ade25 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 1 Aug 2022 12:40:52 -0400 Subject: [PATCH 070/536] Fix up CI on main --- lib/syntax_tree/rake/check_task.rb | 4 +++- lib/syntax_tree/rake/write_task.rb | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/syntax_tree/rake/check_task.rb b/lib/syntax_tree/rake/check_task.rb index edaa1000..afe5013c 100644 --- a/lib/syntax_tree/rake/check_task.rb +++ b/lib/syntax_tree/rake/check_task.rb @@ -64,7 +64,9 @@ def define_task def run_task arguments = ["check"] arguments << "--plugins=#{plugins.join(",")}" if plugins.any? - arguments << "--print-width=#{print_width}" if print_width != DEFAULT_PRINT_WIDTH + if print_width != DEFAULT_PRINT_WIDTH + arguments << "--print-width=#{print_width}" + end SyntaxTree::CLI.run(arguments + Array(source_files)) end diff --git a/lib/syntax_tree/rake/write_task.rb b/lib/syntax_tree/rake/write_task.rb index 83c0e77a..9a9e8330 100644 --- a/lib/syntax_tree/rake/write_task.rb +++ b/lib/syntax_tree/rake/write_task.rb @@ -64,7 +64,9 @@ def define_task def run_task arguments = ["write"] arguments << "--plugins=#{plugins.join(",")}" if plugins.any? - arguments << "--print-width=#{print_width}" if print_width != DEFAULT_PRINT_WIDTH + if print_width != DEFAULT_PRINT_WIDTH + arguments << "--print-width=#{print_width}" + end SyntaxTree::CLI.run(arguments + Array(source_files)) end From bd85cd5cdf631ad85aef3dd06f89aafb4a1aa0af Mon Sep 17 00:00:00 2001 From: Mo Lawson Date: Mon, 1 Aug 2022 13:30:05 -0500 Subject: [PATCH 071/536] Add support for a simple config file As syntax_tree gains wider usage, it'll be very helpful to allow projects to commit their particular configs within their repo for use by any tool that uses `stree` instead of having to create wrapper scripts, rely on rake tasks that come with their own overhead, or manually keep settings up to date among all contributors. This set of changes takes inspiration from sorbet to provide users with a simple config file that's also easy to parse and use within the gem. It's a text file that has the exact options you'd pass on the command line, each on their own line. These are parsed and prepended to the arguments array within `CLI.run`, still allowing for other options to passed from the command line. I decided to restrict this only to the command line options and avoid the source files argument, opting to let other tools pass their own source file from the command line, which is preferable for tools like editor integrations that might interact with a single file at a time. If users want to interact with all of their Ruby files at once, the rake tasks are perfect for providing larger, static patterns of files. And since they use `CLI.run` as well, they'll pick up options from a .streerc file, if present. I also opted for only supporting a single .streerc file at the project root. If there's a need for multiple configs or config locations, this can be easily extended to look up through a directory structure or accept an option for a specific config file location (or even a different filename). Those felt out of scope for this initial support. --- README.md | 14 +++++++++ lib/syntax_tree/cli.rb | 7 +++++ test/cli_test.rb | 71 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 92 insertions(+) diff --git a/README.md b/README.md index 3a3803ec..fb1a49cd 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ It is built with only standard library dependencies. It additionally ships with - [json](#json) - [match](#match) - [write](#write) + - [Configuration](#configuration) - [Library](#library) - [SyntaxTree.read(filepath)](#syntaxtreereadfilepath) - [SyntaxTree.parse(source)](#syntaxtreeparsesource) @@ -231,6 +232,19 @@ To change the print width that you are writing with, specify the `--print-width` stree write --print-width=100 path/to/file.rb ``` +### Configuration + +Any of the above CLI commands can also read configuration options from a `.streerc` file in the directory where the commands are executed. + +This should be a text file with each argument on a separate line. + +```txt +--print-width=100 +--plugins=plugin/trailing_comma +``` + +If this file is present, it will _always_ be used for CLI commands. You can also pass options from the command line as in the examples above. The options in the `.streerc` file are passed to the CLI first, then the arguments from the command line. In the case of exclusive options (e.g. `--print-width`), this means that the command line options override what's in the config file. In the case of options that can take multiple inputs (e.g. `--plugins`), the effect is additive. That is, the plugins passed from the command line will be loaded _in addition to_ the plugins in the config file. + ## Library Syntax Tree can be used as a library to access the syntax tree underlying Ruby source code. diff --git a/lib/syntax_tree/cli.rb b/lib/syntax_tree/cli.rb index cad4fc35..fb2e4554 100644 --- a/lib/syntax_tree/cli.rb +++ b/lib/syntax_tree/cli.rb @@ -4,6 +4,8 @@ module SyntaxTree # Syntax Tree ships with the `stree` CLI, which can be used to inspect and # manipulate Ruby code. This module is responsible for powering that CLI. module CLI + CONFIG_FILE = ".streerc" + # A utility wrapper around colored strings in the output. class Color attr_reader :value, :code @@ -269,6 +271,11 @@ def run(argv) name, *arguments = argv print_width = DEFAULT_PRINT_WIDTH + config_file = File.join(Dir.pwd, CONFIG_FILE) + if File.readable?(config_file) + arguments.unshift(*File.readlines(config_file, chomp: true)) + end + while arguments.first&.start_with?("--") case (argument = arguments.shift) when /^--plugins=(.+)$/ diff --git a/test/cli_test.rb b/test/cli_test.rb index 31e4b7e2..9bc237fb 100644 --- a/test/cli_test.rb +++ b/test/cli_test.rb @@ -180,6 +180,77 @@ def test_language_server $stdout = prev_stdout end + def test_config_file + config_file = File.join(Dir.pwd, SyntaxTree::CLI::CONFIG_FILE) + config = <<~TXT + --print-width=100 + --plugins=plugin + TXT + File.write(config_file, config) + + Dir.mktmpdir do |directory| + Dir.mkdir(File.join(directory, "syntax_tree")) + $:.unshift(directory) + + File.write( + File.join(directory, "syntax_tree", "plugin.rb"), + "puts 'Hello, world!'" + ) + + file = Tempfile.new(%w[test- .rb]) + contents = "#{"a" * 40} + #{"b" * 40}\n" + file.write(contents) + + result = run_cli("format", file: file) + assert_equal("Hello, world!\n#{contents}", result.stdio) + end + ensure + FileUtils.rm(config_file) + end + + def test_print_width_args_with_config_file + config_file = File.join(Dir.pwd, SyntaxTree::CLI::CONFIG_FILE) + File.write(config_file, "--print-width=100") + + contents = "#{"a" * 40} + #{"b" * 40}\n" + + file = Tempfile.new(%w[test- .rb]) + file.write(contents) + result = run_cli("check", file: file) + assert_includes(result.stdio, "match") + + file = Tempfile.new(%w[test- .rb]) + file.write(contents) + result = run_cli("check", "--print-width=82", file: file) + assert_includes(result.stderr, "expected") + ensure + FileUtils.rm(config_file) + end + + def test_plugin_args_with_config_file + config_file = File.join(Dir.pwd, SyntaxTree::CLI::CONFIG_FILE) + File.write(config_file, "--plugins=hello_plugin") + + Dir.mktmpdir do |directory| + Dir.mkdir(File.join(directory, "syntax_tree")) + $:.unshift(directory) + + File.write( + File.join(directory, "syntax_tree", "hello_plugin.rb"), + "puts 'Hello, world!'" + ) + File.write( + File.join(directory, "syntax_tree", "bye_plugin.rb"), + "puts 'Bye, world!'" + ) + + result = run_cli("format", "--plugins=bye_plugin") + assert_equal("Hello, world!\nBye, world!\ntest\n", result.stdio) + end + ensure + FileUtils.rm(config_file) + end + private Result = Struct.new(:status, :stdio, :stderr, keyword_init: true) From 4b9a3b7c64ef2ef6a180ed7e2d52e9471475ec95 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 1 Aug 2022 16:54:17 -0400 Subject: [PATCH 072/536] Fix up tests on main --- test/cli_test.rb | 155 +++++++++++++++++++++++------------------------ 1 file changed, 77 insertions(+), 78 deletions(-) diff --git a/test/cli_test.rb b/test/cli_test.rb index 9bc237fb..4514d104 100644 --- a/test/cli_test.rb +++ b/test/cli_test.rb @@ -20,7 +20,7 @@ def test_handler file = Tempfile.new(%w[test- .test]) file.puts("test") - result = run_cli("ast", file: file) + result = run_cli("ast", contents: file) assert_equal("\"test\\n\" + \"test\\n\"\n", result.stdio) ensure SyntaxTree::HANDLERS.delete(".test") @@ -32,10 +32,7 @@ def test_ast end def test_ast_syntax_error - file = Tempfile.new(%w[test- .rb]) - file.puts("foo\n<>\nbar\n") - - result = run_cli("ast", file: file) + result = run_cli("ast", contents: "foo\n<>\nbar\n") assert_includes(result.stderr, "syntax error") end @@ -45,18 +42,13 @@ def test_check end def test_check_unformatted - file = Tempfile.new(%w[test- .rb]) - file.write("foo") - - result = run_cli("check", file: file) + result = run_cli("check", contents: "foo") assert_includes(result.stderr, "expected") end def test_check_print_width - file = Tempfile.new(%w[test- .rb]) - file.write("#{"a" * 40} + #{"b" * 40}\n") - - result = run_cli("check", "--print-width=100", file: file) + contents = "#{"a" * 40} + #{"b" * 40}\n" + result = run_cli("check", "--print-width=100", contents: contents) assert_includes(result.stdio, "match") end @@ -104,15 +96,12 @@ def test_write file = Tempfile.new(%w[test- .test]) filepath = file.path - result = run_cli("write", file: file) + result = run_cli("write", contents: file) assert_includes(result.stdio, filepath) end def test_write_syntax_tree - file = Tempfile.new(%w[test- .rb]) - file.write("<>") - - result = run_cli("write", file: file) + result = run_cli("write", contents: "<>") assert_includes(result.stderr, "syntax error") end @@ -146,19 +135,15 @@ def test_no_arguments_no_tty def test_generic_error SyntaxTree.stub(:format, ->(*) { raise }) do result = run_cli("format") + refute_equal(0, result.status) end end def test_plugins - Dir.mktmpdir do |directory| - Dir.mkdir(File.join(directory, "syntax_tree")) - $:.unshift(directory) + with_plugin_directory do |directory| + directory.plugin("plugin", "puts 'Hello, world!'") - File.write( - File.join(directory, "syntax_tree", "plugin.rb"), - "puts 'Hello, world!'" - ) result = run_cli("format", "--plugins=plugin") assert_equal("Hello, world!\ntest\n", result.stdio) @@ -181,85 +166,67 @@ def test_language_server end def test_config_file - config_file = File.join(Dir.pwd, SyntaxTree::CLI::CONFIG_FILE) config = <<~TXT --print-width=100 --plugins=plugin TXT - File.write(config_file, config) - Dir.mktmpdir do |directory| - Dir.mkdir(File.join(directory, "syntax_tree")) - $:.unshift(directory) + with_config_file(config) do + with_plugin_directory do |directory| + directory.plugin("plugin", "puts 'Hello, world!'") - File.write( - File.join(directory, "syntax_tree", "plugin.rb"), - "puts 'Hello, world!'" - ) + contents = "#{"a" * 40} + #{"b" * 40}\n" + result = run_cli("format", contents: contents) - file = Tempfile.new(%w[test- .rb]) - contents = "#{"a" * 40} + #{"b" * 40}\n" - file.write(contents) - - result = run_cli("format", file: file) - assert_equal("Hello, world!\n#{contents}", result.stdio) + assert_equal("Hello, world!\n#{contents}", result.stdio) + end end - ensure - FileUtils.rm(config_file) end def test_print_width_args_with_config_file - config_file = File.join(Dir.pwd, SyntaxTree::CLI::CONFIG_FILE) - File.write(config_file, "--print-width=100") + with_config_file("--print-width=100") do + result = run_cli("check", contents: "#{"a" * 40} + #{"b" * 40}\n") - contents = "#{"a" * 40} + #{"b" * 40}\n" + assert_includes(result.stdio, "match") + end + end - file = Tempfile.new(%w[test- .rb]) - file.write(contents) - result = run_cli("check", file: file) - assert_includes(result.stdio, "match") + def test_print_width_args_with_config_file_override + with_config_file("--print-width=100") do + contents = "#{"a" * 40} + #{"b" * 40}\n" + result = run_cli("check", "--print-width=82", contents: contents) - file = Tempfile.new(%w[test- .rb]) - file.write(contents) - result = run_cli("check", "--print-width=82", file: file) - assert_includes(result.stderr, "expected") - ensure - FileUtils.rm(config_file) + assert_includes(result.stderr, "expected") + end end def test_plugin_args_with_config_file - config_file = File.join(Dir.pwd, SyntaxTree::CLI::CONFIG_FILE) - File.write(config_file, "--plugins=hello_plugin") + with_config_file("--plugins=hello") do + with_plugin_directory do |directory| + directory.plugin("hello", "puts 'Hello, world!'") + directory.plugin("goodbye", "puts 'Bye, world!'") - Dir.mktmpdir do |directory| - Dir.mkdir(File.join(directory, "syntax_tree")) - $:.unshift(directory) + result = run_cli("format", "--plugins=goodbye") - File.write( - File.join(directory, "syntax_tree", "hello_plugin.rb"), - "puts 'Hello, world!'" - ) - File.write( - File.join(directory, "syntax_tree", "bye_plugin.rb"), - "puts 'Bye, world!'" - ) - - result = run_cli("format", "--plugins=bye_plugin") - assert_equal("Hello, world!\nBye, world!\ntest\n", result.stdio) + assert_equal("Hello, world!\nBye, world!\ntest\n", result.stdio) + end end - ensure - FileUtils.rm(config_file) end private Result = Struct.new(:status, :stdio, :stderr, keyword_init: true) - def run_cli(command, *args, file: nil) - if file.nil? - file = Tempfile.new(%w[test- .rb]) - file.puts("test") - end + def run_cli(command, *args, contents: :default) + file = + case contents + when :default + Tempfile.new(%w[test- .rb]).tap { |file| file.puts("test") } + when String + Tempfile.new(%w[test- .rb]).tap { |file| file.write(contents) } + else + contents + end file.rewind @@ -272,5 +239,37 @@ def run_cli(command, *args, file: nil) file.close file.unlink end + + def with_config_file(contents) + filepath = File.join(Dir.pwd, SyntaxTree::CLI::CONFIG_FILE) + File.write(filepath, contents) + + yield + ensure + FileUtils.rm(filepath) + end + + class PluginDirectory + attr_reader :directory + + def initialize(directory) + @directory = directory + end + + def plugin(name, contents) + File.write(File.join(directory, "#{name}.rb"), contents) + end + end + + def with_plugin_directory + Dir.mktmpdir do |directory| + $:.unshift(directory) + + plugin_directory = File.join(directory, "syntax_tree") + Dir.mkdir(plugin_directory) + + yield PluginDirectory.new(plugin_directory) + end + end end end From 282eb1d5cebdd222f7ee5da0aa470a379956e633 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 1 Aug 2022 16:56:18 -0400 Subject: [PATCH 073/536] Fix rubocop violations on main --- test/cli_test.rb | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/test/cli_test.rb b/test/cli_test.rb index 4514d104..48a07f54 100644 --- a/test/cli_test.rb +++ b/test/cli_test.rb @@ -218,7 +218,7 @@ def test_plugin_args_with_config_file Result = Struct.new(:status, :stdio, :stderr, keyword_init: true) def run_cli(command, *args, contents: :default) - file = + tempfile = case contents when :default Tempfile.new(%w[test- .rb]).tap { |file| file.puts("test") } @@ -228,16 +228,18 @@ def run_cli(command, *args, contents: :default) contents end - file.rewind + tempfile.rewind status = nil stdio, stderr = - capture_io { status = SyntaxTree::CLI.run([command, *args, file.path]) } + capture_io do + status = SyntaxTree::CLI.run([command, *args, tempfile.path]) + end Result.new(status: status, stdio: stdio, stderr: stderr) ensure - file.close - file.unlink + tempfile.close + tempfile.unlink end def with_config_file(contents) From ddd64064e92fed94bfe5e0fc7f41f704ae5bb04e Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 2 Aug 2022 08:57:39 -0400 Subject: [PATCH 074/536] Fix up CLI tests on main --- test/cli_test.rb | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/test/cli_test.rb b/test/cli_test.rb index 48a07f54..21991e53 100644 --- a/test/cli_test.rb +++ b/test/cli_test.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require_relative "test_helper" +require "securerandom" module SyntaxTree class CLITest < Minitest::Test @@ -142,9 +143,8 @@ def test_generic_error def test_plugins with_plugin_directory do |directory| - directory.plugin("plugin", "puts 'Hello, world!'") - - result = run_cli("format", "--plugins=plugin") + plugin = directory.plugin("puts 'Hello, world!'") + result = run_cli("format", "--plugins=#{plugin}") assert_equal("Hello, world!\ntest\n", result.stdio) end @@ -166,15 +166,14 @@ def test_language_server end def test_config_file - config = <<~TXT - --print-width=100 - --plugins=plugin - TXT - - with_config_file(config) do - with_plugin_directory do |directory| - directory.plugin("plugin", "puts 'Hello, world!'") + with_plugin_directory do |directory| + plugin = directory.plugin("puts 'Hello, world!'") + config = <<~TXT + --print-width=100 + --plugins=#{plugin} + TXT + with_config_file(config) do contents = "#{"a" * 40} + #{"b" * 40}\n" result = run_cli("format", contents: contents) @@ -201,12 +200,12 @@ def test_print_width_args_with_config_file_override end def test_plugin_args_with_config_file - with_config_file("--plugins=hello") do - with_plugin_directory do |directory| - directory.plugin("hello", "puts 'Hello, world!'") - directory.plugin("goodbye", "puts 'Bye, world!'") + with_plugin_directory do |directory| + plugin1 = directory.plugin("puts 'Hello, world!'") - result = run_cli("format", "--plugins=goodbye") + with_config_file("--plugins=#{plugin1}") do + plugin2 = directory.plugin("puts 'Bye, world!'") + result = run_cli("format", "--plugins=#{plugin2}") assert_equal("Hello, world!\nBye, world!\ntest\n", result.stdio) end @@ -258,8 +257,10 @@ def initialize(directory) @directory = directory end - def plugin(name, contents) + def plugin(contents) + name = SecureRandom.hex File.write(File.join(directory, "#{name}.rb"), contents) + name end end From dd07d310692f3a4584f53999adcd6d3d583ccfb6 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 2 Aug 2022 09:06:50 -0400 Subject: [PATCH 075/536] Bump to version 3.3.0 --- CHANGELOG.md | 10 +++++++++- Gemfile.lock | 2 +- lib/syntax_tree/version.rb | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6634f331..cf774ba8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +## [3.3.0] - 2022-08-02 + +### Added + +- [#123](https://github.com/ruby-syntax-tree/syntax_tree/pull/123) - Allow the rake tasks to configure print width. +- [#125](https://github.com/ruby-syntax-tree/syntax_tree/pull/125) - Add support for an `.streerc` file in the current working directory to configure the CLI. + ## [3.2.1] - 2022-07-22 ### Changed @@ -312,7 +319,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - 🎉 Initial release! 🎉 -[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.2.1...HEAD +[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.3.0...HEAD +[3.3.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.2.1...v3.3.0 [3.2.1]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.2.0...v3.2.1 [3.2.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.1.0...v3.2.0 [3.1.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.0.1...v3.1.0 diff --git a/Gemfile.lock b/Gemfile.lock index 0fbbbf84..55a6b335 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - syntax_tree (3.2.1) + syntax_tree (3.3.0) prettier_print GEM diff --git a/lib/syntax_tree/version.rb b/lib/syntax_tree/version.rb index f920098f..6bc508fe 100644 --- a/lib/syntax_tree/version.rb +++ b/lib/syntax_tree/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SyntaxTree - VERSION = "3.2.1" + VERSION = "3.3.0" end From 5318cc02d52d4765f0bded400ed281932100b601 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 4 Aug 2022 17:31:17 +0000 Subject: [PATCH 076/536] Bump rubocop from 1.32.0 to 1.33.0 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.32.0 to 1.33.0. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.32.0...v1.33.0) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 55a6b335..46051d08 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -19,7 +19,7 @@ GEM rake (13.0.6) regexp_parser (2.5.0) rexml (3.2.5) - rubocop (1.32.0) + rubocop (1.33.0) json (~> 2.3) parallel (~> 1.10) parser (>= 3.1.0.0) From 9063741b3ddb020915a35ff41c79770c48ac03c1 Mon Sep 17 00:00:00 2001 From: Peter Krutz Date: Fri, 5 Aug 2022 13:13:47 -0700 Subject: [PATCH 077/536] switch between ruby and haml formatters --- lib/syntax_tree/language_server.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/syntax_tree/language_server.rb b/lib/syntax_tree/language_server.rb index 41b80af1..41da9a4f 100644 --- a/lib/syntax_tree/language_server.rb +++ b/lib/syntax_tree/language_server.rb @@ -56,7 +56,7 @@ def run store.delete(uri) in { method: "textDocument/formatting", id:, params: { textDocument: { uri: } } } contents = store[uri] - write(id: id, result: contents ? [format(store[uri])] : nil) + write(id: id, result: contents ? [format(store[uri], uri.split(".").last)] : nil) in { method: "textDocument/inlayHint", id:, params: { textDocument: { uri: } } } contents = store[uri] write(id: id, result: contents ? inlay_hints(store[uri]) : nil) @@ -86,7 +86,7 @@ def capabilities } end - def format(source) + def format(source, file_extension) { range: { start: { @@ -98,7 +98,7 @@ def format(source) character: 0 } }, - newText: SyntaxTree.format(source, print_width) + newText: file_extension == "haml" ? SyntaxTree::Haml.format(source, print_width) : SyntaxTree.format(source, print_width) } end @@ -119,3 +119,4 @@ def write(value) end end end + From 0369b32ae1ada0c4c3882fb492b74e1f8da9e530 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 9 Aug 2022 18:32:48 +0000 Subject: [PATCH 078/536] Bump rubocop from 1.33.0 to 1.34.1 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.33.0 to 1.34.1. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.33.0...v1.34.1) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 46051d08..145d463a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -12,24 +12,24 @@ GEM json (2.6.2) minitest (5.16.2) parallel (1.22.1) - parser (3.1.2.0) + parser (3.1.2.1) ast (~> 2.4.1) prettier_print (0.1.0) rainbow (3.1.1) rake (13.0.6) regexp_parser (2.5.0) rexml (3.2.5) - rubocop (1.33.0) + rubocop (1.34.1) json (~> 2.3) parallel (~> 1.10) - parser (>= 3.1.0.0) + parser (>= 3.1.2.1) rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 1.8, < 3.0) rexml (>= 3.2.5, < 4.0) - rubocop-ast (>= 1.19.1, < 2.0) + rubocop-ast (>= 1.20.0, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 1.4.0, < 3.0) - rubocop-ast (1.19.1) + rubocop-ast (1.21.0) parser (>= 3.1.1.0) ruby-progressbar (1.11.0) simplecov (0.21.2) From 711f477acc0ef4821fbc2259f7b20b12805382a1 Mon Sep 17 00:00:00 2001 From: Peter Krutz Date: Wed, 10 Aug 2022 09:30:47 -0700 Subject: [PATCH 079/536] use syntaxtree handlers --- lib/syntax_tree/language_server.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/syntax_tree/language_server.rb b/lib/syntax_tree/language_server.rb index 41da9a4f..03270e2a 100644 --- a/lib/syntax_tree/language_server.rb +++ b/lib/syntax_tree/language_server.rb @@ -98,7 +98,7 @@ def format(source, file_extension) character: 0 } }, - newText: file_extension == "haml" ? SyntaxTree::Haml.format(source, print_width) : SyntaxTree.format(source, print_width) + newText: SyntaxTree::HANDLERS[".#{file_extension}"].format(source, print_width) } end From 98eb03f080bd2f45ea67b8511eca6e87d886d1cd Mon Sep 17 00:00:00 2001 From: Peter Krutz Date: Wed, 10 Aug 2022 09:31:17 -0700 Subject: [PATCH 080/536] clean up --- lib/syntax_tree/language_server.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/syntax_tree/language_server.rb b/lib/syntax_tree/language_server.rb index 03270e2a..894fc2fd 100644 --- a/lib/syntax_tree/language_server.rb +++ b/lib/syntax_tree/language_server.rb @@ -119,4 +119,3 @@ def write(value) end end end - From 5dcca1fa4ab43bdaa9f3d54049b47cbfa427d7da Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 12 Aug 2022 17:24:41 +0000 Subject: [PATCH 081/536] Bump rubocop from 1.34.1 to 1.35.0 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.34.1 to 1.35.0. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.34.1...v1.35.0) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 145d463a..01b6e801 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -19,14 +19,14 @@ GEM rake (13.0.6) regexp_parser (2.5.0) rexml (3.2.5) - rubocop (1.34.1) + rubocop (1.35.0) json (~> 2.3) parallel (~> 1.10) parser (>= 3.1.2.1) rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 1.8, < 3.0) rexml (>= 3.2.5, < 4.0) - rubocop-ast (>= 1.20.0, < 2.0) + rubocop-ast (>= 1.20.1, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 1.4.0, < 3.0) rubocop-ast (1.21.0) From 318e8704f5f320bc973ca9d2036e595827f7a59c Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 12 Aug 2022 15:57:53 -0400 Subject: [PATCH 082/536] Better error messages on missing tokens --- lib/syntax_tree.rb | 1 + lib/syntax_tree/language_server.rb | 3 +- lib/syntax_tree/parser.rb | 58 +++++++++++++++++++++++++----- test/parser_test.rb | 8 ++++- 4 files changed, 59 insertions(+), 11 deletions(-) diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index 5772b821..88c66369 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +require "delegate" require "etc" require "json" require "pp" diff --git a/lib/syntax_tree/language_server.rb b/lib/syntax_tree/language_server.rb index 894fc2fd..586174f4 100644 --- a/lib/syntax_tree/language_server.rb +++ b/lib/syntax_tree/language_server.rb @@ -98,7 +98,8 @@ def format(source, file_extension) character: 0 } }, - newText: SyntaxTree::HANDLERS[".#{file_extension}"].format(source, print_width) + newText: + SyntaxTree::HANDLERS[".#{file_extension}"].format(source, print_width) } end diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index 6e6e4b1c..8a64bc32 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -57,6 +57,26 @@ def [](byteindex) end end + # This represents all of the tokens coming back from the lexer. It is + # replacing a simple array because it keeps track of the last deleted token + # from the list for better error messages. + class TokenList < SimpleDelegator + attr_reader :last_deleted + + def initialize(object) + super + @last_deleted = nil + end + + def delete(value) + @last_deleted = super || @last_deleted + end + + def delete_at(index) + @last_deleted = super + end + end + # [String] the source being parsed attr_reader :source @@ -124,7 +144,7 @@ def initialize(source, *) # Most of the time, when a parser event consumes one of these events, it # will be deleted from the list. So ideally, this list stays pretty short # over the course of parsing a source string. - @tokens = [] + @tokens = TokenList.new([]) # Here we're going to build up a list of SingleByteString or # MultiByteString objects. They're each going to represent a string in the @@ -174,6 +194,33 @@ def current_column line[column].to_i - line.start end + # Returns the current location that is being looked at for the parser for + # the purpose of locating the error. + def find_token_error(location) + if location + # If we explicitly passed a location into this find_token_error method, + # that means that's the source of the error, so we'll use that + # information for our error object. + lineno = location.start_line + [lineno, location.start_char - line_counts[lineno - 1].start] + elsif lineno && column + # If there is a line number associated with the current ripper state, + # then we'll use that information to generate the error. + [lineno, column] + elsif (location = tokens.last_deleted&.location) + # If we've already deleted a token from the list of tokens that we are + # consuming, then we'll fall back to that token's location. + lineno = location.start_line + [lineno, location.start_char - line_counts[lineno - 1].start] + else + # Finally, it's possible that when we hit this error the parsing thread + # for ripper has died. In that case, lineno and column both return nil. + # So we're just going to set it to line 1, column 0 in the hopes that + # that makes any sense. + [1, 0] + end + end + # As we build up a list of tokens, we'll periodically need to go backwards # and find the ones that we've already hit in order to determine the # location information for nodes that use them. For example, if you have a @@ -201,14 +248,7 @@ def find_token(type, value = :any, consume: true, location: nil) unless index token = value == :any ? type.name.split("::", 2).last : value message = "Cannot find expected #{token}" - - if location - lineno = location.start_line - column = location.start_char - line_counts[lineno - 1].start - raise ParseError.new(message, lineno, column) - else - raise ParseError.new(message, lineno, column) - end + raise ParseError.new(message, *find_token_error(location)) end tokens.delete_at(index) diff --git a/test/parser_test.rb b/test/parser_test.rb index b36c1a5f..d0c475c1 100644 --- a/test/parser_test.rb +++ b/test/parser_test.rb @@ -32,7 +32,13 @@ def test_parses_ripper_methods end def test_errors_on_missing_token_with_location - assert_raises(Parser::ParseError) { SyntaxTree.parse("\"foo") } + error = assert_raises(Parser::ParseError) { SyntaxTree.parse("f+\"foo") } + assert_equal(2, error.column) + end + + def test_errors_on_missing_end_with_location + error = assert_raises(Parser::ParseError) { SyntaxTree.parse("foo do 1") } + assert_equal(4, error.column) end def test_errors_on_missing_token_without_location From 8fcd7355b13829fa38168a07721a189e45847dae Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 12 Aug 2022 16:13:54 -0400 Subject: [PATCH 083/536] Fix up formatting --- lib/syntax_tree/language_server.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/syntax_tree/language_server.rb b/lib/syntax_tree/language_server.rb index 586174f4..95041b35 100644 --- a/lib/syntax_tree/language_server.rb +++ b/lib/syntax_tree/language_server.rb @@ -86,7 +86,9 @@ def capabilities } end - def format(source, file_extension) + def format(source, extension) + text = SyntaxTree::HANDLERS[".#{extension}"].format(source, print_width) + { range: { start: { @@ -98,8 +100,7 @@ def format(source, file_extension) character: 0 } }, - newText: - SyntaxTree::HANDLERS[".#{file_extension}"].format(source, print_width) + newText: text } end From 1c9b4e7c8fba4c5b089fc51efe628b09141c4cd9 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 12 Aug 2022 16:25:58 -0400 Subject: [PATCH 084/536] Fix rubocop violations --- .rubocop.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.rubocop.yml b/.rubocop.yml index 8cf5f209..f6ffbcd0 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -44,7 +44,7 @@ Style/ExplicitBlockArgument: Enabled: false Style/FormatString: - EnforcedStyle: percent + Enabled: false Style/GuardClause: Enabled: false From 3155f8ae7fe8445b88137f8da9580f574018f128 Mon Sep 17 00:00:00 2001 From: Wassim Metallaoui Date: Sat, 13 Aug 2022 08:01:56 -0500 Subject: [PATCH 085/536] docs: add information on neovim and vim and neovim support --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index fb1a49cd..1e646104 100644 --- a/README.md +++ b/README.md @@ -548,9 +548,11 @@ inherit_gem: syntax_tree: config/rubocop.yml ``` -### VSCode +### Editors -To integrate Syntax Tree into VSCode, you should use the official VSCode extension [ruby-syntax-tree/vscode-syntax-tree](https://github.com/ruby-syntax-tree/vscode-syntax-tree). +* Neovim - formatting via the LSP server can be configured using [nvim-lspconfig](https://github.com/neovim/nvim-lspconfig). +* Vim - format via the CLI using [ALE (Asynchronous Lint Engine)](https://github.com/dense-analysis/ale). +* Visual Studio Code - use the official extension [ruby-syntax-tree/vscode-syntax-tree](https://github.com/ruby-syntax-tree/vscode-syntax-tree). ## Contributing From 3ecc988b66dbe770ed4f3552eb5baaab38eec522 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Sun, 14 Aug 2022 12:48:19 -0400 Subject: [PATCH 086/536] Correct the TOC --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 1e646104..c456d76e 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ It is built with only standard library dependencies. It additionally ships with - [Integration](#integration) - [Rake](#rake) - [RuboCop](#rubocop) - - [VSCode](#vscode) + - [Editors](#editors) - [Contributing](#contributing) - [License](#license) @@ -550,9 +550,9 @@ inherit_gem: ### Editors -* Neovim - formatting via the LSP server can be configured using [nvim-lspconfig](https://github.com/neovim/nvim-lspconfig). -* Vim - format via the CLI using [ALE (Asynchronous Lint Engine)](https://github.com/dense-analysis/ale). -* Visual Studio Code - use the official extension [ruby-syntax-tree/vscode-syntax-tree](https://github.com/ruby-syntax-tree/vscode-syntax-tree). +* [Neovim](https://neovim.io/) - [neovim/nvim-lspconfig](https://github.com/neovim/nvim-lspconfig). +* [Vim](https://www.vim.org/) - [dense-analysis/ale](https://github.com/dense-analysis/ale). +* [VSCode](https://code.visualstudio.com/) - [ruby-syntax-tree/vscode-syntax-tree](https://github.com/ruby-syntax-tree/vscode-syntax-tree). ## Contributing From d1ff2cf18af95b103f16e7036409d228e6512958 Mon Sep 17 00:00:00 2001 From: Vinicius Stock Date: Mon, 15 Aug 2022 14:09:54 -0400 Subject: [PATCH 087/536] Consume end tokens on begins with rescue or ensure Co-authored-by: Kevin Newton --- lib/syntax_tree/parser.rb | 3 +-- test/fixtures/def_endless.rb | 8 ++++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index 8a64bc32..3824b6b3 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -717,8 +717,7 @@ def on_begin(bodystmt) else keyword = find_token(Kw, "begin") end_location = - if bodystmt.rescue_clause || bodystmt.ensure_clause || - bodystmt.else_clause + if bodystmt.else_clause bodystmt.location else find_token(Kw, "end").location diff --git a/test/fixtures/def_endless.rb b/test/fixtures/def_endless.rb index dbac88bb..15ea518b 100644 --- a/test/fixtures/def_endless.rb +++ b/test/fixtures/def_endless.rb @@ -18,3 +18,11 @@ def self.foo() = bar def self.foo = bar % # >= 3.1.0 def self.foo = bar baz +% +begin + true +rescue StandardError + false +end + +def foo? = true From c17269802d6347befba50ecd0242a76ce2bb15cb Mon Sep 17 00:00:00 2001 From: Chris Salvato Date: Tue, 16 Aug 2022 13:34:47 -0600 Subject: [PATCH 088/536] Updates README with docs on globbing --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index c456d76e..d4f07901 100644 --- a/README.md +++ b/README.md @@ -245,6 +245,16 @@ This should be a text file with each argument on a separate line. If this file is present, it will _always_ be used for CLI commands. You can also pass options from the command line as in the examples above. The options in the `.streerc` file are passed to the CLI first, then the arguments from the command line. In the case of exclusive options (e.g. `--print-width`), this means that the command line options override what's in the config file. In the case of options that can take multiple inputs (e.g. `--plugins`), the effect is additive. That is, the plugins passed from the command line will be loaded _in addition to_ the plugins in the config file. +### Globbing + +When running commands with `stree` in the CLI, the globs must follow the Ruby-specific globbing syntax as specified in the docs for [Dir](https://ruby-doc.org/core-2.6.3/Dir.html#method-c-glob). To ensure consistent file matching across environments (e.g. CI vs. local development) it's safest to enclose the glob in quotes. + +For example, if you are in a Rails app and want to ignore the `db/schema.rb` file but check all other Ruby files and the `Gemfile`, you can use the following syntax: + +```shell +stree check '**/{[!schema]}*.rb' 'Gemfile' +``` + ## Library Syntax Tree can be used as a library to access the syntax tree underlying Ruby source code. From db67ea164ea46686d214533971f069a28e4a7ca7 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 16 Aug 2022 16:22:38 -0400 Subject: [PATCH 089/536] More docs --- README.md | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index d4f07901..524aea12 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ It is built with only standard library dependencies. It additionally ships with - [match](#match) - [write](#write) - [Configuration](#configuration) + - [Globbing](#globbing) - [Library](#library) - [SyntaxTree.read(filepath)](#syntaxtreereadfilepath) - [SyntaxTree.parse(source)](#syntaxtreeparsesource) @@ -247,12 +248,20 @@ If this file is present, it will _always_ be used for CLI commands. You can also ### Globbing -When running commands with `stree` in the CLI, the globs must follow the Ruby-specific globbing syntax as specified in the docs for [Dir](https://ruby-doc.org/core-2.6.3/Dir.html#method-c-glob). To ensure consistent file matching across environments (e.g. CI vs. local development) it's safest to enclose the glob in quotes. +When running commands with `stree`, it's common to pass in lists of files. For example: -For example, if you are in a Rails app and want to ignore the `db/schema.rb` file but check all other Ruby files and the `Gemfile`, you can use the following syntax: +```sh +stree write 'lib/*.rb' 'test/*.rb' +``` + +The commands in the CLI accept any number of arguments. This means you _could_ pass `**/*.rb` (note the lack of quotes). This would make your shell expand out the file paths listed according to its own rules. (For example, [here](https://www.gnu.org/software/bash/manual/html_node/Filename-Expansion.html) are the rules for GNU bash.) + +However, it's recommended to instead use quotes, which means that Ruby is responsible for performing the file path expansion instead. This ensures a consistent experience across different environments and shells. The globs must follow the Ruby-specific globbing syntax as specified in the documentation for [Dir](https://ruby-doc.org/core-3.1.1/Dir.html#method-c-glob). + +Baked into this syntax is the ability to provide exceptions to file name patterns as well. For example, if you are in a Rails app and want to exclude files named `schema.rb` but write all other Ruby files, you can use the following syntax: ```shell -stree check '**/{[!schema]}*.rb' 'Gemfile' +stree write '**/{[!schema]}*.rb' ``` ## Library From 4d88fef53743941bf93b44cce24980175d185896 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 16 Aug 2022 17:21:37 -0400 Subject: [PATCH 090/536] Use option parser for CLI --- lib/syntax_tree/cli.rb | 149 ++++++++++++++++++++--------------------- 1 file changed, 74 insertions(+), 75 deletions(-) diff --git a/lib/syntax_tree/cli.rb b/lib/syntax_tree/cli.rb index fb2e4554..9b75e716 100644 --- a/lib/syntax_tree/cli.rb +++ b/lib/syntax_tree/cli.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require "optparse" + module SyntaxTree # Syntax Tree ships with the `stree` CLI, which can be used to inspect and # manipulate Ruby code. This module is responsible for powering that CLI. @@ -70,6 +72,12 @@ def source # The parent action class for the CLI that implements the basics. class Action + attr_reader :options + + def initialize(options) + @options = options + end + def run(item) end @@ -93,15 +101,9 @@ class Check < Action class UnformattedError < StandardError end - attr_reader :print_width - - def initialize(print_width:) - @print_width = print_width - end - def run(item) source = item.source - if source != item.handler.format(source, print_width) + if source != item.handler.format(source, options.print_width) raise UnformattedError end rescue StandardError @@ -124,19 +126,13 @@ class Debug < Action class NonIdempotentFormatError < StandardError end - attr_reader :print_width - - def initialize(print_width:) - @print_width = print_width - end - def run(item) handler = item.handler warning = "[#{Color.yellow("warn")}] #{item.filepath}" - formatted = handler.format(item.source, print_width) + formatted = handler.format(item.source, options.print_width) - if formatted != handler.format(formatted, print_width) + if formatted != handler.format(formatted, options.print_width) raise NonIdempotentFormatError end rescue StandardError @@ -166,14 +162,8 @@ def run(item) # An action of the CLI that formats the input source and prints it out. class Format < Action - attr_reader :print_width - - def initialize(print_width:) - @print_width = print_width - end - def run(item) - puts item.handler.format(item.source, print_width) + puts item.handler.format(item.source, options.print_width) end end @@ -197,18 +187,12 @@ def run(item) # An action of the CLI that formats the input source and writes the # formatted output back to the file. class Write < Action - attr_reader :print_width - - def initialize(print_width:) - @print_width = print_width - end - def run(item) filepath = item.filepath start = Time.now source = item.source - formatted = item.handler.format(source, print_width) + formatted = item.handler.format(source, options.print_width) File.write(filepath, formatted) if filepath != :stdin color = source == formatted ? Color.gray(filepath) : filepath @@ -264,74 +248,89 @@ def run(item) The maximum line width to use when formatting. HELP + # This represents all of the options that can be passed to the CLI. It is + # responsible for parsing the list and then returning the file paths at the + # end. + class Options + attr_reader :print_width + + def initialize(print_width: DEFAULT_PRINT_WIDTH) + @print_width = print_width + end + + def parse(arguments) + parser.parse(arguments) + end + + private + + def parser + OptionParser.new do |opts| + # If there are any plugins specified on the command line, then load + # them by requiring them here. We do this by transforming something + # like + # + # stree format --plugins=haml template.haml + # + # into + # + # require "syntax_tree/haml" + # + opts.on("--plugins=PLUGINS") do |plugins| + plugins.split(",").each { |plugin| require "syntax_tree/#{plugin}" } + end + + # If there is a print width specified on the command line, then + # parse that out here and use it when formatting. + opts.on("--print-width=NUMBER", Integer) do |print_width| + @print_width = print_width + end + end + end + end + class << self # Run the CLI over the given array of strings that make up the arguments # passed to the invocation. def run(argv) name, *arguments = argv - print_width = DEFAULT_PRINT_WIDTH config_file = File.join(Dir.pwd, CONFIG_FILE) if File.readable?(config_file) arguments.unshift(*File.readlines(config_file, chomp: true)) end - while arguments.first&.start_with?("--") - case (argument = arguments.shift) - when /^--plugins=(.+)$/ - # If there are any plugins specified on the command line, then load - # them by requiring them here. We do this by transforming something - # like - # - # stree format --plugins=haml template.haml - # - # into - # - # require "syntax_tree/haml" - # - $1.split(",").each { |plugin| require "syntax_tree/#{plugin}" } - when /^--print-width=(\d+)$/ - # If there is a print width specified on the command line, then - # parse that out here and use it when formatting. - print_width = Integer($1) - else - warn("Unknown CLI option: #{argument}") - warn(HELP) - return 1 - end - end - - case name - when "help" - puts HELP - return 0 - when "lsp" - require "syntax_tree/language_server" - LanguageServer.new(print_width: print_width).run - return 0 - when "version" - puts SyntaxTree::VERSION - return 0 - end + options = Options.new + options.parse(arguments) action = case name when "a", "ast" - AST.new + AST.new(options) when "c", "check" - Check.new(print_width: print_width) + Check.new(options) when "debug" - Debug.new(print_width: print_width) + Debug.new(options) when "doc" - Doc.new + Doc.new(options) + when "help" + puts HELP + return 0 when "j", "json" - Json.new + Json.new(options) + when "lsp" + require "syntax_tree/language_server" + LanguageServer.new(print_width: options.print_width).run + return 0 when "m", "match" - Match.new + Match.new(options) when "f", "format" - Format.new(print_width: print_width) + Format.new(options) + when "version" + puts SyntaxTree::VERSION + return 0 when "w", "write" - Write.new(print_width: print_width) + Write.new(options) else warn(HELP) return 1 From 65ae61cb4f513a38a0c1660ae75268adf30b976d Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 17 Aug 2022 12:00:48 -0400 Subject: [PATCH 091/536] Handle regexp without ending better --- lib/syntax_tree/parser.rb | 11 +++++++++-- test/parser_test.rb | 7 +++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index 3824b6b3..8af0b8ed 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -2837,14 +2837,21 @@ def on_regexp_end(value) # :call-seq: # on_regexp_literal: ( # RegexpContent regexp_content, - # RegexpEnd ending + # (nil | RegexpEnd) ending # ) -> RegexpLiteral def on_regexp_literal(regexp_content, ending) + location = regexp_content.location + + if ending.nil? + message = "Cannot find expected regular expression ending" + raise ParseError.new(message, *find_token_error(location)) + end + RegexpLiteral.new( beginning: regexp_content.beginning, ending: ending.value, parts: regexp_content.parts, - location: regexp_content.location.to(ending.location) + location: location.to(ending.location) ) end diff --git a/test/parser_test.rb b/test/parser_test.rb index d0c475c1..fbff8ec2 100644 --- a/test/parser_test.rb +++ b/test/parser_test.rb @@ -41,6 +41,13 @@ def test_errors_on_missing_end_with_location assert_equal(4, error.column) end + def test_errors_on_missing_regexp_ending + error = + assert_raises(Parser::ParseError) { SyntaxTree.parse("a =~ /foo") } + + assert_equal(5, error.column) + end + def test_errors_on_missing_token_without_location assert_raises(Parser::ParseError) { SyntaxTree.parse(":\"foo") } end From a616551a42638ab57579c42605b68ad0fbe5fe9a Mon Sep 17 00:00:00 2001 From: Chris Salvato Date: Wed, 17 Aug 2022 13:07:41 -0600 Subject: [PATCH 092/536] Removes references to exclusions Exclusions are not possible, so remove the documentation discussing them. --- README.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/README.md b/README.md index 524aea12..859bdf69 100644 --- a/README.md +++ b/README.md @@ -258,12 +258,6 @@ The commands in the CLI accept any number of arguments. This means you _could_ p However, it's recommended to instead use quotes, which means that Ruby is responsible for performing the file path expansion instead. This ensures a consistent experience across different environments and shells. The globs must follow the Ruby-specific globbing syntax as specified in the documentation for [Dir](https://ruby-doc.org/core-3.1.1/Dir.html#method-c-glob). -Baked into this syntax is the ability to provide exceptions to file name patterns as well. For example, if you are in a Rails app and want to exclude files named `schema.rb` but write all other Ruby files, you can use the following syntax: - -```shell -stree write '**/{[!schema]}*.rb' -``` - ## Library Syntax Tree can be used as a library to access the syntax tree underlying Ruby source code. From e224c15ddba69388955dbcabe430ffca1d019508 Mon Sep 17 00:00:00 2001 From: Chris Salvato Date: Wed, 17 Aug 2022 13:14:04 -0600 Subject: [PATCH 093/536] Reintroduces exceptions in glob docs Exceptions are possible, just not very intuitive. Updated the docs to reflect this. --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 859bdf69..9f25b0e7 100644 --- a/README.md +++ b/README.md @@ -258,6 +258,12 @@ The commands in the CLI accept any number of arguments. This means you _could_ p However, it's recommended to instead use quotes, which means that Ruby is responsible for performing the file path expansion instead. This ensures a consistent experience across different environments and shells. The globs must follow the Ruby-specific globbing syntax as specified in the documentation for [Dir](https://ruby-doc.org/core-3.1.1/Dir.html#method-c-glob). +Baked into this syntax is the ability to provide exceptions to file name patterns as well. For example, if you are in a Rails app and want to exclude files named `schema.rb` but write all other Ruby files, you can use the following syntax: + +```shell +stree write "**/{[!schema]*,*}.rb" +``` + ## Library Syntax Tree can be used as a library to access the syntax tree underlying Ruby source code. From b17324e519cecfbcb89d3da59adb87c806b0b734 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 19 Aug 2022 11:31:28 -0400 Subject: [PATCH 094/536] Split out config file logic --- lib/syntax_tree/cli.rb | 35 +++++++++++++++++++++++++----- lib/syntax_tree/language_server.rb | 4 ++++ test/cli_test.rb | 2 +- 3 files changed, 34 insertions(+), 7 deletions(-) diff --git a/lib/syntax_tree/cli.rb b/lib/syntax_tree/cli.rb index 9b75e716..a9ebdef7 100644 --- a/lib/syntax_tree/cli.rb +++ b/lib/syntax_tree/cli.rb @@ -6,8 +6,6 @@ module SyntaxTree # Syntax Tree ships with the `stree` CLI, which can be used to inspect and # manipulate Ruby code. This module is responsible for powering that CLI. module CLI - CONFIG_FILE = ".streerc" - # A utility wrapper around colored strings in the output. class Color attr_reader :value, :code @@ -289,16 +287,41 @@ def parser end end + # We allow a minimal configuration file to act as additional command line + # arguments to the CLI. Each line of the config file should be a new + # argument, as in: + # + # --plugins=plugin/single_quote + # --print-width=100 + # + # When invoking the CLI, we will read this config file and then parse it if + # it exists in the current working directory. + class ConfigFile + FILENAME = ".streerc" + + attr_reader :filepath + + def initialize + @filepath = File.join(Dir.pwd, FILENAME) + end + + def exists? + File.readable?(filepath) + end + + def arguments + exists? ? File.readlines(filepath, chomp: true) : [] + end + end + class << self # Run the CLI over the given array of strings that make up the arguments # passed to the invocation. def run(argv) name, *arguments = argv - config_file = File.join(Dir.pwd, CONFIG_FILE) - if File.readable?(config_file) - arguments.unshift(*File.readlines(config_file, chomp: true)) - end + config_file = ConfigFile.new + arguments.unshift(*config_file.arguments) options = Options.new options.parse(arguments) diff --git a/lib/syntax_tree/language_server.rb b/lib/syntax_tree/language_server.rb index 95041b35..16e94534 100644 --- a/lib/syntax_tree/language_server.rb +++ b/lib/syntax_tree/language_server.rb @@ -119,5 +119,9 @@ def write(value) output.print("Content-Length: #{response.bytesize}\r\n\r\n#{response}") output.flush end + + def log(message) + write(method: "window/logMessage", params: { type: 4, message: message }) + end end end diff --git a/test/cli_test.rb b/test/cli_test.rb index 21991e53..b48ea575 100644 --- a/test/cli_test.rb +++ b/test/cli_test.rb @@ -242,7 +242,7 @@ def run_cli(command, *args, contents: :default) end def with_config_file(contents) - filepath = File.join(Dir.pwd, SyntaxTree::CLI::CONFIG_FILE) + filepath = File.join(Dir.pwd, SyntaxTree::CLI::ConfigFile::FILENAME) File.write(filepath, contents) yield From a12fc3f8096e1d6abc6cda016e7451bcfed005e0 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 19 Aug 2022 11:41:57 -0400 Subject: [PATCH 095/536] Bump to v3.4.0 --- CHANGELOG.md | 16 +++++++++++++++- Gemfile.lock | 2 +- lib/syntax_tree/version.rb | 2 +- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf774ba8..70d430d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,19 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +## [3.4.0] - 2022-08-19 + +### Added + +- [#127](https://github.com/ruby-syntax-tree/syntax_tree/pull/127) - Allow the language server to handle other file extensions if it is activated for those extensions. +- [#133](https://github.com/ruby-syntax-tree/syntax_tree/pull/133) - Add documentation on supporting vim and neovim. + +### Changed + +- [#132](https://github.com/ruby-syntax-tree/syntax_tree/pull/132) - Provide better error messages when end quotes and end keywords are missing from tokens. +- [#134](https://github.com/ruby-syntax-tree/syntax_tree/pull/134) - Ensure the correct `end` keyword is getting removed by `begin..rescue` clauses. +- [#137](https://github.com/ruby-syntax-tree/syntax_tree/pull/137) - Better support regular expressions with no ending token. + ## [3.3.0] - 2022-08-02 ### Added @@ -319,7 +332,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - 🎉 Initial release! 🎉 -[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.3.0...HEAD +[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.4.0...HEAD +[3.4.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.3.0...v3.4.0 [3.3.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.2.1...v3.3.0 [3.2.1]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.2.0...v3.2.1 [3.2.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.1.0...v3.2.0 diff --git a/Gemfile.lock b/Gemfile.lock index 01b6e801..dd10aacb 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - syntax_tree (3.3.0) + syntax_tree (3.4.0) prettier_print GEM diff --git a/lib/syntax_tree/version.rb b/lib/syntax_tree/version.rb index 6bc508fe..c5675bac 100644 --- a/lib/syntax_tree/version.rb +++ b/lib/syntax_tree/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SyntaxTree - VERSION = "3.3.0" + VERSION = "3.4.0" end From a5db93d2f8541024ae2027fe3551eec7b796bc79 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 19 Aug 2022 12:06:25 -0400 Subject: [PATCH 096/536] Use q.format for Formatter.format When you're formatting an individual node, it's useful to push it onto the parent stack to make sure every child node can format properly. --- lib/syntax_tree/formatter.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/syntax_tree/formatter.rb b/lib/syntax_tree/formatter.rb index 6efad8d8..c52e45ad 100644 --- a/lib/syntax_tree/formatter.rb +++ b/lib/syntax_tree/formatter.rb @@ -43,10 +43,10 @@ def initialize( end def self.format(source, node) - formatter = new(source, []) - node.format(formatter) - formatter.flush - formatter.output.join + q = new(source, []) + q.format(node) + q.flush + q.output.join end def format(node, stackable: true) From 76493517da0e385cb1b41938b6e4f1b30ecbfc86 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 19 Aug 2022 17:33:12 +0000 Subject: [PATCH 097/536] Bump minitest from 5.16.2 to 5.16.3 Bumps [minitest](https://github.com/seattlerb/minitest) from 5.16.2 to 5.16.3. - [Release notes](https://github.com/seattlerb/minitest/releases) - [Changelog](https://github.com/minitest/minitest/blob/master/History.rdoc) - [Commits](https://github.com/seattlerb/minitest/compare/v5.16.2...v5.16.3) --- updated-dependencies: - dependency-name: minitest dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index dd10aacb..92a993e2 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -10,7 +10,7 @@ GEM ast (2.4.2) docile (1.4.0) json (2.6.2) - minitest (5.16.2) + minitest (5.16.3) parallel (1.22.1) parser (3.1.2.1) ast (~> 2.4.1) From 37b22f8f46b064c1184be3fb9d6d061fdbcbc791 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Aug 2022 17:35:46 +0000 Subject: [PATCH 098/536] Bump rubocop from 1.35.0 to 1.35.1 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.35.0 to 1.35.1. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.35.0...v1.35.1) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 92a993e2..ef016f9c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -19,7 +19,7 @@ GEM rake (13.0.6) regexp_parser (2.5.0) rexml (3.2.5) - rubocop (1.35.0) + rubocop (1.35.1) json (~> 2.3) parallel (~> 1.10) parser (>= 3.1.2.1) From 87d928c1b5dd440d71b1fdb9576ec382100d793e Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 23 Aug 2022 16:54:18 -0400 Subject: [PATCH 099/536] Fix right assignment token management --- lib/syntax_tree/parser.rb | 7 ++++++- test/fixtures/rassign.rb | 3 +++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index 8af0b8ed..ed9de499 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -910,7 +910,12 @@ def on_case(value, consequent) location: keyword.location.to(consequent.location) ) else - operator = find_token(Kw, "in", consume: false) || find_token(Op, "=>") + operator = + if (keyword = find_token(Kw, "in", consume: false)) + tokens.delete(keyword) + else + find_token(Op, "=>") + end RAssign.new( value: value, diff --git a/test/fixtures/rassign.rb b/test/fixtures/rassign.rb index ce749550..3db52b18 100644 --- a/test/fixtures/rassign.rb +++ b/test/fixtures/rassign.rb @@ -20,3 +20,6 @@ ConstantConstantConstant, ConstantConstantConstant ] +% +a in Integer +b => [Integer => c] From d728d250889785033aa9887e1459088f7ba5aaf1 Mon Sep 17 00:00:00 2001 From: Ufuk Kayserilioglu Date: Fri, 26 Aug 2022 01:21:39 +0300 Subject: [PATCH 100/536] Rename STDINItem to ScriptItem --- lib/syntax_tree/cli.rb | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/lib/syntax_tree/cli.rb b/lib/syntax_tree/cli.rb index a9ebdef7..7abfebd5 100644 --- a/lib/syntax_tree/cli.rb +++ b/lib/syntax_tree/cli.rb @@ -53,18 +53,24 @@ def source end end - # An item of work that corresponds to the stdin content. - class STDINItem + # An item of work that corresponds to a script content passed via the command line. + class ScriptItem + FILEPATH = :script + + def initialize(source) + @source = source + end + def handler HANDLERS[".rb"] end def filepath - :stdin + FILEPATH end def source - $stdin.read + @source end end @@ -191,7 +197,7 @@ def run(item) source = item.source formatted = item.handler.format(source, options.print_width) - File.write(filepath, formatted) if filepath != :stdin + File.write(filepath, formatted) if FileItem === item color = source == formatted ? Color.gray(filepath) : filepath delta = ((Time.now - start) * 1000).round @@ -380,7 +386,7 @@ def run(argv) end end else - queue << STDINItem.new + queue << ScriptItem.new($stdin.read) end # At the end, we're going to return whether or not this worker ever From 1be1965f3e8c83157a57691c535342cbcaea743e Mon Sep 17 00:00:00 2001 From: Ufuk Kayserilioglu Date: Fri, 26 Aug 2022 01:36:25 +0300 Subject: [PATCH 101/536] Add inline script option --- lib/syntax_tree/cli.rb | 37 +++++++++++++++++++++++++------------ test/cli_test.rb | 10 ++++++++++ 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/lib/syntax_tree/cli.rb b/lib/syntax_tree/cli.rb index 7abfebd5..65c71f4d 100644 --- a/lib/syntax_tree/cli.rb +++ b/lib/syntax_tree/cli.rb @@ -212,25 +212,25 @@ def run(item) # The help message displayed if the input arguments are not correctly # ordered or formatted. HELP = <<~HELP - #{Color.bold("stree ast [--plugins=...] [--print-width=NUMBER] FILE")} + #{Color.bold("stree ast [--plugins=...] [--print-width=NUMBER] [-e SCRIPT] FILE")} Print out the AST corresponding to the given files - #{Color.bold("stree check [--plugins=...] [--print-width=NUMBER] FILE")} + #{Color.bold("stree check [--plugins=...] [--print-width=NUMBER] [-e SCRIPT] FILE")} Check that the given files are formatted as syntax tree would format them - #{Color.bold("stree debug [--plugins=...] [--print-width=NUMBER] FILE")} + #{Color.bold("stree debug [--plugins=...] [--print-width=NUMBER] [-e SCRIPT] FILE")} Check that the given files can be formatted idempotently - #{Color.bold("stree doc [--plugins=...] FILE")} + #{Color.bold("stree doc [--plugins=...] [-e SCRIPT] FILE")} Print out the doc tree that would be used to format the given files - #{Color.bold("stree format [--plugins=...] [--print-width=NUMBER] FILE")} + #{Color.bold("stree format [--plugins=...] [--print-width=NUMBER] [-e SCRIPT] FILE")} Print out the formatted version of the given files - #{Color.bold("stree json [--plugins=...] FILE")} + #{Color.bold("stree json [--plugins=...] [-e SCRIPT] FILE")} Print out the JSON representation of the given files - #{Color.bold("stree match [--plugins=...] FILE")} + #{Color.bold("stree match [--plugins=...] [-e SCRIPT] FILE")} Print out a pattern-matching Ruby expression that would match the given files #{Color.bold("stree help")} @@ -242,7 +242,7 @@ def run(item) #{Color.bold("stree version")} Output the current version of syntax tree - #{Color.bold("stree write [--plugins=...] [--print-width=NUMBER] FILE")} + #{Color.bold("stree write [--plugins=...] [--print-width=NUMBER] [-e SCRIPT] FILE")} Read, format, and write back the source of the given files --plugins=... @@ -250,20 +250,24 @@ def run(item) --print-width=NUMBER The maximum line width to use when formatting. + + -e SCRIPT + Parse an inline Ruby string. HELP # This represents all of the options that can be passed to the CLI. It is # responsible for parsing the list and then returning the file paths at the # end. class Options - attr_reader :print_width + attr_reader :print_width, :scripts def initialize(print_width: DEFAULT_PRINT_WIDTH) @print_width = print_width + @scripts = [] end def parse(arguments) - parser.parse(arguments) + parser.parse!(arguments) end private @@ -289,6 +293,12 @@ def parser opts.on("--print-width=NUMBER", Integer) do |print_width| @print_width = print_width end + + # If there is a script specified on the command line, then parse + # it and add it to the list of scripts to run. + opts.on("-e SCRIPT") do |script| + @scripts << script + end end end end @@ -367,7 +377,7 @@ def run(argv) # If we're not reading from stdin and the user didn't supply and # filepaths to be read, then we exit with the usage message. - if $stdin.tty? && arguments.empty? + if $stdin.tty? && arguments.empty? && options.scripts.empty? warn(HELP) return 1 end @@ -377,7 +387,7 @@ def run(argv) # If we're reading from stdin, then we'll just add the stdin object to # the queue. Otherwise, we'll add each of the filepaths to the queue. - if $stdin.tty? || arguments.any? + if $stdin.tty? && (arguments.any? || options.scripts.any?) arguments.each do |pattern| Dir .glob(pattern) @@ -385,6 +395,9 @@ def run(argv) queue << FileItem.new(filepath) if File.file?(filepath) end end + options.scripts.each do |script| + queue << ScriptItem.new(script) + end else queue << ScriptItem.new($stdin.read) end diff --git a/test/cli_test.rb b/test/cli_test.rb index b48ea575..4e8959e2 100644 --- a/test/cli_test.rb +++ b/test/cli_test.rb @@ -133,6 +133,16 @@ def test_no_arguments_no_tty $stdin = stdin end + def test_inline_script + stdio, = capture_io { SyntaxTree::CLI.run(["format", "-e", "1+1"]) } + assert_equal("1 + 1\n", stdio) + end + + def test_multiple_inline_scripts + stdio, = capture_io { SyntaxTree::CLI.run(["format", "-e", "1+1", "-e", "2+2"]) } + assert_equal("1 + 1\n2 + 2\n", stdio) + end + def test_generic_error SyntaxTree.stub(:format, ->(*) { raise }) do result = run_cli("format") From 4a1ac2fdb10067d0403a82f04b463af4227d1c27 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 23 Aug 2022 17:05:46 -0400 Subject: [PATCH 102/536] Support Ruby 2.7.0 --- .github/workflows/main.yml | 2 +- Rakefile | 17 +++++++++++---- lib/syntax_tree/cli.rb | 35 +++++++++++++++++++----------- lib/syntax_tree/formatter.rb | 12 +++++++--- lib/syntax_tree/node.rb | 15 ++++++++----- lib/syntax_tree/rake/check_task.rb | 13 ++++++++++- lib/syntax_tree/rake/write_task.rb | 13 ++++++++++- syntax_tree.gemspec | 2 +- test/cli_test.rb | 4 ++-- test/fixtures/args_forward.rb | 2 +- test/fixtures/hshptn.rb | 2 +- test/fixtures/params.rb | 2 +- test/node_test.rb | 20 +++++++++-------- test/test_helper.rb | 2 +- 14 files changed, 96 insertions(+), 45 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ed3c51fd..d707f33c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -8,7 +8,7 @@ jobs: fail-fast: false matrix: ruby: - - '2.7' + - '2.7.0' - '3.0' - '3.1' - head diff --git a/Rakefile b/Rakefile index 6ba17fe9..4973d45e 100644 --- a/Rakefile +++ b/Rakefile @@ -12,8 +12,17 @@ end task default: :test -SOURCE_FILES = - FileList[%w[Gemfile Rakefile syntax_tree.gemspec lib/**/*.rb test/*.rb]] +configure = ->(task) do + task.source_files = + FileList[%w[Gemfile Rakefile syntax_tree.gemspec lib/**/*.rb test/*.rb]] -SyntaxTree::Rake::CheckTask.new { |t| t.source_files = SOURCE_FILES } -SyntaxTree::Rake::WriteTask.new { |t| t.source_files = SOURCE_FILES } + # Since Syntax Tree supports back to Ruby 2.7.0, we need to make sure that we + # format our code such that it's compatible with that version. This actually + # has very little effect on the output, the only change at the moment is that + # Ruby < 2.7.3 didn't allow a newline before the closing brace of a hash + # pattern. + task.target_ruby_version = Gem::Version.new("2.7.0") +end + +SyntaxTree::Rake::CheckTask.new(&configure) +SyntaxTree::Rake::WriteTask.new(&configure) diff --git a/lib/syntax_tree/cli.rb b/lib/syntax_tree/cli.rb index 65c71f4d..2de20e78 100644 --- a/lib/syntax_tree/cli.rb +++ b/lib/syntax_tree/cli.rb @@ -53,10 +53,13 @@ def source end end - # An item of work that corresponds to a script content passed via the command line. + # An item of work that corresponds to a script content passed via the + # command line. class ScriptItem FILEPATH = :script + attr_reader :source + def initialize(source) @source = source end @@ -68,10 +71,6 @@ def handler def filepath FILEPATH end - - def source - @source - end end # The parent action class for the CLI that implements the basics. @@ -197,7 +196,7 @@ def run(item) source = item.source formatted = item.handler.format(source, options.print_width) - File.write(filepath, formatted) if FileItem === item + File.write(filepath, formatted) if item.filepath != :script color = source == formatted ? Color.gray(filepath) : filepath delta = ((Time.now - start) * 1000).round @@ -259,13 +258,19 @@ def run(item) # responsible for parsing the list and then returning the file paths at the # end. class Options - attr_reader :print_width, :scripts + attr_reader :plugins, :print_width, :scripts, :target_ruby_version def initialize(print_width: DEFAULT_PRINT_WIDTH) + @plugins = [] @print_width = print_width @scripts = [] + @target_ruby_version = nil end + # TODO: This function causes a couple of side-effects that I really don't + # like to have here. It mutates the global state by requiring the plugins, + # and mutates the global options hash by adding the target ruby version. + # That should be done on a config-by-config basis, not here. def parse(arguments) parser.parse!(arguments) end @@ -285,7 +290,8 @@ def parser # require "syntax_tree/haml" # opts.on("--plugins=PLUGINS") do |plugins| - plugins.split(",").each { |plugin| require "syntax_tree/#{plugin}" } + @plugins = plugins.split(",") + @plugins.each { |plugin| require "syntax_tree/#{plugin}" } end # If there is a print width specified on the command line, then @@ -296,8 +302,13 @@ def parser # If there is a script specified on the command line, then parse # it and add it to the list of scripts to run. - opts.on("-e SCRIPT") do |script| - @scripts << script + opts.on("-e SCRIPT") { |script| @scripts << script } + + # If there is a target ruby version specified on the command line, + # parse that out and use it when formatting. + opts.on("--target-ruby-version=VERSION") do |version| + @target_ruby_version = Gem::Version.new(version) + Formatter::OPTIONS[:target_ruby_version] = @target_ruby_version end end end @@ -395,9 +406,7 @@ def run(argv) queue << FileItem.new(filepath) if File.file?(filepath) end end - options.scripts.each do |script| - queue << ScriptItem.new(script) - end + options.scripts.each { |script| queue << ScriptItem.new(script) } else queue << ScriptItem.new($stdin.read) end diff --git a/lib/syntax_tree/formatter.rb b/lib/syntax_tree/formatter.rb index c52e45ad..4c7a00db 100644 --- a/lib/syntax_tree/formatter.rb +++ b/lib/syntax_tree/formatter.rb @@ -14,7 +14,11 @@ class Formatter < PrettierPrint # Note that we're keeping this in a global-ish hash instead of just # overriding methods on classes so that other plugins can reference this if # necessary. For example, the RBS plugin references the quote style. - OPTIONS = { quote: "\"", trailing_comma: false } + OPTIONS = { + quote: "\"", + trailing_comma: false, + target_ruby_version: Gem::Version.new(RUBY_VERSION) + } COMMENT_PRIORITY = 1 HEREDOC_PRIORITY = 2 @@ -23,14 +27,15 @@ class Formatter < PrettierPrint # These options are overridden in plugins to we need to make sure they are # available here. - attr_reader :quote, :trailing_comma + attr_reader :quote, :trailing_comma, :target_ruby_version alias trailing_comma? trailing_comma def initialize( source, *args, quote: OPTIONS[:quote], - trailing_comma: OPTIONS[:trailing_comma] + trailing_comma: OPTIONS[:trailing_comma], + target_ruby_version: OPTIONS[:target_ruby_version] ) super(*args) @@ -40,6 +45,7 @@ def initialize( # Memoizing these values per formatter to make access faster. @quote = quote @trailing_comma = trailing_comma + @target_ruby_version = target_ruby_version end def self.format(source, node) diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index d7b6d6cf..47c534d1 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -2132,8 +2132,7 @@ def format(q) in [ Paren[ contents: { - body: [ArrayLiteral[contents: { parts: [_, _, *] }] => array] - } + body: [ArrayLiteral[contents: { parts: [_, _, *] }] => array] } ] ] # Here we have a single argument that is a set of parentheses wrapping @@ -5116,8 +5115,13 @@ def format(q) q.breakable contents.call end - q.breakable - q.text("}") + + if q.target_ruby_version < Gem::Version.new("2.7.3") + q.text(" }") + else + q.breakable + q.text("}") + end end end end @@ -5204,8 +5208,7 @@ def call(q, node) false in { statements: { body: [truthy] }, - consequent: Else[statements: { body: [falsy] }] - } + consequent: Else[statements: { body: [falsy] }] } ternaryable?(truthy) && ternaryable?(falsy) else false diff --git a/lib/syntax_tree/rake/check_task.rb b/lib/syntax_tree/rake/check_task.rb index afe5013c..48247718 100644 --- a/lib/syntax_tree/rake/check_task.rb +++ b/lib/syntax_tree/rake/check_task.rb @@ -39,16 +39,22 @@ class CheckTask < ::Rake::TaskLib # Defaults to 80. attr_accessor :print_width + # The target Ruby version to use for formatting. + # Defaults to Gem::Version.new(RUBY_VERSION). + attr_accessor :target_ruby_version + def initialize( name = :"stree:check", source_files = ::Rake::FileList["lib/**/*.rb"], plugins = [], - print_width = DEFAULT_PRINT_WIDTH + print_width = DEFAULT_PRINT_WIDTH, + target_ruby_version = Gem::Version.new(RUBY_VERSION) ) @name = name @source_files = source_files @plugins = plugins @print_width = print_width + @target_ruby_version = target_ruby_version yield self if block_given? define_task @@ -64,10 +70,15 @@ def define_task def run_task arguments = ["check"] arguments << "--plugins=#{plugins.join(",")}" if plugins.any? + if print_width != DEFAULT_PRINT_WIDTH arguments << "--print-width=#{print_width}" end + if target_ruby_version != Gem::Version.new(RUBY_VERSION) + arguments << "--target-ruby-version=#{target_ruby_version}" + end + SyntaxTree::CLI.run(arguments + Array(source_files)) end end diff --git a/lib/syntax_tree/rake/write_task.rb b/lib/syntax_tree/rake/write_task.rb index 9a9e8330..69ce97e7 100644 --- a/lib/syntax_tree/rake/write_task.rb +++ b/lib/syntax_tree/rake/write_task.rb @@ -39,16 +39,22 @@ class WriteTask < ::Rake::TaskLib # Defaults to 80. attr_accessor :print_width + # The target Ruby version to use for formatting. + # Defaults to Gem::Version.new(RUBY_VERSION). + attr_accessor :target_ruby_version + def initialize( name = :"stree:write", source_files = ::Rake::FileList["lib/**/*.rb"], plugins = [], - print_width = DEFAULT_PRINT_WIDTH + print_width = DEFAULT_PRINT_WIDTH, + target_ruby_version = Gem::Version.new(RUBY_VERSION) ) @name = name @source_files = source_files @plugins = plugins @print_width = print_width + @target_ruby_version = target_ruby_version yield self if block_given? define_task @@ -64,10 +70,15 @@ def define_task def run_task arguments = ["write"] arguments << "--plugins=#{plugins.join(",")}" if plugins.any? + if print_width != DEFAULT_PRINT_WIDTH arguments << "--print-width=#{print_width}" end + if target_ruby_version != Gem::Version.new(RUBY_VERSION) + arguments << "--target-ruby-version=#{target_ruby_version}" + end + SyntaxTree::CLI.run(arguments + Array(source_files)) end end diff --git a/syntax_tree.gemspec b/syntax_tree.gemspec index 820a61a0..2b461dfd 100644 --- a/syntax_tree.gemspec +++ b/syntax_tree.gemspec @@ -19,7 +19,7 @@ Gem::Specification.new do |spec| .reject { |f| f.match(%r{^(test|spec|features)/}) } end - spec.required_ruby_version = ">= 2.7.3" + spec.required_ruby_version = ">= 2.7.0" spec.bindir = "exe" spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } diff --git a/test/cli_test.rb b/test/cli_test.rb index 4e8959e2..aec8f820 100644 --- a/test/cli_test.rb +++ b/test/cli_test.rb @@ -134,12 +134,12 @@ def test_no_arguments_no_tty end def test_inline_script - stdio, = capture_io { SyntaxTree::CLI.run(["format", "-e", "1+1"]) } + stdio, = capture_io { SyntaxTree::CLI.run(%w[format -e 1+1]) } assert_equal("1 + 1\n", stdio) end def test_multiple_inline_scripts - stdio, = capture_io { SyntaxTree::CLI.run(["format", "-e", "1+1", "-e", "2+2"]) } + stdio, = capture_io { SyntaxTree::CLI.run(%w[format -e 1+1 -e 2+2]) } assert_equal("1 + 1\n2 + 2\n", stdio) end diff --git a/test/fixtures/args_forward.rb b/test/fixtures/args_forward.rb index 5ba618a8..cc538f44 100644 --- a/test/fixtures/args_forward.rb +++ b/test/fixtures/args_forward.rb @@ -1,4 +1,4 @@ -% +% # >= 2.7.3 def foo(...) bar(:baz, ...) end diff --git a/test/fixtures/hshptn.rb b/test/fixtures/hshptn.rb index 505336b8..02d1cf75 100644 --- a/test/fixtures/hshptn.rb +++ b/test/fixtures/hshptn.rb @@ -30,7 +30,7 @@ case foo in **bar end -% +% # >= 2.7.3 case foo in { foo:, # comment1 diff --git a/test/fixtures/params.rb b/test/fixtures/params.rb index 67b6ec90..551aa9a5 100644 --- a/test/fixtures/params.rb +++ b/test/fixtures/params.rb @@ -16,7 +16,7 @@ def foo(*) % def foo(*rest) end -% +% # >= 2.7.3 def foo(...) end % diff --git a/test/node_test.rb b/test/node_test.rb index 30776f9d..07c2fe26 100644 --- a/test/node_test.rb +++ b/test/node_test.rb @@ -104,16 +104,18 @@ def test_arg_star end end - def test_args_forward - source = <<~SOURCE - def get(...) - request(:GET, ...) - end - SOURCE + guard_version("2.7.3") do + def test_args_forward + source = <<~SOURCE + def get(...) + request(:GET, ...) + end + SOURCE - at = location(lines: 2..2, chars: 29..32) - assert_node(ArgsForward, source, at: at) do |node| - node.bodystmt.statements.body.first.arguments.arguments.parts.last + at = location(lines: 2..2, chars: 29..32) + assert_node(ArgsForward, source, at: at) do |node| + node.bodystmt.statements.body.first.arguments.arguments.parts.last + end end end diff --git a/test/test_helper.rb b/test/test_helper.rb index 895fbc82..80e514f0 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -26,7 +26,7 @@ def initialize @called = nil end - def method_missing(called, ...) + def method_missing(called, *, **) @called = called end end From f79754f63d78db19a6c90d832139b33724f594bb Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 23 Aug 2022 17:05:46 -0400 Subject: [PATCH 103/536] Support Ruby 2.7.0 --- .github/workflows/auto-merge.yml | 22 ++++++++++++++++++++++ .github/workflows/main.yml | 19 +------------------ 2 files changed, 23 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/auto-merge.yml diff --git a/.github/workflows/auto-merge.yml b/.github/workflows/auto-merge.yml new file mode 100644 index 00000000..9b28abf4 --- /dev/null +++ b/.github/workflows/auto-merge.yml @@ -0,0 +1,22 @@ +name: Dependabot auto-merge +on: pull_request + +permissions: + contents: write + pull-requests: write + +jobs: + dependabot: + runs-on: ubuntu-latest + if: ${{ github.actor == 'dependabot[bot]' }} + steps: + - name: Dependabot metadata + id: metadata + uses: dependabot/fetch-metadata@v1.3.3 + with: + github-token: "${{ secrets.GITHUB_TOKEN }}" + - name: Enable auto-merge for Dependabot PRs + run: gh pr merge --auto --merge "$PR_URL" + env: + PR_URL: ${{github.event.pull_request.html_url}} + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d707f33c..d35471fa 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,7 +1,7 @@ name: Main on: - push -- pull_request_target +- pull_request jobs: ci: strategy: @@ -40,20 +40,3 @@ jobs: run: | bundle exec rake stree:check bundle exec rubocop - - automerge: - name: AutoMerge - needs: - - ci - - check - runs-on: ubuntu-latest - if: github.event_name == 'pull_request_target' && github.actor == 'dependabot[bot]' - steps: - - uses: actions/github-script@v3 - with: - script: | - github.pulls.merge({ - owner: context.payload.repository.owner.login, - repo: context.payload.repository.name, - pull_number: context.payload.pull_request.number - }) From 75cc00a769c28a49ec42e35b9a09bfbe175d8264 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 26 Aug 2022 11:43:34 -0400 Subject: [PATCH 104/536] Bump to v3.5.0 --- CHANGELOG.md | 15 ++++++++++++++- Gemfile.lock | 2 +- lib/syntax_tree/version.rb | 2 +- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70d430d2..9884a16b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,18 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +## [3.5.0] - 2022-08-26 + +### Added + +- [#148](https://github.com/ruby-syntax-tree/syntax_tree/pull/148) - Support Ruby 2.7.0 (previously we only supported back to 2.7.3). +- [#152](https://github.com/ruby-syntax-tree/syntax_tree/pull/152) - Support the `-e` inline script option for the `stree` CLI. + +### Changed + +- [#141](https://github.com/ruby-syntax-tree/syntax_tree/pull/141) - Use `q.format` for `SyntaxTree.format` so that the main node gets pushed onto the stack for checking parent nodes. +- [#147](https://github.com/ruby-syntax-tree/syntax_tree/pull/147) - Fix rightward assignment token management such that `in` and `=>` stay the same regardless of their context. + ## [3.4.0] - 2022-08-19 ### Added @@ -332,7 +344,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - 🎉 Initial release! 🎉 -[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.4.0...HEAD +[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.5.0...HEAD +[3.5.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.4.0...v3.5.0 [3.4.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.3.0...v3.4.0 [3.3.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.2.1...v3.3.0 [3.2.1]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.2.0...v3.2.1 diff --git a/Gemfile.lock b/Gemfile.lock index ef016f9c..1d69d297 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - syntax_tree (3.4.0) + syntax_tree (3.5.0) prettier_print GEM diff --git a/lib/syntax_tree/version.rb b/lib/syntax_tree/version.rb index c5675bac..42aa2b6c 100644 --- a/lib/syntax_tree/version.rb +++ b/lib/syntax_tree/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SyntaxTree - VERSION = "3.4.0" + VERSION = "3.5.0" end From 3708b7c149570aeecceae8c83f5f98dca5e0fb36 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 26 Aug 2022 11:56:22 -0400 Subject: [PATCH 105/536] More attempts to fix CI --- .github/workflows/gh-pages.yml | 6 ++++- lib/syntax_tree/cli.rb | 40 ++++++++++++++++++++++++++++------ test/cli_test.rb | 24 ++++++++++++++------ 3 files changed, 55 insertions(+), 15 deletions(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 8adc1b45..a8ce8d15 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -1,5 +1,9 @@ name: Github Pages (rdoc) -on: [push] +on: + push: + branches: + - main + jobs: build-and-deploy: runs-on: ubuntu-latest diff --git a/lib/syntax_tree/cli.rb b/lib/syntax_tree/cli.rb index 2de20e78..ca3647f5 100644 --- a/lib/syntax_tree/cli.rb +++ b/lib/syntax_tree/cli.rb @@ -51,13 +51,15 @@ def handler def source handler.read(filepath) end + + def writable? + File.writable?(filepath) + end end # An item of work that corresponds to a script content passed via the # command line. class ScriptItem - FILEPATH = :script - attr_reader :source def initialize(source) @@ -69,7 +71,30 @@ def handler end def filepath - FILEPATH + :script + end + + def writable? + false + end + end + + # An item of work that correspond to the content passed in via stdin. + class STDINItem + def handler + HANDLERS[".rb"] + end + + def filepath + :stdin + end + + def source + $stdin.read + end + + def writable? + false end end @@ -196,7 +221,7 @@ def run(item) source = item.source formatted = item.handler.format(source, options.print_width) - File.write(filepath, formatted) if item.filepath != :script + File.write(filepath, formatted) if item.writable? color = source == formatted ? Color.gray(filepath) : filepath delta = ((Time.now - start) * 1000).round @@ -386,7 +411,7 @@ def run(argv) return 1 end - # If we're not reading from stdin and the user didn't supply and + # If we're not reading from stdin and the user didn't supply any # filepaths to be read, then we exit with the usage message. if $stdin.tty? && arguments.empty? && options.scripts.empty? warn(HELP) @@ -403,12 +428,13 @@ def run(argv) Dir .glob(pattern) .each do |filepath| - queue << FileItem.new(filepath) if File.file?(filepath) + queue << FileItem.new(filepath) if File.readable?(filepath) end end + options.scripts.each { |script| queue << ScriptItem.new(script) } else - queue << ScriptItem.new($stdin.read) + queue << STDINItem.new end # At the end, we're going to return whether or not this worker ever diff --git a/test/cli_test.rb b/test/cli_test.rb index aec8f820..6743f759 100644 --- a/test/cli_test.rb +++ b/test/cli_test.rb @@ -117,7 +117,7 @@ def test_help_default end def test_no_arguments - $stdin.stub(:tty?, true) do + with_tty do *, stderr = capture_io { SyntaxTree::CLI.run(["check"]) } assert_includes(stderr, "stree help") end @@ -134,13 +134,17 @@ def test_no_arguments_no_tty end def test_inline_script - stdio, = capture_io { SyntaxTree::CLI.run(%w[format -e 1+1]) } - assert_equal("1 + 1\n", stdio) + with_tty do + stdio, = capture_io { SyntaxTree::CLI.run(%w[format -e 1+1]) } + assert_equal("1 + 1\n", stdio) + end end def test_multiple_inline_scripts - stdio, = capture_io { SyntaxTree::CLI.run(%w[format -e 1+1 -e 2+2]) } - assert_equal("1 + 1\n2 + 2\n", stdio) + with_tty do + stdio, = capture_io { SyntaxTree::CLI.run(%w[format -e 1+1 -e 2+2]) } + assert_equal("1 + 1\n2 + 2\n", stdio) + end end def test_generic_error @@ -241,8 +245,10 @@ def run_cli(command, *args, contents: :default) status = nil stdio, stderr = - capture_io do - status = SyntaxTree::CLI.run([command, *args, tempfile.path]) + with_tty do + capture_io do + status = SyntaxTree::CLI.run([command, *args, tempfile.path]) + end end Result.new(status: status, stdio: stdio, stderr: stderr) @@ -251,6 +257,10 @@ def run_cli(command, *args, contents: :default) tempfile.unlink end + def with_tty(&block) + $stdin.stub(:tty?, true, &block) + end + def with_config_file(contents) filepath = File.join(Dir.pwd, SyntaxTree::CLI::ConfigFile::FILENAME) File.write(filepath, contents) From 6a898eabc389d8dccca78ed1b0849062956ce3a3 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 26 Aug 2022 12:13:58 -0400 Subject: [PATCH 106/536] GitHub pages new workflow --- .github/workflows/gh-pages.yml | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index a8ce8d15..e818de22 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -5,26 +5,34 @@ on: - main jobs: - build-and-deploy: + build: runs-on: ubuntu-latest steps: - - name: Checkout 🛎️ + - name: Checkout uses: actions/checkout@master - - - name: Set up Ruby 💎 + - name: Set up Ruby uses: ruby/setup-ruby@v1 with: bundler-cache: true ruby-version: '3.1' - - - name: Install rdoc and generate docs 🔧 + - name: Generate docs run: | gem install rdoc - rdoc --main README.md --op rdocs --exclude={Gemfile,Rakefile,"coverage/*","vendor/*","bin/*","test/*","tmp/*"} - cp -r doc rdocs/doc + rdoc --main README.md --op _site --exclude={Gemfile,Rakefile,"coverage/*","vendor/*","bin/*","test/*","tmp/*"} + cp -r doc _site/doc + - name: Upload artifact 🚀 + uses: actions/upload-pages-artifact@main - - name: Deploy 🚀 - uses: peaceiris/actions-gh-pages@v3 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: ./rdocs + deploy: + runs-on: ubuntu-latest + needs: build + permissions: + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v1 From b5c60eae9a5f94a0f4d35d205879a5549e5bef3d Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 26 Aug 2022 12:18:49 -0400 Subject: [PATCH 107/536] More GitHub pages rework --- .github/workflows/gh-pages.yml | 36 ++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index e818de22..fe1917d7 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -1,15 +1,33 @@ -name: Github Pages (rdoc) +name: Deploy rdoc to GitHub Pages + on: push: branches: - - main + - $default-branch + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages +permissions: + contents: read + pages: write + id-token: write + +# Allow one concurrent deployment +concurrency: + group: "pages" + cancel-in-progress: true jobs: + # Build job build: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@master + uses: actions/checkout@v3 + - name: Setup Pages + uses: actions/configure-pages@v2 - name: Set up Ruby uses: ruby/setup-ruby@v1 with: @@ -20,18 +38,16 @@ jobs: gem install rdoc rdoc --main README.md --op _site --exclude={Gemfile,Rakefile,"coverage/*","vendor/*","bin/*","test/*","tmp/*"} cp -r doc _site/doc - - name: Upload artifact 🚀 - uses: actions/upload-pages-artifact@main + - name: Upload artifact + uses: actions/upload-pages-artifact@v1 + # Deployment job deploy: - runs-on: ubuntu-latest - needs: build - permissions: - pages: write - id-token: write environment: name: github-pages url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + needs: build steps: - name: Deploy to GitHub Pages id: deployment From 3c6918936d19e8bd8a93f9c066d7a49f673bbc4c Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 26 Aug 2022 12:20:48 -0400 Subject: [PATCH 108/536] Triggering GitHub actions From b49c28aca515ab1242ec4a039aac057902ee2701 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 26 Aug 2022 12:24:07 -0400 Subject: [PATCH 109/536] Trigger based on main branch --- .github/workflows/gh-pages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index fe1917d7..fc02f2fe 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -3,7 +3,7 @@ name: Deploy rdoc to GitHub Pages on: push: branches: - - $default-branch + - main # Allows you to run this workflow manually from the Actions tab workflow_dispatch: From 31279b8f8ba2ef14b5bfaba591f8f2de0b48e198 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 26 Aug 2022 12:41:35 -0400 Subject: [PATCH 110/536] More documentation on how to pass in content --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9f25b0e7..0aa1e79b 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,9 @@ bundle exec stree version ## CLI -Syntax Tree ships with the `stree` CLI, which can be used to inspect and manipulate Ruby code. Below are listed all of the commands built into the CLI that you can use. Note that for all commands that operate on files, you can also pass in content through STDIN. +Syntax Tree ships with the `stree` CLI, which can be used to inspect and manipulate Ruby code. Below are listed all of the commands built into the CLI that you can use. + +For many commands, file paths are accepted after the configuration options. For all of these commands, you can alternatively pass in content through STDIN or through the `-e` option to specify an inline script. ### ast From 066dc61237900341dfda7ce165ce870b0ec2bb83 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 26 Aug 2022 13:06:09 -0400 Subject: [PATCH 111/536] Document additional plugins --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0aa1e79b..47545502 100644 --- a/README.md +++ b/README.md @@ -502,9 +502,12 @@ In this case, whenever the CLI encounters a filepath that ends with the given ex Below are listed all of the "official" language plugins hosted under the same GitHub organization, which can be used as references for how to implement other plugins. +* [bf](https://github.com/ruby-syntax-tree/syntax_tree-bf) for the [brainf*** language](https://esolangs.org/wiki/Brainfuck). +* [css](https://github.com/ruby-syntax-tree/syntax_tree-css) for the [CSS stylesheet language](https://www.w3.org/Style/CSS/). * [haml](https://github.com/ruby-syntax-tree/syntax_tree-haml) for the [Haml template language](https://haml.info/). -* [json](https://github.com/ruby-syntax-tree/syntax_tree-json) for JSON. +* [json](https://github.com/ruby-syntax-tree/syntax_tree-json) for the [JSON notation language](https://www.json.org/). * [rbs](https://github.com/ruby-syntax-tree/syntax_tree-rbs) for the [RBS type language](https://github.com/ruby/rbs). +* [xml](https://github.com/ruby-syntax-tree/syntax_tree-xml) for the [XML markup language](https://www.w3.org/XML/). ## Integration From 22ea12e52821279082705d9ab7958c813175d000 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Sep 2022 17:26:47 +0000 Subject: [PATCH 112/536] Bump rubocop from 1.35.1 to 1.36.0 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.35.1 to 1.36.0. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.35.1...v1.36.0) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 1d69d297..12c78eba 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -19,7 +19,7 @@ GEM rake (13.0.6) regexp_parser (2.5.0) rexml (3.2.5) - rubocop (1.35.1) + rubocop (1.36.0) json (~> 2.3) parallel (~> 1.10) parser (>= 3.1.2.1) From 9d09cd4ea4a7ef8a3a4dbe4dd1347406f4ffc825 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 19 Sep 2022 10:49:06 -0400 Subject: [PATCH 113/536] Support ignoring certain filepaths in CLI/rake --- README.md | 25 +++++++-- lib/syntax_tree/cli.rb | 19 ++++++- lib/syntax_tree/rake/check_task.rb | 71 +++---------------------- lib/syntax_tree/rake/task.rb | 85 ++++++++++++++++++++++++++++++ lib/syntax_tree/rake/write_task.rb | 73 +++---------------------- test/cli_test.rb | 6 +++ 6 files changed, 144 insertions(+), 135 deletions(-) create mode 100644 lib/syntax_tree/rake/task.rb diff --git a/README.md b/README.md index 47545502..afb65843 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ It is built with only standard library dependencies. It additionally ships with - [textDocument/inlayHint](#textdocumentinlayhint) - [syntaxTree/visualizing](#syntaxtreevisualizing) - [Plugins](#plugins) - - [Configuration](#configuration) + - [Customization](#customization) - [Languages](#languages) - [Integration](#integration) - [Rake](#rake) @@ -235,6 +235,12 @@ To change the print width that you are writing with, specify the `--print-width` stree write --print-width=100 path/to/file.rb ``` +To ignore certain files from a glob (in order to make it easier to specify the filepaths), you can pass the `--ignore-files` option as an additional glob, as in: + +```sh +stree write --ignore-files='db/**/*.rb' '**/*.rb' +``` + ### Configuration Any of the above CLI commands can also read configuration options from a `.streerc` file in the directory where the commands are executed. @@ -475,11 +481,11 @@ The language server additionally includes this custom request to return a textua ## Plugins -You can register additional configuration and additional languages that can flow through the same CLI with Syntax Tree's plugin system. When invoking the CLI, you pass through the list of plugins with the `--plugins` options to the commands that accept them. They should be a comma-delimited list. When the CLI first starts, it will require the files corresponding to those names. +You can register additional customization and additional languages that can flow through the same CLI with Syntax Tree's plugin system. When invoking the CLI, you pass through the list of plugins with the `--plugins` options to the commands that accept them. They should be a comma-delimited list. When the CLI first starts, it will require the files corresponding to those names. -### Configuration +### Customization -To register additional configuration, define a file somewhere in your load path named `syntax_tree/my_plugin`. Then when invoking the CLI, you will pass `--plugins=my_plugin`. To require multiple, separate them by a comma. In this way, you can modify Syntax Tree however you would like. Some plugins ship with Syntax Tree itself. They are: +To register additional customization, define a file somewhere in your load path named `syntax_tree/my_plugin`. Then when invoking the CLI, you will pass `--plugins=my_plugin`. To require multiple, separate them by a comma. In this way, you can modify Syntax Tree however you would like. Some plugins ship with Syntax Tree itself. They are: * `plugin/single_quotes` - This will change all of your string literals to use single quotes instead of the default double quotes. * `plugin/trailing_comma` - This will put trailing commas into multiline array literals, hash literals, and method calls that can support trailing commas. @@ -543,6 +549,17 @@ SyntaxTree::Rake::WriteTask.new do |t| end ``` +#### `ignore_files` + +If you want to ignore certain file patterns when running the command, you can pass the `ignore_files` option. This will be checked with `File.fnmatch?` against each filepath that the command would be run against. For example: + +```ruby +SyntaxTree::Rake::WriteTask.new do |t| + t.source_files = "**/*.rb" + t.ignore_files = "db/**/*.rb" +end +``` + #### `print_width` If you want to use a different print width from the default (80), you can pass that to the `print_width` field, as in: diff --git a/lib/syntax_tree/cli.rb b/lib/syntax_tree/cli.rb index ca3647f5..a364cd34 100644 --- a/lib/syntax_tree/cli.rb +++ b/lib/syntax_tree/cli.rb @@ -283,9 +283,14 @@ def run(item) # responsible for parsing the list and then returning the file paths at the # end. class Options - attr_reader :plugins, :print_width, :scripts, :target_ruby_version + attr_reader :ignore_files, + :plugins, + :print_width, + :scripts, + :target_ruby_version def initialize(print_width: DEFAULT_PRINT_WIDTH) + @ignore_files = "" @plugins = [] @print_width = print_width @scripts = [] @@ -304,6 +309,13 @@ def parse(arguments) def parser OptionParser.new do |opts| + # If there is a glob specified to ignore, then we'll track that here. + # Any of the CLI commands that operate on filenames will then ignore + # this set of files. + opts.on("--ignore-files=GLOB") do |glob| + @ignore_files = glob.match(/\A'(.*)'\z/) ? $1 : glob + end + # If there are any plugins specified on the command line, then load # them by requiring them here. We do this by transforming something # like @@ -428,7 +440,10 @@ def run(argv) Dir .glob(pattern) .each do |filepath| - queue << FileItem.new(filepath) if File.readable?(filepath) + if File.readable?(filepath) && + !File.fnmatch?(options.ignore_files, filepath) + queue << FileItem.new(filepath) + end end end diff --git a/lib/syntax_tree/rake/check_task.rb b/lib/syntax_tree/rake/check_task.rb index 48247718..5b441a5b 100644 --- a/lib/syntax_tree/rake/check_task.rb +++ b/lib/syntax_tree/rake/check_task.rb @@ -1,10 +1,6 @@ # frozen_string_literal: true -require "rake" -require "rake/tasklib" - -require "syntax_tree" -require "syntax_tree/cli" +require_relative "task" module SyntaxTree module Rake @@ -12,74 +8,21 @@ module Rake # # Example: # - # require 'syntax_tree/rake/check_task' + # require "syntax_tree/rake/check_task" # # SyntaxTree::Rake::CheckTask.new do |t| - # t.source_files = '{app,config,lib}/**/*.rb' + # t.source_files = "{app,config,lib}/**/*.rb" # end # # This will create task that can be run with: # - # rake stree_check + # rake stree:check # - class CheckTask < ::Rake::TaskLib - # Name of the task. - # Defaults to :"stree:check". - attr_accessor :name - - # Glob pattern to match source files. - # Defaults to 'lib/**/*.rb'. - attr_accessor :source_files - - # The set of plugins to require. - # Defaults to []. - attr_accessor :plugins - - # Max line length. - # Defaults to 80. - attr_accessor :print_width - - # The target Ruby version to use for formatting. - # Defaults to Gem::Version.new(RUBY_VERSION). - attr_accessor :target_ruby_version - - def initialize( - name = :"stree:check", - source_files = ::Rake::FileList["lib/**/*.rb"], - plugins = [], - print_width = DEFAULT_PRINT_WIDTH, - target_ruby_version = Gem::Version.new(RUBY_VERSION) - ) - @name = name - @source_files = source_files - @plugins = plugins - @print_width = print_width - @target_ruby_version = target_ruby_version - - yield self if block_given? - define_task - end - + class CheckTask < Task private - def define_task - desc "Runs `stree check` over source files" - task(name) { run_task } - end - - def run_task - arguments = ["check"] - arguments << "--plugins=#{plugins.join(",")}" if plugins.any? - - if print_width != DEFAULT_PRINT_WIDTH - arguments << "--print-width=#{print_width}" - end - - if target_ruby_version != Gem::Version.new(RUBY_VERSION) - arguments << "--target-ruby-version=#{target_ruby_version}" - end - - SyntaxTree::CLI.run(arguments + Array(source_files)) + def command + "check" end end end diff --git a/lib/syntax_tree/rake/task.rb b/lib/syntax_tree/rake/task.rb new file mode 100644 index 00000000..ea228e8f --- /dev/null +++ b/lib/syntax_tree/rake/task.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +require "rake" +require "rake/tasklib" + +require "syntax_tree" +require "syntax_tree/cli" + +module SyntaxTree + module Rake + # A parent Rake task that runs a command on a set of source files. + class Task < ::Rake::TaskLib + # Name of the task. + attr_accessor :name + + # Glob pattern to match source files. + # Defaults to 'lib/**/*.rb'. + attr_accessor :source_files + + # The set of plugins to require. + # Defaults to []. + attr_accessor :plugins + + # Max line length. + # Defaults to 80. + attr_accessor :print_width + + # The target Ruby version to use for formatting. + # Defaults to Gem::Version.new(RUBY_VERSION). + attr_accessor :target_ruby_version + + # Glob pattern to ignore source files. + # Defaults to ''. + attr_accessor :ignore_files + + def initialize( + name = :"stree:#{command}", + source_files = ::Rake::FileList["lib/**/*.rb"], + plugins = [], + print_width = DEFAULT_PRINT_WIDTH, + target_ruby_version = Gem::Version.new(RUBY_VERSION), + ignore_files = "" + ) + @name = name + @source_files = source_files + @plugins = plugins + @print_width = print_width + @target_ruby_version = target_ruby_version + @ignore_files = ignore_files + + yield self if block_given? + define_task + end + + private + + # This method needs to be overridden in the child tasks. + def command + raise NotImplementedError + end + + def define_task + desc "Runs `stree #{command}` over source files" + task(name) { run_task } + end + + def run_task + arguments = [command] + arguments << "--plugins=#{plugins.join(",")}" if plugins.any? + + if print_width != DEFAULT_PRINT_WIDTH + arguments << "--print-width=#{print_width}" + end + + if target_ruby_version != Gem::Version.new(RUBY_VERSION) + arguments << "--target-ruby-version=#{target_ruby_version}" + end + + arguments << "--ignore-files=#{ignore_files}" if ignore_files != "" + + SyntaxTree::CLI.run(arguments + Array(source_files)) + end + end + end +end diff --git a/lib/syntax_tree/rake/write_task.rb b/lib/syntax_tree/rake/write_task.rb index 69ce97e7..8037792e 100644 --- a/lib/syntax_tree/rake/write_task.rb +++ b/lib/syntax_tree/rake/write_task.rb @@ -1,85 +1,28 @@ # frozen_string_literal: true -require "rake" -require "rake/tasklib" - -require "syntax_tree" -require "syntax_tree/cli" +require_relative "task" module SyntaxTree module Rake - # A Rake task that runs format on a set of source files. + # A Rake task that runs write on a set of source files. # # Example: # - # require 'syntax_tree/rake/write_task' + # require "syntax_tree/rake/write_task" # # SyntaxTree::Rake::WriteTask.new do |t| - # t.source_files = '{app,config,lib}/**/*.rb' + # t.source_files = "{app,config,lib}/**/*.rb" # end # # This will create task that can be run with: # - # rake stree_write + # rake stree:write # - class WriteTask < ::Rake::TaskLib - # Name of the task. - # Defaults to :"stree:write". - attr_accessor :name - - # Glob pattern to match source files. - # Defaults to 'lib/**/*.rb'. - attr_accessor :source_files - - # The set of plugins to require. - # Defaults to []. - attr_accessor :plugins - - # Max line length. - # Defaults to 80. - attr_accessor :print_width - - # The target Ruby version to use for formatting. - # Defaults to Gem::Version.new(RUBY_VERSION). - attr_accessor :target_ruby_version - - def initialize( - name = :"stree:write", - source_files = ::Rake::FileList["lib/**/*.rb"], - plugins = [], - print_width = DEFAULT_PRINT_WIDTH, - target_ruby_version = Gem::Version.new(RUBY_VERSION) - ) - @name = name - @source_files = source_files - @plugins = plugins - @print_width = print_width - @target_ruby_version = target_ruby_version - - yield self if block_given? - define_task - end - + class WriteTask < Task private - def define_task - desc "Runs `stree write` over source files" - task(name) { run_task } - end - - def run_task - arguments = ["write"] - arguments << "--plugins=#{plugins.join(",")}" if plugins.any? - - if print_width != DEFAULT_PRINT_WIDTH - arguments << "--print-width=#{print_width}" - end - - if target_ruby_version != Gem::Version.new(RUBY_VERSION) - arguments << "--target-ruby-version=#{target_ruby_version}" - end - - SyntaxTree::CLI.run(arguments + Array(source_files)) + def command + "write" end end end diff --git a/test/cli_test.rb b/test/cli_test.rb index 6743f759..de09b093 100644 --- a/test/cli_test.rb +++ b/test/cli_test.rb @@ -32,6 +32,12 @@ def test_ast assert_includes(result.stdio, "ident \"test\"") end + def test_ast_ignore + result = run_cli("ast", "--ignore-files='*/test*'") + assert_equal(0, result.status) + assert_empty(result.stdio) + end + def test_ast_syntax_error result = run_cli("ast", contents: "foo\n<>\nbar\n") assert_includes(result.stderr, "syntax error") From 695918e84d5bdc912492db9013110c841da31109 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 19 Sep 2022 11:09:53 -0400 Subject: [PATCH 114/536] Bump to v3.6.0 --- CHANGELOG.md | 9 ++++++++- Gemfile.lock | 2 +- lib/syntax_tree/basic_visitor.rb | 2 +- lib/syntax_tree/version.rb | 2 +- test/visitor_test.rb | 2 +- 5 files changed, 12 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9884a16b..07c2d8b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +## [3.6.0] - 2022-09-19 + +### Added + +- [#158](https://github.com/ruby-syntax-tree/syntax_tree/pull/158) - Support the ability to pass `--ignore-files` to the CLI and the Rake tasks to ignore a certain pattern of files. + ## [3.5.0] - 2022-08-26 ### Added @@ -344,7 +350,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - 🎉 Initial release! 🎉 -[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.5.0...HEAD +[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.6.0...HEAD +[3.6.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.5.0...v3.6.0 [3.5.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.4.0...v3.5.0 [3.4.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.3.0...v3.4.0 [3.3.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.2.1...v3.3.0 diff --git a/Gemfile.lock b/Gemfile.lock index 12c78eba..6e5f3096 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - syntax_tree (3.5.0) + syntax_tree (3.6.0) prettier_print GEM diff --git a/lib/syntax_tree/basic_visitor.rb b/lib/syntax_tree/basic_visitor.rb index 9e6a84c1..34b7876e 100644 --- a/lib/syntax_tree/basic_visitor.rb +++ b/lib/syntax_tree/basic_visitor.rb @@ -35,7 +35,7 @@ def corrections # In some setups with Ruby you can turn off DidYouMean, so we're going to # respect that setting here. - if defined?(DidYouMean) && DidYouMean.method_defined?(:correct_error) + if defined?(DidYouMean.correct_error) DidYouMean.correct_error(VisitMethodError, self) end end diff --git a/lib/syntax_tree/version.rb b/lib/syntax_tree/version.rb index 42aa2b6c..5883bbdf 100644 --- a/lib/syntax_tree/version.rb +++ b/lib/syntax_tree/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SyntaxTree - VERSION = "3.5.0" + VERSION = "3.6.0" end diff --git a/test/visitor_test.rb b/test/visitor_test.rb index 27bad364..74f3df75 100644 --- a/test/visitor_test.rb +++ b/test/visitor_test.rb @@ -40,7 +40,7 @@ def initialize end end - if defined?(DidYouMean) && DidYouMean.method_defined?(:correct_error) + if defined?(DidYouMean.correct_error) def test_visit_method_correction error = assert_raises { Visitor.visit_method(:visit_binar) } message = From 1ea066bb3a6fa3864a287a48ccab26d194ed8242 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 28 Sep 2022 14:52:53 -0400 Subject: [PATCH 115/536] Fix CLI checking for content Previously, we were checking if $stdin was a TTY to determine if there was content to be read. As it turns out, this isn't really a good indicator, as content could always come later, and some folks run stree in CI when $stdin is not a TTY and still pass filenames. Instead, we now check if no filenames were passed, and in that case we attempt to read from $stdin. --- lib/syntax_tree/cli.rb | 13 +++---------- test/cli_test.rb | 29 ++++++----------------------- 2 files changed, 9 insertions(+), 33 deletions(-) diff --git a/lib/syntax_tree/cli.rb b/lib/syntax_tree/cli.rb index a364cd34..f3564e29 100644 --- a/lib/syntax_tree/cli.rb +++ b/lib/syntax_tree/cli.rb @@ -423,19 +423,12 @@ def run(argv) return 1 end - # If we're not reading from stdin and the user didn't supply any - # filepaths to be read, then we exit with the usage message. - if $stdin.tty? && arguments.empty? && options.scripts.empty? - warn(HELP) - return 1 - end - # We're going to build up a queue of items to process. queue = Queue.new - # If we're reading from stdin, then we'll just add the stdin object to - # the queue. Otherwise, we'll add each of the filepaths to the queue. - if $stdin.tty? && (arguments.any? || options.scripts.any?) + # If there are any arguments or scripts, then we'll add those to the + # queue. Otherwise we'll read the content off STDIN. + if arguments.any? || options.scripts.any? arguments.each do |pattern| Dir .glob(pattern) diff --git a/test/cli_test.rb b/test/cli_test.rb index de09b093..3734e734 100644 --- a/test/cli_test.rb +++ b/test/cli_test.rb @@ -123,13 +123,6 @@ def test_help_default end def test_no_arguments - with_tty do - *, stderr = capture_io { SyntaxTree::CLI.run(["check"]) } - assert_includes(stderr, "stree help") - end - end - - def test_no_arguments_no_tty stdin = $stdin $stdin = StringIO.new("1+1") @@ -140,17 +133,13 @@ def test_no_arguments_no_tty end def test_inline_script - with_tty do - stdio, = capture_io { SyntaxTree::CLI.run(%w[format -e 1+1]) } - assert_equal("1 + 1\n", stdio) - end + stdio, = capture_io { SyntaxTree::CLI.run(%w[format -e 1+1]) } + assert_equal("1 + 1\n", stdio) end def test_multiple_inline_scripts - with_tty do - stdio, = capture_io { SyntaxTree::CLI.run(%w[format -e 1+1 -e 2+2]) } - assert_equal("1 + 1\n2 + 2\n", stdio) - end + stdio, = capture_io { SyntaxTree::CLI.run(%w[format -e 1+1 -e 2+2]) } + assert_equal("1 + 1\n2 + 2\n", stdio) end def test_generic_error @@ -251,10 +240,8 @@ def run_cli(command, *args, contents: :default) status = nil stdio, stderr = - with_tty do - capture_io do - status = SyntaxTree::CLI.run([command, *args, tempfile.path]) - end + capture_io do + status = SyntaxTree::CLI.run([command, *args, tempfile.path]) end Result.new(status: status, stdio: stdio, stderr: stderr) @@ -263,10 +250,6 @@ def run_cli(command, *args, contents: :default) tempfile.unlink end - def with_tty(&block) - $stdin.stub(:tty?, true, &block) - end - def with_config_file(contents) filepath = File.join(Dir.pwd, SyntaxTree::CLI::ConfigFile::FILENAME) File.write(filepath, contents) From 51d24efc9110d7458d78657ceb6cc436815eb0e3 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 28 Sep 2022 15:05:53 -0400 Subject: [PATCH 116/536] Handle parse errors more gracefully in the language server --- lib/syntax_tree/language_server.rb | 34 ++++++++++++++++++------------ test/language_server_test.rb | 18 ++++++++++++++++ 2 files changed, 38 insertions(+), 14 deletions(-) diff --git a/lib/syntax_tree/language_server.rb b/lib/syntax_tree/language_server.rb index 16e94534..d2714b5c 100644 --- a/lib/syntax_tree/language_server.rb +++ b/lib/syntax_tree/language_server.rb @@ -56,10 +56,10 @@ def run store.delete(uri) in { method: "textDocument/formatting", id:, params: { textDocument: { uri: } } } contents = store[uri] - write(id: id, result: contents ? [format(store[uri], uri.split(".").last)] : nil) + write(id: id, result: contents ? format(contents, uri.split(".").last) : nil) in { method: "textDocument/inlayHint", id:, params: { textDocument: { uri: } } } contents = store[uri] - write(id: id, result: contents ? inlay_hints(store[uri]) : nil) + write(id: id, result: contents ? inlay_hints(contents) : nil) in { method: "syntaxTree/visualizing", id:, params: { textDocument: { uri: } } } write(id: id, result: PP.pp(SyntaxTree.parse(store[uri]), +"")) in { method: %r{\$/.+} } @@ -89,19 +89,25 @@ def capabilities def format(source, extension) text = SyntaxTree::HANDLERS[".#{extension}"].format(source, print_width) - { - range: { - start: { - line: 0, - character: 0 + [ + { + range: { + start: { + line: 0, + character: 0 + }, + end: { + line: source.lines.size + 1, + character: 0 + } }, - end: { - line: source.lines.size + 1, - character: 0 - } - }, - newText: text - } + newText: text + } + ] + rescue Parser::ParseError + # If there is a parse error, then we're not going to return any formatting + # changes for this source. + nil end def inlay_hints(source) diff --git a/test/language_server_test.rb b/test/language_server_test.rb index 31062e87..466bf737 100644 --- a/test/language_server_test.rb +++ b/test/language_server_test.rb @@ -120,6 +120,24 @@ def test_formatting end end + def test_formatting_failure + messages = [ + Initialize.new(1), + TextDocumentDidOpen.new("file:///path/to/file.rb", "<>"), + TextDocumentFormatting.new(2, "file:///path/to/file.rb"), + Shutdown.new(3) + ] + + case run_server(messages) + in [ + { id: 1, result: { capabilities: Hash } }, + { id: 2, result: }, + { id: 3, result: {} } + ] + assert_nil(result) + end + end + def test_formatting_print_width contents = "#{"a" * 40} + #{"b" * 40}\n" messages = [ From 2a28cd9ecb4b481f05be3a3f638eb764995743d6 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 28 Sep 2022 15:14:19 -0400 Subject: [PATCH 117/536] Bump to v3.6.1 --- CHANGELOG.md | 10 +++++++++- Gemfile.lock | 2 +- lib/syntax_tree/version.rb | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07c2d8b2..91d1f2c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +## [3.6.1] - 2022-09-28 + +### Changed + +- [#161](https://github.com/ruby-syntax-tree/syntax_tree/pull/161) - Previously, we were checking if STDIN was a TTY to determine if there was content to be read. Instead, we now check if no filenames were passed, and in that case we attempt to read from STDIN. This should fix errors users were experiencing in non-TTY environments like CI. +- [#162](https://github.com/ruby-syntax-tree/syntax_tree/pull/162) - Parse errors shouldn't crash the language server anymore. + ## [3.6.0] - 2022-09-19 ### Added @@ -350,7 +357,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - 🎉 Initial release! 🎉 -[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.6.0...HEAD +[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.6.1...HEAD +[3.6.1]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.6.0...v3.6.1 [3.6.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.5.0...v3.6.0 [3.5.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.4.0...v3.5.0 [3.4.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.3.0...v3.4.0 diff --git a/Gemfile.lock b/Gemfile.lock index 6e5f3096..9d60382c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - syntax_tree (3.6.0) + syntax_tree (3.6.1) prettier_print GEM diff --git a/lib/syntax_tree/version.rb b/lib/syntax_tree/version.rb index 5883bbdf..e7bf7655 100644 --- a/lib/syntax_tree/version.rb +++ b/lib/syntax_tree/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SyntaxTree - VERSION = "3.6.0" + VERSION = "3.6.1" end From 791120ce6d19137b8790f0335631b9d68ba3b222 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 4 Oct 2022 07:37:57 -0400 Subject: [PATCH 118/536] Fix inline comments on loop and conditional predicates --- lib/syntax_tree/node.rb | 2 ++ lib/syntax_tree/parser.rb | 45 +++++++++++++++++++++++---------------- test/fixtures/elsif.rb | 5 +++++ test/fixtures/for.rb | 4 ++++ test/fixtures/if.rb | 4 ++++ test/fixtures/lambda.rb | 4 ++++ test/fixtures/unless.rb | 4 ++++ test/fixtures/until.rb | 4 ++++ test/fixtures/while.rb | 4 ++++ 9 files changed, 58 insertions(+), 18 deletions(-) diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 47c534d1..e3d203a7 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -6004,6 +6004,8 @@ def format(q) q.group(0, "->") do if params.is_a?(Paren) q.format(params) unless params.contents.empty? + elsif params.empty? && params.comments.any? + q.format(params) elsif !params.empty? q.group do q.text("(") diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index ed9de499..7e46e856 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -302,8 +302,8 @@ def find_next_statement_start(position) def on_BEGIN(statements) lbrace = find_token(LBrace) rbrace = find_token(RBrace) - start_char = find_next_statement_start(lbrace.location.end_char) + start_char = find_next_statement_start(lbrace.location.end_char) statements.bind( start_char, start_char - line_counts[lbrace.location.start_line - 1].start, @@ -340,8 +340,8 @@ def on_CHAR(value) def on_END(statements) lbrace = find_token(LBrace) rbrace = find_token(RBrace) - start_char = find_next_statement_start(lbrace.location.end_char) + start_char = find_next_statement_start(lbrace.location.end_char) statements.bind( start_char, start_char - line_counts[lbrace.location.start_line - 1].start, @@ -831,8 +831,8 @@ def on_brace_block(block_var, statements) lbrace = find_token(LBrace) rbrace = find_token(RBrace) location = (block_var || lbrace).location - start_char = find_next_statement_start(location.end_char) + start_char = find_next_statement_start(location.end_char) statements.bind( start_char, start_char - line_counts[location.start_line - 1].start, @@ -1329,8 +1329,8 @@ def on_else(statements) node = tokens[index] ending = node.value == "end" ? tokens.delete_at(index) : node - start_char = find_next_statement_start(keyword.location.end_char) + start_char = find_next_statement_start(keyword.location.end_char) statements.bind( start_char, start_char - line_counts[keyword.location.start_line - 1].start, @@ -1355,9 +1355,10 @@ def on_elsif(predicate, statements, consequent) beginning = find_token(Kw, "elsif") ending = consequent || find_token(Kw, "end") + start_char = find_next_statement_start(predicate.location.end_char) statements.bind( - predicate.location.end_char, - predicate.location.end_column, + start_char, + start_char - line_counts[predicate.location.start_line - 1].start, ending.location.start_char, ending.location.start_column ) @@ -1598,9 +1599,12 @@ def on_for(index, collection, statements) tokens.delete(keyword) end + start_char = + find_next_statement_start((keyword || collection).location.end_char) statements.bind( - (keyword || collection).location.end_char, - (keyword || collection).location.end_column, + start_char, + start_char - + line_counts[(keyword || collection).location.end_line - 1].start, ending.location.start_char, ending.location.start_column ) @@ -1778,9 +1782,10 @@ def on_if(predicate, statements, consequent) beginning = find_token(Kw, "if") ending = consequent || find_token(Kw, "end") + start_char = find_next_statement_start(predicate.location.end_char) statements.bind( - predicate.location.end_char, - predicate.location.end_column, + start_char, + start_char - line_counts[predicate.location.end_line - 1].start, ending.location.start_char, ending.location.start_column ) @@ -2024,9 +2029,10 @@ def on_lambda(params, statements) closing = find_token(Kw, "end") end + start_char = find_next_statement_start(opening.location.end_char) statements.bind( - opening.location.end_char, - opening.location.end_column, + start_char, + start_char - line_counts[opening.location.end_line - 1].start, closing.location.start_char, closing.location.start_column ) @@ -3456,9 +3462,10 @@ def on_unless(predicate, statements, consequent) beginning = find_token(Kw, "unless") ending = consequent || find_token(Kw, "end") + start_char = find_next_statement_start(predicate.location.end_char) statements.bind( - predicate.location.end_char, - predicate.location.end_column, + start_char, + start_char - line_counts[predicate.location.end_line - 1].start, ending.location.start_char, ending.location.start_column ) @@ -3498,9 +3505,10 @@ def on_until(predicate, statements) end # Update the Statements location information + start_char = find_next_statement_start(predicate.location.end_char) statements.bind( - predicate.location.end_char, - predicate.location.end_column, + start_char, + start_char - line_counts[predicate.location.end_line - 1].start, ending.location.start_char, ending.location.start_column ) @@ -3633,9 +3641,10 @@ def on_while(predicate, statements) end # Update the Statements location information + start_char = find_next_statement_start(predicate.location.end_char) statements.bind( - predicate.location.end_char, - predicate.location.end_column, + start_char, + start_char - line_counts[predicate.location.end_line - 1].start, ending.location.start_char, ending.location.start_column ) diff --git a/test/fixtures/elsif.rb b/test/fixtures/elsif.rb index 2e4cd831..e0dd2bd6 100644 --- a/test/fixtures/elsif.rb +++ b/test/fixtures/elsif.rb @@ -17,3 +17,8 @@ else qyz end +% +if true +elsif false # comment1 + # comment2 +end diff --git a/test/fixtures/for.rb b/test/fixtures/for.rb index 62b207ee..1346a367 100644 --- a/test/fixtures/for.rb +++ b/test/fixtures/for.rb @@ -38,3 +38,7 @@ for foo, in [[foo, bar]] foo end +% +for foo in bar # comment1 + # comment2 +end diff --git a/test/fixtures/if.rb b/test/fixtures/if.rb index 1963d301..cfd6a882 100644 --- a/test/fixtures/if.rb +++ b/test/fixtures/if.rb @@ -63,3 +63,7 @@ if (x = x + 1).to_i x end +% +if true # comment1 + # comment2 +end diff --git a/test/fixtures/lambda.rb b/test/fixtures/lambda.rb index d0cc6f9b..5dba3be3 100644 --- a/test/fixtures/lambda.rb +++ b/test/fixtures/lambda.rb @@ -76,3 +76,7 @@ ) ) do end +% +-> do # comment1 + # comment2 +end diff --git a/test/fixtures/unless.rb b/test/fixtures/unless.rb index c66b16bf..2d5038c1 100644 --- a/test/fixtures/unless.rb +++ b/test/fixtures/unless.rb @@ -32,3 +32,7 @@ unless foo a ? b : c end +% +unless true # comment1 + # comment2 +end diff --git a/test/fixtures/until.rb b/test/fixtures/until.rb index 778e3fb0..f3ef5202 100644 --- a/test/fixtures/until.rb +++ b/test/fixtures/until.rb @@ -23,3 +23,7 @@ until (foo += 1) foo end +% +until true # comment1 + # comment2 +end diff --git a/test/fixtures/while.rb b/test/fixtures/while.rb index 1404f07d..9415135a 100644 --- a/test/fixtures/while.rb +++ b/test/fixtures/while.rb @@ -23,3 +23,7 @@ while (foo += 1) foo end +% +while true # comment1 + # comment2 +end From bcdf200aec51a2097d516d0cffd1a9ab17db3ab6 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 4 Oct 2022 08:24:52 -0400 Subject: [PATCH 119/536] Handle formatting blocks outside of tree --- lib/syntax_tree/node.rb | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index e3d203a7..7ecd69ff 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -1955,11 +1955,12 @@ def format(q) # If the receiver of this block a Command or CommandCall node, then there # are no parentheses around the arguments to that command, so we need to # break the block. - receiver = q.parent.call - if receiver.is_a?(Command) || receiver.is_a?(CommandCall) + case q.parent + in { call: Command | CommandCall } q.break_parent format_break(q, break_opening, break_closing) return + else end q.group do @@ -1978,16 +1979,26 @@ def unchangeable_bounds?(q) # If we hit a statements, then we're safe to use whatever since we # know for certain we're going to get split over multiple lines # anyway. - break false if parent.is_a?(Statements) || parent.is_a?(ArgParen) - - [Command, CommandCall].include?(parent.class) + case parent + in Statements | ArgParen + break false + in Command | CommandCall + true + else + false + end end end # If we're a sibling of a control-flow keyword, then we're going to have to # use the do..end bounds. def forced_do_end_bounds?(q) - [Break, Next, Return, Super].include?(q.parent.call.class) + case q.parent + in { call: Break | Next | Return | Super } + true + else + false + end end # If we're the predicate of a loop or conditional, then we're going to have @@ -2314,7 +2325,8 @@ def comments end def format(q) - if operator == :"::" || (operator.is_a?(Op) && operator.value == "::") + case operator + in :"::" | Op[value: "::"] q.text(".") else operator.format(q) From 99492653c851a40ea07d9be61c01d866bf3723cd Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 4 Oct 2022 12:00:48 -0400 Subject: [PATCH 120/536] Bump to version v3.6.2 --- CHANGELOG.md | 10 +++++++++- Gemfile.lock | 2 +- lib/syntax_tree/version.rb | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 91d1f2c1..79145063 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +## [3.6.2] - 2022-10-04 + +### Changed + +- [#165](https://github.com/ruby-syntax-tree/syntax_tree/pull/165) - Conditionals (`if`/`unless`), loops (`for`/`while`/`until`) and lambdas all had issues when comments immediately succeeded the declaration of the node where the comment could potentially be dropped. That has now been fixed. +- [#166](https://github.com/ruby-syntax-tree/syntax_tree/pull/166) - Blocks can now be formatted even if they are the top node of the tree. Previously they were looking to their parent for some additional metadata, so we now handle the case where the parent is nil. + ## [3.6.1] - 2022-09-28 ### Changed @@ -357,7 +364,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - 🎉 Initial release! 🎉 -[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.6.1...HEAD +[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.6.2...HEAD +[3.6.2]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.6.1...v3.6.2 [3.6.1]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.6.0...v3.6.1 [3.6.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.5.0...v3.6.0 [3.5.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.4.0...v3.5.0 diff --git a/Gemfile.lock b/Gemfile.lock index 9d60382c..62b8559b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - syntax_tree (3.6.1) + syntax_tree (3.6.2) prettier_print GEM diff --git a/lib/syntax_tree/version.rb b/lib/syntax_tree/version.rb index e7bf7655..850facb2 100644 --- a/lib/syntax_tree/version.rb +++ b/lib/syntax_tree/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SyntaxTree - VERSION = "3.6.1" + VERSION = "3.6.2" end From 666e7c8c4cfce50f0e540acd18d909fc11d72d7d Mon Sep 17 00:00:00 2001 From: Vinicius Stock Date: Fri, 7 Oct 2022 15:02:31 -0400 Subject: [PATCH 121/536] Raise ParseError on missing end for else statement --- lib/syntax_tree/parser.rb | 5 +++++ test/parser_test.rb | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index 7e46e856..94ce115a 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -1327,6 +1327,11 @@ def on_else(statements) token.is_a?(Kw) && %w[end ensure].include?(token.value) end + if index.nil? + message = "Cannot find expected else ending" + raise ParseError.new(message, *find_token_error(keyword.location)) + end + node = tokens[index] ending = node.value == "end" ? tokens.delete_at(index) : node diff --git a/test/parser_test.rb b/test/parser_test.rb index fbff8ec2..6048cf11 100644 --- a/test/parser_test.rb +++ b/test/parser_test.rb @@ -55,5 +55,15 @@ def test_errors_on_missing_token_without_location def test_handles_strings_with_non_terminated_embedded_expressions assert_raises(Parser::ParseError) { SyntaxTree.parse('"#{"') } end + + def test_errors_on_else_missing_two_ends + assert_raises(Parser::ParseError) { SyntaxTree.parse(<<~RUBY) } + def foo + if something + else + call do + end + RUBY + end end end From cb72efc779aa8e9fdc812fc81b495c7998332f21 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 11 Oct 2022 17:03:26 -0400 Subject: [PATCH 122/536] Bump to v3.6.3 --- CHANGELOG.md | 9 ++++++++- Gemfile.lock | 2 +- lib/syntax_tree/version.rb | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 79145063..f0ba115e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +## [3.6.3] - 2022-10-11 + +### Changed + +- [#167](https://github.com/ruby-syntax-tree/syntax_tree/pull/167) - Change the error encountered when an `else` node does not have an associated `end` token to be a parse error. + ## [3.6.2] - 2022-10-04 ### Changed @@ -364,7 +370,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - 🎉 Initial release! 🎉 -[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.6.2...HEAD +[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.6.3...HEAD +[3.6.3]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.6.2...v3.6.3 [3.6.2]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.6.1...v3.6.2 [3.6.1]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.6.0...v3.6.1 [3.6.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.5.0...v3.6.0 diff --git a/Gemfile.lock b/Gemfile.lock index 62b8559b..6415fcb0 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - syntax_tree (3.6.2) + syntax_tree (3.6.3) prettier_print GEM diff --git a/lib/syntax_tree/version.rb b/lib/syntax_tree/version.rb index 850facb2..ec6dcd3e 100644 --- a/lib/syntax_tree/version.rb +++ b/lib/syntax_tree/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SyntaxTree - VERSION = "3.6.2" + VERSION = "3.6.3" end From aaa76434ac4760764b4c7bad6b3dfb0bb97c18e6 Mon Sep 17 00:00:00 2001 From: Jake Zimmerman Date: Thu, 13 Oct 2022 15:20:11 -0700 Subject: [PATCH 123/536] Allow passing `--ignore-files` multiple times --- lib/syntax_tree/cli.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/syntax_tree/cli.rb b/lib/syntax_tree/cli.rb index f3564e29..b839d562 100644 --- a/lib/syntax_tree/cli.rb +++ b/lib/syntax_tree/cli.rb @@ -290,7 +290,7 @@ class Options :target_ruby_version def initialize(print_width: DEFAULT_PRINT_WIDTH) - @ignore_files = "" + @ignore_files = [] @plugins = [] @print_width = print_width @scripts = [] @@ -313,7 +313,7 @@ def parser # Any of the CLI commands that operate on filenames will then ignore # this set of files. opts.on("--ignore-files=GLOB") do |glob| - @ignore_files = glob.match(/\A'(.*)'\z/) ? $1 : glob + @ignore_files << (glob.match(/\A'(.*)'\z/) ? $1 : glob) end # If there are any plugins specified on the command line, then load @@ -434,7 +434,7 @@ def run(argv) .glob(pattern) .each do |filepath| if File.readable?(filepath) && - !File.fnmatch?(options.ignore_files, filepath) + options.ignore_files.none? { File.fnmatch?(_1, filepath) } queue << FileItem.new(filepath) end end From 3150301e24cd496c57428c5daa146580b4fe3aa6 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Sat, 15 Oct 2022 21:18:47 -0400 Subject: [PATCH 124/536] Tons of various performance tweaks --- bin/profile | 11 +- lib/syntax_tree.rb | 4 +- lib/syntax_tree/formatter.rb | 60 +++- lib/syntax_tree/node.rb | 598 ++++++++++++++++++++--------------- lib/syntax_tree/parser.rb | 160 +++++++--- test/cli_test.rb | 2 +- test/node_test.rb | 2 +- test/quotes_test.rb | 15 + 8 files changed, 539 insertions(+), 313 deletions(-) create mode 100644 test/quotes_test.rb diff --git a/bin/profile b/bin/profile index 0a1b6ade..15bd28ae 100755 --- a/bin/profile +++ b/bin/profile @@ -6,22 +6,21 @@ require "bundler/inline" gemfile do source "https://rubygems.org" gem "stackprof" + gem "prettier_print" end $:.unshift(File.expand_path("../lib", __dir__)) require "syntax_tree" -GC.disable - StackProf.run(mode: :cpu, out: "tmp/profile.dump", raw: true) do - filepath = File.expand_path("../lib/syntax_tree/node.rb", __dir__) - SyntaxTree.format(File.read(filepath)) + Dir[File.join(RbConfig::CONFIG["libdir"], "**/*.rb")].each do |filepath| + SyntaxTree.format(SyntaxTree.read(filepath)) + end end -GC.enable - File.open("tmp/flamegraph.html", "w") do |file| report = Marshal.load(IO.binread("tmp/profile.dump")) + StackProf::Report.new(report).print_text StackProf::Report.new(report).print_d3_flamegraph(file) end diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index 88c66369..29ed048c 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -1,6 +1,5 @@ # frozen_string_literal: true -require "delegate" require "etc" require "json" require "pp" @@ -10,7 +9,6 @@ require_relative "syntax_tree/formatter" require_relative "syntax_tree/node" -require_relative "syntax_tree/parser" require_relative "syntax_tree/version" require_relative "syntax_tree/basic_visitor" @@ -20,6 +18,8 @@ require_relative "syntax_tree/visitor/match_visitor" require_relative "syntax_tree/visitor/pretty_print_visitor" +require_relative "syntax_tree/parser" + # Syntax Tree is a suite of tools built on top of the internal CRuby parser. It # provides the ability to generate a syntax tree from source, as well as the # tools necessary to inspect and manipulate that syntax tree. It can be used to diff --git a/lib/syntax_tree/formatter.rb b/lib/syntax_tree/formatter.rb index 4c7a00db..dc124fbc 100644 --- a/lib/syntax_tree/formatter.rb +++ b/lib/syntax_tree/formatter.rb @@ -4,6 +4,26 @@ module SyntaxTree # A slightly enhanced PP that knows how to format recursively including # comments. class Formatter < PrettierPrint + # It's very common to use seplist with ->(q) { q.breakable_return }. We wrap + # that pattern into an object to cut down on having to create a bunch of + # lambdas all over the place. + class BreakableReturnSeparator + def call(q) + q.breakable_return + end + end + + # Similar to the previous, it's common to ->(q) { q.breakable_space }. We + # also wrap that pattern into an object to cut down on lambdas. + class BreakableSpaceSeparator + def call(q) + q.breakable_space + end + end + + BREAKABLE_RETURN_SEPARATOR = BreakableReturnSeparator.new + BREAKABLE_SPACE_SEPARATOR = BreakableSpaceSeparator.new + # We want to minimize as much as possible the number of options that are # available in syntax tree. For the most part, if users want non-default # formatting, they should override the format methods on the specific nodes @@ -75,8 +95,7 @@ def format(node, stackable: true) doc = if leading.last&.ignore? range = source[node.location.start_char...node.location.end_char] - separator = -> { breakable(indent: false, force: true) } - seplist(range.split(/\r?\n/, -1), separator) { |line| text(line) } + seplist(range.split(/\r?\n/, -1), Formatter::BREAKABLE_RETURN_SEPARATOR) { |line| text(line) } else node.format(self) end @@ -108,5 +127,42 @@ def parent def parents stack[0...-1].reverse_each end + + # This is a simplified version of prettyprint's group. It doesn't provide + # any of the more advanced options because we don't need them and they take + # up expensive computation time. + def group + contents = [] + doc = Group.new(0, contents: contents) + + groups << doc + target << doc + + with_target(contents) { yield } + groups.pop + doc + end + + # A similar version to the super, except that it calls back into the + # separator proc with the instance of `self`. + def seplist(list, sep = nil, iter_method = :each) # :yield: element + first = true + list.__send__(iter_method) do |*v| + if first + first = false + elsif sep + sep.call(self) + else + comma_breakable + end + yield(*v) + end + end + + # This is a much simplified version of prettyprint's text. It avoids + # calculating width by pushing the string directly onto the target. + def text(string) + target << string + end end end diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 7ecd69ff..2aa51fd8 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -177,10 +177,10 @@ def format(q) q.text("BEGIN ") q.format(lbrace) q.indent do - q.breakable + q.breakable_space q.format(statements) end - q.breakable + q.breakable_space q.text("}") end end @@ -280,10 +280,10 @@ def format(q) q.text("END ") q.format(lbrace) q.indent do - q.breakable + q.breakable_space q.format(statements) end - q.breakable + q.breakable_space q.text("}") end end @@ -327,10 +327,8 @@ def deconstruct_keys(_keys) def format(q) q.text("__END__") - q.breakable(force: true) - - separator = -> { q.breakable(indent: false, force: true) } - q.seplist(value.split(/\r?\n/, -1), separator) { |line| q.text(line) } + q.breakable_force + q.seplist(value.split(/\r?\n/, -1), Formatter::BREAKABLE_RETURN_SEPARATOR) { |line| q.text(line) } end end @@ -412,7 +410,7 @@ def format(q) q.format(left_argument, stackable: false) q.group do q.nest(keyword.length) do - q.breakable(force: left_argument.comments.any?) + left_argument.comments.any? ? q.breakable_force : q.breakable_space q.format(AliasArgumentFormatter.new(right), stackable: false) end end @@ -476,10 +474,10 @@ def format(q) if index q.indent do - q.breakable("") + q.breakable_empty q.format(index) end - q.breakable("") + q.breakable_empty end q.text("]") @@ -537,10 +535,10 @@ def format(q) if index q.indent do - q.breakable("") + q.breakable_empty q.format(index) end - q.breakable("") + q.breakable_empty end q.text("]") @@ -593,14 +591,16 @@ def format(q) return end - q.group(0, "(", ")") do + q.text("(") + q.group do q.indent do - q.breakable("") + q.breakable_empty q.format(arguments) q.if_break { q.text(",") } if q.trailing_comma? && trailing_comma? end - q.breakable("") + q.breakable_empty end + q.text(")") end private @@ -800,10 +800,11 @@ def initialize(contents) end def format(q) - q.group(0, "%w[", "]") do + q.text("%w[") + q.group do q.indent do - q.breakable("") - q.seplist(contents.parts, -> { q.breakable }) do |part| + q.breakable_empty + q.seplist(contents.parts, Formatter::BREAKABLE_SPACE_SEPARATOR) do |part| if part.is_a?(StringLiteral) q.format(part.parts.first) else @@ -811,8 +812,9 @@ def format(q) end end end - q.breakable("") + q.breakable_empty end + q.text("]") end end @@ -826,15 +828,17 @@ def initialize(contents) end def format(q) - q.group(0, "%i[", "]") do + q.text("%i[") + q.group do q.indent do - q.breakable("") - q.seplist(contents.parts, -> { q.breakable }) do |part| + q.breakable_empty + q.seplist(contents.parts, Formatter::BREAKABLE_SPACE_SEPARATOR) do |part| q.format(part.value) end end - q.breakable("") + q.breakable_empty end + q.text("]") end end @@ -861,6 +865,13 @@ def format(q) # # provided the line length was hit between `bar` and `baz`. class VarRefsFormatter + class Separator + def call(q) + q.text(",") + q.fill_breakable + end + end + # [Args] the contents of the array attr_reader :contents @@ -869,20 +880,16 @@ def initialize(contents) end def format(q) - q.group(0, "[", "]") do + q.text("[") + q.group do q.indent do - q.breakable("") - - separator = -> do - q.text(",") - q.fill_breakable - end - - q.seplist(contents.parts, separator) { |part| q.format(part) } + q.breakable_empty + q.seplist(contents.parts, Separator.new) { |part| q.format(part) } q.if_break { q.text(",") } if q.trailing_comma? end - q.breakable("") + q.breakable_empty end + q.text("]") end end @@ -902,11 +909,11 @@ def format(q) q.text("[") q.indent do lbracket.comments.each do |comment| - q.breakable(force: true) + q.breakable_force comment.format(q) end end - q.breakable(force: true) + q.breakable_force q.text("]") end end @@ -973,13 +980,13 @@ def format(q) if contents q.indent do - q.breakable("") + q.breakable_empty q.format(contents) q.if_break { q.text(",") } if q.trailing_comma? end end - q.breakable("") + q.breakable_empty q.text("]") end end @@ -1127,7 +1134,7 @@ def format(q) q.format(constant) if constant q.text("[") q.indent do - q.breakable("") + q.breakable_empty parts = [*requireds] parts << RestFormatter.new(rest) if rest @@ -1135,7 +1142,7 @@ def format(q) q.seplist(parts) { |part| q.format(part) } end - q.breakable("") + q.breakable_empty q.text("]") end end @@ -1145,13 +1152,13 @@ def format(q) module AssignFormatting def self.skip_indent?(value) case value - in ArrayLiteral | HashLiteral | Heredoc | Lambda | QSymbols | QWords | - Symbols | Words + when ArrayLiteral, HashLiteral, Heredoc, Lambda, QSymbols, QWords, + Symbols, Words true - in Call[receiver:] - skip_indent?(receiver) - in DynaSymbol[quote:] - quote.start_with?("%s") + when Call + skip_indent?(value.receiver) + when DynaSymbol + value.quote.start_with?("%s") else false end @@ -1206,7 +1213,7 @@ def format(q) q.format(value) else q.indent do - q.breakable + q.breakable_space q.format(value) end end @@ -1277,7 +1284,7 @@ def format_contents(q) q.format(value) else q.indent do - q.breakable + q.breakable_space q.format(value) end end @@ -1544,12 +1551,12 @@ def format(q) unless bodystmt.empty? q.indent do - q.breakable(force: true) unless bodystmt.statements.empty? + q.breakable_force unless bodystmt.statements.empty? q.format(bodystmt) end end - q.breakable(force: true) + q.breakable_force q.text("end") end end @@ -1592,10 +1599,10 @@ def format(q) q.text("^(") q.nest(1) do q.indent do - q.breakable("") + q.breakable_empty q.format(statement) end - q.breakable("") + q.breakable_empty q.text(")") end end @@ -1669,7 +1676,7 @@ def format(q) q.text(operator.to_s) q.indent do - q.breakable(power ? "" : " ") + power ? q.breakable_empty : q.breakable_space q.format(right) end end @@ -1716,15 +1723,29 @@ def deconstruct_keys(_keys) { params: params, locals: locals, location: location, comments: comments } end + # Within the pipes of the block declaration, we don't want any spaces. So + # we'll separate the parameters with a comma and space but no breakables. + class Separator + def call(q) + q.text(", ") + end + end + + # We'll keep a single instance of this separator around for all block vars + # to cut down on allocations. + SEPARATOR = Separator.new + def format(q) - q.group(0, "|", "|") do + q.text("|") + q.group do q.remove_breaks(q.format(params)) if locals.any? q.text("; ") - q.seplist(locals, -> { q.text(", ") }) { |local| q.format(local) } + q.seplist(locals, SEPARATOR) { |local| q.format(local) } end end + q.text("|") end end @@ -1816,10 +1837,8 @@ def bind(start_char, start_column, end_char, end_column) end_column: end_column ) - parts = [rescue_clause, else_clause, ensure_clause] - # Here we're going to determine the bounds for the statements - consequent = parts.compact.first + consequent = rescue_clause || else_clause || ensure_clause statements.bind( start_char, start_column, @@ -1829,7 +1848,7 @@ def bind(start_char, start_column, end_char, end_column) # Next we're going to determine the rescue clause if there is one if rescue_clause - consequent = parts.drop(1).compact.first + consequent = else_clause || ensure_clause rescue_clause.bind_end( consequent ? consequent.location.start_char : end_char, consequent ? consequent.location.start_column : end_column @@ -1868,26 +1887,26 @@ def format(q) if rescue_clause q.nest(-2) do - q.breakable(force: true) + q.breakable_force q.format(rescue_clause) end end if else_clause q.nest(-2) do - q.breakable(force: true) + q.breakable_force q.format(else_keyword) end unless else_clause.empty? - q.breakable(force: true) + q.breakable_force q.format(else_clause) end end if ensure_clause q.nest(-2) do - q.breakable(force: true) + q.breakable_force q.format(ensure_clause) end end @@ -2004,22 +2023,16 @@ def forced_do_end_bounds?(q) # If we're the predicate of a loop or conditional, then we're going to have # to go with the {..} bounds. def forced_brace_bounds?(q) - parents = q.parents.to_a - parents.each_with_index.any? do |parent, index| - # If we hit certain breakpoints then we know we're safe. - break false if [Paren, Statements].include?(parent.class) - - [ - If, - IfMod, - IfOp, - Unless, - UnlessMod, - While, - WhileMod, - Until, - UntilMod - ].include?(parent.class) && parent.predicate == parents[index - 1] + previous = nil + q.parents.any? do |parent| + case parent + when Paren, Statements + # If we hit certain breakpoints then we know we're safe. + return false + when If, IfMod, IfOp, Unless, UnlessMod, While, WhileMod, Until, UntilMod + return true if parent.predicate == previous + previous = parent + end end end @@ -2034,12 +2047,12 @@ def format_break(q, opening, closing) unless statements.empty? q.indent do - q.breakable + q.breakable_space q.format(statements) end end - q.breakable + q.breakable_space q.text(closing) end @@ -2048,17 +2061,17 @@ def format_flat(q, opening, closing) q.format(BlockOpenFormatter.new(opening, block_open), stackable: false) if node.block_var - q.breakable + q.breakable_space q.format(node.block_var) - q.breakable + q.breakable_space end if statements.empty? q.text(" ") if opening == "do" else - q.breakable unless node.block_var + q.breakable_space unless node.block_var q.format(statements) - q.breakable + q.breakable_space end q.text(closing) @@ -2241,20 +2254,20 @@ def format(q) def format_array_contents(q, array) q.if_break { q.text("[") } q.indent do - q.breakable("") + q.breakable_empty q.format(array.contents) end - q.breakable("") + q.breakable_empty q.if_break { q.text("]") } end def format_arguments(q, opening, closing) q.if_break { q.text(opening) } q.indent do - q.breakable(" ") + q.breakable_space q.format(node.arguments) end - q.breakable("") + q.breakable_empty q.if_break { q.text(closing) } end @@ -2446,7 +2459,7 @@ def format_chain(q, children) in Call # If we're at a Call node and not a MethodAddBlock node in the # chain then we're going to add a newline so it indents properly. - q.breakable("") + q.breakable_empty else end @@ -2530,7 +2543,7 @@ def format_child( # them out here since we're bypassing the normal comment printing. if child.comments.any? && !skip_comments child.comments.each do |comment| - comment.inline? ? q.text(" ") : q.breakable + comment.inline? ? q.text(" ") : q.breakable_space comment.format(q) end @@ -2605,7 +2618,8 @@ def format(q) # If we're at the top of a call chain, then we're going to do some # specialized printing in case we can print it nicely. We _only_ do this # at the top of the chain to avoid weird recursion issues. - if !CallChainFormatter.chained?(q.parent) && + if !ENV["STREE_SKIP_CALL_CHAIN"] && + !CallChainFormatter.chained?(q.parent) && CallChainFormatter.chained?(receiver) q.group do q @@ -2642,7 +2656,7 @@ def format_contents(q) q.group do q.indent do if receiver.comments.any? || call_operator.comments.any? - q.breakable(force: true) + q.breakable_force end if call_operator.comments.empty? @@ -2719,9 +2733,9 @@ def format(q) q.format(value) end - q.breakable(force: true) + q.breakable_force q.format(consequent) - q.breakable(force: true) + q.breakable_force q.text("end") end @@ -2788,7 +2802,7 @@ def format(q) else q.group do q.indent do - q.breakable + q.breakable_space q.format(pattern) end end @@ -2887,7 +2901,7 @@ def format(q) if bodystmt.empty? q.group do declaration.call - q.breakable(force: true) + q.breakable_force q.text("end") end else @@ -2895,11 +2909,11 @@ def format(q) declaration.call q.indent do - q.breakable(force: true) + q.breakable_force q.format(bodystmt) end - q.breakable(force: true) + q.breakable_force q.text("end") end end @@ -3069,7 +3083,7 @@ def format(q) if message.comments.any?(&:leading?) q.format(CallOperatorFormatter.new(operator), stackable: false) q.indent do - q.breakable("") + q.breakable_empty q.format(message) end else @@ -3155,7 +3169,7 @@ def trailing? end def ignore? - value[1..].strip == "stree-ignore" + value.match?(/\A#\s*stree-ignore\s*\z/) end def comments @@ -3455,12 +3469,12 @@ def format(q) unless bodystmt.empty? q.indent do - q.breakable(force: true) + q.breakable_force q.format(bodystmt) end end - q.breakable(force: true) + q.breakable_force q.text("end") end end @@ -3549,7 +3563,7 @@ def format(q) q.text(" =") q.group do q.indent do - q.breakable + q.breakable_space q.format(statement) end end @@ -3590,13 +3604,15 @@ def deconstruct_keys(_keys) end def format(q) - q.group(0, "defined?(", ")") do + q.text("defined?(") + q.group do q.indent do - q.breakable("") + q.breakable_empty q.format(value) end - q.breakable("") + q.breakable_empty end + q.text(")") end end @@ -3678,12 +3694,12 @@ def format(q) unless bodystmt.empty? q.indent do - q.breakable(force: true) + q.breakable_force q.format(bodystmt) end end - q.breakable(force: true) + q.breakable_force q.text("end") end end @@ -3948,12 +3964,12 @@ def deconstruct_keys(_keys) def format(q) opening_quote, closing_quote = quotes(q) - q.group(0, opening_quote, closing_quote) do + q.text(opening_quote) + q.group do parts.each do |part| if part.is_a?(TStringContent) value = Quotes.normalize(part.value, closing_quote) - separator = -> { q.breakable(force: true, indent: false) } - q.seplist(value.split(/\r?\n/, -1), separator) do |text| + q.seplist(value.split(/\r?\n/, -1), Formatter::BREAKABLE_RETURN_SEPARATOR) do |text| q.text(text) end else @@ -3961,6 +3977,7 @@ def format(q) end end end + q.text(closing_quote) end private @@ -4056,7 +4073,7 @@ def format(q) unless statements.empty? q.indent do - q.breakable(force: true) + q.breakable_force q.format(statements) end end @@ -4126,14 +4143,14 @@ def format(q) unless statements.empty? q.indent do - q.breakable(force: true) + q.breakable_force q.format(statements) end end if consequent q.group do - q.breakable(force: true) + q.breakable_force q.format(consequent) end end @@ -4329,7 +4346,7 @@ def format(q) unless statements.empty? q.indent do - q.breakable(force: true) + q.breakable_force q.format(statements) end end @@ -4588,7 +4605,7 @@ def format(q) q.text("[") q.indent do - q.breakable("") + q.breakable_empty q.text("*") q.format(left) @@ -4601,7 +4618,7 @@ def format(q) q.format(right) end - q.breakable("") + q.breakable_empty q.text("]") end end @@ -4663,12 +4680,12 @@ def format(q) unless statements.empty? q.indent do - q.breakable(force: true) + q.breakable_force q.format(statements) end end - q.breakable(force: true) + q.breakable_force q.text("end") end end @@ -4731,11 +4748,11 @@ def format(q) q.text("{") q.indent do lbrace.comments.each do |comment| - q.breakable(force: true) + q.breakable_force comment.format(q) end end - q.breakable(force: true) + q.breakable_force q.text("}") end end @@ -4800,14 +4817,14 @@ def format_contents(q) q.format(lbrace) if assocs.empty? - q.breakable("") + q.breakable_empty else q.indent do - q.breakable + q.breakable_space q.seplist(assocs) { |assoc| q.format(assoc) } q.if_break { q.text(",") } if q.trailing_comma? end - q.breakable + q.breakable_space end q.text("}") @@ -4873,22 +4890,32 @@ def deconstruct_keys(_keys) } end - def format(q) - # This is a very specific behavior where you want to force a newline, but - # don't want to force the break parent. - breakable = -> { q.breakable(indent: false, force: :skip_break_parent) } + # This is a very specific behavior where you want to force a newline, but + # don't want to force the break parent. + class Separator + DOC = PrettierPrint::Breakable.new(" ", 1, indent: false, force: true) + def call(q) + q.target << DOC + end + end + + # We're going to keep an instance around so we don't have to allocate a new + # one every time we format a heredoc. + SEPARATOR = Separator.new + + def format(q) q.group do q.format(beginning) q.line_suffix(priority: Formatter::HEREDOC_PRIORITY) do q.group do - breakable.call + SEPARATOR.call(q) parts.each do |part| if part.is_a?(TStringContent) texts = part.value.split(/\r?\n/, -1) - q.seplist(texts, breakable) { |text| q.text(text) } + q.seplist(texts, SEPARATOR) { |text| q.text(text) } else q.format(part) end @@ -5097,10 +5124,10 @@ def format(q) q.format(constant) q.text("[") q.indent do - q.breakable("") + q.breakable_empty contents.call end - q.breakable("") + q.breakable_empty q.text("]") end return @@ -5124,14 +5151,14 @@ def format(q) q.group do q.text("{") q.indent do - q.breakable + q.breakable_space contents.call end if q.target_ruby_version < Gem::Version.new("2.7.3") q.text(" }") else - q.breakable + q.breakable_space q.text("}") end end @@ -5188,8 +5215,12 @@ def self.call(parent) queue = [parent] while (node = queue.shift) - return true if [Assign, MAssign, OpAssign].include?(node.class) - queue += node.child_nodes.compact + case node + when Assign, MAssign, OpAssign + return true + else + node.child_nodes.each { |child| queue << child if child } + end end false @@ -5311,17 +5342,17 @@ def format_break(q, force:) unless node.statements.empty? q.indent do - q.breakable(force: force) + force ? q.breakable_force : q.breakable_space q.format(node.statements) end end if node.consequent - q.breakable(force: force) + force ? q.breakable_force : q.breakable_space q.format(node.consequent) end - q.breakable(force: force) + force ? q.breakable_force : q.breakable_space q.text("end") end @@ -5333,11 +5364,11 @@ def format_ternary(q) q.nest(keyword.length + 1) { q.format(node.predicate) } q.indent do - q.breakable + q.breakable_space q.format(node.statements) end - q.breakable + q.breakable_space q.group do q.format(node.consequent.keyword) q.indent do @@ -5351,7 +5382,7 @@ def format_ternary(q) end end - q.breakable + q.breakable_space q.text("end") end .if_flat do @@ -5507,19 +5538,19 @@ def format_break(q) q.nest("if ".length) { q.format(predicate) } q.indent do - q.breakable + q.breakable_space q.format(truthy) end - q.breakable + q.breakable_space q.text("else") q.indent do - q.breakable + q.breakable_space q.format(falsy) end - q.breakable + q.breakable_space q.text("end") end end @@ -5529,11 +5560,11 @@ def format_flat(q) q.text(" ?") q.indent do - q.breakable + q.breakable_space q.format(truthy) q.text(" :") - q.breakable + q.breakable_space q.format(falsy) end end @@ -5566,10 +5597,10 @@ def format_break(q) q.text("#{keyword} ") q.nest(keyword.length + 1) { q.format(node.predicate) } q.indent do - q.breakable + q.breakable_space q.format(node.statement) end - q.breakable + q.breakable_space q.text("end") end @@ -5720,13 +5751,13 @@ def format(q) unless statements.empty? q.indent do - q.breakable(force: true) + q.breakable_force q.format(statements) end end if consequent - q.breakable(force: true) + q.breakable_force q.format(consequent) end end @@ -6013,7 +6044,8 @@ def deconstruct_keys(_keys) end def format(q) - q.group(0, "->") do + q.text("->") + q.group do if params.is_a?(Paren) q.format(params) unless params.contents.empty? elsif params.empty? && params.comments.any? @@ -6039,10 +6071,10 @@ def format(q) unless statements.empty? q.indent do - q.breakable + q.breakable_space q.format(statements) end - q.breakable + q.breakable_space end q.text("}") @@ -6051,12 +6083,12 @@ def format(q) unless statements.empty? q.indent do - q.breakable + q.breakable_space q.format(statements) end end - q.breakable + q.breakable_space q.text("end") end end @@ -6123,7 +6155,7 @@ def format(q) if locals.any? q.text("; ") - q.seplist(locals, -> { q.text(", ") }) { |local| q.format(local) } + q.seplist(locals, BlockVar::SEPARATOR) { |local| q.format(local) } end end end @@ -6277,7 +6309,7 @@ def format(q) q.group { q.format(target) } q.text(" =") q.indent do - q.breakable + q.breakable_space q.format(value) end end @@ -6323,7 +6355,8 @@ def format(q) # If we're at the top of a call chain, then we're going to do some # specialized printing in case we can print it nicely. We _only_ do this # at the top of the chain to avoid weird recursion issues. - if !CallChainFormatter.chained?(q.parent) && + if !ENV["STREE_SKIP_CALL_CHAIN"] && + !CallChainFormatter.chained?(q.parent) && CallChainFormatter.chained?(call) q.group do q @@ -6431,15 +6464,17 @@ def format(q) q.format(contents) q.text(",") if comma else - q.group(0, "(", ")") do + q.text("(") + q.group do q.indent do - q.breakable("") + q.breakable_empty q.format(contents) end q.text(",") if comma - q.breakable("") + q.breakable_empty end + q.text(")") end end end @@ -6496,7 +6531,7 @@ def format(q) if bodystmt.empty? q.group do declaration.call - q.breakable(force: true) + q.breakable_force q.text("end") end else @@ -6504,11 +6539,11 @@ def format(q) declaration.call q.indent do - q.breakable(force: true) + q.breakable_force q.format(bodystmt) end - q.breakable(force: true) + q.breakable_force q.text("end") end end @@ -6696,7 +6731,7 @@ def format(q) q.format(value) else q.indent do - q.breakable + q.breakable_space q.format(value) end end @@ -6767,10 +6802,10 @@ def self.break(q) q.text("(") q.indent do - q.breakable("") + q.breakable_empty yield end - q.breakable("") + q.breakable_empty q.text(")") end end @@ -6970,12 +7005,16 @@ def format(q) if ![Def, Defs, DefEndless].include?(q.parent.class) || parts.empty? q.nest(0, &contents) else - q.group(0, "(", ")") do - q.indent do - q.breakable("") - contents.call + q.nest(0) do + q.text("(") + q.group do + q.indent do + q.breakable_empty + contents.call + end + q.breakable_empty end - q.breakable("") + q.text(")") end end end @@ -7029,12 +7068,12 @@ def format(q) if contents && (!contents.is_a?(Params) || !contents.empty?) q.indent do - q.breakable("") + q.breakable_empty q.format(contents) end end - q.breakable("") + q.breakable_empty q.text(")") end end @@ -7108,7 +7147,7 @@ def format(q) # We're going to put a newline on the end so that it always has one unless # it ends with the special __END__ syntax. In that case we want to # replicate the text exactly so we will just let it be. - q.breakable(force: true) unless statements.body.last.is_a?(EndContent) + q.breakable_force unless statements.body.last.is_a?(EndContent) end end @@ -7160,15 +7199,17 @@ def format(q) closing = Quotes.matching(opening[2]) end - q.group(0, opening, closing) do + q.text(opening) + q.group do q.indent do - q.breakable("") - q.seplist(elements, -> { q.breakable }) do |element| + q.breakable_empty + q.seplist(elements, Formatter::BREAKABLE_SPACE_SEPARATOR) do |element| q.format(element) end end - q.breakable("") + q.breakable_empty end + q.text(closing) end end @@ -7251,15 +7292,17 @@ def format(q) closing = Quotes.matching(opening[2]) end - q.group(0, opening, closing) do + q.text(opening) + q.group do q.indent do - q.breakable("") - q.seplist(elements, -> { q.breakable }) do |element| + q.breakable_empty + q.seplist(elements, Formatter::BREAKABLE_SPACE_SEPARATOR) do |element| q.format(element) end end - q.breakable("") + q.breakable_empty end + q.text(closing) end end @@ -7781,13 +7824,13 @@ def format(q) unless statements.empty? q.indent do - q.breakable(force: true) + q.breakable_force q.format(statements) end end if consequent - q.breakable(force: true) + q.breakable_force q.format(consequent) end end @@ -7835,19 +7878,21 @@ def deconstruct_keys(_keys) end def format(q) - q.group(0, "begin", "end") do + q.text("begin") + q.group do q.indent do - q.breakable(force: true) + q.breakable_force q.format(statement) end - q.breakable(force: true) + q.breakable_force q.text("rescue StandardError") q.indent do - q.breakable(force: true) + q.breakable_force q.format(value) end - q.breakable(force: true) + q.breakable_force end + q.text("end") end end @@ -8066,14 +8111,16 @@ def deconstruct_keys(_keys) end def format(q) - q.group(0, "class << ", "end") do + q.text("class << ") + q.group do q.format(target) q.indent do - q.breakable(force: true) + q.breakable_force q.format(bodystmt) end - q.breakable(force: true) + q.breakable_force end + q.text("end") end end @@ -8179,37 +8226,34 @@ def format(q) end end - access_controls = - Hash.new do |hash, node| - hash[node] = node.is_a?(VCall) && - %w[private protected public].include?(node.value.value) - end - - body.each_with_index do |statement, index| + previous = nil + body.each do |statement| next if statement.is_a?(VoidStmt) if line.nil? q.format(statement) elsif (statement.location.start_line - line) > 1 - q.breakable(force: true) - q.breakable(force: true) + q.breakable_force + q.breakable_force q.format(statement) - elsif access_controls[statement] || access_controls[body[index - 1]] - q.breakable(force: true) - q.breakable(force: true) + elsif (statement.is_a?(VCall) && statement.access_control?) || + (previous.is_a?(VCall) && previous.access_control?) + q.breakable_force + q.breakable_force q.format(statement) elsif statement.location.start_line != line - q.breakable(force: true) + q.breakable_force q.format(statement) elsif !q.parent.is_a?(StringEmbExpr) - q.breakable(force: true) + q.breakable_force q.format(statement) else q.text("; ") q.format(statement) end - + line = statement.location.end_line + previous = statement end end @@ -8327,7 +8371,7 @@ def format(q) q.format(left) q.text(" \\") q.indent do - q.breakable(force: true) + q.breakable_force q.format(right) end end @@ -8413,15 +8457,21 @@ def format(q) # same line in the source, then we're going to leave them in place and # assume that's the way the developer wanted this expression # represented. - q.remove_breaks(q.group(0, '#{', "}") { q.format(statements) }) + q.remove_breaks( + q.group do + q.text('#{') + q.format(statements) + q.text("}") + end + ) else q.group do q.text('#{') q.indent do - q.breakable("") + q.breakable_empty q.format(statements) end - q.breakable("") + q.breakable_empty q.text("}") end end @@ -8479,12 +8529,12 @@ def format(q) [quote, quote] end - q.group(0, opening_quote, closing_quote) do + q.text(opening_quote) + q.group do parts.each do |part| if part.is_a?(TStringContent) value = Quotes.normalize(part.value, closing_quote) - separator = -> { q.breakable(force: true, indent: false) } - q.seplist(value.split(/\r?\n/, -1), separator) do |text| + q.seplist(value.split(/\r?\n/, -1), Formatter::BREAKABLE_RETURN_SEPARATOR) do |text| q.text(text) end else @@ -8492,6 +8542,7 @@ def format(q) end end end + q.text(closing_quote) end end @@ -8698,15 +8749,17 @@ def format(q) closing = Quotes.matching(opening[2]) end - q.group(0, opening, closing) do + q.text(opening) + q.group do q.indent do - q.breakable("") - q.seplist(elements, -> { q.breakable }) do |element| + q.breakable_empty + q.seplist(elements, Formatter::BREAKABLE_SPACE_SEPARATOR) do |element| q.format(element) end end - q.breakable("") + q.breakable_empty end + q.text(closing) end end @@ -9031,27 +9084,26 @@ def deconstruct_keys(_keys) end def format(q) - parent = q.parents.take(2)[1] - ternary = - (parent.is_a?(If) || parent.is_a?(Unless)) && - Ternaryable.call(q, parent) - q.text("not") if parentheses q.text("(") - elsif ternary - q.if_break { q.text(" ") }.if_flat { q.text("(") } - else - q.text(" ") - end - - q.format(statement) if statement - - if parentheses + q.format(statement) if statement q.text(")") - elsif ternary - q.if_flat { q.text(")") } + else + parent = q.parents.take(2)[1] + ternary = + (parent.is_a?(If) || parent.is_a?(Unless)) && + Ternaryable.call(q, parent) + + if ternary + q.if_break { q.text(" ") }.if_flat { q.text("(") } + q.format(statement) if statement + q.if_flat { q.text(")") } if ternary + else + q.text(" ") + q.format(statement) if statement + end end end end @@ -9316,10 +9368,10 @@ def format_break(q) q.text("#{keyword} ") q.nest(keyword.length + 1) { q.format(node.predicate) } q.indent do - q.breakable("") + q.breakable_empty q.format(statements) end - q.breakable("") + q.breakable_empty q.text("end") end end @@ -9372,7 +9424,7 @@ def format(q) q.group do q.text(keyword) q.nest(keyword.length) { q.format(predicate) } - q.breakable(force: true) + q.breakable_force q.text("end") end else @@ -9572,6 +9624,27 @@ def deconstruct_keys(_keys) def format(q) q.format(value) end + + # Oh man I hate this so much. Basically, ripper doesn't provide enough + # functionality to actually know where pins are within an expression. So we + # have to walk the tree ourselves and insert more information. In doing so, + # we have to replace this node by a pinned node when necessary. + # + # To be clear, this method should just not exist. It's not good. It's a + # place of shame. But it's necessary for now, so I'm keeping it. + def pin(parent) + replace = PinnedVarRef.new(value: value, location: location) + + parent.deconstruct_keys([]).each do |key, value| + if value == self + parent.instance_variable_set(:"@#{key}", replace) + break + elsif value.is_a?(Array) && (index = value.index(self)) + parent.public_send(key)[index] = replace + break + end + end + end end # PinnedVarRef represents a pinned variable reference within a pattern @@ -9653,6 +9726,10 @@ def deconstruct_keys(_keys) def format(q) q.format(value) end + + def access_control? + @access_control ||= %w[private protected public].include?(value.value) + end end # VoidStmt represents an empty lexical block of code. @@ -9742,6 +9819,22 @@ def deconstruct_keys(_keys) } end + # We have a special separator here for when clauses which causes them to + # fill as much of the line as possible as opposed to everything breaking + # into its own line as soon as you hit the print limit. + class Separator + def call(q) + q.group do + q.text(",") + q.breakable_space + end + end + end + + # We're going to keep a single instance of this separator around so we don't + # have to allocate a new one every time we format a when clause. + SEPARATOR = Separator.new + def format(q) keyword = "when " @@ -9752,8 +9845,7 @@ def format(q) if arguments.comments.any? q.format(arguments) else - separator = -> { q.group { q.comma_breakable } } - q.seplist(arguments.parts, separator) { |part| q.format(part) } + q.seplist(arguments.parts, SEPARATOR) { |part| q.format(part) } end # Very special case here. If you're inside of a when clause and the @@ -9768,13 +9860,13 @@ def format(q) unless statements.empty? q.indent do - q.breakable(force: true) + q.breakable_force q.format(statements) end end if consequent - q.breakable(force: true) + q.breakable_force q.format(consequent) end end @@ -9829,7 +9921,7 @@ def format(q) q.group do q.text(keyword) q.nest(keyword.length) { q.format(predicate) } - q.breakable(force: true) + q.breakable_force q.text("end") end else @@ -9995,15 +10087,17 @@ def format(q) closing = Quotes.matching(opening[2]) end - q.group(0, opening, closing) do + q.text(opening) + q.group do q.indent do - q.breakable("") - q.seplist(elements, -> { q.breakable }) do |element| + q.breakable_empty + q.seplist(elements, Formatter::BREAKABLE_SPACE_SEPARATOR) do |element| q.format(element) end end - q.breakable("") + q.breakable_empty end + q.text(closing) end end @@ -10147,10 +10241,10 @@ def format(q) else q.if_break { q.text("(") }.if_flat { q.text(" ") } q.indent do - q.breakable("") + q.breakable_empty q.format(arguments) end - q.breakable("") + q.breakable_empty q.if_break { q.text(")") } end end diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index 94ce115a..9ca26155 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -60,29 +60,46 @@ def [](byteindex) # This represents all of the tokens coming back from the lexer. It is # replacing a simple array because it keeps track of the last deleted token # from the list for better error messages. - class TokenList < SimpleDelegator - attr_reader :last_deleted + class TokenList + attr_reader :tokens, :last_deleted - def initialize(object) - super + def initialize + @tokens = [] @last_deleted = nil end + def <<(token) + tokens << token + end + + def [](index) + tokens[index] + end + + def any?(&block) + tokens.any?(&block) + end + + def reverse_each(&block) + tokens.reverse_each(&block) + end + + def rindex(&block) + tokens.rindex(&block) + end + def delete(value) - @last_deleted = super || @last_deleted + @last_deleted = tokens.delete(value) || @last_deleted end def delete_at(index) - @last_deleted = super + @last_deleted = tokens.delete_at(index) end end # [String] the source being parsed attr_reader :source - # [Array[ String ]] the list of lines in the source - attr_reader :lines - # [Array[ SingleByteString | MultiByteString ]] the list of objects that # represent the start of each line in character offsets attr_reader :line_counts @@ -105,12 +122,6 @@ def initialize(source, *) # example. @source = source - # Similarly, we keep the lines of the source string around to be able to - # check if certain lines contain certain characters. For example, we'll - # use this to generate the content that goes after the __END__ keyword. - # Or we'll use this to check if a comment has other content on its line. - @lines = source.split(/\r?\n/) - # This is the full set of comments that have been found by the parser. # It's a running list. At the end of every block of statements, they will # go in and attempt to grab any comments that are on their own line and @@ -144,7 +155,7 @@ def initialize(source, *) # Most of the time, when a parser event consumes one of these events, it # will be deleted from the list. So ideally, this list stays pretty short # over the course of parsing a source string. - @tokens = TokenList.new([]) + @tokens = TokenList.new # Here we're going to build up a list of SingleByteString or # MultiByteString objects. They're each going to represent a string in the @@ -283,13 +294,18 @@ def find_colon2_before(const) # By finding the next non-space character, we can make sure that the bounds # of the statement list are correct. def find_next_statement_start(position) - remaining = source[position..] - - if remaining.sub(/\A +/, "")[0] == "#" - return position + remaining.index("\n") + maximum = source.length + + position.upto(maximum) do |pound_index| + case source[pound_index] + when "#" + return source.index("\n", pound_index + 1) || maximum + when " " + # continue + else + return position + end end - - position end # -------------------------------------------------------------------------- @@ -567,6 +583,56 @@ def on_array(contents) end end + # Ugh... I really do not like this class. Basically, ripper doesn't provide + # enough information about where pins are located in the tree. It only gives + # events for ^ ops and var_ref nodes. You have to piece it together + # yourself. + # + # Note that there are edge cases here that we straight up do not address, + # because I honestly think it's going to be faster to write a new parser + # than to address them. For example, this will not work properly: + # + # foo in ^((bar = 0; bar; baz)) + # + # If someone actually does something like that, we'll have to find another + # way to make this work. + class PinVisitor < Visitor + attr_reader :pins, :stack + + def initialize(pins) + @pins = pins + @stack = [] + end + + def visit(node) + return if pins.empty? + stack << node + super + stack.pop + end + + def visit_var_ref(node) + pins.shift + node.pin(stack[-2]) + end + + def self.visit(node, tokens) + start_char = node.location.start_char + allocated = [] + + tokens.reverse_each do |token| + char = token.location.start_char + break if char <= start_char + + if token.is_a?(Op) && token.value == "^" + allocated.unshift(tokens.delete(token)) + end + end + + new(allocated).visit(node) if allocated.any? + end + end + # :call-seq: # on_aryptn: ( # (nil | VarRef) constant, @@ -917,12 +983,15 @@ def on_case(value, consequent) find_token(Op, "=>") end - RAssign.new( + node = RAssign.new( value: value, operator: operator, pattern: consequent, location: value.location.to(consequent.location) ) + + PinVisitor.visit(node, tokens) + node end end @@ -1004,20 +1073,20 @@ def on_command_call(receiver, operator, message, arguments) # :call-seq: # on_comment: (String value) -> Comment def on_comment(value) - line = lineno - comment = - Comment.new( - value: value.chomp, - inline: value.strip != lines[line - 1].strip, - location: - Location.token( - line: line, - char: char_pos, - column: current_column, - size: value.size - 1 - ) + char = char_pos + location = + Location.token( + line: lineno, + char: char, + column: current_column, + size: value.size - 1 ) + index = source.rindex(/[^\t ]/, char - 1) if char != 0 + inline = index && (source[index] != "\n") + comment = + Comment.new(value: value.chomp, inline: inline, location: location) + @comments << comment comment end @@ -1878,12 +1947,15 @@ def on_in(pattern, statements, consequent) ending.location.start_column ) - In.new( + node = In.new( pattern: pattern, statements: statements, consequent: consequent, location: beginning.location.to(ending.location) ) + + PinVisitor.visit(node, tokens) + node end # :call-seq: @@ -2551,13 +2623,13 @@ def on_period(value) # :call-seq: # on_program: (Statements statements) -> Program def on_program(statements) - last_column = source.length - line_counts[lines.length - 1].start + last_column = source.length - line_counts.last.start location = Location.new( start_line: 1, start_char: 0, start_column: 0, - end_line: lines.length, + end_line: line_counts.length - 1, end_char: source.length, end_column: last_column ) @@ -3569,17 +3641,7 @@ def on_var_field(value) # :call-seq: # on_var_ref: ((Const | CVar | GVar | Ident | IVar | Kw) value) -> VarRef def on_var_ref(value) - pin = find_token(Op, "^", consume: false) - - if pin && pin.location.start_char == value.location.start_char - 1 - tokens.delete(pin) - PinnedVarRef.new( - value: value, - location: pin.location.to(value.location) - ) - else - VarRef.new(value: value, location: value.location) - end + VarRef.new(value: value, location: value.location) end # :call-seq: diff --git a/test/cli_test.rb b/test/cli_test.rb index 3734e734..03293333 100644 --- a/test/cli_test.rb +++ b/test/cli_test.rb @@ -139,7 +139,7 @@ def test_inline_script def test_multiple_inline_scripts stdio, = capture_io { SyntaxTree::CLI.run(%w[format -e 1+1 -e 2+2]) } - assert_equal("1 + 1\n2 + 2\n", stdio) + assert_equal(["1 + 1", "2 + 2"], stdio.split("\n").sort) end def test_generic_error diff --git a/test/node_test.rb b/test/node_test.rb index 07c2fe26..1a5af125 100644 --- a/test/node_test.rb +++ b/test/node_test.rb @@ -951,7 +951,7 @@ def test_var_field guard_version("3.1.0") do def test_pinned_var_ref source = "foo in ^bar" - at = location(chars: 7..11) + at = location(chars: 8..11) assert_node(PinnedVarRef, source, at: at, &:pattern) end diff --git a/test/quotes_test.rb b/test/quotes_test.rb new file mode 100644 index 00000000..2e2e0243 --- /dev/null +++ b/test/quotes_test.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require_relative "test_helper" + +module SyntaxTree + class QuotesTest < Minitest::Test + def test_normalize + content = "'aaa' \"bbb\" \\'ccc\\' \\\"ddd\\\"" + enclosing = "\"" + + result = Quotes.normalize(content, enclosing) + assert_equal "'aaa' \\\"bbb\\\" \\'ccc\\' \\\"ddd\\\"", result + end + end +end From ed0c4754aee57a80a23d030f30df76df6d710435 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Sat, 15 Oct 2022 21:45:43 -0400 Subject: [PATCH 125/536] Split up find_token options into more explicit variants --- lib/syntax_tree/parser.rb | 327 ++++++++++++++++++++------------------ 1 file changed, 173 insertions(+), 154 deletions(-) diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index 9ca26155..132780b6 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -244,28 +244,49 @@ def find_token_error(location) # "module" (which would happen to be the innermost keyword). Then the outer # one would only be able to grab the first one. In this way all of the # tokens act as their own stack. - def find_token(type, value = :any, consume: true, location: nil) + # + # If we're expecting to be able to find a token and consume it, but can't + # actually find it, then we need to raise an error. This is _usually_ caused + # by a syntax error in the source that we're printing. It could also be + # caused by accidentally attempting to consume a token twice by two + # different parser event handlers. + def find_token(type) + index = tokens.rindex { |token| token.is_a?(type) } + tokens[index] if index + end + + def find_token_value(type, value) index = tokens.rindex do |token| - token.is_a?(type) && (value == :any || (token.value == value)) + token.is_a?(type) && (token.value == value) end - if consume - # If we're expecting to be able to find a token and consume it, but - # can't actually find it, then we need to raise an error. This is - # _usually_ caused by a syntax error in the source that we're printing. - # It could also be caused by accidentally attempting to consume a token - # twice by two different parser event handlers. - unless index - token = value == :any ? type.name.split("::", 2).last : value - message = "Cannot find expected #{token}" - raise ParseError.new(message, *find_token_error(location)) + tokens[index] if index + end + + def consume_token(type, location: nil) + index = tokens.rindex { |token| token.is_a?(type) } + + unless index + message = "Cannot find expected #{type.name.split("::", 2).last}" + raise ParseError.new(message, *find_token_error(location)) + end + + tokens.delete_at(index) + end + + def consume_token_value(type, value) + index = + tokens.rindex do |token| + token.is_a?(type) && (token.value == value) end - tokens.delete_at(index) - elsif index - tokens[index] + unless index + message = "Cannot find expected #{value}" + raise ParseError.new(message, *find_token_error(nil)) end + + tokens.delete_at(index) end # A helper function to find a :: operator. We do special handling instead of @@ -316,8 +337,8 @@ def find_next_statement_start(position) # :call-seq: # on_BEGIN: (Statements statements) -> BEGINBlock def on_BEGIN(statements) - lbrace = find_token(LBrace) - rbrace = find_token(RBrace) + lbrace = consume_token(LBrace) + rbrace = consume_token(RBrace) start_char = find_next_statement_start(lbrace.location.end_char) statements.bind( @@ -327,7 +348,7 @@ def on_BEGIN(statements) rbrace.location.start_column ) - keyword = find_token(Kw, "BEGIN") + keyword = consume_token_value(Kw, "BEGIN") BEGINBlock.new( lbrace: lbrace, @@ -354,8 +375,8 @@ def on_CHAR(value) # :call-seq: # on_END: (Statements statements) -> ENDBlock def on_END(statements) - lbrace = find_token(LBrace) - rbrace = find_token(RBrace) + lbrace = consume_token(LBrace) + rbrace = consume_token(RBrace) start_char = find_next_statement_start(lbrace.location.end_char) statements.bind( @@ -365,7 +386,7 @@ def on_END(statements) rbrace.location.start_column ) - keyword = find_token(Kw, "END") + keyword = consume_token_value(Kw, "END") ENDBlock.new( lbrace: lbrace, @@ -396,7 +417,7 @@ def on___end__(value) # (DynaSymbol | SymbolLiteral) right # ) -> Alias def on_alias(left, right) - keyword = find_token(Kw, "alias") + keyword = consume_token_value(Kw, "alias") Alias.new( left: left, @@ -408,8 +429,8 @@ def on_alias(left, right) # :call-seq: # on_aref: (untyped collection, (nil | Args) index) -> ARef def on_aref(collection, index) - find_token(LBracket) - rbracket = find_token(RBracket) + consume_token(LBracket) + rbracket = consume_token(RBracket) ARef.new( collection: collection, @@ -424,8 +445,8 @@ def on_aref(collection, index) # (nil | Args) index # ) -> ARefField def on_aref_field(collection, index) - find_token(LBracket) - rbracket = find_token(RBracket) + consume_token(LBracket) + rbracket = consume_token(RBracket) ARefField.new( collection: collection, @@ -443,8 +464,8 @@ def on_aref_field(collection, index) # (nil | Args | ArgsForward) arguments # ) -> ArgParen def on_arg_paren(arguments) - lparen = find_token(LParen) - rparen = find_token(RParen) + lparen = consume_token(LParen) + rparen = consume_token(RParen) # If the arguments exceed the ending of the parentheses, then we know we # have a heredoc in the arguments, and we need to use the bounds of the @@ -489,7 +510,7 @@ def on_args_add_block(arguments, block) # First, see if there is an & operator that could potentially be # associated with the block part of this args_add_block. If there is not, # then just return the arguments. - operator = find_token(Op, "&", consume: false) + operator = find_token_value(Op, "&") return arguments unless operator # If there are any arguments and the operator we found from the list is @@ -521,7 +542,7 @@ def on_args_add_block(arguments, block) # :call-seq: # on_args_add_star: (Args arguments, untyped star) -> Args def on_args_add_star(arguments, argument) - beginning = find_token(Op, "*") + beginning = consume_token_value(Op, "*") ending = argument || beginning location = @@ -543,7 +564,7 @@ def on_args_add_star(arguments, argument) # :call-seq: # on_args_forward: () -> ArgsForward def on_args_forward - op = find_token(Op, "...") + op = consume_token_value(Op, "...") ArgsForward.new(value: op.value, location: op.location) end @@ -563,8 +584,8 @@ def on_args_new # ArrayLiteral | QSymbols | QWords | Symbols | Words def on_array(contents) if !contents || contents.is_a?(Args) - lbracket = find_token(LBracket) - rbracket = find_token(RBracket) + lbracket = consume_token(LBracket) + rbracket = consume_token(RBracket) ArrayLiteral.new( lbracket: lbracket, @@ -573,7 +594,7 @@ def on_array(contents) ) else tstring_end = - find_token(TStringEnd, location: contents.beginning.location) + consume_token(TStringEnd, location: contents.beginning.location) contents.class.new( beginning: contents.beginning, @@ -649,7 +670,7 @@ def on_aryptn(constant, requireds, rest, posts) # of the various parts. location = if parts.empty? - find_token(LBracket).location.to(find_token(RBracket).location) + consume_token(LBracket).location.to(consume_token(RBracket).location) else parts[0].location.to(parts[-1].location) end @@ -710,7 +731,7 @@ def on_assoc_new(key, value) # :call-seq: # on_assoc_splat: (untyped value) -> AssocSplat def on_assoc_splat(value) - operator = find_token(Op, "**") + operator = consume_token_value(Op, "**") AssocSplat.new( value: value, @@ -770,23 +791,23 @@ def on_bare_assoc_hash(assocs) # :call-seq: # on_begin: (untyped bodystmt) -> Begin | PinnedBegin def on_begin(bodystmt) - pin = find_token(Op, "^", consume: false) + pin = find_token_value(Op, "^") if pin && pin.location.start_char < bodystmt.location.start_char tokens.delete(pin) - find_token(LParen) + consume_token(LParen) - rparen = find_token(RParen) + rparen = consume_token(RParen) location = pin.location.to(rparen.location) PinnedBegin.new(statement: bodystmt, location: location) else - keyword = find_token(Kw, "begin") + keyword = consume_token_value(Kw, "begin") end_location = if bodystmt.else_clause bodystmt.location else - find_token(Kw, "end").location + consume_token_value(Kw, "end").location end bodystmt.bind( @@ -861,7 +882,7 @@ def on_block_var(params, locals) # :call-seq: # on_blockarg: (Ident name) -> BlockArg def on_blockarg(name) - operator = find_token(Op, "&") + operator = consume_token_value(Op, "&") location = operator.location location = location.to(name.location) if name @@ -880,7 +901,7 @@ def on_bodystmt(statements, rescue_clause, else_clause, ensure_clause) BodyStmt.new( statements: statements, rescue_clause: rescue_clause, - else_keyword: else_clause && find_token(Kw, "else"), + else_keyword: else_clause && consume_token_value(Kw, "else"), else_clause: else_clause, ensure_clause: ensure_clause, location: @@ -894,8 +915,8 @@ def on_bodystmt(statements, rescue_clause, else_clause, ensure_clause) # Statements statements # ) -> BraceBlock def on_brace_block(block_var, statements) - lbrace = find_token(LBrace) - rbrace = find_token(RBrace) + lbrace = consume_token(LBrace) + rbrace = consume_token(RBrace) location = (block_var || lbrace).location start_char = find_next_statement_start(location.end_char) @@ -930,7 +951,7 @@ def on_brace_block(block_var, statements) # :call-seq: # on_break: (Args arguments) -> Break def on_break(arguments) - keyword = find_token(Kw, "break") + keyword = consume_token_value(Kw, "break") location = keyword.location location = location.to(arguments.location) if arguments.parts.any? @@ -966,7 +987,7 @@ def on_call(receiver, operator, message) # :call-seq: # on_case: (untyped value, untyped consequent) -> Case | RAssign def on_case(value, consequent) - if (keyword = find_token(Kw, "case", consume: false)) + if (keyword = find_token_value(Kw, "case")) tokens.delete(keyword) Case.new( @@ -977,10 +998,10 @@ def on_case(value, consequent) ) else operator = - if (keyword = find_token(Kw, "in", consume: false)) + if (keyword = find_token_value(Kw, "in")) tokens.delete(keyword) else - find_token(Op, "=>") + consume_token_value(Op, "=>") end node = RAssign.new( @@ -1002,8 +1023,8 @@ def on_case(value, consequent) # BodyStmt bodystmt # ) -> ClassDeclaration def on_class(constant, superclass, bodystmt) - beginning = find_token(Kw, "class") - ending = find_token(Kw, "end") + beginning = consume_token_value(Kw, "class") + ending = consume_token_value(Kw, "end") location = (superclass || constant).location start_char = find_next_statement_start(location.end_char) @@ -1161,7 +1182,7 @@ def on_def(name, params, bodystmt) # Find the beginning of the method definition, which works for single-line # and normal method definitions. - beginning = find_token(Kw, "def") + beginning = consume_token_value(Kw, "def") # If there aren't any params then we need to correct the params node # location information @@ -1181,7 +1202,7 @@ def on_def(name, params, bodystmt) params = Params.new(location: location) end - ending = find_token(Kw, "end", consume: false) + ending = find_token_value(Kw, "end") if ending tokens.delete(ending) @@ -1219,13 +1240,13 @@ def on_def(name, params, bodystmt) # :call-seq: # on_defined: (untyped value) -> Defined def on_defined(value) - beginning = find_token(Kw, "defined?") + beginning = consume_token_value(Kw, "defined?") ending = value range = beginning.location.end_char...value.location.start_char if source[range].include?("(") - find_token(LParen) - ending = find_token(RParen) + consume_token(LParen) + ending = consume_token(RParen) end Defined.new( @@ -1266,8 +1287,8 @@ def on_defs(target, operator, name, params, bodystmt) params = Params.new(location: location) end - beginning = find_token(Kw, "def") - ending = find_token(Kw, "end", consume: false) + beginning = consume_token_value(Kw, "def") + ending = find_token_value(Kw, "end") if ending tokens.delete(ending) @@ -1307,8 +1328,8 @@ def on_defs(target, operator, name, params, bodystmt) # :call-seq: # on_do_block: (BlockVar block_var, BodyStmt bodystmt) -> DoBlock def on_do_block(block_var, bodystmt) - beginning = find_token(Kw, "do") - ending = find_token(Kw, "end") + beginning = consume_token_value(Kw, "do") + ending = consume_token_value(Kw, "end") location = (block_var || beginning).location start_char = find_next_statement_start(location.end_char) @@ -1330,7 +1351,7 @@ def on_do_block(block_var, bodystmt) # :call-seq: # on_dot2: ((nil | untyped) left, (nil | untyped) right) -> Dot2 def on_dot2(left, right) - operator = find_token(Op, "..") + operator = consume_token_value(Op, "..") beginning = left || operator ending = right || operator @@ -1345,7 +1366,7 @@ def on_dot2(left, right) # :call-seq: # on_dot3: ((nil | untyped) left, (nil | untyped) right) -> Dot3 def on_dot3(left, right) - operator = find_token(Op, "...") + operator = consume_token_value(Op, "...") beginning = left || operator ending = right || operator @@ -1360,10 +1381,10 @@ def on_dot3(left, right) # :call-seq: # on_dyna_symbol: (StringContent string_content) -> DynaSymbol def on_dyna_symbol(string_content) - if find_token(SymBeg, consume: false) + if symbeg = find_token(SymBeg) # A normal dynamic symbol - symbeg = find_token(SymBeg) - tstring_end = find_token(TStringEnd, location: symbeg.location) + tokens.delete(symbeg) + tstring_end = consume_token(TStringEnd, location: symbeg.location) DynaSymbol.new( quote: symbeg.value, @@ -1372,8 +1393,8 @@ def on_dyna_symbol(string_content) ) else # A dynamic symbol as a hash key - tstring_beg = find_token(TStringBeg) - label_end = find_token(LabelEnd) + tstring_beg = consume_token(TStringBeg) + label_end = consume_token(LabelEnd) DynaSymbol.new( parts: string_content.parts, @@ -1386,7 +1407,7 @@ def on_dyna_symbol(string_content) # :call-seq: # on_else: (Statements statements) -> Else def on_else(statements) - keyword = find_token(Kw, "else") + keyword = consume_token_value(Kw, "else") # else can either end with an end keyword (in which case we'll want to # consume that event) or it can end with an ensure keyword (in which case @@ -1426,8 +1447,8 @@ def on_else(statements) # (nil | Elsif | Else) consequent # ) -> Elsif def on_elsif(predicate, statements, consequent) - beginning = find_token(Kw, "elsif") - ending = consequent || find_token(Kw, "end") + beginning = consume_token_value(Kw, "elsif") + ending = consequent || consume_token_value(Kw, "end") start_char = find_next_statement_start(predicate.location.end_char) statements.bind( @@ -1547,11 +1568,11 @@ def on_embvar(value) # :call-seq: # on_ensure: (Statements statements) -> Ensure def on_ensure(statements) - keyword = find_token(Kw, "ensure") + keyword = consume_token_value(Kw, "ensure") # We don't want to consume the :@kw event, because that would break # def..ensure..end chains. - ending = find_token(Kw, "end", consume: false) + ending = find_token_value(Kw, "end") start_char = find_next_statement_start(keyword.location.end_char) statements.bind( start_char, @@ -1573,7 +1594,7 @@ def on_ensure(statements) # :call-seq: # on_excessed_comma: () -> ExcessedComma def on_excessed_comma(*) - comma = find_token(Comma) + comma = consume_token(Comma) ExcessedComma.new(value: comma.value, location: comma.location) end @@ -1626,9 +1647,7 @@ def on_fndptn(constant, left, values, right) # right left parenthesis, or the left splat. We're going to use this to # determine how to find the closing of the pattern, as well as determining # the location of the node. - opening = - find_token(LBracket, consume: false) || - find_token(LParen, consume: false) || left + opening = find_token(LBracket) || find_token(LParen) || left # The closing is based on the opening, which is either the matched # punctuation or the right splat. @@ -1636,10 +1655,10 @@ def on_fndptn(constant, left, values, right) case opening in LBracket tokens.delete(opening) - find_token(RBracket) + consume_token(RBracket) in LParen tokens.delete(opening) - find_token(RParen) + consume_token(RParen) else right end @@ -1660,13 +1679,13 @@ def on_fndptn(constant, left, values, right) # Statements statements # ) -> For def on_for(index, collection, statements) - beginning = find_token(Kw, "for") - in_keyword = find_token(Kw, "in") - ending = find_token(Kw, "end") + beginning = consume_token_value(Kw, "for") + in_keyword = consume_token_value(Kw, "in") + ending = consume_token_value(Kw, "end") # Consume the do keyword if it exists so that it doesn't get confused for # some other block - keyword = find_token(Kw, "do", consume: false) + keyword = find_token_value(Kw, "do") if keyword && keyword.location.start_char > collection.location.end_char && keyword.location.end_char < ending.location.start_char @@ -1714,8 +1733,8 @@ def on_gvar(value) # :call-seq: # on_hash: ((nil | Array[AssocNew | AssocSplat]) assocs) -> HashLiteral def on_hash(assocs) - lbrace = find_token(LBrace) - rbrace = find_token(RBrace) + lbrace = consume_token(LBrace) + rbrace = consume_token(RBrace) HashLiteral.new( lbrace: lbrace, @@ -1799,8 +1818,8 @@ def on_hshptn(constant, keywords, keyword_rest) if keyword_rest # We're doing this to delete the token from the list so that it doesn't # confuse future patterns by thinking they have an extra ** on the end. - find_token(Op, "**") - elsif (token = find_token(Op, "**", consume: false)) + consume_token_value(Op, "**") + elsif (token = find_token_value(Op, "**")) tokens.delete(token) # Create an artificial VarField if we find an extra ** on the end. This @@ -1813,8 +1832,8 @@ def on_hshptn(constant, keywords, keyword_rest) # If there's no constant, there may be braces, so we're going to look for # those to get our bounds. unless constant - lbrace = find_token(LBrace, consume: false) - rbrace = find_token(RBrace, consume: false) + lbrace = find_token(LBrace) + rbrace = find_token(RBrace) if lbrace && rbrace parts = [lbrace, *parts, rbrace] @@ -1853,8 +1872,8 @@ def on_ident(value) # (nil | Elsif | Else) consequent # ) -> If def on_if(predicate, statements, consequent) - beginning = find_token(Kw, "if") - ending = consequent || find_token(Kw, "end") + beginning = consume_token_value(Kw, "if") + ending = consequent || consume_token_value(Kw, "end") start_char = find_next_statement_start(predicate.location.end_char) statements.bind( @@ -1886,7 +1905,7 @@ def on_ifop(predicate, truthy, falsy) # :call-seq: # on_if_mod: (untyped predicate, untyped statement) -> IfMod def on_if_mod(predicate, statement) - find_token(Kw, "if") + consume_token_value(Kw, "if") IfMod.new( statement: statement, @@ -1929,11 +1948,11 @@ def on_in(pattern, statements, consequent) # Here we have a rightward assignment return pattern unless statements - beginning = find_token(Kw, "in") - ending = consequent || find_token(Kw, "end") + beginning = consume_token_value(Kw, "in") + ending = consequent || consume_token_value(Kw, "end") statements_start = pattern - if (token = find_token(Kw, "then", consume: false)) + if (token = find_token_value(Kw, "then")) tokens.delete(token) statements_start = token end @@ -2010,7 +2029,7 @@ def on_kw(value) # :call-seq: # on_kwrest_param: ((nil | Ident) name) -> KwRestParam def on_kwrest_param(name) - location = find_token(Op, "**").location + location = consume_token_value(Op, "**").location location = location.to(name.location) if name KwRestParam.new(name: name, location: location) @@ -2056,7 +2075,7 @@ def on_label_end(value) # (BodyStmt | Statements) statements # ) -> Lambda def on_lambda(params, statements) - beginning = find_token(TLambda) + beginning = consume_token(TLambda) braces = tokens.any? do |token| token.is_a?(TLamBeg) && @@ -2099,11 +2118,11 @@ def on_lambda(params, statements) end if braces - opening = find_token(TLamBeg) - closing = find_token(RBrace) + opening = consume_token(TLamBeg) + closing = consume_token(RBrace) else - opening = find_token(Kw, "do") - closing = find_token(Kw, "end") + opening = consume_token_value(Kw, "do") + closing = consume_token_value(Kw, "end") end start_char = find_next_statement_start(opening.location.end_char) @@ -2334,7 +2353,7 @@ def on_mlhs_add_post(left, right) # (nil | ARefField | Field | Ident | VarField) part # ) -> MLHS def on_mlhs_add_star(mlhs, part) - beginning = find_token(Op, "*") + beginning = consume_token_value(Op, "*") ending = part || beginning location = beginning.location.to(ending.location) @@ -2357,8 +2376,8 @@ def on_mlhs_new # :call-seq: # on_mlhs_paren: ((MLHS | MLHSParen) contents) -> MLHSParen def on_mlhs_paren(contents) - lparen = find_token(LParen) - rparen = find_token(RParen) + lparen = consume_token(LParen) + rparen = consume_token(RParen) comma_range = lparen.location.end_char...rparen.location.start_char contents.comma = true if source[comma_range].strip.end_with?(",") @@ -2375,8 +2394,8 @@ def on_mlhs_paren(contents) # BodyStmt bodystmt # ) -> ModuleDeclaration def on_module(constant, bodystmt) - beginning = find_token(Kw, "module") - ending = find_token(Kw, "end") + beginning = consume_token_value(Kw, "module") + ending = consume_token_value(Kw, "end") start_char = find_next_statement_start(constant.location.end_char) bodystmt.bind( @@ -2415,7 +2434,7 @@ def on_mrhs_add(mrhs, part) # :call-seq: # on_mrhs_add_star: (MRHS mrhs, untyped value) -> MRHS def on_mrhs_add_star(mrhs, value) - beginning = find_token(Op, "*") + beginning = consume_token_value(Op, "*") ending = value || beginning arg_star = @@ -2443,7 +2462,7 @@ def on_mrhs_new_from_args(arguments) # :call-seq: # on_next: (Args arguments) -> Next def on_next(arguments) - keyword = find_token(Kw, "next") + keyword = consume_token_value(Kw, "next") location = keyword.location location = location.to(arguments.location) if arguments.parts.any? @@ -2558,8 +2577,8 @@ def on_params( # :call-seq: # on_paren: (untyped contents) -> Paren def on_paren(contents) - lparen = find_token(LParen) - rparen = find_token(RParen) + lparen = consume_token(LParen) + rparen = consume_token(RParen) if contents.is_a?(Params) location = contents.location @@ -2764,7 +2783,7 @@ def on_qsymbols_beg(value) # :call-seq: # on_qsymbols_new: () -> QSymbols def on_qsymbols_new - beginning = find_token(QSymbolsBeg) + beginning = consume_token(QSymbolsBeg) QSymbols.new( beginning: beginning, @@ -2805,7 +2824,7 @@ def on_qwords_beg(value) # :call-seq: # on_qwords_new: () -> QWords def on_qwords_new - beginning = find_token(QWordsBeg) + beginning = consume_token(QWordsBeg) QWords.new( beginning: beginning, @@ -2870,7 +2889,7 @@ def on_rbracket(value) # :call-seq: # on_redo: () -> Redo def on_redo - keyword = find_token(Kw, "redo") + keyword = consume_token_value(Kw, "redo") Redo.new(value: keyword.value, location: keyword.location) end @@ -2946,7 +2965,7 @@ def on_regexp_literal(regexp_content, ending) # :call-seq: # on_regexp_new: () -> RegexpContent def on_regexp_new - regexp_beg = find_token(RegexpBeg) + regexp_beg = consume_token(RegexpBeg) RegexpContent.new( beginning: regexp_beg.value, @@ -2963,7 +2982,7 @@ def on_regexp_new # (nil | Rescue) consequent # ) -> Rescue def on_rescue(exceptions, variable, statements, consequent) - keyword = find_token(Kw, "rescue") + keyword = consume_token_value(Kw, "rescue") exceptions = exceptions[0] if exceptions.is_a?(Array) last_node = variable || exceptions || keyword @@ -3015,7 +3034,7 @@ def on_rescue(exceptions, variable, statements, consequent) # :call-seq: # on_rescue_mod: (untyped statement, untyped value) -> RescueMod def on_rescue_mod(statement, value) - find_token(Kw, "rescue") + consume_token_value(Kw, "rescue") RescueMod.new( statement: statement, @@ -3027,7 +3046,7 @@ def on_rescue_mod(statement, value) # :call-seq: # on_rest_param: ((nil | Ident) name) -> RestParam def on_rest_param(name) - location = find_token(Op, "*").location + location = consume_token_value(Op, "*").location location = location.to(name.location) if name RestParam.new(name: name, location: location) @@ -3036,7 +3055,7 @@ def on_rest_param(name) # :call-seq: # on_retry: () -> Retry def on_retry - keyword = find_token(Kw, "retry") + keyword = consume_token_value(Kw, "retry") Retry.new(value: keyword.value, location: keyword.location) end @@ -3044,7 +3063,7 @@ def on_retry # :call-seq: # on_return: (Args arguments) -> Return def on_return(arguments) - keyword = find_token(Kw, "return") + keyword = consume_token_value(Kw, "return") Return.new( arguments: arguments, @@ -3055,7 +3074,7 @@ def on_return(arguments) # :call-seq: # on_return0: () -> Return0 def on_return0 - keyword = find_token(Kw, "return") + keyword = consume_token_value(Kw, "return") Return0.new(value: keyword.value, location: keyword.location) end @@ -3082,8 +3101,8 @@ def on_rparen(value) # :call-seq: # on_sclass: (untyped target, BodyStmt bodystmt) -> SClass def on_sclass(target, bodystmt) - beginning = find_token(Kw, "class") - ending = find_token(Kw, "end") + beginning = consume_token_value(Kw, "class") + ending = consume_token_value(Kw, "end") start_char = find_next_statement_start(target.location.end_char) bodystmt.bind( @@ -3181,7 +3200,7 @@ def on_string_content # :call-seq: # on_string_dvar: ((Backref | VarRef) variable) -> StringDVar def on_string_dvar(variable) - embvar = find_token(EmbVar) + embvar = consume_token(EmbVar) StringDVar.new( variable: variable, @@ -3192,8 +3211,8 @@ def on_string_dvar(variable) # :call-seq: # on_string_embexpr: (Statements statements) -> StringEmbExpr def on_string_embexpr(statements) - embexpr_beg = find_token(EmbExprBeg) - embexpr_end = find_token(EmbExprEnd) + embexpr_beg = consume_token(EmbExprBeg) + embexpr_end = consume_token(EmbExprEnd) statements.bind( embexpr_beg.location.end_char, @@ -3234,8 +3253,8 @@ def on_string_literal(string) location: heredoc.location ) else - tstring_beg = find_token(TStringBeg) - tstring_end = find_token(TStringEnd, location: tstring_beg.location) + tstring_beg = consume_token(TStringBeg) + tstring_end = consume_token(TStringEnd, location: tstring_beg.location) location = Location.new( @@ -3261,7 +3280,7 @@ def on_string_literal(string) # :call-seq: # on_super: ((ArgParen | Args) arguments) -> Super def on_super(arguments) - keyword = find_token(Kw, "super") + keyword = consume_token_value(Kw, "super") Super.new( arguments: arguments, @@ -3308,7 +3327,7 @@ def on_symbol(value) # ) -> SymbolLiteral def on_symbol_literal(value) if value.is_a?(SymbolContent) - symbeg = find_token(SymBeg) + symbeg = consume_token(SymBeg) SymbolLiteral.new( value: value.value, @@ -3352,7 +3371,7 @@ def on_symbols_beg(value) # :call-seq: # on_symbols_new: () -> Symbols def on_symbols_new - beginning = find_token(SymbolsBeg) + beginning = consume_token(SymbolsBeg) Symbols.new( beginning: beginning, @@ -3482,13 +3501,13 @@ def on_unary(operator, statement) # We have somewhat special handling of the not operator since if it has # parentheses they don't get reported as a paren node for some reason. - beginning = find_token(Kw, "not") + beginning = consume_token_value(Kw, "not") ending = statement || beginning parentheses = source[beginning.location.end_char] == "(" if parentheses - find_token(LParen) - ending = find_token(RParen) + consume_token(LParen) + ending = consume_token(RParen) end Not.new( @@ -3521,7 +3540,7 @@ def on_unary(operator, statement) # :call-seq: # on_undef: (Array[DynaSymbol | SymbolLiteral] symbols) -> Undef def on_undef(symbols) - keyword = find_token(Kw, "undef") + keyword = consume_token_value(Kw, "undef") Undef.new( symbols: symbols, @@ -3536,8 +3555,8 @@ def on_undef(symbols) # ((nil | Elsif | Else) consequent) # ) -> Unless def on_unless(predicate, statements, consequent) - beginning = find_token(Kw, "unless") - ending = consequent || find_token(Kw, "end") + beginning = consume_token_value(Kw, "unless") + ending = consequent || consume_token_value(Kw, "end") start_char = find_next_statement_start(predicate.location.end_char) statements.bind( @@ -3558,7 +3577,7 @@ def on_unless(predicate, statements, consequent) # :call-seq: # on_unless_mod: (untyped predicate, untyped statement) -> UnlessMod def on_unless_mod(predicate, statement) - find_token(Kw, "unless") + consume_token_value(Kw, "unless") UnlessMod.new( statement: statement, @@ -3570,12 +3589,12 @@ def on_unless_mod(predicate, statement) # :call-seq: # on_until: (untyped predicate, Statements statements) -> Until def on_until(predicate, statements) - beginning = find_token(Kw, "until") - ending = find_token(Kw, "end") + beginning = consume_token_value(Kw, "until") + ending = consume_token_value(Kw, "end") # Consume the do keyword if it exists so that it doesn't get confused for # some other block - keyword = find_token(Kw, "do", consume: false) + keyword = find_token_value(Kw, "do") if keyword && keyword.location.start_char > predicate.location.end_char && keyword.location.end_char < ending.location.start_char tokens.delete(keyword) @@ -3600,7 +3619,7 @@ def on_until(predicate, statements) # :call-seq: # on_until_mod: (untyped predicate, untyped statement) -> UntilMod def on_until_mod(predicate, statement) - find_token(Kw, "until") + consume_token_value(Kw, "until") UntilMod.new( statement: statement, @@ -3612,7 +3631,7 @@ def on_until_mod(predicate, statement) # :call-seq: # on_var_alias: (GVar left, (Backref | GVar) right) -> VarAlias def on_var_alias(left, right) - keyword = find_token(Kw, "alias") + keyword = consume_token_value(Kw, "alias") VarAlias.new( left: left, @@ -3666,11 +3685,11 @@ def on_void_stmt # (nil | Else | When) consequent # ) -> When def on_when(arguments, statements, consequent) - beginning = find_token(Kw, "when") - ending = consequent || find_token(Kw, "end") + beginning = consume_token_value(Kw, "when") + ending = consequent || consume_token_value(Kw, "end") statements_start = arguments - if (token = find_token(Kw, "then", consume: false)) + if (token = find_token_value(Kw, "then")) tokens.delete(token) statements_start = token end @@ -3696,12 +3715,12 @@ def on_when(arguments, statements, consequent) # :call-seq: # on_while: (untyped predicate, Statements statements) -> While def on_while(predicate, statements) - beginning = find_token(Kw, "while") - ending = find_token(Kw, "end") + beginning = consume_token_value(Kw, "while") + ending = consume_token_value(Kw, "end") # Consume the do keyword if it exists so that it doesn't get confused for # some other block - keyword = find_token(Kw, "do", consume: false) + keyword = find_token_value(Kw, "do") if keyword && keyword.location.start_char > predicate.location.end_char && keyword.location.end_char < ending.location.start_char tokens.delete(keyword) @@ -3726,7 +3745,7 @@ def on_while(predicate, statements) # :call-seq: # on_while_mod: (untyped predicate, untyped statement) -> WhileMod def on_while_mod(predicate, statement) - find_token(Kw, "while") + consume_token_value(Kw, "while") WhileMod.new( statement: statement, @@ -3789,7 +3808,7 @@ def on_words_beg(value) # :call-seq: # on_words_new: () -> Words def on_words_new - beginning = find_token(WordsBeg) + beginning = consume_token(WordsBeg) Words.new( beginning: beginning, @@ -3823,7 +3842,7 @@ def on_xstring_new if heredoc && heredoc.beginning.value.include?("`") heredoc.location else - find_token(Backtick).location + consume_token(Backtick).location end XString.new(parts: [], location: location) @@ -3843,7 +3862,7 @@ def on_xstring_literal(xstring) location: heredoc.location ) else - ending = find_token(TStringEnd, location: xstring.location) + ending = consume_token(TStringEnd, location: xstring.location) XStringLiteral.new( parts: xstring.parts, @@ -3855,7 +3874,7 @@ def on_xstring_literal(xstring) # :call-seq: # on_yield: ((Args | Paren) arguments) -> Yield def on_yield(arguments) - keyword = find_token(Kw, "yield") + keyword = consume_token_value(Kw, "yield") Yield.new( arguments: arguments, @@ -3866,7 +3885,7 @@ def on_yield(arguments) # :call-seq: # on_yield0: () -> Yield0 def on_yield0 - keyword = find_token(Kw, "yield") + keyword = consume_token_value(Kw, "yield") Yield0.new(value: keyword.value, location: keyword.location) end @@ -3874,7 +3893,7 @@ def on_yield0 # :call-seq: # on_zsuper: () -> ZSuper def on_zsuper - keyword = find_token(Kw, "super") + keyword = consume_token_value(Kw, "super") ZSuper.new(value: keyword.value, location: keyword.location) end From 840ebabc7a245ae9f23d7136a39482aef4be8367 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Sat, 15 Oct 2022 23:26:50 -0400 Subject: [PATCH 126/536] Even more performance tweaks --- lib/syntax_tree.rb | 10 ++ lib/syntax_tree/node.rb | 86 +++++++++----- lib/syntax_tree/parser.rb | 231 +++++++++++++++++++------------------- test/interface_test.rb | 4 + 4 files changed, 187 insertions(+), 144 deletions(-) diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index 29ed048c..da84273c 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -20,6 +20,16 @@ require_relative "syntax_tree/parser" +# We rely on Symbol#name being available, which is only available in Ruby 3.0+. +# In case we're running on an older Ruby version, we polyfill it here. +unless :+.respond_to?(:name) + class Symbol + def name + to_s.freeze + end + end +end + # Syntax Tree is a suite of tools built on top of the internal CRuby parser. It # provides the ability to generate a syntax tree from source, as well as the # tools necessary to inspect and manipulate that syntax tree. It can be used to diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 2aa51fd8..24d35985 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -606,12 +606,14 @@ def format(q) private def trailing_comma? - case arguments - in Args[parts: [*, ArgBlock]] + return false unless arguments.is_a?(Args) + parts = arguments.parts + + if parts.last&.is_a?(ArgBlock) # If the last argument is a block, then we can't put a trailing comma # after it without resulting in a syntax error. false - in Args[parts: [Command | CommandCall]] + elsif parts.length == 1 && (part = parts.first) && (part.is_a?(Command) || part.is_a?(CommandCall)) # If the only argument is a command or command call, then a trailing # comma would be parsed as part of that expression instead of on this # one, so we don't want to add a trailing comma. @@ -1668,13 +1670,11 @@ def format(q) q.text(" ") unless power if operator == :<< - q.text(operator.to_s) - q.text(" ") + q.text("<< ") q.format(right) else q.group do - q.text(operator.to_s) - + q.text(operator.name) q.indent do power ? q.breakable_empty : q.breakable_space q.format(right) @@ -1974,12 +1974,11 @@ def format(q) # If the receiver of this block a Command or CommandCall node, then there # are no parentheses around the arguments to that command, so we need to # break the block. - case q.parent - in { call: Command | CommandCall } + case q.parent.call + when Command, CommandCall q.break_parent format_break(q, break_opening, break_closing) return - else end q.group do @@ -1999,9 +1998,9 @@ def unchangeable_bounds?(q) # know for certain we're going to get split over multiple lines # anyway. case parent - in Statements | ArgParen + when Statements, ArgParen break false - in Command | CommandCall + when Command, CommandCall true else false @@ -2012,8 +2011,8 @@ def unchangeable_bounds?(q) # If we're a sibling of a control-flow keyword, then we're going to have to # use the do..end bounds. def forced_do_end_bounds?(q) - case q.parent - in { call: Break | Next | Return | Super } + case q.parent.call + when Break, Next, Return, Super true else false @@ -2997,15 +2996,31 @@ def format(q) private def align(q, node, &block) - case node.arguments - in Args[parts: [Def | Defs | DefEndless]] - q.text(" ") - yield - in Args[parts: [IfOp]] - q.if_flat { q.text(" ") } - yield - in Args[parts: [Command => command]] - align(q, command, &block) + arguments = node.arguments + + if arguments.is_a?(Args) + parts = arguments.parts + + if parts.size == 1 + part = parts.first + + case part + when Def, Defs, DefEndless + q.text(" ") + yield + when IfOp + q.if_flat { q.text(" ") } + yield + when Command + align(q, part, &block) + else + q.text(" ") + q.nest(message.value.length + 1) { yield } + end + else + q.text(" ") + q.nest(message.value.length + 1) { yield } + end else q.text(" ") q.nest(message.value.length + 1) { yield } @@ -3092,13 +3107,16 @@ def format(q) end end - case arguments - in Args[parts: [IfOp]] - q.if_flat { q.text(" ") } - q.format(arguments) - in Args - q.text(" ") - q.nest(argument_alignment(q, doc)) { q.format(arguments) } + if arguments + parts = arguments.parts + + if parts.length == 1 && parts.first.is_a?(IfOp) + q.if_flat { q.text(" ") } + q.format(arguments) + else + q.text(" ") + q.nest(argument_alignment(q, doc)) { q.format(arguments) } + end else # If there are no arguments, print nothing. end @@ -5861,11 +5879,15 @@ class Kw < Node # [String] the value of the keyword attr_reader :value + # [Symbol] the symbol version of the value + attr_reader :name + # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments def initialize(value:, location:, comments: []) @value = value + @name = value.to_sym @location = location @comments = comments end @@ -6645,11 +6667,15 @@ class Op < Node # [String] the operator attr_reader :value + # [Symbol] the symbol version of the value + attr_reader :name + # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments def initialize(value:, location:, comments: []) @value = value + @name = value.to_sym @location = location @comments = comments end diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index 132780b6..3245efa1 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -164,7 +164,7 @@ def initialize(source, *) @line_counts = [] last_index = 0 - @source.lines.each do |line| + @source.each_line do |line| @line_counts << if line.size == line.bytesize SingleByteString.new(last_index) else @@ -250,42 +250,48 @@ def find_token_error(location) # by a syntax error in the source that we're printing. It could also be # caused by accidentally attempting to consume a token twice by two # different parser event handlers. + def find_token(type) index = tokens.rindex { |token| token.is_a?(type) } tokens[index] if index end - def find_token_value(type, value) - index = - tokens.rindex do |token| - token.is_a?(type) && (token.value == value) - end - + def find_keyword(name) + index = tokens.rindex { |token| token.is_a?(Kw) && (token.name == name) } tokens[index] if index end - def consume_token(type, location: nil) - index = tokens.rindex { |token| token.is_a?(type) } + def find_operator(name) + index = tokens.rindex { |token| token.is_a?(Op) && (token.name == name) } + tokens[index] if index + end - unless index - message = "Cannot find expected #{type.name.split("::", 2).last}" - raise ParseError.new(message, *find_token_error(location)) - end + def consume_error(name, location) + message = "Cannot find expected #{name}" + raise ParseError.new(message, *find_token_error(location)) + end + def consume_token(type) + index = tokens.rindex { |token| token.is_a?(type) } + consume_error(type.name.split("::", 2).last, nil) unless index tokens.delete_at(index) end - def consume_token_value(type, value) - index = - tokens.rindex do |token| - token.is_a?(type) && (token.value == value) - end + def consume_tstring_end(location) + index = tokens.rindex { |token| token.is_a?(TStringEnd) } + consume_error("string ending", location) unless index + tokens.delete_at(index) + end - unless index - message = "Cannot find expected #{value}" - raise ParseError.new(message, *find_token_error(nil)) - end + def consume_keyword(name) + index = tokens.rindex { |token| token.is_a?(Kw) && (token.name == name) } + consume_error(name, nil) unless index + tokens.delete_at(index) + end + def consume_operator(name) + index = tokens.rindex { |token| token.is_a?(Op) && (token.name == name) } + consume_error(name, nil) unless index tokens.delete_at(index) end @@ -348,7 +354,7 @@ def on_BEGIN(statements) rbrace.location.start_column ) - keyword = consume_token_value(Kw, "BEGIN") + keyword = consume_keyword(:BEGIN) BEGINBlock.new( lbrace: lbrace, @@ -386,7 +392,7 @@ def on_END(statements) rbrace.location.start_column ) - keyword = consume_token_value(Kw, "END") + keyword = consume_keyword(:END) ENDBlock.new( lbrace: lbrace, @@ -417,7 +423,7 @@ def on___end__(value) # (DynaSymbol | SymbolLiteral) right # ) -> Alias def on_alias(left, right) - keyword = consume_token_value(Kw, "alias") + keyword = consume_keyword(:alias) Alias.new( left: left, @@ -510,7 +516,7 @@ def on_args_add_block(arguments, block) # First, see if there is an & operator that could potentially be # associated with the block part of this args_add_block. If there is not, # then just return the arguments. - operator = find_token_value(Op, "&") + operator = find_operator(:&) return arguments unless operator # If there are any arguments and the operator we found from the list is @@ -542,7 +548,7 @@ def on_args_add_block(arguments, block) # :call-seq: # on_args_add_star: (Args arguments, untyped star) -> Args def on_args_add_star(arguments, argument) - beginning = consume_token_value(Op, "*") + beginning = consume_operator(:*) ending = argument || beginning location = @@ -564,7 +570,7 @@ def on_args_add_star(arguments, argument) # :call-seq: # on_args_forward: () -> ArgsForward def on_args_forward - op = consume_token_value(Op, "...") + op = consume_operator(:"...") ArgsForward.new(value: op.value, location: op.location) end @@ -593,8 +599,7 @@ def on_array(contents) location: lbracket.location.to(rbracket.location) ) else - tstring_end = - consume_token(TStringEnd, location: contents.beginning.location) + tstring_end = consume_tstring_end(contents.beginning.location) contents.class.new( beginning: contents.beginning, @@ -731,7 +736,7 @@ def on_assoc_new(key, value) # :call-seq: # on_assoc_splat: (untyped value) -> AssocSplat def on_assoc_splat(value) - operator = consume_token_value(Op, "**") + operator = consume_operator(:**) AssocSplat.new( value: value, @@ -791,7 +796,7 @@ def on_bare_assoc_hash(assocs) # :call-seq: # on_begin: (untyped bodystmt) -> Begin | PinnedBegin def on_begin(bodystmt) - pin = find_token_value(Op, "^") + pin = find_operator(:^) if pin && pin.location.start_char < bodystmt.location.start_char tokens.delete(pin) @@ -802,12 +807,12 @@ def on_begin(bodystmt) PinnedBegin.new(statement: bodystmt, location: location) else - keyword = consume_token_value(Kw, "begin") + keyword = consume_keyword(:begin) end_location = if bodystmt.else_clause bodystmt.location else - consume_token_value(Kw, "end").location + consume_keyword(:end).location end bodystmt.bind( @@ -833,13 +838,11 @@ def on_binary(left, operator, right) # Here, we're going to search backward for the token that's between the # two operands that matches the operator so we can delete it from the # list. + range = (left.location.end_char + 1)...right.location.start_char index = tokens.rindex do |token| - location = token.location - - token.is_a?(Op) && token.value == operator.to_s && - location.start_char > left.location.end_char && - location.end_char < right.location.start_char + token.is_a?(Op) && token.name == operator && + range.cover?(token.location.start_char) end tokens.delete_at(index) if index @@ -882,7 +885,7 @@ def on_block_var(params, locals) # :call-seq: # on_blockarg: (Ident name) -> BlockArg def on_blockarg(name) - operator = consume_token_value(Op, "&") + operator = consume_operator(:&) location = operator.location location = location.to(name.location) if name @@ -901,7 +904,7 @@ def on_bodystmt(statements, rescue_clause, else_clause, ensure_clause) BodyStmt.new( statements: statements, rescue_clause: rescue_clause, - else_keyword: else_clause && consume_token_value(Kw, "else"), + else_keyword: else_clause && consume_keyword(:else), else_clause: else_clause, ensure_clause: ensure_clause, location: @@ -951,7 +954,7 @@ def on_brace_block(block_var, statements) # :call-seq: # on_break: (Args arguments) -> Break def on_break(arguments) - keyword = consume_token_value(Kw, "break") + keyword = consume_keyword(:break) location = keyword.location location = location.to(arguments.location) if arguments.parts.any? @@ -987,7 +990,7 @@ def on_call(receiver, operator, message) # :call-seq: # on_case: (untyped value, untyped consequent) -> Case | RAssign def on_case(value, consequent) - if (keyword = find_token_value(Kw, "case")) + if (keyword = find_keyword(:case)) tokens.delete(keyword) Case.new( @@ -998,10 +1001,10 @@ def on_case(value, consequent) ) else operator = - if (keyword = find_token_value(Kw, "in")) + if (keyword = find_keyword(:in)) tokens.delete(keyword) else - consume_token_value(Op, "=>") + consume_operator(:"=>") end node = RAssign.new( @@ -1023,8 +1026,8 @@ def on_case(value, consequent) # BodyStmt bodystmt # ) -> ClassDeclaration def on_class(constant, superclass, bodystmt) - beginning = consume_token_value(Kw, "class") - ending = consume_token_value(Kw, "end") + beginning = consume_keyword(:class) + ending = consume_keyword(:end) location = (superclass || constant).location start_char = find_next_statement_start(location.end_char) @@ -1182,7 +1185,7 @@ def on_def(name, params, bodystmt) # Find the beginning of the method definition, which works for single-line # and normal method definitions. - beginning = consume_token_value(Kw, "def") + beginning = consume_keyword(:def) # If there aren't any params then we need to correct the params node # location information @@ -1202,7 +1205,7 @@ def on_def(name, params, bodystmt) params = Params.new(location: location) end - ending = find_token_value(Kw, "end") + ending = find_keyword(:end) if ending tokens.delete(ending) @@ -1240,7 +1243,7 @@ def on_def(name, params, bodystmt) # :call-seq: # on_defined: (untyped value) -> Defined def on_defined(value) - beginning = consume_token_value(Kw, "defined?") + beginning = consume_keyword(:defined?) ending = value range = beginning.location.end_char...value.location.start_char @@ -1287,8 +1290,8 @@ def on_defs(target, operator, name, params, bodystmt) params = Params.new(location: location) end - beginning = consume_token_value(Kw, "def") - ending = find_token_value(Kw, "end") + beginning = consume_keyword(:def) + ending = find_keyword(:end) if ending tokens.delete(ending) @@ -1328,8 +1331,8 @@ def on_defs(target, operator, name, params, bodystmt) # :call-seq: # on_do_block: (BlockVar block_var, BodyStmt bodystmt) -> DoBlock def on_do_block(block_var, bodystmt) - beginning = consume_token_value(Kw, "do") - ending = consume_token_value(Kw, "end") + beginning = consume_keyword(:do) + ending = consume_keyword(:end) location = (block_var || beginning).location start_char = find_next_statement_start(location.end_char) @@ -1351,7 +1354,7 @@ def on_do_block(block_var, bodystmt) # :call-seq: # on_dot2: ((nil | untyped) left, (nil | untyped) right) -> Dot2 def on_dot2(left, right) - operator = consume_token_value(Op, "..") + operator = consume_operator(:"..") beginning = left || operator ending = right || operator @@ -1366,7 +1369,7 @@ def on_dot2(left, right) # :call-seq: # on_dot3: ((nil | untyped) left, (nil | untyped) right) -> Dot3 def on_dot3(left, right) - operator = consume_token_value(Op, "...") + operator = consume_operator(:"...") beginning = left || operator ending = right || operator @@ -1384,7 +1387,7 @@ def on_dyna_symbol(string_content) if symbeg = find_token(SymBeg) # A normal dynamic symbol tokens.delete(symbeg) - tstring_end = consume_token(TStringEnd, location: symbeg.location) + tstring_end = consume_tstring_end(symbeg.location) DynaSymbol.new( quote: symbeg.value, @@ -1407,7 +1410,7 @@ def on_dyna_symbol(string_content) # :call-seq: # on_else: (Statements statements) -> Else def on_else(statements) - keyword = consume_token_value(Kw, "else") + keyword = consume_keyword(:else) # else can either end with an end keyword (in which case we'll want to # consume that event) or it can end with an ensure keyword (in which case @@ -1447,8 +1450,8 @@ def on_else(statements) # (nil | Elsif | Else) consequent # ) -> Elsif def on_elsif(predicate, statements, consequent) - beginning = consume_token_value(Kw, "elsif") - ending = consequent || consume_token_value(Kw, "end") + beginning = consume_keyword(:elsif) + ending = consequent || consume_keyword(:end) start_char = find_next_statement_start(predicate.location.end_char) statements.bind( @@ -1568,11 +1571,11 @@ def on_embvar(value) # :call-seq: # on_ensure: (Statements statements) -> Ensure def on_ensure(statements) - keyword = consume_token_value(Kw, "ensure") + keyword = consume_keyword(:ensure) # We don't want to consume the :@kw event, because that would break # def..ensure..end chains. - ending = find_token_value(Kw, "end") + ending = find_keyword(:end) start_char = find_next_statement_start(keyword.location.end_char) statements.bind( start_char, @@ -1679,13 +1682,13 @@ def on_fndptn(constant, left, values, right) # Statements statements # ) -> For def on_for(index, collection, statements) - beginning = consume_token_value(Kw, "for") - in_keyword = consume_token_value(Kw, "in") - ending = consume_token_value(Kw, "end") + beginning = consume_keyword(:for) + in_keyword = consume_keyword(:in) + ending = consume_keyword(:end) # Consume the do keyword if it exists so that it doesn't get confused for # some other block - keyword = find_token_value(Kw, "do") + keyword = find_keyword(:do) if keyword && keyword.location.start_char > collection.location.end_char && keyword.location.end_char < ending.location.start_char @@ -1818,8 +1821,8 @@ def on_hshptn(constant, keywords, keyword_rest) if keyword_rest # We're doing this to delete the token from the list so that it doesn't # confuse future patterns by thinking they have an extra ** on the end. - consume_token_value(Op, "**") - elsif (token = find_token_value(Op, "**")) + consume_operator(:**) + elsif (token = find_operator(:**)) tokens.delete(token) # Create an artificial VarField if we find an extra ** on the end. This @@ -1872,8 +1875,8 @@ def on_ident(value) # (nil | Elsif | Else) consequent # ) -> If def on_if(predicate, statements, consequent) - beginning = consume_token_value(Kw, "if") - ending = consequent || consume_token_value(Kw, "end") + beginning = consume_keyword(:if) + ending = consequent || consume_keyword(:end) start_char = find_next_statement_start(predicate.location.end_char) statements.bind( @@ -1905,7 +1908,7 @@ def on_ifop(predicate, truthy, falsy) # :call-seq: # on_if_mod: (untyped predicate, untyped statement) -> IfMod def on_if_mod(predicate, statement) - consume_token_value(Kw, "if") + consume_keyword(:if) IfMod.new( statement: statement, @@ -1948,11 +1951,11 @@ def on_in(pattern, statements, consequent) # Here we have a rightward assignment return pattern unless statements - beginning = consume_token_value(Kw, "in") - ending = consequent || consume_token_value(Kw, "end") + beginning = consume_keyword(:in) + ending = consequent || consume_keyword(:end) statements_start = pattern - if (token = find_token_value(Kw, "then")) + if (token = find_keyword(:then)) tokens.delete(token) statements_start = token end @@ -2029,7 +2032,7 @@ def on_kw(value) # :call-seq: # on_kwrest_param: ((nil | Ident) name) -> KwRestParam def on_kwrest_param(name) - location = consume_token_value(Op, "**").location + location = consume_operator(:**).location location = location.to(name.location) if name KwRestParam.new(name: name, location: location) @@ -2121,8 +2124,8 @@ def on_lambda(params, statements) opening = consume_token(TLamBeg) closing = consume_token(RBrace) else - opening = consume_token_value(Kw, "do") - closing = consume_token_value(Kw, "end") + opening = consume_keyword(:do) + closing = consume_keyword(:end) end start_char = find_next_statement_start(opening.location.end_char) @@ -2353,7 +2356,7 @@ def on_mlhs_add_post(left, right) # (nil | ARefField | Field | Ident | VarField) part # ) -> MLHS def on_mlhs_add_star(mlhs, part) - beginning = consume_token_value(Op, "*") + beginning = consume_operator(:*) ending = part || beginning location = beginning.location.to(ending.location) @@ -2394,8 +2397,8 @@ def on_mlhs_paren(contents) # BodyStmt bodystmt # ) -> ModuleDeclaration def on_module(constant, bodystmt) - beginning = consume_token_value(Kw, "module") - ending = consume_token_value(Kw, "end") + beginning = consume_keyword(:module) + ending = consume_keyword(:end) start_char = find_next_statement_start(constant.location.end_char) bodystmt.bind( @@ -2434,7 +2437,7 @@ def on_mrhs_add(mrhs, part) # :call-seq: # on_mrhs_add_star: (MRHS mrhs, untyped value) -> MRHS def on_mrhs_add_star(mrhs, value) - beginning = consume_token_value(Op, "*") + beginning = consume_operator(:*) ending = value || beginning arg_star = @@ -2462,7 +2465,7 @@ def on_mrhs_new_from_args(arguments) # :call-seq: # on_next: (Args arguments) -> Next def on_next(arguments) - keyword = consume_token_value(Kw, "next") + keyword = consume_keyword(:next) location = keyword.location location = location.to(arguments.location) if arguments.parts.any? @@ -2889,7 +2892,7 @@ def on_rbracket(value) # :call-seq: # on_redo: () -> Redo def on_redo - keyword = consume_token_value(Kw, "redo") + keyword = consume_keyword(:redo) Redo.new(value: keyword.value, location: keyword.location) end @@ -2982,7 +2985,7 @@ def on_regexp_new # (nil | Rescue) consequent # ) -> Rescue def on_rescue(exceptions, variable, statements, consequent) - keyword = consume_token_value(Kw, "rescue") + keyword = consume_keyword(:rescue) exceptions = exceptions[0] if exceptions.is_a?(Array) last_node = variable || exceptions || keyword @@ -3034,7 +3037,7 @@ def on_rescue(exceptions, variable, statements, consequent) # :call-seq: # on_rescue_mod: (untyped statement, untyped value) -> RescueMod def on_rescue_mod(statement, value) - consume_token_value(Kw, "rescue") + consume_keyword(:rescue) RescueMod.new( statement: statement, @@ -3046,7 +3049,7 @@ def on_rescue_mod(statement, value) # :call-seq: # on_rest_param: ((nil | Ident) name) -> RestParam def on_rest_param(name) - location = consume_token_value(Op, "*").location + location = consume_operator(:*).location location = location.to(name.location) if name RestParam.new(name: name, location: location) @@ -3055,7 +3058,7 @@ def on_rest_param(name) # :call-seq: # on_retry: () -> Retry def on_retry - keyword = consume_token_value(Kw, "retry") + keyword = consume_keyword(:retry) Retry.new(value: keyword.value, location: keyword.location) end @@ -3063,7 +3066,7 @@ def on_retry # :call-seq: # on_return: (Args arguments) -> Return def on_return(arguments) - keyword = consume_token_value(Kw, "return") + keyword = consume_keyword(:return) Return.new( arguments: arguments, @@ -3074,7 +3077,7 @@ def on_return(arguments) # :call-seq: # on_return0: () -> Return0 def on_return0 - keyword = consume_token_value(Kw, "return") + keyword = consume_keyword(:return) Return0.new(value: keyword.value, location: keyword.location) end @@ -3101,8 +3104,8 @@ def on_rparen(value) # :call-seq: # on_sclass: (untyped target, BodyStmt bodystmt) -> SClass def on_sclass(target, bodystmt) - beginning = consume_token_value(Kw, "class") - ending = consume_token_value(Kw, "end") + beginning = consume_keyword(:class) + ending = consume_keyword(:end) start_char = find_next_statement_start(target.location.end_char) bodystmt.bind( @@ -3254,7 +3257,7 @@ def on_string_literal(string) ) else tstring_beg = consume_token(TStringBeg) - tstring_end = consume_token(TStringEnd, location: tstring_beg.location) + tstring_end = consume_tstring_end(tstring_beg.location) location = Location.new( @@ -3280,7 +3283,7 @@ def on_string_literal(string) # :call-seq: # on_super: ((ArgParen | Args) arguments) -> Super def on_super(arguments) - keyword = consume_token_value(Kw, "super") + keyword = consume_keyword(:super) Super.new( arguments: arguments, @@ -3501,7 +3504,7 @@ def on_unary(operator, statement) # We have somewhat special handling of the not operator since if it has # parentheses they don't get reported as a paren node for some reason. - beginning = consume_token_value(Kw, "not") + beginning = consume_keyword(:not) ending = statement || beginning parentheses = source[beginning.location.end_char] == "(" @@ -3540,7 +3543,7 @@ def on_unary(operator, statement) # :call-seq: # on_undef: (Array[DynaSymbol | SymbolLiteral] symbols) -> Undef def on_undef(symbols) - keyword = consume_token_value(Kw, "undef") + keyword = consume_keyword(:undef) Undef.new( symbols: symbols, @@ -3555,8 +3558,8 @@ def on_undef(symbols) # ((nil | Elsif | Else) consequent) # ) -> Unless def on_unless(predicate, statements, consequent) - beginning = consume_token_value(Kw, "unless") - ending = consequent || consume_token_value(Kw, "end") + beginning = consume_keyword(:unless) + ending = consequent || consume_keyword(:end) start_char = find_next_statement_start(predicate.location.end_char) statements.bind( @@ -3577,7 +3580,7 @@ def on_unless(predicate, statements, consequent) # :call-seq: # on_unless_mod: (untyped predicate, untyped statement) -> UnlessMod def on_unless_mod(predicate, statement) - consume_token_value(Kw, "unless") + consume_keyword(:unless) UnlessMod.new( statement: statement, @@ -3589,12 +3592,12 @@ def on_unless_mod(predicate, statement) # :call-seq: # on_until: (untyped predicate, Statements statements) -> Until def on_until(predicate, statements) - beginning = consume_token_value(Kw, "until") - ending = consume_token_value(Kw, "end") + beginning = consume_keyword(:until) + ending = consume_keyword(:end) # Consume the do keyword if it exists so that it doesn't get confused for # some other block - keyword = find_token_value(Kw, "do") + keyword = find_keyword(:do) if keyword && keyword.location.start_char > predicate.location.end_char && keyword.location.end_char < ending.location.start_char tokens.delete(keyword) @@ -3619,7 +3622,7 @@ def on_until(predicate, statements) # :call-seq: # on_until_mod: (untyped predicate, untyped statement) -> UntilMod def on_until_mod(predicate, statement) - consume_token_value(Kw, "until") + consume_keyword(:until) UntilMod.new( statement: statement, @@ -3631,7 +3634,7 @@ def on_until_mod(predicate, statement) # :call-seq: # on_var_alias: (GVar left, (Backref | GVar) right) -> VarAlias def on_var_alias(left, right) - keyword = consume_token_value(Kw, "alias") + keyword = consume_keyword(:alias) VarAlias.new( left: left, @@ -3685,11 +3688,11 @@ def on_void_stmt # (nil | Else | When) consequent # ) -> When def on_when(arguments, statements, consequent) - beginning = consume_token_value(Kw, "when") - ending = consequent || consume_token_value(Kw, "end") + beginning = consume_keyword(:when) + ending = consequent || consume_keyword(:end) statements_start = arguments - if (token = find_token_value(Kw, "then")) + if (token = find_keyword(:then)) tokens.delete(token) statements_start = token end @@ -3715,12 +3718,12 @@ def on_when(arguments, statements, consequent) # :call-seq: # on_while: (untyped predicate, Statements statements) -> While def on_while(predicate, statements) - beginning = consume_token_value(Kw, "while") - ending = consume_token_value(Kw, "end") + beginning = consume_keyword(:while) + ending = consume_keyword(:end) # Consume the do keyword if it exists so that it doesn't get confused for # some other block - keyword = find_token_value(Kw, "do") + keyword = find_keyword(:do) if keyword && keyword.location.start_char > predicate.location.end_char && keyword.location.end_char < ending.location.start_char tokens.delete(keyword) @@ -3745,7 +3748,7 @@ def on_while(predicate, statements) # :call-seq: # on_while_mod: (untyped predicate, untyped statement) -> WhileMod def on_while_mod(predicate, statement) - consume_token_value(Kw, "while") + consume_keyword(:while) WhileMod.new( statement: statement, @@ -3862,7 +3865,7 @@ def on_xstring_literal(xstring) location: heredoc.location ) else - ending = consume_token(TStringEnd, location: xstring.location) + ending = consume_tstring_end(xstring.location) XStringLiteral.new( parts: xstring.parts, @@ -3874,7 +3877,7 @@ def on_xstring_literal(xstring) # :call-seq: # on_yield: ((Args | Paren) arguments) -> Yield def on_yield(arguments) - keyword = consume_token_value(Kw, "yield") + keyword = consume_keyword(:yield) Yield.new( arguments: arguments, @@ -3885,7 +3888,7 @@ def on_yield(arguments) # :call-seq: # on_yield0: () -> Yield0 def on_yield0 - keyword = consume_token_value(Kw, "yield") + keyword = consume_keyword(:yield) Yield0.new(value: keyword.value, location: keyword.location) end @@ -3893,7 +3896,7 @@ def on_yield0 # :call-seq: # on_zsuper: () -> ZSuper def on_zsuper - keyword = consume_token_value(Kw, "super") + keyword = consume_keyword(:super) ZSuper.new(value: keyword.value, location: keyword.location) end diff --git a/test/interface_test.rb b/test/interface_test.rb index 49a74e92..5086680e 100644 --- a/test/interface_test.rb +++ b/test/interface_test.rb @@ -54,8 +54,12 @@ def instantiate(klass) case klass.name when "SyntaxTree::Binary" klass.new(**params, operator: :+) + when "SyntaxTree::Kw" + klass.new(**params, value: "kw") when "SyntaxTree::Label" klass.new(**params, value: "label:") + when "SyntaxTree::Op" + klass.new(**params, value: "+") when "SyntaxTree::RegexpLiteral" klass.new(**params, ending: "/") when "SyntaxTree::Statements" From b5d226b4f24e225e72d62b7d4fb3152e9b7aa0b7 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 17 Oct 2022 09:54:32 -0400 Subject: [PATCH 127/536] Move lambdas into their own methods, smarter token finding in on_args_add_block --- lib/syntax_tree/node.rb | 102 ++++++++++++++++++++++---------------- lib/syntax_tree/parser.rb | 25 ++++++---- 2 files changed, 72 insertions(+), 55 deletions(-) diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 24d35985..eccf0638 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -2885,27 +2885,15 @@ def deconstruct_keys(_keys) end def format(q) - declaration = -> do - q.group do - q.text("class ") - q.format(constant) - - if superclass - q.text(" < ") - q.format(superclass) - end - end - end - if bodystmt.empty? q.group do - declaration.call + format_declaration(q) q.breakable_force q.text("end") end else q.group do - declaration.call + format_declaration(q) q.indent do q.breakable_force @@ -2917,6 +2905,20 @@ def format(q) end end end + + private + + def format_declaration(q) + q.group do + q.text("class ") + q.format(constant) + + if superclass + q.text(" < ") + q.format(superclass) + end + end + end end # Comma represents the use of the , operator. @@ -5122,18 +5124,7 @@ def deconstruct_keys(_keys) def format(q) parts = keywords.map { |(key, value)| KeywordFormatter.new(key, value) } parts << KeywordRestFormatter.new(keyword_rest) if keyword_rest - nested = PATTERNS.include?(q.parent.class) - contents = -> do - q.group { q.seplist(parts) { |part| q.format(part, stackable: false) } } - - # If there isn't a constant, and there's a blank keyword_rest, then we - # have an plain ** that needs to have a `then` after it in order to - # parse correctly on the next parse. - if !constant && keyword_rest && keyword_rest.value.nil? && !nested - q.text(" then") - end - end # If there is a constant, we're going to format to have the constant name # first and then use brackets. @@ -5143,7 +5134,7 @@ def format(q) q.text("[") q.indent do q.breakable_empty - contents.call + format_contents(q, parts, nested) end q.breakable_empty q.text("]") @@ -5160,7 +5151,7 @@ def format(q) # If there's only one pair, then we'll just print the contents provided # we're not inside another pattern. if !nested && parts.size == 1 - contents.call + format_contents(q, parts, nested) return end @@ -5170,7 +5161,7 @@ def format(q) q.text("{") q.indent do q.breakable_space - contents.call + format_contents(q, parts, nested) end if q.target_ruby_version < Gem::Version.new("2.7.3") @@ -5181,6 +5172,19 @@ def format(q) end end end + + private + + def format_contents(q, parts, nested) + q.group { q.seplist(parts) { |part| q.format(part, stackable: false) } } + + # If there isn't a constant, and there's a blank keyword_rest, then we + # have an plain ** that needs to have a `then` after it in order to + # parse correctly on the next parse. + if !constant && keyword_rest && keyword_rest.value.nil? && !nested + q.text(" then") + end + end end # The list of nodes that represent patterns inside of pattern matching so that @@ -6543,22 +6547,15 @@ def deconstruct_keys(_keys) end def format(q) - declaration = -> do - q.group do - q.text("module ") - q.format(constant) - end - end - if bodystmt.empty? q.group do - declaration.call + format_declaration(q) q.breakable_force q.text("end") end else q.group do - declaration.call + format_declaration(q) q.indent do q.breakable_force @@ -6570,6 +6567,15 @@ def format(q) end end end + + private + + def format_declaration(q) + q.group do + q.text("module ") + q.format(constant) + end + end end # MRHS represents the values that are being assigned on the right-hand side of @@ -7023,27 +7029,35 @@ def format(q) parts << KeywordRestFormatter.new(keyword_rest) if keyword_rest parts << block if block - contents = -> do - q.seplist(parts) { |part| q.format(part) } - q.format(rest) if rest.is_a?(ExcessedComma) + if parts.empty? + q.nest(0) { format_contents(q, parts) } + return end - if ![Def, Defs, DefEndless].include?(q.parent.class) || parts.empty? - q.nest(0, &contents) - else + case q.parent + when Def, Defs, DefEndless q.nest(0) do q.text("(") q.group do q.indent do q.breakable_empty - contents.call + format_contents(q, parts) end q.breakable_empty end q.text(")") end + else + q.nest(0) { format_contents(q, parts) } end end + + private + + def format_contents(q, parts) + q.seplist(parts) { |part| q.format(part) } + q.format(rest) if rest.is_a?(ExcessedComma) + end end # Paren represents using balanced parentheses in a couple places in a Ruby diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index 3245efa1..78a6f84b 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -513,23 +513,26 @@ def on_args_add(arguments, argument) # (false | untyped) block # ) -> Args def on_args_add_block(arguments, block) + end_char = arguments.parts.any? && arguments.location.end_char + # First, see if there is an & operator that could potentially be # associated with the block part of this args_add_block. If there is not, # then just return the arguments. - operator = find_operator(:&) - return arguments unless operator - - # If there are any arguments and the operator we found from the list is - # not after them, then we're going to return the arguments as-is because - # we're looking at an & that occurs before the arguments are done. - if arguments.parts.any? && - operator.location.start_char < arguments.location.end_char - return arguments - end + index = + tokens.rindex do |token| + # If there are any arguments and the operator we found from the list + # is not after them, then we're going to return the arguments as-is + # because we're looking at an & that occurs before the arguments are + # done. + return arguments if end_char && token.location.start_char < end_char + token.is_a?(Op) && (token.name == :&) + end + + return arguments unless index # Now we know we have an & operator, so we're going to delete it from the # list of tokens to make sure it doesn't get confused with anything else. - tokens.delete(operator) + operator = tokens.delete_at(index) # Construct the location that represents the block argument. location = operator.location From 7b6dbeb3edea077f38369240f5a0dc4d1e0390ae Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 17 Oct 2022 11:16:46 -0400 Subject: [PATCH 128/536] More micro-optimizations --- bin/ybench | 17 +++ lib/syntax_tree/node.rb | 239 ++++++++++++++++++++++------------------ 2 files changed, 151 insertions(+), 105 deletions(-) create mode 100755 bin/ybench diff --git a/bin/ybench b/bin/ybench new file mode 100755 index 00000000..9dcc45aa --- /dev/null +++ b/bin/ybench @@ -0,0 +1,17 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "bundler/inline" + +gemfile do + source "https://rubygems.org" + gem "benchmark-ips" +end + +string = "a" * 1000 + "\n" + +Benchmark.ips do |x| + x.report("chomp") { string.chomp } + x.report("chop") { string.chop } + x.compare! +end diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index eccf0638..b3babfb5 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -1413,17 +1413,21 @@ class Labels def format_key(q, key) case key - in Label + when Label q.format(key) - in SymbolLiteral + when SymbolLiteral q.format(key.value) q.text(":") - in DynaSymbol[parts: [TStringContent[value: LABEL] => part]] - q.format(part) - q.text(":") - in DynaSymbol - q.format(key) - q.text(":") + when DynaSymbol + parts = key.parts + + if parts.length == 1 && (part = parts.first) && part.is_a?(TStringContent) && part.value.match?(LABEL) + q.format(part) + q.text(":") + else + q.format(key) + q.text(":") + end end end end @@ -1433,8 +1437,7 @@ class Rockets def format_key(q, key) case key when Label - q.text(":") - q.text(key.value.chomp(":")) + q.text(":#{key.value.chomp(":")}") when DynaSymbol q.text(":") q.format(key) @@ -2145,105 +2148,128 @@ def format(q) q.group do q.text(keyword) - case node.arguments.parts - in [] + parts = node.arguments.parts + length = parts.length + + if length == 0 # Here there are no arguments at all, so we're not going to print # anything. This would be like if we had: # # break # - in [ - Paren[ - contents: { - body: [ArrayLiteral[contents: { parts: [_, _, *] }] => array] } - ] - ] - # Here we have a single argument that is a set of parentheses wrapping - # an array literal that has at least 2 elements. We're going to print - # the contents of the array directly. This would be like if we had: - # - # break([1, 2, 3]) - # - # which we will print as: - # - # break 1, 2, 3 - # - q.text(" ") - format_array_contents(q, array) - in [Paren[contents: { body: [ArrayLiteral => statement] }]] - # Here we have a single argument that is a set of parentheses wrapping - # an array literal that has 0 or 1 elements. We're going to skip the - # parentheses but print the array itself. This would be like if we - # had: - # - # break([1]) - # - # which we will print as: - # - # break [1] - # - q.text(" ") - q.format(statement) - in [Paren[contents: { body: [statement] }]] if skip_parens?(statement) - # Here we have a single argument that is a set of parentheses that - # themselves contain a single statement. That statement is a simple - # value that we can skip the parentheses for. This would be like if we - # had: - # - # break(1) - # - # which we will print as: - # - # break 1 - # - q.text(" ") - q.format(statement) - in [Paren => part] - # Here we have a single argument that is a set of parentheses. We're - # going to print the parentheses themselves as if they were the set of - # arguments. This would be like if we had: - # - # break(foo.bar) - # - q.format(part) - in [ArrayLiteral[contents: { parts: [_, _, *] }] => array] - # Here there is a single argument that is an array literal with at - # least two elements. We skip directly into the array literal's - # elements in order to print the contents. This would be like if we - # had: - # - # break [1, 2, 3] - # - # which we will print as: - # - # break 1, 2, 3 - # - q.text(" ") - format_array_contents(q, array) - in [ArrayLiteral => part] - # Here there is a single argument that is an array literal with 0 or 1 - # elements. In this case we're going to print the array as it is - # because skipping the brackets would change the remaining. This would - # be like if we had: - # - # break [] - # break [1] - # - q.text(" ") - q.format(part) - in [_] - # Here there is a single argument that hasn't matched one of our - # previous cases. We're going to print the argument as it is. This - # would be like if we had: - # - # break foo - # - format_arguments(q, "(", ")") - else + elsif length >= 2 # If there are multiple arguments, format them all. If the line is # going to break into multiple, then use brackets to start and end the # expression. format_arguments(q, " [", "]") + else + # If we get here, then we're formatting a single argument to the flow + # control keyword. + part = parts.first + + case part + when Paren + statements = part.contents.body + + if statements.length == 1 + statement = statements.first + + if statement.is_a?(ArrayLiteral) + contents = statement.contents + + if contents && contents.parts.length >= 2 + # Here we have a single argument that is a set of parentheses + # wrapping an array literal that has at least 2 elements. + # We're going to print the contents of the array directly. + # This would be like if we had: + # + # break([1, 2, 3]) + # + # which we will print as: + # + # break 1, 2, 3 + # + q.text(" ") + format_array_contents(q, statement) + else + # Here we have a single argument that is a set of parentheses + # wrapping an array literal that has 0 or 1 elements. We're + # going to skip the parentheses but print the array itself. + # This would be like if we had: + # + # break([1]) + # + # which we will print as: + # + # break [1] + # + q.text(" ") + q.format(statement) + end + elsif skip_parens?(statement) + # Here we have a single argument that is a set of parentheses + # that themselves contain a single statement. That statement is + # a simple value that we can skip the parentheses for. This + # would be like if we had: + # + # break(1) + # + # which we will print as: + # + # break 1 + # + q.text(" ") + q.format(statement) + else + # Here we have a single argument that is a set of parentheses. + # We're going to print the parentheses themselves as if they + # were the set of arguments. This would be like if we had: + # + # break(foo.bar) + # + q.format(part) + end + else + q.format(part) + end + when ArrayLiteral + contents = part.contents + + if contents && contents.parts.length >= 2 + # Here there is a single argument that is an array literal with at + # least two elements. We skip directly into the array literal's + # elements in order to print the contents. This would be like if + # we had: + # + # break [1, 2, 3] + # + # which we will print as: + # + # break 1, 2, 3 + # + q.text(" ") + format_array_contents(q, part) + else + # Here there is a single argument that is an array literal with 0 + # or 1 elements. In this case we're going to print the array as it + # is because skipping the brackets would change the remaining. + # This would be like if we had: + # + # break [] + # break [1] + # + q.text(" ") + q.format(part) + end + else + # Here there is a single argument that hasn't matched one of our + # previous cases. We're going to print the argument as it is. This + # would be like if we had: + # + # break foo + # + format_arguments(q, "(", ")") + end end end end @@ -3791,15 +3817,18 @@ def initialize(operator, node) end def format(q) - space = [If, IfMod, Unless, UnlessMod].include?(q.parent.class) - left = node.left right = node.right q.format(left) if left - q.text(" ") if space - q.text(operator) - q.text(" ") if space + + case q.parent + when If, IfMod, Unless, UnlessMod + q.text(" #{operator} ") + else + q.text(operator) + end + q.format(right) if right end end From 929726bda1371541101ff40e6c91765a0e3aad25 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 17 Oct 2022 12:21:36 -0400 Subject: [PATCH 129/536] Remove remaining pattern matching --- bin/ybench | 17 --- lib/syntax_tree/formatter.rb | 4 + lib/syntax_tree/node.rb | 204 +++++++++++++++++++---------------- lib/syntax_tree/parser.rb | 21 ++-- 4 files changed, 129 insertions(+), 117 deletions(-) delete mode 100755 bin/ybench diff --git a/bin/ybench b/bin/ybench deleted file mode 100755 index 9dcc45aa..00000000 --- a/bin/ybench +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -require "bundler/inline" - -gemfile do - source "https://rubygems.org" - gem "benchmark-ips" -end - -string = "a" * 1000 + "\n" - -Benchmark.ips do |x| - x.report("chomp") { string.chomp } - x.report("chop") { string.chop } - x.compare! -end diff --git a/lib/syntax_tree/formatter.rb b/lib/syntax_tree/formatter.rb index dc124fbc..5fe5e260 100644 --- a/lib/syntax_tree/formatter.rb +++ b/lib/syntax_tree/formatter.rb @@ -120,6 +120,10 @@ def format_each(nodes) nodes.each { |node| format(node) } end + def grandparent + stack[-3] + end + def parent stack[-2] end diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index b3babfb5..cb4fadef 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -2298,10 +2298,15 @@ def format_arguments(q, opening, closing) def skip_parens?(node) case node - in FloatLiteral | Imaginary | Int | RationalLiteral - true - in VarRef[value: Const | CVar | GVar | IVar | Kw] + when FloatLiteral, Imaginary, Int, RationalLiteral true + when VarRef + case node.value + when Const, CVar, GVar, IVar, Kw + true + else + false + end else false end @@ -2364,8 +2369,14 @@ def comments def format(q) case operator - in :"::" | Op[value: "::"] + when :"::" q.text(".") + when Op + if operator.value == "::" + q.text(".") + else + operator.format(q) + end else operator.format(q) end @@ -2401,13 +2412,18 @@ def format(q) # First, walk down the chain until we get to the point where we're not # longer at a chainable node. loop do - case children.last - in Call[receiver: Call] - children << children.last.receiver - in Call[receiver: MethodAddBlock[call: Call]] - children << children.last.receiver - in MethodAddBlock[call: Call] - children << children.last.call + case (child = children.last) + when Call + case (receiver = child.receiver) + when Call + children << receiver + when MethodAddBlock + receiver.call.is_a?(Call) ? children << receiver : break + else + break + end + when MethodAddBlock + child.call.is_a?(Call) ? children << child.call : break else break end @@ -2426,10 +2442,8 @@ def format(q) # nodes. parent = parents[3] if parent.is_a?(DoBlock) - case parent - in MethodAddBlock[call: FCall[value: { value: "sig" }]] + if parent.is_a?(MethodAddBlock) && parent.call.is_a?(FCall) && parent.call.value.value == "sig" threshold = 2 - else end end @@ -2472,20 +2486,17 @@ def format_chain(q, children) skip_operator = false while (child = children.pop) - case child - in Call[ - receiver: Call[message: { value: "where" }], - message: { value: "not" } - ] - # This is very specialized behavior wherein we group - # .where.not calls together because it looks better. For more - # information, see - # https://github.com/prettier/plugin-ruby/issues/862. - in Call - # If we're at a Call node and not a MethodAddBlock node in the - # chain then we're going to add a newline so it indents properly. - q.breakable_empty - else + if child.is_a?(Call) + if child.receiver.is_a?(Call) && child.receiver.message.value == "where" && child.message.value == "not" + # This is very specialized behavior wherein we group + # .where.not calls together because it looks better. For more + # information, see + # https://github.com/prettier/plugin-ruby/issues/862. + else + # If we're at a Call node and not a MethodAddBlock node in the + # chain then we're going to add a newline so it indents properly. + q.breakable_empty + end end format_child( @@ -2498,9 +2509,9 @@ def format_chain(q, children) # If the parent call node has a comment on the message then we need # to print the operator trailing in order to keep it working. - case children.last - in Call[message: { comments: [_, *] }, operator:] - q.format(CallOperatorFormatter.new(operator)) + last_child = children.last + if last_child.is_a?(Call) && last_child.message.comments.any? + q.format(CallOperatorFormatter.new(last_child.operator)) skip_operator = true else skip_operator = false @@ -2515,18 +2526,22 @@ def format_chain(q, children) if empty_except_last case node - in Call + when Call node.format_arguments(q) - in MethodAddBlock[block:] - q.format(block) + when MethodAddBlock + q.format(node.block) end end end def self.chained?(node) + return false if ENV["STREE_FAST_FORMAT"] + case node - in Call | MethodAddBlock[call: Call] + when Call true + when MethodAddBlock + node.call.is_a?(Call) else false end @@ -2538,9 +2553,12 @@ def self.chained?(node) # want to indent the first call. So we'll pop off the first children and # format it separately here. def attach_directly?(node) - [ArrayLiteral, HashLiteral, Heredoc, If, Unless, XStringLiteral].include?( - node.receiver.class - ) + case node.receiver + when ArrayLiteral, HashLiteral, Heredoc, If, Unless, XStringLiteral + true + else + false + end end def format_child( @@ -2552,7 +2570,7 @@ def format_child( ) # First, format the actual contents of the child. case child - in Call + when Call q.group do unless skip_operator q.format(CallOperatorFormatter.new(child.operator)) @@ -2560,7 +2578,7 @@ def format_child( q.format(child.message) if child.message != :call child.format_arguments(q) unless skip_attached end - in MethodAddBlock + when MethodAddBlock q.format(child.block) unless skip_attached end @@ -2643,9 +2661,7 @@ def format(q) # If we're at the top of a call chain, then we're going to do some # specialized printing in case we can print it nicely. We _only_ do this # at the top of the chain to avoid weird recursion issues. - if !ENV["STREE_SKIP_CALL_CHAIN"] && - !CallChainFormatter.chained?(q.parent) && - CallChainFormatter.chained?(receiver) + if CallChainFormatter.chained?(receiver) && !CallChainFormatter.chained?(q.parent) q.group do q .if_break { CallChainFormatter.new(self).format(q) } @@ -2658,9 +2674,9 @@ def format(q) def format_arguments(q) case arguments - in ArgParen + when ArgParen q.format(arguments) - in Args + when Args q.text(" ") q.format(arguments) else @@ -2821,7 +2837,7 @@ def format(q) q.format(operator) case pattern - in AryPtn | FndPtn | HshPtn + when AryPtn, FndPtn, HshPtn q.text(" ") q.format(pattern) else @@ -5286,28 +5302,35 @@ def self.call(parent) module Ternaryable class << self def call(q, node) - case q.parents.take(2)[1] - in Paren[contents: Statements[body: [node]]] - # If this is a conditional inside of a parentheses as the only - # content, then we don't want to transform it into a ternary. - # Presumably the user wanted it to be an explicit conditional because - # there are parentheses around it. So we'll just leave it in place. - false - else - # Otherwise, we're going to check the conditional for certain cases. - case node - in predicate: Assign | Command | CommandCall | MAssign | OpAssign - false - in predicate: Not[parentheses: false] - false - in { - statements: { body: [truthy] }, - consequent: Else[statements: { body: [falsy] }] } - ternaryable?(truthy) && ternaryable?(falsy) - else - false - end + return false if ENV["STREE_FAST_FORMAT"] + + # If this is a conditional inside of a parentheses as the only content, + # then we don't want to transform it into a ternary. Presumably the user + # wanted it to be an explicit conditional because there are parentheses + # around it. So we'll just leave it in place. + grandparent = q.grandparent + if grandparent.is_a?(Paren) && (body = grandparent.contents.body) && body.length == 1 && body.first == node + return false + end + + # Otherwise, we'll check the type of predicate. For certain nodes we + # want to force it to not be a ternary, like if the predicate is an + # assignment because it's hard to read. + case node.predicate + when Assign, Command, CommandCall, MAssign, OpAssign + return false + when Not + return false unless node.predicate.parentheses? end + + # If there's no Else, then this can't be represented as a ternary. + return false unless node.consequent.is_a?(Else) + + truthy_body = node.statements.body + falsy_body = node.consequent.statements.body + + (truthy_body.length == 1) && ternaryable?(truthy_body.first) && + (falsy_body.length == 1) && ternaryable?(falsy_body.first) end private @@ -5316,24 +5339,23 @@ def call(q, node) # parentheses around them. In this case we say they cannot be ternaried # and default instead to breaking them into multiple lines. def ternaryable?(statement) - # This is a list of nodes that should not be allowed to be a part of a - # ternary clause. - no_ternary = [ - Alias, Assign, Break, Command, CommandCall, Heredoc, If, IfMod, IfOp, + case statement + when Alias, Assign, Break, Command, CommandCall, Heredoc, If, IfMod, IfOp, Lambda, MAssign, Next, OpAssign, RescueMod, Return, Return0, Super, Undef, Unless, UnlessMod, Until, UntilMod, VarAlias, VoidStmt, While, WhileMod, Yield, Yield0, ZSuper - ] - - # Here we're going to check that the only statement inside the - # statements node is no a part of our denied list of nodes that can be - # ternaries. - # - # If the user is using one of the lower precedence "and" or "or" - # operators, then we can't use a ternary expression as it would break - # the flow control. - !no_ternary.include?(statement.class) && - !(statement.is_a?(Binary) && %i[and or].include?(statement.operator)) + # This is a list of nodes that should not be allowed to be a part of a + # ternary clause. + false + when Binary + # If the user is using one of the lower precedence "and" or "or" + # operators, then we can't use a ternary expression as it would break + # the flow control. + operator = statement.operator + operator != "and" && operator != "or" + else + true + end end end end @@ -5453,8 +5475,11 @@ def format_ternary(q) end def contains_conditional? - case node - in statements: { body: [If | IfMod | IfOp | Unless | UnlessMod] } + statements = node.statements.body + return false if statements.length != 1 + + case statements.first + when If, IfMod, IfOp, Unless, UnlessMod true else false @@ -6410,9 +6435,7 @@ def format(q) # If we're at the top of a call chain, then we're going to do some # specialized printing in case we can print it nicely. We _only_ do this # at the top of the chain to avoid weird recursion issues. - if !ENV["STREE_SKIP_CALL_CHAIN"] && - !CallChainFormatter.chained?(q.parent) && - CallChainFormatter.chained?(call) + if CallChainFormatter.chained?(call) && !CallChainFormatter.chained?(q.parent) q.group do q .if_break { CallChainFormatter.new(self).format(q) } @@ -9122,6 +9145,7 @@ class Not < Node # [boolean] whether or not parentheses were used attr_reader :parentheses + alias parentheses? parentheses # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments @@ -9160,10 +9184,10 @@ def format(q) q.format(statement) if statement q.text(")") else - parent = q.parents.take(2)[1] + grandparent = q.grandparent ternary = - (parent.is_a?(If) || parent.is_a?(Unless)) && - Ternaryable.call(q, parent) + (grandparent.is_a?(If) || grandparent.is_a?(Unless)) && + Ternaryable.call(q, grandparent) if ternary q.if_break { q.text(" ") }.if_flat { q.text("(") } diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index 78a6f84b..15f8522b 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -689,12 +689,13 @@ def on_aryptn(constant, requireds, rest, posts) if rest.is_a?(VarField) && rest.value.nil? tokens.rindex do |rtoken| case rtoken - in Op[value: "*"] - rest = VarField.new(value: nil, location: rtoken.location) + when Comma break - in Comma - break - else + when Op + if rtoken.value == "*" + rest = VarField.new(value: nil, location: rtoken.location) + break + end end end end @@ -1659,10 +1660,10 @@ def on_fndptn(constant, left, values, right) # punctuation or the right splat. closing = case opening - in LBracket + when LBracket tokens.delete(opening) consume_token(RBracket) - in LParen + when LParen tokens.delete(opening) consume_token(RParen) else @@ -2092,7 +2093,7 @@ def on_lambda(params, statements) # capturing lambda var until 3.2, we need to normalize all of that here. params = case params - in Paren[contents: Params] + when Paren # In this case we've gotten to the <3.2 parentheses wrapping a set of # parameters case. Here we need to manually scan for lambda locals. range = (params.location.start_char + 1)...params.location.end_char @@ -2112,12 +2113,12 @@ def on_lambda(params, statements) location: params.location, comments: params.comments ) - in Params + when Params # In this case we've gotten to the <3.2 plain set of parameters. In # this case there cannot be lambda locals, so we will wrap the # parameters into a lambda var that has no locals. LambdaVar.new(params: params, locals: [], location: params.location) - in LambdaVar + when LambdaVar # In this case we've gotten to 3.2+ lambda var. In this case we don't # need to do anything and can just the value as given. params From b5154ac7efdc625daf71a761a41c3a3009d87328 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 17 Oct 2022 12:40:06 -0400 Subject: [PATCH 130/536] each_line instead of split --- lib/syntax_tree/formatter.rb | 53 ++++++++++---------- lib/syntax_tree/node.rb | 95 ++++++++++++++++++++++++++---------- 2 files changed, 96 insertions(+), 52 deletions(-) diff --git a/lib/syntax_tree/formatter.rb b/lib/syntax_tree/formatter.rb index 5fe5e260..39ed1583 100644 --- a/lib/syntax_tree/formatter.rb +++ b/lib/syntax_tree/formatter.rb @@ -4,26 +4,6 @@ module SyntaxTree # A slightly enhanced PP that knows how to format recursively including # comments. class Formatter < PrettierPrint - # It's very common to use seplist with ->(q) { q.breakable_return }. We wrap - # that pattern into an object to cut down on having to create a bunch of - # lambdas all over the place. - class BreakableReturnSeparator - def call(q) - q.breakable_return - end - end - - # Similar to the previous, it's common to ->(q) { q.breakable_space }. We - # also wrap that pattern into an object to cut down on lambdas. - class BreakableSpaceSeparator - def call(q) - q.breakable_space - end - end - - BREAKABLE_RETURN_SEPARATOR = BreakableReturnSeparator.new - BREAKABLE_SPACE_SEPARATOR = BreakableSpaceSeparator.new - # We want to minimize as much as possible the number of options that are # available in syntax tree. For the most part, if users want non-default # formatting, they should override the format methods on the specific nodes @@ -82,20 +62,39 @@ def format(node, stackable: true) # If there are comments, then we're going to format them around the node # so that they get printed properly. if node.comments.any? - leading, trailing = node.comments.partition(&:leading?) + trailing = [] + last_leading = nil - # Print all comments that were found before the node. - leading.each do |comment| - comment.format(self) - breakable(force: true) + # First, we're going to print all of the comments that were found before + # the node. We'll also gather up any trailing comments that we find. + node.comments.each do |comment| + if comment.leading? + comment.format(self) + breakable(force: true) + last_leading = comment + else + trailing << comment + end end # If the node has a stree-ignore comment right before it, then we're # going to just print out the node as it was seen in the source. doc = - if leading.last&.ignore? + if last_leading&.ignore? range = source[node.location.start_char...node.location.end_char] - seplist(range.split(/\r?\n/, -1), Formatter::BREAKABLE_RETURN_SEPARATOR) { |line| text(line) } + first = true + + range.each_line(chomp: true) do |line| + if first + first = false + else + breakable_return + end + + text(line) + end + + breakable_return if range.end_with?("\n") else node.format(self) end diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index cb4fadef..ce67a135 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -328,7 +328,19 @@ def deconstruct_keys(_keys) def format(q) q.text("__END__") q.breakable_force - q.seplist(value.split(/\r?\n/, -1), Formatter::BREAKABLE_RETURN_SEPARATOR) { |line| q.text(line) } + + first = true + value.each_line(chomp: true) do |line| + if first + first = false + else + q.breakable_return + end + + q.text(line) + end + + q.breakable_return if value.end_with?("\n") end end @@ -792,6 +804,17 @@ def format(q) # [one, two, three] # class ArrayLiteral < Node + # It's very common to use seplist with ->(q) { q.breakable_space }. We wrap + # that pattern into an object to cut down on having to create a bunch of + # lambdas all over the place. + class BreakableSpaceSeparator + def call(q) + q.breakable_space + end + end + + BREAKABLE_SPACE_SEPARATOR = BreakableSpaceSeparator.new + # Formats an array of multiple simple string literals into the %w syntax. class QWordsFormatter # [Args] the contents of the array @@ -806,7 +829,7 @@ def format(q) q.group do q.indent do q.breakable_empty - q.seplist(contents.parts, Formatter::BREAKABLE_SPACE_SEPARATOR) do |part| + q.seplist(contents.parts, BREAKABLE_SPACE_SEPARATOR) do |part| if part.is_a?(StringLiteral) q.format(part.parts.first) else @@ -834,7 +857,7 @@ def format(q) q.group do q.indent do q.breakable_empty - q.seplist(contents.parts, Formatter::BREAKABLE_SPACE_SEPARATOR) do |part| + q.seplist(contents.parts, BREAKABLE_SPACE_SEPARATOR) do |part| q.format(part.value) end end @@ -4034,9 +4057,19 @@ def format(q) parts.each do |part| if part.is_a?(TStringContent) value = Quotes.normalize(part.value, closing_quote) - q.seplist(value.split(/\r?\n/, -1), Formatter::BREAKABLE_RETURN_SEPARATOR) do |text| - q.text(text) + first = true + + value.each_line(chomp: true) do |line| + if first + first = false + else + q.breakable_return + end + + q.text(line) end + + q.breakable_return if value.end_with?("\n") else q.format(part) end @@ -4957,17 +4990,7 @@ def deconstruct_keys(_keys) # This is a very specific behavior where you want to force a newline, but # don't want to force the break parent. - class Separator - DOC = PrettierPrint::Breakable.new(" ", 1, indent: false, force: true) - - def call(q) - q.target << DOC - end - end - - # We're going to keep an instance around so we don't have to allocate a new - # one every time we format a heredoc. - SEPARATOR = Separator.new + SEPARATOR = PrettierPrint::Breakable.new(" ", 1, indent: false, force: true) def format(q) q.group do @@ -4975,12 +4998,24 @@ def format(q) q.line_suffix(priority: Formatter::HEREDOC_PRIORITY) do q.group do - SEPARATOR.call(q) + q.target << SEPARATOR parts.each do |part| if part.is_a?(TStringContent) - texts = part.value.split(/\r?\n/, -1) - q.seplist(texts, SEPARATOR) { |text| q.text(text) } + value = part.value + first = true + + value.each_line(chomp: true) do |line| + if first + first = false + else + q.target << SEPARATOR + end + + q.text(line) + end + + q.target << SEPARATOR if value.end_with?("\n") else q.format(part) end @@ -7295,7 +7330,7 @@ def format(q) q.group do q.indent do q.breakable_empty - q.seplist(elements, Formatter::BREAKABLE_SPACE_SEPARATOR) do |element| + q.seplist(elements, ArrayLiteral::BREAKABLE_SPACE_SEPARATOR) do |element| q.format(element) end end @@ -7388,7 +7423,7 @@ def format(q) q.group do q.indent do q.breakable_empty - q.seplist(elements, Formatter::BREAKABLE_SPACE_SEPARATOR) do |element| + q.seplist(elements, ArrayLiteral::BREAKABLE_SPACE_SEPARATOR) do |element| q.format(element) end end @@ -8626,9 +8661,19 @@ def format(q) parts.each do |part| if part.is_a?(TStringContent) value = Quotes.normalize(part.value, closing_quote) - q.seplist(value.split(/\r?\n/, -1), Formatter::BREAKABLE_RETURN_SEPARATOR) do |text| - q.text(text) + first = true + + value.each_line(chomp: true) do |line| + if first + first = false + else + q.breakable_return + end + + q.text(line) end + + q.breakable_return if value.end_with?("\n") else q.format(part) end @@ -8845,7 +8890,7 @@ def format(q) q.group do q.indent do q.breakable_empty - q.seplist(elements, Formatter::BREAKABLE_SPACE_SEPARATOR) do |element| + q.seplist(elements, ArrayLiteral::BREAKABLE_SPACE_SEPARATOR) do |element| q.format(element) end end @@ -10184,7 +10229,7 @@ def format(q) q.group do q.indent do q.breakable_empty - q.seplist(elements, Formatter::BREAKABLE_SPACE_SEPARATOR) do |element| + q.seplist(elements, ArrayLiteral::BREAKABLE_SPACE_SEPARATOR) do |element| q.format(element) end end From e31ddedb55f68b66baef1ef494d434c395ba1930 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 17 Oct 2022 13:06:35 -0400 Subject: [PATCH 131/536] Update the prettier_print version --- Gemfile.lock | 8 ++++---- syntax_tree.gemspec | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 6415fcb0..76bda432 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -2,7 +2,7 @@ PATH remote: . specs: syntax_tree (3.6.3) - prettier_print + prettier_print (>= 1.0.0) GEM remote: https://rubygems.org/ @@ -14,10 +14,10 @@ GEM parallel (1.22.1) parser (3.1.2.1) ast (~> 2.4.1) - prettier_print (0.1.0) + prettier_print (1.0.0) rainbow (3.1.1) rake (13.0.6) - regexp_parser (2.5.0) + regexp_parser (2.6.0) rexml (3.2.5) rubocop (1.36.0) json (~> 2.3) @@ -38,7 +38,7 @@ GEM simplecov_json_formatter (~> 0.1) simplecov-html (0.12.3) simplecov_json_formatter (0.1.4) - unicode-display_width (2.2.0) + unicode-display_width (2.3.0) PLATFORMS arm64-darwin-21 diff --git a/syntax_tree.gemspec b/syntax_tree.gemspec index 2b461dfd..ec7d57ef 100644 --- a/syntax_tree.gemspec +++ b/syntax_tree.gemspec @@ -25,7 +25,7 @@ Gem::Specification.new do |spec| spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } spec.require_paths = %w[lib] - spec.add_dependency "prettier_print" + spec.add_dependency "prettier_print", ">= 1.0.0" spec.add_development_dependency "bundler" spec.add_development_dependency "minitest" From b117c9b1ae3bef29698598b4f7dcf51340739a6a Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 17 Oct 2022 13:15:21 -0400 Subject: [PATCH 132/536] Fix message checking accidentally introduced when removing pattern matching --- lib/syntax_tree/node.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index ce67a135..cbeca9ae 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -2510,7 +2510,7 @@ def format_chain(q, children) while (child = children.pop) if child.is_a?(Call) - if child.receiver.is_a?(Call) && child.receiver.message.value == "where" && child.message.value == "not" + if child.receiver.is_a?(Call) && (child.receiver.message != :call) && (child.receiver.message.value == "where") && (child.message.value == "not") # This is very specialized behavior wherein we group # .where.not calls together because it looks better. For more # information, see From f5ac5fef12c8736b60560de4d3b92f764b9eafa7 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 17 Oct 2022 13:37:37 -0400 Subject: [PATCH 133/536] Reformat --- lib/syntax_tree/node.rb | 88 ++++++++++++++++++++++----------------- lib/syntax_tree/parser.rb | 26 ++++++------ 2 files changed, 64 insertions(+), 50 deletions(-) diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index cbeca9ae..82d378c6 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -625,7 +625,8 @@ def trailing_comma? # If the last argument is a block, then we can't put a trailing comma # after it without resulting in a syntax error. false - elsif parts.length == 1 && (part = parts.first) && (part.is_a?(Command) || part.is_a?(CommandCall)) + elsif parts.length == 1 && (part = parts.first) && + (part.is_a?(Command) || part.is_a?(CommandCall)) # If the only argument is a command or command call, then a trailing # comma would be parsed as part of that expression instead of on this # one, so we don't want to add a trailing comma. @@ -1444,7 +1445,8 @@ def format_key(q, key) when DynaSymbol parts = key.parts - if parts.length == 1 && (part = parts.first) && part.is_a?(TStringContent) && part.value.match?(LABEL) + if parts.length == 1 && (part = parts.first) && + part.is_a?(TStringContent) && part.value.match?(LABEL) q.format(part) q.text(":") else @@ -2054,7 +2056,8 @@ def forced_brace_bounds?(q) when Paren, Statements # If we hit certain breakpoints then we know we're safe. return false - when If, IfMod, IfOp, Unless, UnlessMod, While, WhileMod, Until, UntilMod + when If, IfMod, IfOp, Unless, UnlessMod, While, WhileMod, Until, + UntilMod return true if parent.predicate == previous previous = parent end @@ -2395,11 +2398,7 @@ def format(q) when :"::" q.text(".") when Op - if operator.value == "::" - q.text(".") - else - operator.format(q) - end + operator.value == "::" ? q.text(".") : operator.format(q) else operator.format(q) end @@ -2465,7 +2464,8 @@ def format(q) # nodes. parent = parents[3] if parent.is_a?(DoBlock) - if parent.is_a?(MethodAddBlock) && parent.call.is_a?(FCall) && parent.call.value.value == "sig" + if parent.is_a?(MethodAddBlock) && parent.call.is_a?(FCall) && + parent.call.value.value == "sig" threshold = 2 end end @@ -2510,7 +2510,10 @@ def format_chain(q, children) while (child = children.pop) if child.is_a?(Call) - if child.receiver.is_a?(Call) && (child.receiver.message != :call) && (child.receiver.message.value == "where") && (child.message.value == "not") + if child.receiver.is_a?(Call) && + (child.receiver.message != :call) && + (child.receiver.message.value == "where") && + (child.message.value == "not") # This is very specialized behavior wherein we group # .where.not calls together because it looks better. For more # information, see @@ -2684,7 +2687,8 @@ def format(q) # If we're at the top of a call chain, then we're going to do some # specialized printing in case we can print it nicely. We _only_ do this # at the top of the chain to avoid weird recursion issues. - if CallChainFormatter.chained?(receiver) && !CallChainFormatter.chained?(q.parent) + if CallChainFormatter.chained?(receiver) && + !CallChainFormatter.chained?(q.parent) q.group do q .if_break { CallChainFormatter.new(self).format(q) } @@ -5344,7 +5348,8 @@ def call(q, node) # wanted it to be an explicit conditional because there are parentheses # around it. So we'll just leave it in place. grandparent = q.grandparent - if grandparent.is_a?(Paren) && (body = grandparent.contents.body) && body.length == 1 && body.first == node + if grandparent.is_a?(Paren) && (body = grandparent.contents.body) && + body.length == 1 && body.first == node return false end @@ -5375,10 +5380,10 @@ def call(q, node) # and default instead to breaking them into multiple lines. def ternaryable?(statement) case statement - when Alias, Assign, Break, Command, CommandCall, Heredoc, If, IfMod, IfOp, - Lambda, MAssign, Next, OpAssign, RescueMod, Return, Return0, Super, - Undef, Unless, UnlessMod, Until, UntilMod, VarAlias, VoidStmt, While, - WhileMod, Yield, Yield0, ZSuper + when Alias, Assign, Break, Command, CommandCall, Heredoc, If, IfMod, + IfOp, Lambda, MAssign, Next, OpAssign, RescueMod, Return, Return0, + Super, Undef, Unless, UnlessMod, Until, UntilMod, VarAlias, + VoidStmt, While, WhileMod, Yield, Yield0, ZSuper # This is a list of nodes that should not be allowed to be a part of a # ternary clause. false @@ -6470,7 +6475,8 @@ def format(q) # If we're at the top of a call chain, then we're going to do some # specialized printing in case we can print it nicely. We _only_ do this # at the top of the chain to avoid weird recursion issues. - if CallChainFormatter.chained?(call) && !CallChainFormatter.chained?(q.parent) + if CallChainFormatter.chained?(call) && + !CallChainFormatter.chained?(q.parent) q.group do q .if_break { CallChainFormatter.new(self).format(q) } @@ -7330,9 +7336,10 @@ def format(q) q.group do q.indent do q.breakable_empty - q.seplist(elements, ArrayLiteral::BREAKABLE_SPACE_SEPARATOR) do |element| - q.format(element) - end + q.seplist( + elements, + ArrayLiteral::BREAKABLE_SPACE_SEPARATOR + ) { |element| q.format(element) } end q.breakable_empty end @@ -7423,9 +7430,10 @@ def format(q) q.group do q.indent do q.breakable_empty - q.seplist(elements, ArrayLiteral::BREAKABLE_SPACE_SEPARATOR) do |element| - q.format(element) - end + q.seplist( + elements, + ArrayLiteral::BREAKABLE_SPACE_SEPARATOR + ) { |element| q.format(element) } end q.breakable_empty end @@ -8378,7 +8386,7 @@ def format(q) q.text("; ") q.format(statement) end - + line = statement.location.end_line previous = statement end @@ -8890,9 +8898,10 @@ def format(q) q.group do q.indent do q.breakable_empty - q.seplist(elements, ArrayLiteral::BREAKABLE_SPACE_SEPARATOR) do |element| - q.format(element) - end + q.seplist( + elements, + ArrayLiteral::BREAKABLE_SPACE_SEPARATOR + ) { |element| q.format(element) } end q.breakable_empty end @@ -9773,15 +9782,17 @@ def format(q) def pin(parent) replace = PinnedVarRef.new(value: value, location: location) - parent.deconstruct_keys([]).each do |key, value| - if value == self - parent.instance_variable_set(:"@#{key}", replace) - break - elsif value.is_a?(Array) && (index = value.index(self)) - parent.public_send(key)[index] = replace - break + parent + .deconstruct_keys([]) + .each do |key, value| + if value == self + parent.instance_variable_set(:"@#{key}", replace) + break + elsif value.is_a?(Array) && (index = value.index(self)) + parent.public_send(key)[index] = replace + break + end end - end end end @@ -10229,9 +10240,10 @@ def format(q) q.group do q.indent do q.breakable_empty - q.seplist(elements, ArrayLiteral::BREAKABLE_SPACE_SEPARATOR) do |element| - q.format(element) - end + q.seplist( + elements, + ArrayLiteral::BREAKABLE_SPACE_SEPARATOR + ) { |element| q.format(element) } end q.breakable_empty end diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index 15f8522b..70f1e2a3 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -1011,12 +1011,13 @@ def on_case(value, consequent) consume_operator(:"=>") end - node = RAssign.new( - value: value, - operator: operator, - pattern: consequent, - location: value.location.to(consequent.location) - ) + node = + RAssign.new( + value: value, + operator: operator, + pattern: consequent, + location: value.location.to(consequent.location) + ) PinVisitor.visit(node, tokens) node @@ -1973,12 +1974,13 @@ def on_in(pattern, statements, consequent) ending.location.start_column ) - node = In.new( - pattern: pattern, - statements: statements, - consequent: consequent, - location: beginning.location.to(ending.location) - ) + node = + In.new( + pattern: pattern, + statements: statements, + consequent: consequent, + location: beginning.location.to(ending.location) + ) PinVisitor.visit(node, tokens) node From a640ea427e0c2eda7e47bdf403d24e3d0ee5abde Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Oct 2022 17:39:58 +0000 Subject: [PATCH 134/536] Bump prettier_print from 0.1.0 to 1.0.0 Bumps [prettier_print](https://github.com/ruby-syntax-tree/prettier_print) from 0.1.0 to 1.0.0. - [Release notes](https://github.com/ruby-syntax-tree/prettier_print/releases) - [Changelog](https://github.com/ruby-syntax-tree/prettier_print/blob/main/CHANGELOG.md) - [Commits](https://github.com/ruby-syntax-tree/prettier_print/compare/v0.1.0...v1.0.0) --- updated-dependencies: - dependency-name: prettier_print dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 6415fcb0..aa2a3d2e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -14,7 +14,7 @@ GEM parallel (1.22.1) parser (3.1.2.1) ast (~> 2.4.1) - prettier_print (0.1.0) + prettier_print (1.0.0) rainbow (3.1.1) rake (13.0.6) regexp_parser (2.5.0) From 7003178faabe5ba3d2703cae92c807e6bfa42b91 Mon Sep 17 00:00:00 2001 From: Vinicius Stock Date: Mon, 17 Oct 2022 13:11:15 -0400 Subject: [PATCH 135/536] Add WithEnvironment module to track locals Co-authored-by: Alexandre Terrasa Co-authored-by: Stan Lo Co-authored-by: Emily Samp Co-authored-by: Kaan Ozkan Co-authored-by: Adison Lampert Co-authored-by: Dirceu Tiegs --- README.md | 42 +- lib/syntax_tree.rb | 2 + lib/syntax_tree/visitor/environment.rb | 81 ++++ lib/syntax_tree/visitor/with_environment.rb | 141 +++++++ test/visitor_with_environment_test.rb | 410 ++++++++++++++++++++ 5 files changed, 666 insertions(+), 10 deletions(-) create mode 100644 lib/syntax_tree/visitor/environment.rb create mode 100644 lib/syntax_tree/visitor/with_environment.rb create mode 100644 test/visitor_with_environment_test.rb diff --git a/README.md b/README.md index afb65843..30c35ac8 100644 --- a/README.md +++ b/README.md @@ -368,16 +368,16 @@ program = SyntaxTree.parse("1 + 1") puts program.construct_keys # SyntaxTree::Program[ -# statements: SyntaxTree::Statements[ -# body: [ -# SyntaxTree::Binary[ -# left: SyntaxTree::Int[value: "1"], -# operator: :+, -# right: SyntaxTree::Int[value: "1"] -# ] -# ] -# ] -# ] +# statements: SyntaxTree::Statements[ +# body: [ +# SyntaxTree::Binary[ +# left: SyntaxTree::Int[value: "1"], +# operator: :+, +# right: SyntaxTree::Int[value: "1"] +# ] +# ] +# ] +# ] ``` ## Visitor @@ -447,6 +447,28 @@ end The visitor defined above will error out unless it's only visiting a `SyntaxTree::Int` node. This is useful in a couple of ways, e.g., if you're trying to define a visitor to handle the whole tree but it's currently a work-in-progress. +### WithEnvironment + +The `WithEnvironment` module can be included in visitors to automatically keep track of local variables and arguments +defined inside each environment. A `current_environment` accessor is made availble to the request, allowing it to find +all usages and definitions of a local. + +```ruby +class MyVisitor < Visitor + include WithEnvironment + + def visit_ident(node) + # find_local will return a Local for any local variables or arguments present in the current environment or nil if + # the identifier is not a local + local = current_environment.find_local(node) + + puts local.type # print the type of the local (:variable or :argument) + puts local.definitions # print the array of locations where this local is defined + puts local.usages # print the array of locations where this local occurs + end +end +``` + ## Language server Syntax Tree additionally ships with a language server conforming to the [language server protocol](https://microsoft.github.io/language-server-protocol/). It can be invoked through the CLI by running: diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index 88c66369..7861ddcd 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -19,6 +19,8 @@ require_relative "syntax_tree/visitor/json_visitor" require_relative "syntax_tree/visitor/match_visitor" require_relative "syntax_tree/visitor/pretty_print_visitor" +require_relative "syntax_tree/visitor/environment" +require_relative "syntax_tree/visitor/with_environment" # Syntax Tree is a suite of tools built on top of the internal CRuby parser. It # provides the ability to generate a syntax tree from source, as well as the diff --git a/lib/syntax_tree/visitor/environment.rb b/lib/syntax_tree/visitor/environment.rb new file mode 100644 index 00000000..dfcf0a80 --- /dev/null +++ b/lib/syntax_tree/visitor/environment.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +module SyntaxTree + # The environment class is used to keep track of local variables and arguments + # inside a particular scope + class Environment + # [Array[Local]] The local variables and arguments defined in this + # environment + attr_reader :locals + + # This class tracks the occurrences of a local variable or argument + class Local + # [Symbol] The type of the local (e.g. :argument, :variable) + attr_reader :type + + # [Array[Location]] The locations of all definitions and assignments of + # this local + attr_reader :definitions + + # [Array[Location]] The locations of all usages of this local + attr_reader :usages + + # initialize: (Symbol type) -> void + def initialize(type) + @type = type + @definitions = [] + @usages = [] + end + + # add_definition: (Location location) -> void + def add_definition(location) + @definitions << location + end + + # add_usage: (Location location) -> void + def add_usage(location) + @usages << location + end + end + + # initialize: (Environment | nil parent) -> void + def initialize(parent = nil) + @locals = {} + @parent = parent + end + + # Adding a local definition will either insert a new entry in the locals + # hash or append a new definition location to an existing local. Notice that + # it's not possible to change the type of a local after it has been + # registered + # add_local_definition: (Ident | Label identifier, Symbol type) -> void + def add_local_definition(identifier, type) + name = identifier.value.delete_suffix(":") + + @locals[name] ||= Local.new(type) + @locals[name].add_definition(identifier.location) + end + + # Adding a local usage will either insert a new entry in the locals + # hash or append a new usage location to an existing local. Notice that + # it's not possible to change the type of a local after it has been + # registered + # add_local_usage: (Ident | Label identifier, Symbol type) -> void + def add_local_usage(identifier, type) + name = identifier.value.delete_suffix(":") + + @locals[name] ||= Local.new(type) + @locals[name].add_usage(identifier.location) + end + + # Try to find the local given its name in this environment or any of its + # parents + # find_local: (String name) -> Local | nil + def find_local(name) + local = @locals[name] + return local unless local.nil? + + @parent&.find_local(name) + end + end +end diff --git a/lib/syntax_tree/visitor/with_environment.rb b/lib/syntax_tree/visitor/with_environment.rb new file mode 100644 index 00000000..62e59c98 --- /dev/null +++ b/lib/syntax_tree/visitor/with_environment.rb @@ -0,0 +1,141 @@ +# frozen_string_literal: true + +module SyntaxTree + # WithEnvironment is a module intended to be included in classes inheriting + # from Visitor. The module overrides a few visit methods to automatically keep + # track of local variables and arguments defined in the current environment. + # Example usage: + # class MyVisitor < Visitor + # include WithEnvironment + # + # def visit_ident(node) + # # Check if we're visiting an identifier for an argument, a local + # variable or something else + # local = current_environment.find_local(node) + # + # if local.type == :argument + # # handle identifiers for arguments + # elsif local.type == :variable + # # handle identifiers for variables + # else + # # handle other identifiers, such as method names + # end + # end + module WithEnvironment + def current_environment + @current_environment ||= Environment.new + end + + def with_new_environment + previous_environment = @current_environment + @current_environment = Environment.new(previous_environment) + yield + ensure + @current_environment = previous_environment + end + + # Visits for nodes that create new environments, such as classes, modules + # and method definitions + def visit_class(node) + with_new_environment { super } + end + + def visit_module(node) + with_new_environment { super } + end + + def visit_method_add_block(node) + with_new_environment { super } + end + + def visit_def(node) + with_new_environment { super } + end + + def visit_defs(node) + with_new_environment { super } + end + + def visit_def_endless(node) + with_new_environment { super } + end + + # Visit for keeping track of local arguments, such as method and block + # arguments + def visit_params(node) + node.requireds.each do |param| + @current_environment.add_local_definition(param, :argument) + end + + node.posts.each do |param| + @current_environment.add_local_definition(param, :argument) + end + + node.keywords.each do |param| + @current_environment.add_local_definition(param.first, :argument) + end + + node.optionals.each do |param| + @current_environment.add_local_definition(param.first, :argument) + end + + super + end + + def visit_rest_param(node) + name = node.name + @current_environment.add_local_definition(name, :argument) if name + + super + end + + def visit_kwrest_param(node) + name = node.name + @current_environment.add_local_definition(name, :argument) if name + + super + end + + def visit_blockarg(node) + name = node.name + @current_environment.add_local_definition(name, :argument) if name + + super + end + + # Visit for keeping track of local variable definitions + def visit_var_field(node) + value = node.value + + if value.is_a?(SyntaxTree::Ident) + @current_environment.add_local_definition(value, :variable) + end + + super + end + + alias visit_pinned_var_ref visit_var_field + + # Visits for keeping track of variable and argument usages + def visit_aref_field(node) + name = node.collection.value + @current_environment.add_local_usage(name, :variable) if name + + super + end + + def visit_var_ref(node) + value = node.value + + if value.is_a?(SyntaxTree::Ident) + definition = @current_environment.find_local(value.value) + + if definition + @current_environment.add_local_usage(value, definition.type) + end + end + + super + end + end +end diff --git a/test/visitor_with_environment_test.rb b/test/visitor_with_environment_test.rb new file mode 100644 index 00000000..915b2143 --- /dev/null +++ b/test/visitor_with_environment_test.rb @@ -0,0 +1,410 @@ +# frozen_string_literal: true + +require_relative "test_helper" + +module SyntaxTree + class VisitorWithEnvironmentTest < Minitest::Test + class Collector < Visitor + include WithEnvironment + + attr_reader :variables, :arguments + + def initialize + @variables = {} + @arguments = {} + end + + def visit_ident(node) + local = current_environment.find_local(node.value) + return unless local + + value = node.value.delete_suffix(":") + + case local.type + when :argument + @arguments[value] = local + when :variable + @variables[value] = local + end + end + + def visit_label(node) + value = node.value.delete_suffix(":") + local = current_environment.find_local(value) + return unless local + + @arguments[value] = node if local.type == :argument + end + end + + def test_collecting_simple_variables + tree = SyntaxTree.parse(<<~RUBY) + def foo + a = 1 + a + end + RUBY + + visitor = Collector.new + visitor.visit(tree) + + assert_equal(1, visitor.variables.length) + + variable = visitor.variables["a"] + assert_equal(1, variable.definitions.length) + assert_equal(1, variable.usages.length) + + assert_equal(2, variable.definitions[0].start_line) + assert_equal(3, variable.usages[0].start_line) + end + + def test_collecting_aref_variables + tree = SyntaxTree.parse(<<~RUBY) + def foo + a = [] + a[1] + end + RUBY + + visitor = Collector.new + visitor.visit(tree) + + assert_equal(1, visitor.variables.length) + + variable = visitor.variables["a"] + assert_equal(1, variable.definitions.length) + assert_equal(1, variable.usages.length) + + assert_equal(2, variable.definitions[0].start_line) + assert_equal(3, variable.usages[0].start_line) + end + + def test_collecting_multi_assign_variables + tree = SyntaxTree.parse(<<~RUBY) + def foo + a, b = [1, 2] + puts a + puts b + end + RUBY + + visitor = Collector.new + visitor.visit(tree) + + assert_equal(2, visitor.variables.length) + + variable_a = visitor.variables["a"] + assert_equal(1, variable_a.definitions.length) + assert_equal(1, variable_a.usages.length) + + assert_equal(2, variable_a.definitions[0].start_line) + assert_equal(3, variable_a.usages[0].start_line) + + variable_b = visitor.variables["b"] + assert_equal(1, variable_b.definitions.length) + assert_equal(1, variable_b.usages.length) + + assert_equal(2, variable_b.definitions[0].start_line) + assert_equal(4, variable_b.usages[0].start_line) + end + + def test_collecting_pattern_matching_variables + tree = SyntaxTree.parse(<<~RUBY) + def foo + case [1, 2] + in Integer => a, Integer + puts a + end + end + RUBY + + visitor = Collector.new + visitor.visit(tree) + + # There are two occurrences, one on line 3 for pinning and one on line 4 + # for reference + assert_equal(1, visitor.variables.length) + + variable = visitor.variables["a"] + + # Assignment a + assert_equal(3, variable.definitions[0].start_line) + assert_equal(4, variable.usages[0].start_line) + end + + def test_collecting_pinned_variables + tree = SyntaxTree.parse(<<~RUBY) + def foo + a = 18 + case [1, 2] + in ^a, *rest + puts a + puts rest + end + end + RUBY + + visitor = Collector.new + visitor.visit(tree) + + assert_equal(2, visitor.variables.length) + + variable_a = visitor.variables["a"] + assert_equal(2, variable_a.definitions.length) + assert_equal(1, variable_a.usages.length) + + assert_equal(2, variable_a.definitions[0].start_line) + assert_equal(4, variable_a.definitions[1].start_line) + assert_equal(5, variable_a.usages[0].start_line) + + variable_rest = visitor.variables["rest"] + assert_equal(1, variable_rest.definitions.length) + assert_equal(4, variable_rest.definitions[0].start_line) + + # Rest is considered a vcall by the parser instead of a var_ref + # assert_equal(1, variable_rest.usages.length) + # assert_equal(6, variable_rest.usages[0].start_line) + end + + if RUBY_VERSION >= "3.1" + def test_collecting_one_line_pattern_matching_variables + tree = SyntaxTree.parse(<<~RUBY) + def foo + [1] => a + puts a + end + RUBY + + visitor = Collector.new + visitor.visit(tree) + + assert_equal(1, visitor.variables.length) + + variable = visitor.variables["a"] + assert_equal(1, variable.definitions.length) + assert_equal(1, variable.usages.length) + + assert_equal(2, variable.definitions[0].start_line) + assert_equal(3, variable.usages[0].start_line) + end + + def test_collecting_endless_method_arguments + tree = SyntaxTree.parse(<<~RUBY) + def foo(a) = puts a + RUBY + + visitor = Collector.new + visitor.visit(tree) + + assert_equal(1, visitor.arguments.length) + + argument = visitor.arguments["a"] + assert_equal(1, argument.definitions.length) + assert_equal(1, argument.usages.length) + + assert_equal(1, argument.definitions[0].start_line) + assert_equal(1, argument.usages[0].start_line) + end + end + + def test_collecting_method_arguments + tree = SyntaxTree.parse(<<~RUBY) + def foo(a) + puts a + end + RUBY + + visitor = Collector.new + visitor.visit(tree) + + assert_equal(1, visitor.arguments.length) + + argument = visitor.arguments["a"] + assert_equal(1, argument.definitions.length) + assert_equal(1, argument.usages.length) + + assert_equal(1, argument.definitions[0].start_line) + assert_equal(2, argument.usages[0].start_line) + end + + def test_collecting_singleton_method_arguments + tree = SyntaxTree.parse(<<~RUBY) + def self.foo(a) + puts a + end + RUBY + + visitor = Collector.new + visitor.visit(tree) + + assert_equal(1, visitor.arguments.length) + + argument = visitor.arguments["a"] + assert_equal(1, argument.definitions.length) + assert_equal(1, argument.usages.length) + + assert_equal(1, argument.definitions[0].start_line) + assert_equal(2, argument.usages[0].start_line) + end + + def test_collecting_method_arguments_all_types + tree = SyntaxTree.parse(<<~RUBY) + def foo(a, b = 1, *c, d, e: 1, **f, &block) + puts a + puts b + puts c + puts d + puts e + puts f + block.call + end + RUBY + + visitor = Collector.new + visitor.visit(tree) + + assert_equal(7, visitor.arguments.length) + + argument_a = visitor.arguments["a"] + assert_equal(1, argument_a.definitions.length) + assert_equal(1, argument_a.usages.length) + assert_equal(1, argument_a.definitions[0].start_line) + assert_equal(2, argument_a.usages[0].start_line) + + argument_b = visitor.arguments["b"] + assert_equal(1, argument_b.definitions.length) + assert_equal(1, argument_b.usages.length) + assert_equal(1, argument_b.definitions[0].start_line) + assert_equal(3, argument_b.usages[0].start_line) + + argument_c = visitor.arguments["c"] + assert_equal(1, argument_c.definitions.length) + assert_equal(1, argument_c.usages.length) + assert_equal(1, argument_c.definitions[0].start_line) + assert_equal(4, argument_c.usages[0].start_line) + + argument_d = visitor.arguments["d"] + assert_equal(1, argument_d.definitions.length) + assert_equal(1, argument_d.usages.length) + assert_equal(1, argument_d.definitions[0].start_line) + assert_equal(5, argument_d.usages[0].start_line) + + argument_e = visitor.arguments["e"] + assert_equal(1, argument_e.definitions.length) + assert_equal(1, argument_e.usages.length) + assert_equal(1, argument_e.definitions[0].start_line) + assert_equal(6, argument_e.usages[0].start_line) + + argument_f = visitor.arguments["f"] + assert_equal(1, argument_f.definitions.length) + assert_equal(1, argument_f.usages.length) + assert_equal(1, argument_f.definitions[0].start_line) + assert_equal(7, argument_f.usages[0].start_line) + + argument_block = visitor.arguments["block"] + assert_equal(1, argument_block.definitions.length) + assert_equal(1, argument_block.usages.length) + assert_equal(1, argument_block.definitions[0].start_line) + assert_equal(8, argument_block.usages[0].start_line) + end + + def test_collecting_block_arguments + tree = SyntaxTree.parse(<<~RUBY) + def foo + [].each do |i| + puts i + end + end + RUBY + + visitor = Collector.new + visitor.visit(tree) + + assert_equal(1, visitor.arguments.length) + + argument = visitor.arguments["i"] + assert_equal(1, argument.definitions.length) + assert_equal(1, argument.usages.length) + assert_equal(2, argument.definitions[0].start_line) + assert_equal(3, argument.usages[0].start_line) + end + + def test_collecting_one_line_block_arguments + tree = SyntaxTree.parse(<<~RUBY) + def foo + [].each { |i| puts i } + end + RUBY + + visitor = Collector.new + visitor.visit(tree) + + assert_equal(1, visitor.arguments.length) + + argument = visitor.arguments["i"] + assert_equal(1, argument.definitions.length) + assert_equal(1, argument.usages.length) + assert_equal(2, argument.definitions[0].start_line) + assert_equal(2, argument.usages[0].start_line) + end + + def test_collecting_shadowed_block_arguments + tree = SyntaxTree.parse(<<~RUBY) + def foo + i = "something" + + [].each do |i| + puts i + end + + i + end + RUBY + + visitor = Collector.new + visitor.visit(tree) + + assert_equal(1, visitor.arguments.length) + assert_equal(1, visitor.variables.length) + + argument = visitor.arguments["i"] + assert_equal(1, argument.definitions.length) + assert_equal(1, argument.usages.length) + assert_equal(4, argument.definitions[0].start_line) + assert_equal(5, argument.usages[0].start_line) + + variable = visitor.variables["i"] + assert_equal(1, variable.definitions.length) + assert_equal(1, variable.usages.length) + assert_equal(2, variable.definitions[0].start_line) + assert_equal(8, variable.usages[0].start_line) + end + + def test_collecting_shadowed_local_variables + tree = SyntaxTree.parse(<<~RUBY) + def foo(a) + puts a + a = 123 + a + end + RUBY + + visitor = Collector.new + visitor.visit(tree) + + # All occurrences are considered arguments, despite overriding the + # argument value + assert_equal(1, visitor.arguments.length) + assert_equal(0, visitor.variables.length) + + argument = visitor.arguments["a"] + assert_equal(2, argument.definitions.length) + assert_equal(2, argument.usages.length) + + assert_equal(1, argument.definitions[0].start_line) + assert_equal(3, argument.definitions[1].start_line) + assert_equal(2, argument.usages[0].start_line) + assert_equal(4, argument.usages[1].start_line) + end + end +end From 95b25842b1e5f2cb6aeb1fd53d1795186352ec97 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 17 Oct 2022 13:47:10 -0400 Subject: [PATCH 136/536] Fix incorrect logic in forced_brace_bounds? translation --- lib/syntax_tree/node.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 82d378c6..8b4e2e1d 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -2059,8 +2059,10 @@ def forced_brace_bounds?(q) when If, IfMod, IfOp, Unless, UnlessMod, While, WhileMod, Until, UntilMod return true if parent.predicate == previous - previous = parent end + + previous = parent + false end end From 89081dfb05266530c83b6b8b17be52f8017bfd24 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 17 Oct 2022 13:53:26 -0400 Subject: [PATCH 137/536] Fix up rubocop violations --- .rubocop.yml | 3 +++ lib/syntax_tree.rb | 2 +- lib/syntax_tree/formatter.rb | 2 +- lib/syntax_tree/node.rb | 16 +++++++++------- lib/syntax_tree/parser.rb | 2 +- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index f6ffbcd0..3323c741 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -55,6 +55,9 @@ Style/IdenticalConditionalBranches: Style/IfInsideElse: Enabled: false +Style/IfWithBooleanLiteralBranches: + Enabled: false + Style/KeywordParametersOrder: Enabled: false diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index da84273c..ed783e47 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -23,7 +23,7 @@ # We rely on Symbol#name being available, which is only available in Ruby 3.0+. # In case we're running on an older Ruby version, we polyfill it here. unless :+.respond_to?(:name) - class Symbol + class Symbol # rubocop:disable Style/Documentation def name to_s.freeze end diff --git a/lib/syntax_tree/formatter.rb b/lib/syntax_tree/formatter.rb index 39ed1583..f878490c 100644 --- a/lib/syntax_tree/formatter.rb +++ b/lib/syntax_tree/formatter.rb @@ -148,7 +148,7 @@ def group # A similar version to the super, except that it calls back into the # separator proc with the instance of `self`. - def seplist(list, sep = nil, iter_method = :each) # :yield: element + def seplist(list, sep = nil, iter_method = :each) first = true list.__send__(iter_method) do |*v| if first diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 8b4e2e1d..d183a9f8 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -621,11 +621,11 @@ def trailing_comma? return false unless arguments.is_a?(Args) parts = arguments.parts - if parts.last&.is_a?(ArgBlock) + if parts.last.is_a?(ArgBlock) # If the last argument is a block, then we can't put a trailing comma # after it without resulting in a syntax error. false - elsif parts.length == 1 && (part = parts.first) && + elsif (parts.length == 1) && (part = parts.first) && (part.is_a?(Command) || part.is_a?(CommandCall)) # If the only argument is a command or command call, then a trailing # comma would be parsed as part of that expression instead of on this @@ -891,6 +891,7 @@ def format(q) # # provided the line length was hit between `bar` and `baz`. class VarRefsFormatter + # The separator for the fill algorithm. class Separator def call(q) q.text(",") @@ -2522,7 +2523,8 @@ def format_chain(q, children) # https://github.com/prettier/plugin-ruby/issues/862. else # If we're at a Call node and not a MethodAddBlock node in the - # chain then we're going to add a newline so it indents properly. + # chain then we're going to add a newline so it indents + # properly. q.breakable_empty end end @@ -2701,6 +2703,8 @@ def format(q) end end + # Print out the arguments to this call. If there are no arguments, then do + #nothing. def format_arguments(q) case arguments when ArgParen @@ -2708,8 +2712,6 @@ def format_arguments(q) when Args q.text(" ") q.format(arguments) - else - # Do nothing if there are no arguments. end end @@ -3180,6 +3182,8 @@ def format(q) end end + # Format the arguments for this command call here. If there are no + # arguments, then print nothing. if arguments parts = arguments.parts @@ -3190,8 +3194,6 @@ def format(q) q.text(" ") q.nest(argument_alignment(q, doc)) { q.format(arguments) } end - else - # If there are no arguments, print nothing. end end end diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index 70f1e2a3..61a7ca57 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -1389,7 +1389,7 @@ def on_dot3(left, right) # :call-seq: # on_dyna_symbol: (StringContent string_content) -> DynaSymbol def on_dyna_symbol(string_content) - if symbeg = find_token(SymBeg) + if (symbeg = find_token(SymBeg)) # A normal dynamic symbol tokens.delete(symbeg) tstring_end = consume_tstring_end(symbeg.location) From 8f15bbb771129cb43587579a3f10679e260fc808 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 17 Oct 2022 13:55:42 -0400 Subject: [PATCH 138/536] Check if this is working on truffleruby --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d35471fa..afd7eb8e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,6 +12,7 @@ jobs: - '3.0' - '3.1' - head + - truffleruby name: CI runs-on: ubuntu-latest env: From aa0573ddc919c18d641b71757571805c780a3d3a Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 17 Oct 2022 13:58:58 -0400 Subject: [PATCH 139/536] Fix incorrect translation of Ternaryable.call --- lib/syntax_tree/node.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index d183a9f8..dcdd0275 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -5396,7 +5396,7 @@ def ternaryable?(statement) # operators, then we can't use a ternary expression as it would break # the flow control. operator = statement.operator - operator != "and" && operator != "or" + operator != :and && operator != :or else true end From 7b2bc9b6d46970b58bf7a3457d23d2ee87d28e6a Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 17 Oct 2022 14:01:12 -0400 Subject: [PATCH 140/536] Remove truffleruby from tests until we finish removing pattern matching --- .github/workflows/main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index afd7eb8e..d35471fa 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,7 +12,6 @@ jobs: - '3.0' - '3.1' - head - - truffleruby name: CI runs-on: ubuntu-latest env: From 66edef87879d7e13d1e34641f5cdd0b92b926405 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 17 Oct 2022 14:19:37 -0400 Subject: [PATCH 141/536] Bump to version 4.0.0 --- CHANGELOG.md | 15 ++++++++++++++- Gemfile.lock | 2 +- lib/syntax_tree/version.rb | 2 +- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f0ba115e..3ed4e458 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,18 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +## [4.0.0] - 2022-10-17 + +### Added + +- [#169](https://github.com/ruby-syntax-tree/syntax_tree/pull/169) - You can now pass `--ignore-files` multiple times. +- [#157](https://github.com/ruby-syntax-tree/syntax_tree/pull/157) - We now support tracking local variable definitions throughout the visitor. This allows you to access scope information while visiting the tree. +- [#170](https://github.com/ruby-syntax-tree/syntax_tree/pull/170) - There is now an undocumented `STREE_FAST_FORMAT` environment variable checked when formatting. It has the effect of turning _off_ formatting call chains and ternaries in special ways. This improves performance quite a bit. I'm leaving it undocumented because ideally we just improve the performance as a whole. This is meant as a stopgap until we get there. + +### Changed + +- [#170](https://github.com/ruby-syntax-tree/syntax_tree/pull/170) - We now require at least version `1.0.0` of `prettier_print`. This is to take advantage of the first-class string support in the doc tree. + ## [3.6.3] - 2022-10-11 ### Changed @@ -370,7 +382,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - 🎉 Initial release! 🎉 -[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.6.3...HEAD +[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v4.0.0...HEAD +[4.0.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.6.3...v4.0.0 [3.6.3]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.6.2...v3.6.3 [3.6.2]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.6.1...v3.6.2 [3.6.1]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.6.0...v3.6.1 diff --git a/Gemfile.lock b/Gemfile.lock index 76bda432..00ae409b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - syntax_tree (3.6.3) + syntax_tree (4.0.0) prettier_print (>= 1.0.0) GEM diff --git a/lib/syntax_tree/version.rb b/lib/syntax_tree/version.rb index ec6dcd3e..8456abd4 100644 --- a/lib/syntax_tree/version.rb +++ b/lib/syntax_tree/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SyntaxTree - VERSION = "3.6.3" + VERSION = "4.0.0" end From 974cdcb25d6f19a033fa5618eb1f14dab249a3ee Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 17 Oct 2022 14:20:38 -0400 Subject: [PATCH 142/536] Update CHANGELOG to mention pattern matching --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ed4e458..c4558185 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ### Changed - [#170](https://github.com/ruby-syntax-tree/syntax_tree/pull/170) - We now require at least version `1.0.0` of `prettier_print`. This is to take advantage of the first-class string support in the doc tree. +- [#170](https://github.com/ruby-syntax-tree/syntax_tree/pull/170) - Pattern matching has been removed from usage internal to this library (excluding the language server). This should hopefully enable runtimes that don't have pattern matching fully implemented yet (e.g., TruffleRuby) to run this gem. ## [3.6.3] - 2022-10-11 From 3605b50c55b3245cdb8e8754c59775b22c479975 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 17 Oct 2022 15:15:34 -0400 Subject: [PATCH 143/536] Use a refinement for Symbol#name instead of a polyfill --- .rubocop.yml | 3 +++ lib/syntax_tree.rb | 10 ---------- lib/syntax_tree/node.rb | 14 ++++++++++++++ 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index 3323c741..4dbeeb33 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -13,6 +13,9 @@ AllCops: Layout/LineLength: Max: 80 +Lint/AmbiguousBlockAssociation: + Enabled: false + Lint/DuplicateBranch: Enabled: false diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index 52ec700b..fbd4fcef 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -22,16 +22,6 @@ require_relative "syntax_tree/parser" -# We rely on Symbol#name being available, which is only available in Ruby 3.0+. -# In case we're running on an older Ruby version, we polyfill it here. -unless :+.respond_to?(:name) - class Symbol # rubocop:disable Style/Documentation - def name - to_s.freeze - end - end -end - # Syntax Tree is a suite of tools built on top of the internal CRuby parser. It # provides the ability to generate a syntax tree from source, as well as the # tools necessary to inspect and manipulate that syntax tree. It can be used to diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index dcdd0275..5162655e 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -1651,6 +1651,20 @@ def format(q) # array << value # class Binary < Node + # Since Binary's operator is a symbol, it's better to use the `name` method + # than to allocate a new string every time. This is a tiny performance + # optimization, but enough that it shows up in the profiler. Adding this in + # for older Ruby versions. + unless :+.respond_to?(:name) + using Module.new { + refine Symbol do + def name + to_s.freeze + end + end + } + end + # [untyped] the left-hand side of the expression attr_reader :left From f924f9db4f422e3cdb02194f251885f48c2fc4bd Mon Sep 17 00:00:00 2001 From: Vinicius Stock Date: Mon, 17 Oct 2022 16:45:19 -0400 Subject: [PATCH 144/536] Fix current_environment usages to use accessor instead of instance variable --- lib/syntax_tree/visitor/with_environment.rb | 22 ++++++++++----------- test/visitor_with_environment_test.rb | 20 +++++++++++++++++++ 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/lib/syntax_tree/visitor/with_environment.rb b/lib/syntax_tree/visitor/with_environment.rb index 62e59c98..ad101e0a 100644 --- a/lib/syntax_tree/visitor/with_environment.rb +++ b/lib/syntax_tree/visitor/with_environment.rb @@ -64,19 +64,19 @@ def visit_def_endless(node) # arguments def visit_params(node) node.requireds.each do |param| - @current_environment.add_local_definition(param, :argument) + current_environment.add_local_definition(param, :argument) end node.posts.each do |param| - @current_environment.add_local_definition(param, :argument) + current_environment.add_local_definition(param, :argument) end node.keywords.each do |param| - @current_environment.add_local_definition(param.first, :argument) + current_environment.add_local_definition(param.first, :argument) end node.optionals.each do |param| - @current_environment.add_local_definition(param.first, :argument) + current_environment.add_local_definition(param.first, :argument) end super @@ -84,21 +84,21 @@ def visit_params(node) def visit_rest_param(node) name = node.name - @current_environment.add_local_definition(name, :argument) if name + current_environment.add_local_definition(name, :argument) if name super end def visit_kwrest_param(node) name = node.name - @current_environment.add_local_definition(name, :argument) if name + current_environment.add_local_definition(name, :argument) if name super end def visit_blockarg(node) name = node.name - @current_environment.add_local_definition(name, :argument) if name + current_environment.add_local_definition(name, :argument) if name super end @@ -108,7 +108,7 @@ def visit_var_field(node) value = node.value if value.is_a?(SyntaxTree::Ident) - @current_environment.add_local_definition(value, :variable) + current_environment.add_local_definition(value, :variable) end super @@ -119,7 +119,7 @@ def visit_var_field(node) # Visits for keeping track of variable and argument usages def visit_aref_field(node) name = node.collection.value - @current_environment.add_local_usage(name, :variable) if name + current_environment.add_local_usage(name, :variable) if name super end @@ -128,10 +128,10 @@ def visit_var_ref(node) value = node.value if value.is_a?(SyntaxTree::Ident) - definition = @current_environment.find_local(value.value) + definition = current_environment.find_local(value.value) if definition - @current_environment.add_local_usage(value, definition.type) + current_environment.add_local_usage(value, definition.type) end end diff --git a/test/visitor_with_environment_test.rb b/test/visitor_with_environment_test.rb index 915b2143..302dbfbe 100644 --- a/test/visitor_with_environment_test.rb +++ b/test/visitor_with_environment_test.rb @@ -406,5 +406,25 @@ def foo(a) assert_equal(2, argument.usages[0].start_line) assert_equal(4, argument.usages[1].start_line) end + + def test_variables_in_the_top_level + tree = SyntaxTree.parse(<<~RUBY) + a = 123 + a + RUBY + + visitor = Collector.new + visitor.visit(tree) + + assert_equal(0, visitor.arguments.length) + assert_equal(1, visitor.variables.length) + + variable = visitor.variables["a"] + assert_equal(1, variable.definitions.length) + assert_equal(1, variable.usages.length) + + assert_equal(1, variable.definitions[0].start_line) + assert_equal(2, variable.usages[0].start_line) + end end end From e47b3b8d50c5eb9be374d6297f68a6d35c23d5cf Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 18 Oct 2022 10:23:01 -0400 Subject: [PATCH 145/536] Update prettier_print requirement --- Gemfile.lock | 6 +++--- syntax_tree.gemspec | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 00ae409b..f713e865 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -2,7 +2,7 @@ PATH remote: . specs: syntax_tree (4.0.0) - prettier_print (>= 1.0.0) + prettier_print (>= 1.0.1) GEM remote: https://rubygems.org/ @@ -14,7 +14,7 @@ GEM parallel (1.22.1) parser (3.1.2.1) ast (~> 2.4.1) - prettier_print (1.0.0) + prettier_print (1.0.1) rainbow (3.1.1) rake (13.0.6) regexp_parser (2.6.0) @@ -29,7 +29,7 @@ GEM rubocop-ast (>= 1.20.1, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 1.4.0, < 3.0) - rubocop-ast (1.21.0) + rubocop-ast (1.22.0) parser (>= 3.1.1.0) ruby-progressbar (1.11.0) simplecov (0.21.2) diff --git a/syntax_tree.gemspec b/syntax_tree.gemspec index ec7d57ef..fe74c7e7 100644 --- a/syntax_tree.gemspec +++ b/syntax_tree.gemspec @@ -25,7 +25,7 @@ Gem::Specification.new do |spec| spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } spec.require_paths = %w[lib] - spec.add_dependency "prettier_print", ">= 1.0.0" + spec.add_dependency "prettier_print", ">= 1.0.1" spec.add_development_dependency "bundler" spec.add_development_dependency "minitest" From 8763c185a621732638cb3299e5b1482280245cbe Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 18 Oct 2022 10:28:46 -0400 Subject: [PATCH 146/536] Bump to v4.0.1 --- CHANGELOG.md | 8 ++++++++ Gemfile.lock | 2 +- lib/syntax_tree/version.rb | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c4558185..dc5dc7d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,14 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +## [4.0.1] - 2022-10-18 + +### Changed + +- [#172](https://github.com/ruby-syntax-tree/syntax_tree/pull/172) - Use a refinement for `Symbol#name` addition so that other runtimes or tools don't get confused by its availability. +- [#173](https://github.com/ruby-syntax-tree/syntax_tree/pull/173) - Fix the `current_environment` usage to use the method instead of the instance variable. +- [#175](https://github.com/ruby-syntax-tree/syntax_tree/pull/175) - Update `prettier_print` requirement since v1.0.0 had a bug with `#breakable_return`. + ## [4.0.0] - 2022-10-17 ### Added diff --git a/Gemfile.lock b/Gemfile.lock index f713e865..f55c7768 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - syntax_tree (4.0.0) + syntax_tree (4.0.1) prettier_print (>= 1.0.1) GEM diff --git a/lib/syntax_tree/version.rb b/lib/syntax_tree/version.rb index 8456abd4..695f4c88 100644 --- a/lib/syntax_tree/version.rb +++ b/lib/syntax_tree/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SyntaxTree - VERSION = "4.0.0" + VERSION = "4.0.1" end From 07ac277f819ec1400994c7240c97d6039acda901 Mon Sep 17 00:00:00 2001 From: Vinicius Stock Date: Tue, 18 Oct 2022 13:40:21 -0400 Subject: [PATCH 147/536] Remove aref_field specific visit from WithEnvironment ARefField nodes end up with regular VarRef nodes inside, so we were actually counting them twice --- lib/syntax_tree/visitor/with_environment.rb | 7 -- test/visitor_with_environment_test.rb | 80 +++++++++++++++++++++ 2 files changed, 80 insertions(+), 7 deletions(-) diff --git a/lib/syntax_tree/visitor/with_environment.rb b/lib/syntax_tree/visitor/with_environment.rb index ad101e0a..6fd30188 100644 --- a/lib/syntax_tree/visitor/with_environment.rb +++ b/lib/syntax_tree/visitor/with_environment.rb @@ -117,13 +117,6 @@ def visit_var_field(node) alias visit_pinned_var_ref visit_var_field # Visits for keeping track of variable and argument usages - def visit_aref_field(node) - name = node.collection.value - current_environment.add_local_usage(name, :variable) if name - - super - end - def visit_var_ref(node) value = node.value diff --git a/test/visitor_with_environment_test.rb b/test/visitor_with_environment_test.rb index 302dbfbe..ea547811 100644 --- a/test/visitor_with_environment_test.rb +++ b/test/visitor_with_environment_test.rb @@ -426,5 +426,85 @@ def test_variables_in_the_top_level assert_equal(1, variable.definitions[0].start_line) assert_equal(2, variable.usages[0].start_line) end + + def test_aref_field + tree = SyntaxTree.parse(<<~RUBY) + object = {} + object["name"] = "something" + RUBY + + visitor = Collector.new + visitor.visit(tree) + + assert_equal(0, visitor.arguments.length) + assert_equal(1, visitor.variables.length) + + variable = visitor.variables["object"] + assert_equal(1, variable.definitions.length) + assert_equal(1, variable.usages.length) + + assert_equal(1, variable.definitions[0].start_line) + assert_equal(2, variable.usages[0].start_line) + end + + def test_aref_on_a_method_call + tree = SyntaxTree.parse(<<~RUBY) + object = MyObject.new + object.attributes["name"] = "something" + RUBY + + visitor = Collector.new + visitor.visit(tree) + + assert_equal(0, visitor.arguments.length) + assert_equal(1, visitor.variables.length) + + variable = visitor.variables["object"] + assert_equal(1, variable.definitions.length) + assert_equal(1, variable.usages.length) + + assert_equal(1, variable.definitions[0].start_line) + assert_equal(2, variable.usages[0].start_line) + end + + def test_aref_with_two_accesses + tree = SyntaxTree.parse(<<~RUBY) + object = MyObject.new + object["first"]["second"] ||= [] + RUBY + + visitor = Collector.new + visitor.visit(tree) + + assert_equal(0, visitor.arguments.length) + assert_equal(1, visitor.variables.length) + + variable = visitor.variables["object"] + assert_equal(1, variable.definitions.length) + assert_equal(1, variable.usages.length) + + assert_equal(1, variable.definitions[0].start_line) + assert_equal(2, variable.usages[0].start_line) + end + + def test_aref_on_a_method_call_with_arguments + tree = SyntaxTree.parse(<<~RUBY) + object = MyObject.new + object.instance_variable_get(:@attributes)[:something] = :other_thing + RUBY + + visitor = Collector.new + visitor.visit(tree) + + assert_equal(0, visitor.arguments.length) + assert_equal(1, visitor.variables.length) + + variable = visitor.variables["object"] + assert_equal(1, variable.definitions.length) + assert_equal(1, variable.usages.length) + + assert_equal(1, variable.definitions[0].start_line) + assert_equal(2, variable.usages[0].start_line) + end end end From 329ce7dc4e5f1af09bd83c0d6345998d0f7b4bb6 Mon Sep 17 00:00:00 2001 From: Vinicius Stock Date: Tue, 18 Oct 2022 13:58:07 -0400 Subject: [PATCH 148/536] Only create fresh environment for a MethodAddBlock block The call part of a MethodAddBlock node occurs in the same environment. Only the block portion of it occurs in a fresh environment. --- lib/syntax_tree/visitor/with_environment.rb | 6 ++++- test/visitor_with_environment_test.rb | 27 +++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/lib/syntax_tree/visitor/with_environment.rb b/lib/syntax_tree/visitor/with_environment.rb index 6fd30188..cc27a671 100644 --- a/lib/syntax_tree/visitor/with_environment.rb +++ b/lib/syntax_tree/visitor/with_environment.rb @@ -44,8 +44,12 @@ def visit_module(node) with_new_environment { super } end + # When we find a method invocation with a block, only the code that happens + # inside of the block needs a fresh environment. The method invocation + # itself happens in the same environment def visit_method_add_block(node) - with_new_environment { super } + visit(node.call) + with_new_environment { visit(node.block) } end def visit_def(node) diff --git a/test/visitor_with_environment_test.rb b/test/visitor_with_environment_test.rb index ea547811..07a73848 100644 --- a/test/visitor_with_environment_test.rb +++ b/test/visitor_with_environment_test.rb @@ -506,5 +506,32 @@ def test_aref_on_a_method_call_with_arguments assert_equal(1, variable.definitions[0].start_line) assert_equal(2, variable.usages[0].start_line) end + + def test_double_aref_on_method_call + tree = SyntaxTree.parse(<<~RUBY) + object = MyObject.new + object["attributes"].find { |a| a["field"] == "expected" }["value"] = "changed" + RUBY + + visitor = Collector.new + visitor.visit(tree) + + assert_equal(1, visitor.arguments.length) + assert_equal(1, visitor.variables.length) + + variable = visitor.variables["object"] + assert_equal(1, variable.definitions.length) + assert_equal(1, variable.usages.length) + + assert_equal(1, variable.definitions[0].start_line) + assert_equal(2, variable.usages[0].start_line) + + argument = visitor.arguments["a"] + assert_equal(1, argument.definitions.length) + assert_equal(1, argument.usages.length) + + assert_equal(2, argument.definitions[0].start_line) + assert_equal(2, argument.usages[0].start_line) + end end end From bd42046385d79c31fdbe32562bd483eaaef18915 Mon Sep 17 00:00:00 2001 From: Vinicius Stock Date: Tue, 18 Oct 2022 14:08:37 -0400 Subject: [PATCH 149/536] Handle nested required arguments Handle arguments of type [].each do |one, (two, three)| end when declaring arguments --- lib/syntax_tree/visitor/with_environment.rb | 16 +++- test/visitor_with_environment_test.rb | 82 +++++++++++++++++++++ 2 files changed, 95 insertions(+), 3 deletions(-) diff --git a/lib/syntax_tree/visitor/with_environment.rb b/lib/syntax_tree/visitor/with_environment.rb index cc27a671..043cbd4c 100644 --- a/lib/syntax_tree/visitor/with_environment.rb +++ b/lib/syntax_tree/visitor/with_environment.rb @@ -67,9 +67,7 @@ def visit_def_endless(node) # Visit for keeping track of local arguments, such as method and block # arguments def visit_params(node) - node.requireds.each do |param| - current_environment.add_local_definition(param, :argument) - end + add_argument_definitions(node.requireds) node.posts.each do |param| current_environment.add_local_definition(param, :argument) @@ -134,5 +132,17 @@ def visit_var_ref(node) super end + + private + + def add_argument_definitions(list) + list.each do |param| + if param.is_a?(SyntaxTree::MLHSParen) + add_argument_definitions(param.contents.parts) + else + current_environment.add_local_definition(param, :argument) + end + end + end end end diff --git a/test/visitor_with_environment_test.rb b/test/visitor_with_environment_test.rb index 07a73848..b37bad16 100644 --- a/test/visitor_with_environment_test.rb +++ b/test/visitor_with_environment_test.rb @@ -533,5 +533,87 @@ def test_double_aref_on_method_call assert_equal(2, argument.definitions[0].start_line) assert_equal(2, argument.usages[0].start_line) end + + def test_nested_arguments + tree = SyntaxTree.parse(<<~RUBY) + [[1, [2, 3]]].each do |one, (two, three)| + one + two + three + end + RUBY + + visitor = Collector.new + visitor.visit(tree) + + assert_equal(3, visitor.arguments.length) + assert_equal(0, visitor.variables.length) + + argument = visitor.arguments["one"] + assert_equal(1, argument.definitions.length) + assert_equal(1, argument.usages.length) + + assert_equal(1, argument.definitions[0].start_line) + assert_equal(2, argument.usages[0].start_line) + + argument = visitor.arguments["two"] + assert_equal(1, argument.definitions.length) + assert_equal(1, argument.usages.length) + + assert_equal(1, argument.definitions[0].start_line) + assert_equal(3, argument.usages[0].start_line) + + argument = visitor.arguments["three"] + assert_equal(1, argument.definitions.length) + assert_equal(1, argument.usages.length) + + assert_equal(1, argument.definitions[0].start_line) + assert_equal(4, argument.usages[0].start_line) + end + + def test_double_nested_arguments + tree = SyntaxTree.parse(<<~RUBY) + [[1, [2, 3]]].each do |one, (two, (three, four))| + one + two + three + four + end + RUBY + + visitor = Collector.new + visitor.visit(tree) + + assert_equal(4, visitor.arguments.length) + assert_equal(0, visitor.variables.length) + + argument = visitor.arguments["one"] + assert_equal(1, argument.definitions.length) + assert_equal(1, argument.usages.length) + + assert_equal(1, argument.definitions[0].start_line) + assert_equal(2, argument.usages[0].start_line) + + argument = visitor.arguments["two"] + assert_equal(1, argument.definitions.length) + assert_equal(1, argument.usages.length) + + assert_equal(1, argument.definitions[0].start_line) + assert_equal(3, argument.usages[0].start_line) + + argument = visitor.arguments["three"] + assert_equal(1, argument.definitions.length) + assert_equal(1, argument.usages.length) + + assert_equal(1, argument.definitions[0].start_line) + assert_equal(4, argument.usages[0].start_line) + + argument = visitor.arguments["four"] + assert_equal(1, argument.definitions.length) + assert_equal(1, argument.usages.length) + + assert_equal(1, argument.definitions[0].start_line) + assert_equal(5, argument.usages[0].start_line) + end end end From c71415ec2ceb31b5a785e082cfc668818b7c8900 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 19 Oct 2022 18:32:07 +0000 Subject: [PATCH 150/536] Bump prettier_print from 1.0.1 to 1.0.2 Bumps [prettier_print](https://github.com/ruby-syntax-tree/prettier_print) from 1.0.1 to 1.0.2. - [Release notes](https://github.com/ruby-syntax-tree/prettier_print/releases) - [Changelog](https://github.com/ruby-syntax-tree/prettier_print/blob/main/CHANGELOG.md) - [Commits](https://github.com/ruby-syntax-tree/prettier_print/compare/v1.0.1...v1.0.2) --- updated-dependencies: - dependency-name: prettier_print dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index f55c7768..2dab3bab 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -14,7 +14,7 @@ GEM parallel (1.22.1) parser (3.1.2.1) ast (~> 2.4.1) - prettier_print (1.0.1) + prettier_print (1.0.2) rainbow (3.1.1) rake (13.0.6) regexp_parser (2.6.0) From 2f8d28056a6b9679c39030937d98aea1f9e5dcb6 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 19 Oct 2022 18:47:57 -0400 Subject: [PATCH 151/536] Bump to version 4.0.2 --- CHANGELOG.md | 10 +++++++++- Gemfile.lock | 4 ++-- lib/syntax_tree/version.rb | 2 +- syntax_tree.gemspec | 2 +- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc5dc7d5..46f47ec9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +## [4.0.2] - 2022-10-19 + +### Changed + +- [#177](https://github.com/ruby-syntax-tree/syntax_tree/pull/177) - Fix up various other issues with the environment visitor addition. + ## [4.0.1] - 2022-10-18 ### Changed @@ -391,7 +397,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - 🎉 Initial release! 🎉 -[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v4.0.0...HEAD +[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v4.0.2...HEAD +[4.0.2]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v4.0.1...v4.0.2 +[4.0.1]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v4.0.0...v4.0.1 [4.0.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.6.3...v4.0.0 [3.6.3]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.6.2...v3.6.3 [3.6.2]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.6.1...v3.6.2 diff --git a/Gemfile.lock b/Gemfile.lock index 2dab3bab..abe983b2 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,8 +1,8 @@ PATH remote: . specs: - syntax_tree (4.0.1) - prettier_print (>= 1.0.1) + syntax_tree (4.0.2) + prettier_print (>= 1.0.2) GEM remote: https://rubygems.org/ diff --git a/lib/syntax_tree/version.rb b/lib/syntax_tree/version.rb index 695f4c88..98d461df 100644 --- a/lib/syntax_tree/version.rb +++ b/lib/syntax_tree/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SyntaxTree - VERSION = "4.0.1" + VERSION = "4.0.2" end diff --git a/syntax_tree.gemspec b/syntax_tree.gemspec index fe74c7e7..c82a8e98 100644 --- a/syntax_tree.gemspec +++ b/syntax_tree.gemspec @@ -25,7 +25,7 @@ Gem::Specification.new do |spec| spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } spec.require_paths = %w[lib] - spec.add_dependency "prettier_print", ">= 1.0.1" + spec.add_dependency "prettier_print", ">= 1.0.2" spec.add_development_dependency "bundler" spec.add_development_dependency "minitest" From 0348249c3e262915e838d4b30e5d1b36d63a6fa4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 20 Oct 2022 17:30:19 +0000 Subject: [PATCH 152/536] Bump rubocop from 1.36.0 to 1.37.0 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.36.0 to 1.37.0. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.36.0...v1.37.0) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index abe983b2..b14b07e5 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -19,14 +19,14 @@ GEM rake (13.0.6) regexp_parser (2.6.0) rexml (3.2.5) - rubocop (1.36.0) + rubocop (1.37.0) json (~> 2.3) parallel (~> 1.10) parser (>= 3.1.2.1) rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 1.8, < 3.0) rexml (>= 3.2.5, < 4.0) - rubocop-ast (>= 1.20.1, < 2.0) + rubocop-ast (>= 1.22.0, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 1.4.0, < 3.0) rubocop-ast (1.22.0) From 7c72062aa6a99e6a154c7cf0282367ec0f79c5e3 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 24 Oct 2022 16:34:55 -0400 Subject: [PATCH 153/536] stree search --- .rubocop.yml | 3 ++ Gemfile.lock | 6 +-- README.md | 24 ++++++++++ lib/syntax_tree.rb | 1 + lib/syntax_tree/cli.rb | 44 +++++++++++++++++-- lib/syntax_tree/node.rb | 12 ++--- lib/syntax_tree/search.rb | 92 +++++++++++++++++++++++++++++++++++++++ test/cli_test.rb | 5 +++ test/search_test.rb | 51 ++++++++++++++++++++++ 9 files changed, 226 insertions(+), 12 deletions(-) create mode 100644 lib/syntax_tree/search.rb create mode 100644 test/search_test.rb diff --git a/.rubocop.yml b/.rubocop.yml index 4dbeeb33..c0892d8a 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -28,6 +28,9 @@ Lint/InterpolationCheck: Lint/MissingSuper: Enabled: false +Lint/RedundantRequireStatement: + Enabled: false + Lint/UnusedMethodArgument: AllowUnusedKeywordArguments: true diff --git a/Gemfile.lock b/Gemfile.lock index b14b07e5..7a227148 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -19,17 +19,17 @@ GEM rake (13.0.6) regexp_parser (2.6.0) rexml (3.2.5) - rubocop (1.37.0) + rubocop (1.37.1) json (~> 2.3) parallel (~> 1.10) parser (>= 3.1.2.1) rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 1.8, < 3.0) rexml (>= 3.2.5, < 4.0) - rubocop-ast (>= 1.22.0, < 2.0) + rubocop-ast (>= 1.23.0, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 1.4.0, < 3.0) - rubocop-ast (1.22.0) + rubocop-ast (1.23.0) parser (>= 3.1.1.0) ruby-progressbar (1.11.0) simplecov (0.21.2) diff --git a/README.md b/README.md index 30c35ac8..c8c51445 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ It is built with only standard library dependencies. It additionally ships with - [format](#format) - [json](#json) - [match](#match) + - [search](#search) - [write](#write) - [Configuration](#configuration) - [Globbing](#globbing) @@ -215,6 +216,29 @@ SyntaxTree::Program[ ] ``` +### search + +This command will search the given filepaths against the specified pattern to find nodes that match. The pattern is a Ruby pattern-matching expression that is matched against each node in the tree. It can optionally be loaded from a file if you specify a filepath as the pattern argument. + +```sh +stree search VarRef path/to/file.rb +``` + +For a file that contains `Foo + Bar` you will receive: + +```ruby +path/to/file.rb:1:0: Foo + Bar +path/to/file.rb:1:6: Foo + Bar +``` + +If you put `VarRef` into a file instead (for example, `query.txt`), you would instead run: + +```sh +stree search query.txt path/to/file.rb +``` + +Note that the output of the `match` CLI command creates a valid pattern that can be used as the input for this command. + ### write This command will format the listed files and write that formatted version back to the source files. Note that this overwrites the original content, to be sure to be using a version control system. diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index fbd4fcef..eef142ff 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -21,6 +21,7 @@ require_relative "syntax_tree/visitor/with_environment" require_relative "syntax_tree/parser" +require_relative "syntax_tree/search" # Syntax Tree is a suite of tools built on top of the internal CRuby parser. It # provides the ability to generate a syntax tree from source, as well as the diff --git a/lib/syntax_tree/cli.rb b/lib/syntax_tree/cli.rb index b839d562..c5eae1bc 100644 --- a/lib/syntax_tree/cli.rb +++ b/lib/syntax_tree/cli.rb @@ -212,6 +212,39 @@ def run(item) end end + # An action of the CLI that searches for the given pattern matching pattern + # in the given files. + class Search < Action + attr_reader :search + + def initialize(query) + query = File.read(query) if File.readable?(query) + @search = SyntaxTree::Search.new(query) + rescue SyntaxTree::Search::UncompilableError => error + warn(error.message) + exit(1) + end + + def run(item) + search.scan(item.handler.parse(item.source)) do |node| + location = node.location + line = location.start_line + + bold_range = + if line == location.end_line + location.start_column...location.end_column + else + location.start_column.. + end + + source = item.source.lines[line - 1].chomp + source[bold_range] = Color.bold(source[bold_range]).to_s + + puts("#{item.filepath}:#{line}:#{location.start_column}: #{source}") + end + end + end + # An action of the CLI that formats the input source and writes the # formatted output back to the file. class Write < Action @@ -263,6 +296,9 @@ def run(item) #{Color.bold("stree lsp [--plugins=...] [--print-width=NUMBER]")} Run syntax tree in language server mode + #{Color.bold("stree search PATTERN [-e SCRIPT] FILE")} + Search for the given pattern in the given files + #{Color.bold("stree version")} Output the current version of syntax tree @@ -400,6 +436,8 @@ def run(argv) Debug.new(options) when "doc" Doc.new(options) + when "f", "format" + Format.new(options) when "help" puts HELP return 0 @@ -411,8 +449,8 @@ def run(argv) return 0 when "m", "match" Match.new(options) - when "f", "format" - Format.new(options) + when "s", "search" + Search.new(arguments.shift) when "version" puts SyntaxTree::VERSION return 0 @@ -434,7 +472,7 @@ def run(argv) .glob(pattern) .each do |filepath| if File.readable?(filepath) && - options.ignore_files.none? { File.fnmatch?(_1, filepath) } + options.ignore_files.none? { File.fnmatch?(_1, filepath) } queue << FileItem.new(filepath) end end diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 5162655e..aa133b7f 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -1657,12 +1657,12 @@ class Binary < Node # for older Ruby versions. unless :+.respond_to?(:name) using Module.new { - refine Symbol do - def name - to_s.freeze - end - end - } + refine Symbol do + def name + to_s.freeze + end + end + } end # [untyped] the left-hand side of the expression diff --git a/lib/syntax_tree/search.rb b/lib/syntax_tree/search.rb new file mode 100644 index 00000000..13378c4e --- /dev/null +++ b/lib/syntax_tree/search.rb @@ -0,0 +1,92 @@ +# frozen_string_literal: true + +module SyntaxTree + # Provides an interface for searching for a pattern of nodes against a + # subtree of an AST. + class Search + class UncompilableError < StandardError + end + + attr_reader :matcher + + def initialize(query) + root = SyntaxTree.parse("case nil\nin #{query}\nend") + @matcher = compile(root.statements.body.first.consequent.pattern) + end + + def scan(root) + return to_enum(__method__, root) unless block_given? + queue = [root] + + until queue.empty? + node = queue.shift + next unless node + + yield node if matcher.call(node) + queue += node.child_nodes + end + end + + private + + def compile(pattern) + case pattern + in Binary[left:, operator: :|, right:] + compiled_left = compile(left) + compiled_right = compile(right) + + ->(node) { compiled_left.call(node) || compiled_right.call(node) } + in Const[value:] if SyntaxTree.const_defined?(value) + clazz = SyntaxTree.const_get(value) + + ->(node) { node.is_a?(clazz) } + in Const[value:] if Object.const_defined?(value) + clazz = Object.const_get(value) + + ->(node) { node.is_a?(clazz) } + in ConstPathRef[parent: VarRef[value: Const[value: "SyntaxTree"]]] + compile(pattern.constant) + in HshPtn[constant:, keywords:, keyword_rest: nil] + compiled_constant = compile(constant) + + preprocessed_keywords = + keywords.to_h do |keyword, value| + raise NoMatchingPatternError unless keyword.is_a?(Label) + [keyword.value.chomp(":").to_sym, compile(value)] + end + + compiled_keywords = ->(node) do + deconstructed = node.deconstruct_keys(preprocessed_keywords.keys) + preprocessed_keywords.all? do |keyword, matcher| + matcher.call(deconstructed[keyword]) + end + end + + ->(node) do + compiled_constant.call(node) && compiled_keywords.call(node) + end + in RegexpLiteral[parts: [TStringContent[value:]]] + regexp = /#{value}/ + + ->(attribute) { regexp.match?(attribute) } + in StringLiteral[parts: [TStringContent[value:]]] + ->(attribute) { attribute == value } + in VarRef[value: Const => value] + compile(value) + end + rescue NoMatchingPatternError + raise UncompilableError, <<~ERROR + Syntax Tree was unable to compile the pattern you provided to search + into a usable expression. It failed on the node within the pattern + matching expression represented by: + + #{PP.pp(pattern, +"").chomp} + + Note that not all syntax supported by Ruby's pattern matching syntax is + also supported by Syntax Tree's code search. If you're using some syntax + that you believe should be supported, please open an issue on the GitHub + repository at https://github.com/ruby-syntax-tree/syntax_tree. + ERROR + end + end +end diff --git a/test/cli_test.rb b/test/cli_test.rb index 03293333..1a037918 100644 --- a/test/cli_test.rb +++ b/test/cli_test.rb @@ -94,6 +94,11 @@ def test_match assert_includes(result.stdio, "SyntaxTree::Program") end + def test_search + result = run_cli("search", "VarRef", contents: "Foo + Bar") + assert_equal(2, result.stdio.lines.length) + end + def test_version result = run_cli("version") assert_includes(result.stdio, SyntaxTree::VERSION.to_s) diff --git a/test/search_test.rb b/test/search_test.rb new file mode 100644 index 00000000..6b030e99 --- /dev/null +++ b/test/search_test.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +require_relative "test_helper" + +module SyntaxTree + class SearchTest < Minitest::Test + def test_search_binary_or + root = SyntaxTree.parse("Foo + Bar + 1") + scanned = Search.new("VarRef | Int").scan(root).to_a + + assert_equal 3, scanned.length + assert_equal "1", scanned.min_by { |node| node.class.name }.value + end + + def test_search_const + root = SyntaxTree.parse("Foo + Bar + Baz") + + scanned = Search.new("VarRef").scan(root).to_a + + assert_equal 3, scanned.length + assert_equal %w[Bar Baz Foo], scanned.map { |node| node.value.value }.sort + end + + def test_search_syntax_tree_const + root = SyntaxTree.parse("Foo + Bar + Baz") + + scanned = Search.new("SyntaxTree::VarRef").scan(root).to_a + + assert_equal 3, scanned.length + end + + def test_search_hash_pattern_string + root = SyntaxTree.parse("Foo + Bar + Baz") + + scanned = Search.new("VarRef[value: Const[value: 'Foo']]").scan(root).to_a + + assert_equal 1, scanned.length + assert_equal "Foo", scanned.first.value.value + end + + def test_search_hash_pattern_regexp + root = SyntaxTree.parse("Foo + Bar + Baz") + + query = "VarRef[value: Const[value: /^Ba/]]" + scanned = Search.new(query).scan(root).to_a + + assert_equal 2, scanned.length + assert_equal %w[Bar Baz], scanned.map { |node| node.value.value }.sort + end + end +end From 515f58985001e393b38b3ba679100537169238aa Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 24 Oct 2022 16:55:03 -0400 Subject: [PATCH 154/536] Bump to v4.1.0 --- CHANGELOG.md | 9 ++++++++- Gemfile.lock | 2 +- lib/syntax_tree/version.rb | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46f47ec9..48a3fbca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +## [4.1.0] - 2022-10-24 + +### Added + +- [#180](https://github.com/ruby-syntax-tree/syntax_tree/pull/180) - The new `stree search` CLI command and the corresponding `SyntaxTree::Search` class for searching for a pattern against a given syntax tree. + ## [4.0.2] - 2022-10-19 ### Changed @@ -397,7 +403,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - 🎉 Initial release! 🎉 -[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v4.0.2...HEAD +[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v4.1.0...HEAD +[4.1.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v4.0.2...v4.1.0 [4.0.2]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v4.0.1...v4.0.2 [4.0.1]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v4.0.0...v4.0.1 [4.0.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.6.3...v4.0.0 diff --git a/Gemfile.lock b/Gemfile.lock index 7a227148..195e2226 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - syntax_tree (4.0.2) + syntax_tree (4.1.0) prettier_print (>= 1.0.2) GEM diff --git a/lib/syntax_tree/version.rb b/lib/syntax_tree/version.rb index 98d461df..36843ea9 100644 --- a/lib/syntax_tree/version.rb +++ b/lib/syntax_tree/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SyntaxTree - VERSION = "4.0.2" + VERSION = "4.1.0" end From 78eea51cc15eabfc41ea719cb8ad7ef794a839e4 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 24 Oct 2022 17:00:15 -0400 Subject: [PATCH 155/536] Fix README syntax highlighting --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c8c51445..9b1e681d 100644 --- a/README.md +++ b/README.md @@ -226,7 +226,7 @@ stree search VarRef path/to/file.rb For a file that contains `Foo + Bar` you will receive: -```ruby +``` path/to/file.rb:1:0: Foo + Bar path/to/file.rb:1:6: Foo + Bar ``` From 0c5ebad2511938bb346472adcf5c14c9a5fd6f9f Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 25 Oct 2022 10:28:16 -0400 Subject: [PATCH 156/536] Support even more syntax --- .gitignore | 1 + lib/syntax_tree/search.rb | 57 ++++++++++++++++++++++++++++++++------- 2 files changed, 49 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 2838e82b..69755243 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ /vendor/ test.rb +query.txt diff --git a/lib/syntax_tree/search.rb b/lib/syntax_tree/search.rb index 13378c4e..7326272b 100644 --- a/lib/syntax_tree/search.rb +++ b/lib/syntax_tree/search.rb @@ -29,13 +29,37 @@ def scan(root) private + def combine_and(left, right) + ->(node) { left.call(node) && right.call(node) } + end + + def combine_or(left, right) + ->(node) { left.call(node) || right.call(node) } + end + def compile(pattern) case pattern - in Binary[left:, operator: :|, right:] - compiled_left = compile(left) - compiled_right = compile(right) + in AryPtn[constant:, requireds:, rest: nil, posts: []] + compiled_constant = compile(constant) if constant + + preprocessed = requireds.map { |required| compile(required) } - ->(node) { compiled_left.call(node) || compiled_right.call(node) } + compiled_requireds = ->(node) do + deconstructed = node.deconstruct + + deconstructed.length == preprocessed.length && + preprocessed.zip(deconstructed).all? do |(matcher, value)| + matcher.call(value) + end + end + + if compiled_constant + combine_and(compiled_constant, compiled_requireds) + else + compiled_requireds + end + in Binary[left:, operator: :|, right:] + combine_or(compile(left), compile_right) in Const[value:] if SyntaxTree.const_defined?(value) clazz = SyntaxTree.const_get(value) @@ -46,33 +70,48 @@ def compile(pattern) ->(node) { node.is_a?(clazz) } in ConstPathRef[parent: VarRef[value: Const[value: "SyntaxTree"]]] compile(pattern.constant) + in DynaSymbol[parts: [TStringContent[value:]]] + symbol = value.to_sym + + ->(attribute) { attribute == value } in HshPtn[constant:, keywords:, keyword_rest: nil] compiled_constant = compile(constant) - preprocessed_keywords = + preprocessed = keywords.to_h do |keyword, value| raise NoMatchingPatternError unless keyword.is_a?(Label) [keyword.value.chomp(":").to_sym, compile(value)] end compiled_keywords = ->(node) do - deconstructed = node.deconstruct_keys(preprocessed_keywords.keys) - preprocessed_keywords.all? do |keyword, matcher| + deconstructed = node.deconstruct_keys(preprocessed.keys) + + preprocessed.all? do |keyword, matcher| matcher.call(deconstructed[keyword]) end end - ->(node) do - compiled_constant.call(node) && compiled_keywords.call(node) + if compiled_constant + combine_and(compiled_constant, compiled_keywords) + else + compiled_keywords end in RegexpLiteral[parts: [TStringContent[value:]]] regexp = /#{value}/ ->(attribute) { regexp.match?(attribute) } + in StringLiteral[parts: []] + ->(attribute) { attribute == "" } in StringLiteral[parts: [TStringContent[value:]]] ->(attribute) { attribute == value } + in SymbolLiteral[value:] + symbol = value.value.to_sym + + ->(attribute) { attribute == symbol } in VarRef[value: Const => value] compile(value) + in VarRef[value: Kw[value: "nil"]] + ->(attribute) { attribute.nil? } end rescue NoMatchingPatternError raise UncompilableError, <<~ERROR From bfa8c399cce95fa72406d9f54e672950f336bfea Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 25 Oct 2022 10:58:21 -0400 Subject: [PATCH 157/536] Add an expr CLI command --- lib/syntax_tree.rb | 1 + lib/syntax_tree/cli.rb | 33 +++++++- lib/syntax_tree/pattern.rb | 168 +++++++++++++++++++++++++++++++++++++ lib/syntax_tree/search.rb | 113 +------------------------ test/cli_test.rb | 5 ++ test/search_test.rb | 57 ++++++++----- 6 files changed, 241 insertions(+), 136 deletions(-) create mode 100644 lib/syntax_tree/pattern.rb diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index eef142ff..3979a976 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -21,6 +21,7 @@ require_relative "syntax_tree/visitor/with_environment" require_relative "syntax_tree/parser" +require_relative "syntax_tree/pattern" require_relative "syntax_tree/search" # Syntax Tree is a suite of tools built on top of the internal CRuby parser. It diff --git a/lib/syntax_tree/cli.rb b/lib/syntax_tree/cli.rb index c5eae1bc..b847e059 100644 --- a/lib/syntax_tree/cli.rb +++ b/lib/syntax_tree/cli.rb @@ -188,6 +188,20 @@ def run(item) end end + # An action of the CLI that outputs a pattern-matching Ruby expression that + # would match the first expression of the input given. + class Expr < Action + def run(item) + case item.handler.parse(item.source) + in Program[statements: Statements[body: [expression]]] + puts expression.construct_keys + else + warn("The input to `stree expr` must be a single expression.") + exit(1) + end + end + end + # An action of the CLI that formats the input source and prints it out. class Format < Action def run(item) @@ -219,10 +233,15 @@ class Search < Action def initialize(query) query = File.read(query) if File.readable?(query) - @search = SyntaxTree::Search.new(query) - rescue SyntaxTree::Search::UncompilableError => error - warn(error.message) - exit(1) + pattern = + begin + Pattern.new(query).compile + rescue Pattern::CompilationError => error + warn(error.message) + exit(1) + end + + @search = SyntaxTree::Search.new(pattern) end def run(item) @@ -281,6 +300,10 @@ def run(item) #{Color.bold("stree doc [--plugins=...] [-e SCRIPT] FILE")} Print out the doc tree that would be used to format the given files + #{Color.bold("stree expr [-e SCRIPT] FILE")} + Print out a pattern-matching Ruby expression that would match the first + expression of the given files + #{Color.bold("stree format [--plugins=...] [--print-width=NUMBER] [-e SCRIPT] FILE")} Print out the formatted version of the given files @@ -436,6 +459,8 @@ def run(argv) Debug.new(options) when "doc" Doc.new(options) + when "e", "expr" + Expr.new(options) when "f", "format" Format.new(options) when "help" diff --git a/lib/syntax_tree/pattern.rb b/lib/syntax_tree/pattern.rb new file mode 100644 index 00000000..5ae78a2e --- /dev/null +++ b/lib/syntax_tree/pattern.rb @@ -0,0 +1,168 @@ +# frozen_string_literal: true + +module SyntaxTree + # A pattern is an object that wraps a Ruby pattern matching expression. The + # expression would normally be passed to an `in` clause within a `case` + # expression or a rightward assignment expression. For example, in the + # following snippet: + # + # case node + # in Const[value: "SyntaxTree"] + # end + # + # the pattern is the `Const[value: "SyntaxTree"]` expression. Within Syntax + # Tree, every node generates these kinds of expressions using the + # #construct_keys method. + # + # The pattern gets compiled into an object that responds to call by running + # the #compile method. This method itself will run back through Syntax Tree to + # parse the expression into a tree, then walk the tree to generate the + # necessary callable objects. For example, if you wanted to compile the + # expression above into a callable, you would: + # + # callable = SyntaxTree::Pattern.new("Const[value: 'SyntaxTree']").compile + # callable.call(node) + # + # The callable object returned by #compile is guaranteed to respond to #call + # with a single argument, which is the node to match against. It also is + # guaranteed to respond to #===, which means it itself can be used in a `case` + # expression, as in: + # + # case node + # when callable + # end + # + # If the query given to the initializer cannot be compiled into a valid + # matcher (either because of a syntax error or because it is using syntax we + # do not yet support) then a SyntaxTree::Pattern::CompilationError will be + # raised. + class Pattern + class CompilationError < StandardError + def initialize(repr) + super(<<~ERROR) + Syntax Tree was unable to compile the pattern you provided to search + into a usable expression. It failed on to understand the node + represented by: + + #{repr} + + Note that not all syntax supported by Ruby's pattern matching syntax + is also supported by Syntax Tree's code search. If you're using some + syntax that you believe should be supported, please open an issue on + GitHub at https://github.com/ruby-syntax-tree/syntax_tree/issues/new. + ERROR + end + end + + attr_reader :query + + def initialize(query) + @query = query + end + + def compile + program = + begin + SyntaxTree.parse("case nil\nin #{query}\nend") + rescue Parser::ParseError + raise CompilationError, query + end + + compile_node(program.statements.body.first.consequent.pattern) + end + + private + + def combine_and(left, right) + ->(node) { left.call(node) && right.call(node) } + end + + def combine_or(left, right) + ->(node) { left.call(node) || right.call(node) } + end + + def compile_node(node) + case node + in AryPtn[constant:, requireds:, rest: nil, posts: []] + compiled_constant = compile_node(constant) if constant + + preprocessed = requireds.map { |required| compile_node(required) } + + compiled_requireds = ->(node) do + deconstructed = node.deconstruct + + deconstructed.length == preprocessed.length && + preprocessed.zip(deconstructed).all? do |(matcher, value)| + matcher.call(value) + end + end + + if compiled_constant + combine_and(compiled_constant, compiled_requireds) + else + compiled_requireds + end + in Binary[left:, operator: :|, right:] + combine_or(compile_node(left), compile_node(right)) + in Const[value:] if SyntaxTree.const_defined?(value) + clazz = SyntaxTree.const_get(value) + + ->(node) { node.is_a?(clazz) } + in Const[value:] if Object.const_defined?(value) + clazz = Object.const_get(value) + + ->(node) { node.is_a?(clazz) } + in ConstPathRef[parent: VarRef[value: Const[value: "SyntaxTree"]], constant:] + compile_node(constant) + in DynaSymbol[parts: []] + symbol = "".to_sym + + ->(node) { node == symbol } + in DynaSymbol[parts: [TStringContent[value:]]] + symbol = value.to_sym + + ->(attribute) { attribute == value } + in HshPtn[constant:, keywords:, keyword_rest: nil] + compiled_constant = compile_node(constant) + + preprocessed = + keywords.to_h do |keyword, value| + raise NoMatchingPatternError unless keyword.is_a?(Label) + [keyword.value.chomp(":").to_sym, compile_node(value)] + end + + compiled_keywords = ->(node) do + deconstructed = node.deconstruct_keys(preprocessed.keys) + + preprocessed.all? do |keyword, matcher| + matcher.call(deconstructed[keyword]) + end + end + + if compiled_constant + combine_and(compiled_constant, compiled_keywords) + else + compiled_keywords + end + in RegexpLiteral[parts: [TStringContent[value:]]] + regexp = /#{value}/ + + ->(attribute) { regexp.match?(attribute) } + in StringLiteral[parts: []] + ->(attribute) { attribute == "" } + in StringLiteral[parts: [TStringContent[value:]]] + ->(attribute) { attribute == value } + in SymbolLiteral[value:] + symbol = value.value.to_sym + + ->(attribute) { attribute == symbol } + in VarRef[value: Const => value] + compile_node(value) + in VarRef[value: Kw[value: "nil"]] + ->(attribute) { attribute.nil? } + end + rescue NoMatchingPatternError + raise CompilationError, PP.pp(node, +"").chomp + end + end +end diff --git a/lib/syntax_tree/search.rb b/lib/syntax_tree/search.rb index 7326272b..9fd52ba1 100644 --- a/lib/syntax_tree/search.rb +++ b/lib/syntax_tree/search.rb @@ -4,14 +4,10 @@ module SyntaxTree # Provides an interface for searching for a pattern of nodes against a # subtree of an AST. class Search - class UncompilableError < StandardError - end - - attr_reader :matcher + attr_reader :pattern - def initialize(query) - root = SyntaxTree.parse("case nil\nin #{query}\nend") - @matcher = compile(root.statements.body.first.consequent.pattern) + def initialize(pattern) + @pattern = pattern end def scan(root) @@ -22,110 +18,9 @@ def scan(root) node = queue.shift next unless node - yield node if matcher.call(node) + yield node if pattern.call(node) queue += node.child_nodes end end - - private - - def combine_and(left, right) - ->(node) { left.call(node) && right.call(node) } - end - - def combine_or(left, right) - ->(node) { left.call(node) || right.call(node) } - end - - def compile(pattern) - case pattern - in AryPtn[constant:, requireds:, rest: nil, posts: []] - compiled_constant = compile(constant) if constant - - preprocessed = requireds.map { |required| compile(required) } - - compiled_requireds = ->(node) do - deconstructed = node.deconstruct - - deconstructed.length == preprocessed.length && - preprocessed.zip(deconstructed).all? do |(matcher, value)| - matcher.call(value) - end - end - - if compiled_constant - combine_and(compiled_constant, compiled_requireds) - else - compiled_requireds - end - in Binary[left:, operator: :|, right:] - combine_or(compile(left), compile_right) - in Const[value:] if SyntaxTree.const_defined?(value) - clazz = SyntaxTree.const_get(value) - - ->(node) { node.is_a?(clazz) } - in Const[value:] if Object.const_defined?(value) - clazz = Object.const_get(value) - - ->(node) { node.is_a?(clazz) } - in ConstPathRef[parent: VarRef[value: Const[value: "SyntaxTree"]]] - compile(pattern.constant) - in DynaSymbol[parts: [TStringContent[value:]]] - symbol = value.to_sym - - ->(attribute) { attribute == value } - in HshPtn[constant:, keywords:, keyword_rest: nil] - compiled_constant = compile(constant) - - preprocessed = - keywords.to_h do |keyword, value| - raise NoMatchingPatternError unless keyword.is_a?(Label) - [keyword.value.chomp(":").to_sym, compile(value)] - end - - compiled_keywords = ->(node) do - deconstructed = node.deconstruct_keys(preprocessed.keys) - - preprocessed.all? do |keyword, matcher| - matcher.call(deconstructed[keyword]) - end - end - - if compiled_constant - combine_and(compiled_constant, compiled_keywords) - else - compiled_keywords - end - in RegexpLiteral[parts: [TStringContent[value:]]] - regexp = /#{value}/ - - ->(attribute) { regexp.match?(attribute) } - in StringLiteral[parts: []] - ->(attribute) { attribute == "" } - in StringLiteral[parts: [TStringContent[value:]]] - ->(attribute) { attribute == value } - in SymbolLiteral[value:] - symbol = value.value.to_sym - - ->(attribute) { attribute == symbol } - in VarRef[value: Const => value] - compile(value) - in VarRef[value: Kw[value: "nil"]] - ->(attribute) { attribute.nil? } - end - rescue NoMatchingPatternError - raise UncompilableError, <<~ERROR - Syntax Tree was unable to compile the pattern you provided to search - into a usable expression. It failed on the node within the pattern - matching expression represented by: - - #{PP.pp(pattern, +"").chomp} - - Note that not all syntax supported by Ruby's pattern matching syntax is - also supported by Syntax Tree's code search. If you're using some syntax - that you believe should be supported, please open an issue on the GitHub - repository at https://github.com/ruby-syntax-tree/syntax_tree. - ERROR - end end end diff --git a/test/cli_test.rb b/test/cli_test.rb index 1a037918..b4ef0afc 100644 --- a/test/cli_test.rb +++ b/test/cli_test.rb @@ -79,6 +79,11 @@ def test_doc assert_includes(result.stdio, "test") end + def test_expr + result = run_cli("expr") + assert_includes(result.stdio, "SyntaxTree::Ident") + end + def test_format result = run_cli("format") assert_equal("test\n", result.stdio) diff --git a/test/search_test.rb b/test/search_test.rb index 6b030e99..314142e3 100644 --- a/test/search_test.rb +++ b/test/search_test.rb @@ -5,47 +5,58 @@ module SyntaxTree class SearchTest < Minitest::Test def test_search_binary_or - root = SyntaxTree.parse("Foo + Bar + 1") - scanned = Search.new("VarRef | Int").scan(root).to_a + results = search("Foo + Bar + 1", "VarRef | Int") - assert_equal 3, scanned.length - assert_equal "1", scanned.min_by { |node| node.class.name }.value + assert_equal 3, results.length + assert_equal "1", results.min_by { |node| node.class.name }.value end def test_search_const - root = SyntaxTree.parse("Foo + Bar + Baz") + results = search("Foo + Bar + Baz", "VarRef") - scanned = Search.new("VarRef").scan(root).to_a - - assert_equal 3, scanned.length - assert_equal %w[Bar Baz Foo], scanned.map { |node| node.value.value }.sort + assert_equal 3, results.length + assert_equal %w[Bar Baz Foo], results.map { |node| node.value.value }.sort end def test_search_syntax_tree_const - root = SyntaxTree.parse("Foo + Bar + Baz") - - scanned = Search.new("SyntaxTree::VarRef").scan(root).to_a + results = search("Foo + Bar + Baz", "SyntaxTree::VarRef") - assert_equal 3, scanned.length + assert_equal 3, results.length end def test_search_hash_pattern_string - root = SyntaxTree.parse("Foo + Bar + Baz") - - scanned = Search.new("VarRef[value: Const[value: 'Foo']]").scan(root).to_a + results = search("Foo + Bar + Baz", "VarRef[value: Const[value: 'Foo']]") - assert_equal 1, scanned.length - assert_equal "Foo", scanned.first.value.value + assert_equal 1, results.length + assert_equal "Foo", results.first.value.value end def test_search_hash_pattern_regexp - root = SyntaxTree.parse("Foo + Bar + Baz") + results = search("Foo + Bar + Baz", "VarRef[value: Const[value: /^Ba/]]") + + assert_equal 2, results.length + assert_equal %w[Bar Baz], results.map { |node| node.value.value }.sort + end + + def test_search_string_empty + results = search("''", "StringLiteral[parts: []]") + + assert_equal 1, results.length + end + + def test_search_symbol_empty + results = search(":''", "DynaSymbol[parts: []]") + + assert_equal 1, results.length + end + + private - query = "VarRef[value: Const[value: /^Ba/]]" - scanned = Search.new(query).scan(root).to_a + def search(source, query) + pattern = Pattern.new(query).compile + program = SyntaxTree.parse(source) - assert_equal 2, scanned.length - assert_equal %w[Bar Baz], scanned.map { |node| node.value.value }.sort + Search.new(pattern).scan(program).to_a end end end From 02b3c496e816e1b2a93e6af287131c4f50daa766 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 25 Oct 2022 12:38:55 -0400 Subject: [PATCH 158/536] Documentation for search --- README.md | 24 +++++++++++++++++++ lib/syntax_tree.rb | 6 +++++ lib/syntax_tree/pattern.rb | 48 +++++++++++++++++++++----------------- 3 files changed, 56 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 9b1e681d..368c9361 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ It is built with only standard library dependencies. It additionally ships with - [CLI](#cli) - [ast](#ast) - [check](#check) + - [expr](#expr) - [format](#format) - [json](#json) - [match](#match) @@ -26,6 +27,7 @@ It is built with only standard library dependencies. It additionally ships with - [SyntaxTree.read(filepath)](#syntaxtreereadfilepath) - [SyntaxTree.parse(source)](#syntaxtreeparsesource) - [SyntaxTree.format(source)](#syntaxtreeformatsource) + - [SyntaxTree.search(source, query, &block)](#syntaxtreesearchsource-query-block) - [Nodes](#nodes) - [child_nodes](#child_nodes) - [Pattern matching](#pattern-matching) @@ -129,6 +131,24 @@ To change the print width that you are checking against, specify the `--print-wi stree check --print-width=100 path/to/file.rb ``` +### expr + +This command will output a Ruby case-match expression that would match correctly against the first expression of the input. + +```sh +stree expr path/to/file.rb +``` + +For a file that contains `1 + 1`, you will receive: + +```ruby +SyntaxTree::Binary[ + left: SyntaxTree::Int[value: "1"], + operator: :+, + right: SyntaxTree::Int[value: "1"] +] +``` + ### format This command will output the formatted version of each of the listed files. Importantly, it will not write that content back to the source files. It is meant to display the formatted version only. @@ -312,6 +332,10 @@ This function takes an input string containing Ruby code and returns the syntax This function takes an input string containing Ruby code, parses it into its underlying syntax tree, and formats it back out to a string. You can optionally pass a second argument to this method as well that is the maximum width to print. It defaults to `80`. +### SyntaxTree.search(source, query, &block) + +This function takes an input string containing Ruby code, an input string containing a valid Ruby `in` clause expression that can be used to match against nodes in the tree (can be generated using `stree expr`, `stree match`, or `Node#construct_keys`), and a block. Each node that matches the given query will be yielded to the block. The block will receive the node as its only argument. + ## Nodes There are many different node types in the syntax tree. They are meant to be treated as immutable structs containing links to child nodes with minimal logic contained within their implementation. However, for the most part they all respond to a certain set of APIs, listed below. diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index 3979a976..df2f43a9 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -75,4 +75,10 @@ def self.read(filepath) File.read(filepath, encoding: encoding) end + + # Searches through the given source using the given pattern and yields each + # node in the tree that matches the pattern to the given block. + def self.search(source, query, &block) + Search.new(Pattern.new(query).compile).scan(parse(source), &block) + end end diff --git a/lib/syntax_tree/pattern.rb b/lib/syntax_tree/pattern.rb index 5ae78a2e..aa558361 100644 --- a/lib/syntax_tree/pattern.rb +++ b/lib/syntax_tree/pattern.rb @@ -37,6 +37,8 @@ module SyntaxTree # do not yet support) then a SyntaxTree::Pattern::CompilationError will be # raised. class Pattern + # Raised when the query given to a pattern is either invalid Ruby syntax or + # is using syntax that we don't yet support. class CompilationError < StandardError def initialize(repr) super(<<~ERROR) @@ -76,27 +78,27 @@ def compile def combine_and(left, right) ->(node) { left.call(node) && right.call(node) } end - + def combine_or(left, right) ->(node) { left.call(node) || right.call(node) } end - def compile_node(node) - case node + def compile_node(root) + case root in AryPtn[constant:, requireds:, rest: nil, posts: []] compiled_constant = compile_node(constant) if constant - + preprocessed = requireds.map { |required| compile_node(required) } - + compiled_requireds = ->(node) do deconstructed = node.deconstruct - + deconstructed.length == preprocessed.length && - preprocessed.zip(deconstructed).all? do |(matcher, value)| - matcher.call(value) - end + preprocessed + .zip(deconstructed) + .all? { |(matcher, value)| matcher.call(value) } end - + if compiled_constant combine_and(compiled_constant, compiled_requireds) else @@ -106,39 +108,41 @@ def compile_node(node) combine_or(compile_node(left), compile_node(right)) in Const[value:] if SyntaxTree.const_defined?(value) clazz = SyntaxTree.const_get(value) - + ->(node) { node.is_a?(clazz) } in Const[value:] if Object.const_defined?(value) clazz = Object.const_get(value) - + ->(node) { node.is_a?(clazz) } - in ConstPathRef[parent: VarRef[value: Const[value: "SyntaxTree"]], constant:] + in ConstPathRef[ + parent: VarRef[value: Const[value: "SyntaxTree"]], constant: + ] compile_node(constant) in DynaSymbol[parts: []] - symbol = "".to_sym + symbol = :"" ->(node) { node == symbol } in DynaSymbol[parts: [TStringContent[value:]]] symbol = value.to_sym - + ->(attribute) { attribute == value } in HshPtn[constant:, keywords:, keyword_rest: nil] compiled_constant = compile_node(constant) - + preprocessed = keywords.to_h do |keyword, value| raise NoMatchingPatternError unless keyword.is_a?(Label) [keyword.value.chomp(":").to_sym, compile_node(value)] end - + compiled_keywords = ->(node) do deconstructed = node.deconstruct_keys(preprocessed.keys) - + preprocessed.all? do |keyword, matcher| matcher.call(deconstructed[keyword]) end end - + if compiled_constant combine_and(compiled_constant, compiled_keywords) else @@ -146,7 +150,7 @@ def compile_node(node) end in RegexpLiteral[parts: [TStringContent[value:]]] regexp = /#{value}/ - + ->(attribute) { regexp.match?(attribute) } in StringLiteral[parts: []] ->(attribute) { attribute == "" } @@ -154,7 +158,7 @@ def compile_node(node) ->(attribute) { attribute == value } in SymbolLiteral[value:] symbol = value.value.to_sym - + ->(attribute) { attribute == symbol } in VarRef[value: Const => value] compile_node(value) @@ -162,7 +166,7 @@ def compile_node(node) ->(attribute) { attribute.nil? } end rescue NoMatchingPatternError - raise CompilationError, PP.pp(node, +"").chomp + raise CompilationError, PP.pp(root, +"").chomp end end end From bd9a1c3d58026f84f57d8b012f9a7963e7de30e8 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 25 Oct 2022 13:23:30 -0400 Subject: [PATCH 159/536] Bump to v4.2.0 --- CHANGELOG.md | 14 +++++++++++++- Gemfile.lock | 2 +- lib/syntax_tree/version.rb | 2 +- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 48a3fbca..bbaf044e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,17 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +## [4.2.0] - 2022-10-25 + +### Added + +- [#182](https://github.com/ruby-syntax-tree/syntax_tree/pull/182) - The new `stree expr` CLI command will function similarly to the `stree match` CLI command except that it only outputs the first expression of the program. +- [#182](https://github.com/ruby-syntax-tree/syntax_tree/pull/182) - Added the `SyntaxTree::Pattern` class for compiling `in` expressions into procs. + +### Changed + +- [#182](https://github.com/ruby-syntax-tree/syntax_tree/pull/182) - Much more syntax is now supported by the search command. + ## [4.1.0] - 2022-10-24 ### Added @@ -403,7 +414,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - 🎉 Initial release! 🎉 -[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v4.1.0...HEAD +[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v4.2.0...HEAD +[4.2.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v4.1.0...v4.2.0 [4.1.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v4.0.2...v4.1.0 [4.0.2]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v4.0.1...v4.0.2 [4.0.1]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v4.0.0...v4.0.1 diff --git a/Gemfile.lock b/Gemfile.lock index 195e2226..339de160 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - syntax_tree (4.1.0) + syntax_tree (4.2.0) prettier_print (>= 1.0.2) GEM diff --git a/lib/syntax_tree/version.rb b/lib/syntax_tree/version.rb index 36843ea9..0b68a850 100644 --- a/lib/syntax_tree/version.rb +++ b/lib/syntax_tree/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SyntaxTree - VERSION = "4.1.0" + VERSION = "4.2.0" end From 9cef7af6611b9d306f43f5b54fa381b61cc29ba4 Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Fri, 28 Oct 2022 14:11:55 +0200 Subject: [PATCH 160/536] Make the test suite pass on TruffleRuby --- Rakefile | 8 +++++++- lib/syntax_tree/cli.rb | 6 +++--- test/cli_test.rb | 2 ++ test/location_test.rb | 12 ++++-------- test/node_test.rb | 7 +++---- test/test_helper.rb | 22 +++++++++++++--------- 6 files changed, 32 insertions(+), 25 deletions(-) diff --git a/Rakefile b/Rakefile index 4973d45e..6de81bd8 100644 --- a/Rakefile +++ b/Rakefile @@ -7,7 +7,13 @@ require "syntax_tree/rake_tasks" Rake::TestTask.new(:test) do |t| t.libs << "test" t.libs << "lib" - t.test_files = FileList["test/**/*_test.rb"] + test_files = FileList["test/**/*_test.rb"] + if RUBY_ENGINE == "truffleruby" + # language_server.rb uses pattern matching + test_files -= FileList["test/language_server/*_test.rb"] + test_files -= FileList["test/language_server_test.rb"] + end + t.test_files = test_files end task default: :test diff --git a/lib/syntax_tree/cli.rb b/lib/syntax_tree/cli.rb index b847e059..be0bb793 100644 --- a/lib/syntax_tree/cli.rb +++ b/lib/syntax_tree/cli.rb @@ -192,9 +192,9 @@ def run(item) # would match the first expression of the input given. class Expr < Action def run(item) - case item.handler.parse(item.source) - in Program[statements: Statements[body: [expression]]] - puts expression.construct_keys + program = item.handler.parse(item.source) + if Program === program and expressions = program.statements.body and expressions.size == 1 + puts expressions.first.construct_keys else warn("The input to `stree expr` must be a single expression.") exit(1) diff --git a/test/cli_test.rb b/test/cli_test.rb index b4ef0afc..b5316d7f 100644 --- a/test/cli_test.rb +++ b/test/cli_test.rb @@ -148,6 +148,7 @@ def test_inline_script end def test_multiple_inline_scripts + skip if RUBY_ENGINE == "truffleruby" # Relies on a thread-safe StringIO stdio, = capture_io { SyntaxTree::CLI.run(%w[format -e 1+1 -e 2+2]) } assert_equal(["1 + 1", "2 + 2"], stdio.split("\n").sort) end @@ -172,6 +173,7 @@ def test_plugins def test_language_server prev_stdin = $stdin prev_stdout = $stdout + skip unless SUPPORTS_PATTERN_MATCHING request = { method: "shutdown" }.merge(jsonrpc: "2.0").to_json $stdin = diff --git a/test/location_test.rb b/test/location_test.rb index 2a697281..26831fb1 100644 --- a/test/location_test.rb +++ b/test/location_test.rb @@ -14,19 +14,15 @@ def test_lines def test_deconstruct location = Location.fixed(line: 1, char: 0, column: 0) - case location - in [start_line, 0, 0, *] - assert_equal(1, start_line) - end + assert_equal(1, location.start_line) + assert_equal(0, location.start_char) + assert_equal(0, location.start_column) end def test_deconstruct_keys location = Location.fixed(line: 1, char: 0, column: 0) - case location - in start_line: - assert_equal(1, start_line) - end + assert_equal(1, location.start_line) end end end diff --git a/test/node_test.rb b/test/node_test.rb index 1a5af125..ce26f9ea 100644 --- a/test/node_test.rb +++ b/test/node_test.rb @@ -759,10 +759,9 @@ def test_program program = parser.parse refute(parser.error?) - case program - in statements: { body: [statement] } - assert_kind_of(VCall, statement) - end + statements = program.statements.body + assert_equal 1, statements.size + assert_kind_of(VCall, statements.first) json = JSON.parse(program.to_json) io = StringIO.new diff --git a/test/test_helper.rb b/test/test_helper.rb index 80e514f0..c421d8ee 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -17,6 +17,8 @@ require "pp" require "minitest/autorun" +SUPPORTS_PATTERN_MATCHING = RUBY_ENGINE != "truffleruby" + module SyntaxTree module Assertions class Recorder @@ -67,15 +69,17 @@ def assert_syntax_tree(node) refute_includes(json, "#<") assert_equal(type, JSON.parse(json)["type"]) - # Get a match expression from the node, then assert that it can in fact - # match the node. - # rubocop:disable all - assert(eval(<<~RUBY)) - case node - in #{node.construct_keys} - true - end - RUBY + if SUPPORTS_PATTERN_MATCHING + # Get a match expression from the node, then assert that it can in fact + # match the node. + # rubocop:disable all + assert(eval(<<~RUBY)) + case node + in #{node.construct_keys} + true + end + RUBY + end end Minitest::Test.include(self) From 0c8b5e3215d3c2a8f882c13c71f183f125f999db Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Fri, 28 Oct 2022 14:11:10 +0200 Subject: [PATCH 161/536] Replace pattern matching in lib/syntax_tree/pattern.rb * For performance and compatibility with Rubies not supporting it yet. --- lib/syntax_tree/pattern.rb | 61 +++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 31 deletions(-) diff --git a/lib/syntax_tree/pattern.rb b/lib/syntax_tree/pattern.rb index aa558361..15b6a0a9 100644 --- a/lib/syntax_tree/pattern.rb +++ b/lib/syntax_tree/pattern.rb @@ -84,11 +84,11 @@ def combine_or(left, right) end def compile_node(root) - case root - in AryPtn[constant:, requireds:, rest: nil, posts: []] + if AryPtn === root and root.rest.nil? and root.posts.empty? + constant = root.constant compiled_constant = compile_node(constant) if constant - preprocessed = requireds.map { |required| compile_node(required) } + preprocessed = root.requireds.map { |required| compile_node(required) } compiled_requireds = ->(node) do deconstructed = node.deconstruct @@ -104,34 +104,32 @@ def compile_node(root) else compiled_requireds end - in Binary[left:, operator: :|, right:] - combine_or(compile_node(left), compile_node(right)) - in Const[value:] if SyntaxTree.const_defined?(value) - clazz = SyntaxTree.const_get(value) + elsif Binary === root and root.operator == :| + combine_or(compile_node(root.left), compile_node(root.right)) + elsif Const === root and SyntaxTree.const_defined?(root.value) + clazz = SyntaxTree.const_get(root.value) ->(node) { node.is_a?(clazz) } - in Const[value:] if Object.const_defined?(value) - clazz = Object.const_get(value) + elsif Const === root and Object.const_defined?(root.value) + clazz = Object.const_get(root.value) ->(node) { node.is_a?(clazz) } - in ConstPathRef[ - parent: VarRef[value: Const[value: "SyntaxTree"]], constant: - ] - compile_node(constant) - in DynaSymbol[parts: []] + elsif ConstPathRef === root and VarRef === root.parent and Const === root.parent.value and root.parent.value.value == "SyntaxTree" + compile_node(root.constant) + elsif DynaSymbol === root and root.parts.empty? symbol = :"" ->(node) { node == symbol } - in DynaSymbol[parts: [TStringContent[value:]]] - symbol = value.to_sym + elsif DynaSymbol === root and parts = root.parts and parts.size == 1 and TStringContent === parts[0] + symbol = parts[0].value.to_sym ->(attribute) { attribute == value } - in HshPtn[constant:, keywords:, keyword_rest: nil] - compiled_constant = compile_node(constant) + elsif HshPtn === root and root.keyword_rest.nil? + compiled_constant = compile_node(root.constant) preprocessed = - keywords.to_h do |keyword, value| - raise NoMatchingPatternError unless keyword.is_a?(Label) + root.keywords.to_h do |keyword, value| + raise CompilationError, PP.pp(root, +"").chomp unless keyword.is_a?(Label) [keyword.value.chomp(":").to_sym, compile_node(value)] end @@ -148,25 +146,26 @@ def compile_node(root) else compiled_keywords end - in RegexpLiteral[parts: [TStringContent[value:]]] - regexp = /#{value}/ + elsif RegexpLiteral === root and parts = root.parts and parts.size == 1 and TStringContent === parts[0] + regexp = /#{parts[0].value}/ ->(attribute) { regexp.match?(attribute) } - in StringLiteral[parts: []] + elsif StringLiteral === root and root.parts.empty? ->(attribute) { attribute == "" } - in StringLiteral[parts: [TStringContent[value:]]] + elsif StringLiteral === root and parts = root.parts and parts.size == 1 and TStringContent === parts[0] + value = parts[0].value ->(attribute) { attribute == value } - in SymbolLiteral[value:] - symbol = value.value.to_sym + elsif SymbolLiteral === root + symbol = root.value.value.to_sym ->(attribute) { attribute == symbol } - in VarRef[value: Const => value] - compile_node(value) - in VarRef[value: Kw[value: "nil"]] + elsif VarRef === root and Const === root.value + compile_node(root.value) + elsif VarRef === root and Kw === root.value and root.value.value.nil? ->(attribute) { attribute.nil? } + else + raise CompilationError, PP.pp(root, +"").chomp end - rescue NoMatchingPatternError - raise CompilationError, PP.pp(root, +"").chomp end end end From 12f3e8691f192a2e20baad867d46f0c2e5832011 Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Fri, 28 Oct 2022 15:05:27 +0200 Subject: [PATCH 162/536] Fix test for DynaSymbol --- lib/syntax_tree/pattern.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/syntax_tree/pattern.rb b/lib/syntax_tree/pattern.rb index 15b6a0a9..f56e8a1b 100644 --- a/lib/syntax_tree/pattern.rb +++ b/lib/syntax_tree/pattern.rb @@ -123,7 +123,7 @@ def compile_node(root) elsif DynaSymbol === root and parts = root.parts and parts.size == 1 and TStringContent === parts[0] symbol = parts[0].value.to_sym - ->(attribute) { attribute == value } + ->(node) { node == symbol } elsif HshPtn === root and root.keyword_rest.nil? compiled_constant = compile_node(root.constant) From 48c659cfa5fe9b4c82939e4196e5aac8bf97f5e9 Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Fri, 28 Oct 2022 15:06:21 +0200 Subject: [PATCH 163/536] Add TruffleRuby in CI --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d35471fa..7bbdedc7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,6 +12,7 @@ jobs: - '3.0' - '3.1' - head + - truffleruby-head name: CI runs-on: ubuntu-latest env: From ab6e669635df9cfea52bdf98d8f2d72efb1099c0 Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Fri, 28 Oct 2022 15:19:35 +0200 Subject: [PATCH 164/536] Run in --verbose mode to see progress in CI --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7bbdedc7..9f95cc9d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -17,6 +17,7 @@ jobs: runs-on: ubuntu-latest env: CI: true + TESTOPTS: --verbose steps: - uses: actions/checkout@master - uses: ruby/setup-ruby@v1 From 42bb2a44fc81a3e36c0506fc7d842258eae7a641 Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Fri, 28 Oct 2022 15:21:05 +0200 Subject: [PATCH 165/536] Run bundle exec rake stree:write --- lib/syntax_tree/cli.rb | 3 ++- lib/syntax_tree/pattern.rb | 17 ++++++++++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/lib/syntax_tree/cli.rb b/lib/syntax_tree/cli.rb index be0bb793..518b58a6 100644 --- a/lib/syntax_tree/cli.rb +++ b/lib/syntax_tree/cli.rb @@ -193,7 +193,8 @@ def run(item) class Expr < Action def run(item) program = item.handler.parse(item.source) - if Program === program and expressions = program.statements.body and expressions.size == 1 + if Program === program and expressions = program.statements.body and + expressions.size == 1 puts expressions.first.construct_keys else warn("The input to `stree expr` must be a single expression.") diff --git a/lib/syntax_tree/pattern.rb b/lib/syntax_tree/pattern.rb index f56e8a1b..c612c4ea 100644 --- a/lib/syntax_tree/pattern.rb +++ b/lib/syntax_tree/pattern.rb @@ -114,13 +114,16 @@ def compile_node(root) clazz = Object.const_get(root.value) ->(node) { node.is_a?(clazz) } - elsif ConstPathRef === root and VarRef === root.parent and Const === root.parent.value and root.parent.value.value == "SyntaxTree" + elsif ConstPathRef === root and VarRef === root.parent and + Const === root.parent.value and + root.parent.value.value == "SyntaxTree" compile_node(root.constant) elsif DynaSymbol === root and root.parts.empty? symbol = :"" ->(node) { node == symbol } - elsif DynaSymbol === root and parts = root.parts and parts.size == 1 and TStringContent === parts[0] + elsif DynaSymbol === root and parts = root.parts and parts.size == 1 and + TStringContent === parts[0] symbol = parts[0].value.to_sym ->(node) { node == symbol } @@ -129,7 +132,9 @@ def compile_node(root) preprocessed = root.keywords.to_h do |keyword, value| - raise CompilationError, PP.pp(root, +"").chomp unless keyword.is_a?(Label) + unless keyword.is_a?(Label) + raise CompilationError, PP.pp(root, +"").chomp + end [keyword.value.chomp(":").to_sym, compile_node(value)] end @@ -146,13 +151,15 @@ def compile_node(root) else compiled_keywords end - elsif RegexpLiteral === root and parts = root.parts and parts.size == 1 and TStringContent === parts[0] + elsif RegexpLiteral === root and parts = root.parts and + parts.size == 1 and TStringContent === parts[0] regexp = /#{parts[0].value}/ ->(attribute) { regexp.match?(attribute) } elsif StringLiteral === root and root.parts.empty? ->(attribute) { attribute == "" } - elsif StringLiteral === root and parts = root.parts and parts.size == 1 and TStringContent === parts[0] + elsif StringLiteral === root and parts = root.parts and + parts.size == 1 and TStringContent === parts[0] value = parts[0].value ->(attribute) { attribute == value } elsif SymbolLiteral === root From 570a2d134d74024ca17d557e843d5b6720c1e9dd Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Fri, 28 Oct 2022 15:33:54 +0200 Subject: [PATCH 166/536] Do not run idempotency_test.rb on TruffleRuby * It's too slow as it includes all Ruby files from stdlib, gems and more. --- test/idempotency_test.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/idempotency_test.rb b/test/idempotency_test.rb index 1f560db2..76116572 100644 --- a/test/idempotency_test.rb +++ b/test/idempotency_test.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -return unless ENV["CI"] +return unless ENV["CI"] and RUBY_ENGINE != "truffleruby" require_relative "test_helper" module SyntaxTree From b4f41326d9e2cac0af6ac4e222755ee8257e24be Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 28 Oct 2022 10:18:45 -0400 Subject: [PATCH 167/536] Fix up style violations --- .rubocop.yml | 3 + lib/syntax_tree/cli.rb | 4 +- lib/syntax_tree/pattern.rb | 271 ++++++++++++++++++++++++++----------- test/idempotency_test.rb | 2 +- 4 files changed, 196 insertions(+), 84 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index c0892d8a..27efc39a 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -46,6 +46,9 @@ Naming/MethodParameterName: Naming/RescuedExceptionsVariableName: PreferredName: error +Style/CaseEquality: + Enabled: false + Style/ExplicitBlockArgument: Enabled: false diff --git a/lib/syntax_tree/cli.rb b/lib/syntax_tree/cli.rb index 518b58a6..62e8ab68 100644 --- a/lib/syntax_tree/cli.rb +++ b/lib/syntax_tree/cli.rb @@ -193,8 +193,8 @@ def run(item) class Expr < Action def run(item) program = item.handler.parse(item.source) - if Program === program and expressions = program.statements.body and - expressions.size == 1 + + if (expressions = program.statements.body) && expressions.size == 1 puts expressions.first.construct_keys else warn("The input to `stree expr` must be a single expression.") diff --git a/lib/syntax_tree/pattern.rb b/lib/syntax_tree/pattern.rb index c612c4ea..439d573f 100644 --- a/lib/syntax_tree/pattern.rb +++ b/lib/syntax_tree/pattern.rb @@ -75,103 +75,212 @@ def compile private + # Shortcut for combining two procs into one that returns true if both return + # true. def combine_and(left, right) - ->(node) { left.call(node) && right.call(node) } + ->(other) { left.call(other) && right.call(other) } end + # Shortcut for combining two procs into one that returns true if either + # returns true. def combine_or(left, right) - ->(node) { left.call(node) || right.call(node) } + ->(other) { left.call(other) || right.call(other) } end - def compile_node(root) - if AryPtn === root and root.rest.nil? and root.posts.empty? - constant = root.constant - compiled_constant = compile_node(constant) if constant + # Raise an error because the given node is not supported. + def compile_error(node) + raise CompilationError, PP.pp(node, +"").chomp + end - preprocessed = root.requireds.map { |required| compile_node(required) } + # There are a couple of nodes (string literals, dynamic symbols, and regexp) + # that contain list of parts. This can include plain string content, + # interpolated expressions, and interpolated variables. We only support + # plain string content, so this method will extract out the plain string + # content if it is the only element in the list. + def extract_string(node) + parts = node.parts - compiled_requireds = ->(node) do - deconstructed = node.deconstruct + if parts.length == 1 && (part = parts.first) && part.is_a?(TStringContent) + part.value + end + end - deconstructed.length == preprocessed.length && - preprocessed - .zip(deconstructed) - .all? { |(matcher, value)| matcher.call(value) } - end + # in [foo, bar, baz] + def compile_aryptn(node) + compile_error(node) if !node.rest.nil? || node.posts.any? - if compiled_constant - combine_and(compiled_constant, compiled_requireds) - else - compiled_requireds - end - elsif Binary === root and root.operator == :| - combine_or(compile_node(root.left), compile_node(root.right)) - elsif Const === root and SyntaxTree.const_defined?(root.value) - clazz = SyntaxTree.const_get(root.value) - - ->(node) { node.is_a?(clazz) } - elsif Const === root and Object.const_defined?(root.value) - clazz = Object.const_get(root.value) - - ->(node) { node.is_a?(clazz) } - elsif ConstPathRef === root and VarRef === root.parent and - Const === root.parent.value and - root.parent.value.value == "SyntaxTree" - compile_node(root.constant) - elsif DynaSymbol === root and root.parts.empty? + constant = node.constant + compiled_constant = compile_node(constant) if constant + + preprocessed = node.requireds.map { |required| compile_node(required) } + + compiled_requireds = ->(other) do + deconstructed = other.deconstruct + + deconstructed.length == preprocessed.length && + preprocessed + .zip(deconstructed) + .all? { |(matcher, value)| matcher.call(value) } + end + + if compiled_constant + combine_and(compiled_constant, compiled_requireds) + else + compiled_requireds + end + end + + # in foo | bar + def compile_binary(node) + compile_error(node) if node.operator != :| + + combine_or(compile_node(node.left), compile_node(node.right)) + end + + # in Ident + # in String + def compile_const(node) + value = node.value + + if SyntaxTree.const_defined?(value) + clazz = SyntaxTree.const_get(value) + + ->(other) { clazz === other } + elsif Object.const_defined?(value) + clazz = Object.const_get(value) + + ->(other) { clazz === other } + else + compile_error(node) + end + end + + # in SyntaxTree::Ident + def compile_const_path_ref(node) + parent = node.parent + compile_error(node) if !parent.is_a?(VarRef) || !parent.value.is_a?(Const) + + if parent.value.value == "SyntaxTree" + compile_node(node.constant) + else + compile_error(node) + end + end + + # in :"" + # in :"foo" + def compile_dyna_symbol(node) + if node.parts.empty? symbol = :"" - ->(node) { node == symbol } - elsif DynaSymbol === root and parts = root.parts and parts.size == 1 and - TStringContent === parts[0] - symbol = parts[0].value.to_sym - - ->(node) { node == symbol } - elsif HshPtn === root and root.keyword_rest.nil? - compiled_constant = compile_node(root.constant) - - preprocessed = - root.keywords.to_h do |keyword, value| - unless keyword.is_a?(Label) - raise CompilationError, PP.pp(root, +"").chomp - end - [keyword.value.chomp(":").to_sym, compile_node(value)] - end - - compiled_keywords = ->(node) do - deconstructed = node.deconstruct_keys(preprocessed.keys) - - preprocessed.all? do |keyword, matcher| - matcher.call(deconstructed[keyword]) - end + ->(other) { symbol === other } + elsif (value = extract_string(node)) + symbol = value.to_sym + + ->(other) { symbol === other } + else + compile_error(root) + end + end + + # in Ident[value: String] + # in { value: String } + def compile_hshptn(node) + compile_error(node) unless node.keyword_rest.nil? + compiled_constant = compile_node(node.constant) if node.constant + + preprocessed = + node.keywords.to_h do |keyword, value| + compile_error(node) unless keyword.is_a?(Label) + [keyword.value.chomp(":").to_sym, compile_node(value)] end - if compiled_constant - combine_and(compiled_constant, compiled_keywords) - else - compiled_keywords + compiled_keywords = ->(other) do + deconstructed = other.deconstruct_keys(preprocessed.keys) + + preprocessed.all? do |keyword, matcher| + matcher.call(deconstructed[keyword]) end - elsif RegexpLiteral === root and parts = root.parts and - parts.size == 1 and TStringContent === parts[0] - regexp = /#{parts[0].value}/ - - ->(attribute) { regexp.match?(attribute) } - elsif StringLiteral === root and root.parts.empty? - ->(attribute) { attribute == "" } - elsif StringLiteral === root and parts = root.parts and - parts.size == 1 and TStringContent === parts[0] - value = parts[0].value - ->(attribute) { attribute == value } - elsif SymbolLiteral === root - symbol = root.value.value.to_sym - - ->(attribute) { attribute == symbol } - elsif VarRef === root and Const === root.value - compile_node(root.value) - elsif VarRef === root and Kw === root.value and root.value.value.nil? - ->(attribute) { attribute.nil? } + end + + if compiled_constant + combine_and(compiled_constant, compiled_keywords) + else + compiled_keywords + end + end + + # in /foo/ + def compile_regexp_literal(node) + if (value = extract_string(node)) + regexp = /#{value}/ + + ->(attribute) { regexp === attribute } + else + compile_error(node) + end + end + + # in "" + # in "foo" + def compile_string_literal(node) + if node.parts.empty? + ->(attribute) { "" === attribute } + elsif (value = extract_string(node)) + ->(attribute) { value === attribute } + else + compile_error(node) + end + end + + # in :+ + # in :foo + def compile_symbol_literal(node) + symbol = node.value.value.to_sym + + ->(attribute) { symbol === attribute } + end + + # in Foo + # in nil + def compile_var_ref(node) + value = node.value + + if value.is_a?(Const) + compile_node(value) + elsif value.is_a?(Kw) && value.value.nil? + ->(attribute) { nil === attribute } + else + compile_error(node) + end + end + + # Compile any kind of node. Dispatch out to the individual compilation + # methods based on the type of node. + def compile_node(node) + case node + when AryPtn + compile_aryptn(node) + when Binary + compile_binary(node) + when Const + compile_const(node) + when ConstPathRef + compile_const_path_ref(node) + when DynaSymbol + compile_dyna_symbol(node) + when HshPtn + compile_hshptn(node) + when RegexpLiteral + compile_regexp_literal(node) + when StringLiteral + compile_string_literal(node) + when SymbolLiteral + compile_symbol_literal(node) + when VarRef + compile_var_ref(node) else - raise CompilationError, PP.pp(root, +"").chomp + compile_error(node) end end end diff --git a/test/idempotency_test.rb b/test/idempotency_test.rb index 76116572..32d9d196 100644 --- a/test/idempotency_test.rb +++ b/test/idempotency_test.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -return unless ENV["CI"] and RUBY_ENGINE != "truffleruby" +return if !ENV["CI"] || RUBY_ENGINE == "truffleruby" require_relative "test_helper" module SyntaxTree From 7405a3a78e54e20b5f5859ed9e0d840c2be826f6 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 28 Oct 2022 10:33:45 -0400 Subject: [PATCH 168/536] Remove pattern matching entirely --- Rakefile | 8 +- lib/syntax_tree/language_server.rb | 81 ++++++++--- test/cli_test.rb | 1 - test/language_server_test.rb | 208 +++++++++++++---------------- test/test_helper.rb | 4 +- 5 files changed, 160 insertions(+), 142 deletions(-) diff --git a/Rakefile b/Rakefile index 6de81bd8..4973d45e 100644 --- a/Rakefile +++ b/Rakefile @@ -7,13 +7,7 @@ require "syntax_tree/rake_tasks" Rake::TestTask.new(:test) do |t| t.libs << "test" t.libs << "lib" - test_files = FileList["test/**/*_test.rb"] - if RUBY_ENGINE == "truffleruby" - # language_server.rb uses pattern matching - test_files -= FileList["test/language_server/*_test.rb"] - test_files -= FileList["test/language_server_test.rb"] - end - t.test_files = test_files + t.test_files = FileList["test/**/*_test.rb"] end task default: :test diff --git a/lib/syntax_tree/language_server.rb b/lib/syntax_tree/language_server.rb index d2714b5c..c2265c32 100644 --- a/lib/syntax_tree/language_server.rb +++ b/lib/syntax_tree/language_server.rb @@ -13,6 +13,50 @@ module SyntaxTree # stree lsp # class LanguageServer + # This is a small module that effectively mirrors pattern matching. We're + # using it so that we can support truffleruby without having to ignore the + # language server. + module Request + # Represents a hash pattern. + class Shape + attr_reader :values + + def initialize(values) + @values = values + end + + def ===(other) + values.all? do |key, value| + value == :any ? other.key?(key) : value === other[key] + end + end + end + + # Represents an array pattern. + class Tuple + attr_reader :values + + def initialize(values) + @values = values + end + + def ===(other) + values.each_with_index.all? { |value, index| value === other[index] } + end + end + + def self.[](value) + case value + when Array + Tuple.new(value.map { |child| self[child] }) + when Hash + Shape.new(value.transform_values { |child| self[child] }) + else + value + end + end + end + attr_reader :input, :output, :print_width def initialize( @@ -39,30 +83,33 @@ def run # stree-ignore case request - in { method: "initialize", id: } + when Request[method: "initialize", id: :any] store.clear - write(id: id, result: { capabilities: capabilities }) - in { method: "initialized" } + write(id: request[:id], result: { capabilities: capabilities }) + when Request[method: "initialized"] # ignored - in { method: "shutdown" } # tolerate missing ID to be a good citizen + when Request[method: "shutdown"] # tolerate missing ID to be a good citizen store.clear write(id: request[:id], result: {}) return - in { method: "textDocument/didChange", params: { textDocument: { uri: }, contentChanges: [{ text: }, *] } } - store[uri] = text - in { method: "textDocument/didOpen", params: { textDocument: { uri:, text: } } } - store[uri] = text - in { method: "textDocument/didClose", params: { textDocument: { uri: } } } - store.delete(uri) - in { method: "textDocument/formatting", id:, params: { textDocument: { uri: } } } + when Request[method: "textDocument/didChange", params: { textDocument: { uri: :any }, contentChanges: [{ text: :any }] }] + store[request.dig(:params, :textDocument, :uri)] = request.dig(:params, :contentChanges, 0, :text) + when Request[method: "textDocument/didOpen", params: { textDocument: { uri: :any, text: :any } }] + store[request.dig(:params, :textDocument, :uri)] = request.dig(:params, :textDocument, :text) + when Request[method: "textDocument/didClose", params: { textDocument: { uri: :any } }] + store.delete(request.dig(:params, :textDocument, :uri)) + when Request[method: "textDocument/formatting", id: :any, params: { textDocument: { uri: :any } }] + uri = request.dig(:params, :textDocument, :uri) contents = store[uri] - write(id: id, result: contents ? format(contents, uri.split(".").last) : nil) - in { method: "textDocument/inlayHint", id:, params: { textDocument: { uri: } } } + write(id: request[:id], result: contents ? format(contents, uri.split(".").last) : nil) + when Request[method: "textDocument/inlayHint", id: :any, params: { textDocument: { uri: :any } }] + uri = request.dig(:params, :textDocument, :uri) contents = store[uri] - write(id: id, result: contents ? inlay_hints(contents) : nil) - in { method: "syntaxTree/visualizing", id:, params: { textDocument: { uri: } } } - write(id: id, result: PP.pp(SyntaxTree.parse(store[uri]), +"")) - in { method: %r{\$/.+} } + write(id: request[:id], result: contents ? inlay_hints(contents) : nil) + when Request[method: "syntaxTree/visualizing", id: :any, params: { textDocument: { uri: :any } }] + uri = request.dig(:params, :textDocument, :uri) + write(id: request[:id], result: PP.pp(SyntaxTree.parse(store[uri]), +"")) + when Request[method: %r{\$/.+}] # ignored else raise ArgumentError, "Unhandled: #{request}" diff --git a/test/cli_test.rb b/test/cli_test.rb index b5316d7f..c00fb338 100644 --- a/test/cli_test.rb +++ b/test/cli_test.rb @@ -173,7 +173,6 @@ def test_plugins def test_language_server prev_stdin = $stdin prev_stdout = $stdout - skip unless SUPPORTS_PATTERN_MATCHING request = { method: "shutdown" }.merge(jsonrpc: "2.0").to_json $stdin = diff --git a/test/language_server_test.rb b/test/language_server_test.rb index 466bf737..8e1ed9a7 100644 --- a/test/language_server_test.rb +++ b/test/language_server_test.rb @@ -4,6 +4,7 @@ require "syntax_tree/language_server" module SyntaxTree + # stree-ignore class LanguageServerTest < Minitest::Test class Initialize < Struct.new(:id) def to_hash @@ -21,12 +22,7 @@ class TextDocumentDidOpen < Struct.new(:uri, :text) def to_hash { method: "textDocument/didOpen", - params: { - textDocument: { - uri: uri, - text: text - } - } + params: { textDocument: { uri: uri, text: text } } } end end @@ -36,9 +32,7 @@ def to_hash { method: "textDocument/didChange", params: { - textDocument: { - uri: uri - }, + textDocument: { uri: uri }, contentChanges: [{ text: text }] } } @@ -49,11 +43,7 @@ class TextDocumentDidClose < Struct.new(:uri) def to_hash { method: "textDocument/didClose", - params: { - textDocument: { - uri: uri - } - } + params: { textDocument: { uri: uri } } } end end @@ -63,11 +53,7 @@ def to_hash { method: "textDocument/formatting", id: id, - params: { - textDocument: { - uri: uri - } - } + params: { textDocument: { uri: uri } } } end end @@ -77,11 +63,7 @@ def to_hash { method: "textDocument/inlayHint", id: id, - params: { - textDocument: { - uri: uri - } - } + params: { textDocument: { uri: uri } } } end end @@ -91,75 +73,71 @@ def to_hash { method: "syntaxTree/visualizing", id: id, - params: { - textDocument: { - uri: uri - } - } + params: { textDocument: { uri: uri } } } end end def test_formatting - messages = [ + responses = run_server([ Initialize.new(1), TextDocumentDidOpen.new("file:///path/to/file.rb", "class Foo; end"), TextDocumentDidChange.new("file:///path/to/file.rb", "class Bar; end"), TextDocumentFormatting.new(2, "file:///path/to/file.rb"), TextDocumentDidClose.new("file:///path/to/file.rb"), Shutdown.new(3) - ] - - case run_server(messages) - in [ - { id: 1, result: { capabilities: Hash } }, - { id: 2, result: [{ newText: new_text }] }, - { id: 3, result: {} } - ] - assert_equal("class Bar\nend\n", new_text) - end + ]) + + shape = LanguageServer::Request[[ + { id: 1, result: { capabilities: Hash } }, + { id: 2, result: [{ newText: :any }] }, + { id: 3, result: {} } + ]] + + assert_operator(shape, :===, responses) + assert_equal("class Bar\nend\n", responses.dig(1, :result, 0, :newText)) end def test_formatting_failure - messages = [ + responses = run_server([ Initialize.new(1), TextDocumentDidOpen.new("file:///path/to/file.rb", "<>"), TextDocumentFormatting.new(2, "file:///path/to/file.rb"), Shutdown.new(3) - ] - - case run_server(messages) - in [ - { id: 1, result: { capabilities: Hash } }, - { id: 2, result: }, - { id: 3, result: {} } - ] - assert_nil(result) - end + ]) + + shape = LanguageServer::Request[[ + { id: 1, result: { capabilities: Hash } }, + { id: 2, result: :any }, + { id: 3, result: {} } + ]] + + assert_operator(shape, :===, responses) + assert_nil(responses.dig(1, :result)) end def test_formatting_print_width contents = "#{"a" * 40} + #{"b" * 40}\n" - messages = [ + responses = run_server([ Initialize.new(1), TextDocumentDidOpen.new("file:///path/to/file.rb", contents), TextDocumentFormatting.new(2, "file:///path/to/file.rb"), TextDocumentDidClose.new("file:///path/to/file.rb"), Shutdown.new(3) - ] - - case run_server(messages, print_width: 100) - in [ - { id: 1, result: { capabilities: Hash } }, - { id: 2, result: [{ newText: new_text }] }, - { id: 3, result: {} } - ] - assert_equal(contents, new_text) - end + ], print_width: 100) + + shape = LanguageServer::Request[[ + { id: 1, result: { capabilities: Hash } }, + { id: 2, result: [{ newText: :any }] }, + { id: 3, result: {} } + ]] + + assert_operator(shape, :===, responses) + assert_equal(contents, responses.dig(1, :result, 0, :newText)) end def test_inlay_hint - messages = [ + responses = run_server([ Initialize.new(1), TextDocumentDidOpen.new("file:///path/to/file.rb", <<~RUBY), begin @@ -169,37 +147,37 @@ def test_inlay_hint RUBY TextDocumentInlayHint.new(2, "file:///path/to/file.rb"), Shutdown.new(3) - ] - - case run_server(messages) - in [ - { id: 1, result: { capabilities: Hash } }, - { id: 2, result: hints }, - { id: 3, result: {} } - ] - assert_equal(3, hints.length) - end + ]) + + shape = LanguageServer::Request[[ + { id: 1, result: { capabilities: Hash } }, + { id: 2, result: :any }, + { id: 3, result: {} } + ]] + + assert_operator(shape, :===, responses) + assert_equal(3, responses.dig(1, :result).size) end def test_visualizing - messages = [ + responses = run_server([ Initialize.new(1), TextDocumentDidOpen.new("file:///path/to/file.rb", "1 + 2"), SyntaxTreeVisualizing.new(2, "file:///path/to/file.rb"), Shutdown.new(3) - ] - - case run_server(messages) - in [ - { id: 1, result: { capabilities: Hash } }, - { id: 2, result: }, - { id: 3, result: {} } - ] - assert_equal( - "(program (statements ((binary (int \"1\") + (int \"2\")))))\n", - result - ) - end + ]) + + shape = LanguageServer::Request[[ + { id: 1, result: { capabilities: Hash } }, + { id: 2, result: :any }, + { id: 3, result: {} } + ]] + + assert_operator(shape, :===, responses) + assert_equal( + "(program (statements ((binary (int \"1\") + (int \"2\")))))\n", + responses.dig(1, :result) + ) end def test_reading_file @@ -207,20 +185,20 @@ def test_reading_file file.write("class Foo; end") file.rewind - messages = [ + responses = run_server([ Initialize.new(1), TextDocumentFormatting.new(2, "file://#{file.path}"), Shutdown.new(3) - ] - - case run_server(messages) - in [ - { id: 1, result: { capabilities: Hash } }, - { id: 2, result: [{ newText: new_text }] }, - { id: 3, result: {} } - ] - assert_equal("class Foo\nend\n", new_text) - end + ]) + + shape = LanguageServer::Request[[ + { id: 1, result: { capabilities: Hash } }, + { id: 2, result: [{ newText: :any }] }, + { id: 3, result: {} } + ]] + + assert_operator(shape, :===, responses) + assert_equal("class Foo\nend\n", responses.dig(1, :result, 0, :newText)) end end @@ -231,29 +209,30 @@ def test_bogus_request end def test_clean_shutdown - messages = [Initialize.new(1), Shutdown.new(2)] + responses = run_server([Initialize.new(1), Shutdown.new(2)]) - case run_server(messages) - in [{ id: 1, result: { capabilities: Hash } }, { id: 2, result: {} }] - assert_equal(true, true) - end + shape = LanguageServer::Request[[ + { id: 1, result: { capabilities: Hash } }, + { id: 2, result: {} } + ]] + + assert_operator(shape, :===, responses) end def test_file_that_does_not_exist - messages = [ + responses = run_server([ Initialize.new(1), TextDocumentFormatting.new(2, "file:///path/to/file.rb"), Shutdown.new(3) - ] - - case run_server(messages) - in [ - { id: 1, result: { capabilities: Hash } }, - { id: 2, result: nil }, - { id: 3, result: {} } - ] - assert_equal(true, true) - end + ]) + + shape = LanguageServer::Request[[ + { id: 1, result: { capabilities: Hash } }, + { id: 2, result: :any }, + { id: 3, result: {} } + ]] + + assert_operator(shape, :===, responses) end private @@ -281,6 +260,7 @@ def run_server(messages, print_width: DEFAULT_PRINT_WIDTH) output: output, print_width: print_width ).run + read(output.tap(&:rewind)) end end diff --git a/test/test_helper.rb b/test/test_helper.rb index c421d8ee..c46022ae 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -17,8 +17,6 @@ require "pp" require "minitest/autorun" -SUPPORTS_PATTERN_MATCHING = RUBY_ENGINE != "truffleruby" - module SyntaxTree module Assertions class Recorder @@ -69,7 +67,7 @@ def assert_syntax_tree(node) refute_includes(json, "#<") assert_equal(type, JSON.parse(json)["type"]) - if SUPPORTS_PATTERN_MATCHING + if RUBY_ENGINE != "truffleruby" # Get a match expression from the node, then assert that it can in fact # match the node. # rubocop:disable all From 57f5a98d807e261fc945b4c8bbb3ce6fd7f603ad Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 28 Oct 2022 10:43:28 -0400 Subject: [PATCH 169/536] Exit with exit status on rake --- lib/syntax_tree/rake/task.rb | 2 +- test/rake_test.rb | 30 ++++++++++++++++++------------ 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/lib/syntax_tree/rake/task.rb b/lib/syntax_tree/rake/task.rb index ea228e8f..e9a20433 100644 --- a/lib/syntax_tree/rake/task.rb +++ b/lib/syntax_tree/rake/task.rb @@ -78,7 +78,7 @@ def run_task arguments << "--ignore-files=#{ignore_files}" if ignore_files != "" - SyntaxTree::CLI.run(arguments + Array(source_files)) + abort if SyntaxTree::CLI.run(arguments + Array(source_files)) != 0 end end end diff --git a/test/rake_test.rb b/test/rake_test.rb index 57364859..bd315cc6 100644 --- a/test/rake_test.rb +++ b/test/rake_test.rb @@ -6,30 +6,36 @@ module SyntaxTree module Rake class CheckTaskTest < Minitest::Test - Invoke = Struct.new(:args) + Invocation = Struct.new(:args) def test_check_task source_files = "{app,config,lib}/**/*.rb" CheckTask.new { |t| t.source_files = source_files } - invoke = nil - SyntaxTree::CLI.stub(:run, ->(args) { invoke = Invoke.new(args) }) do - ::Rake::Task["stree:check"].invoke - end - - assert_equal(["check", source_files], invoke.args) + invocation = invoke("stree:check") + assert_equal(["check", source_files], invocation.args) end def test_write_task source_files = "{app,config,lib}/**/*.rb" WriteTask.new { |t| t.source_files = source_files } - invoke = nil - SyntaxTree::CLI.stub(:run, ->(args) { invoke = Invoke.new(args) }) do - ::Rake::Task["stree:write"].invoke - end + invocation = invoke("stree:write") + assert_equal(["write", source_files], invocation.args) + end - assert_equal(["write", source_files], invoke.args) + private + + def invoke(task_name) + invocation = nil + stub = ->(args) { invocation = Invocation.new(args) } + + begin + SyntaxTree::CLI.stub(:run, stub) { ::Rake::Task[task_name].invoke } + flunk + rescue SystemExit + invocation + end end end end From 15514e5d9f28d4859466ed61677390353f0d04bf Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 28 Oct 2022 10:52:10 -0400 Subject: [PATCH 170/536] Remove pattern matching from inlay hints --- lib/syntax_tree/language_server/inlay_hints.rb | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/syntax_tree/language_server/inlay_hints.rb b/lib/syntax_tree/language_server/inlay_hints.rb index 12c10230..dfd63b8d 100644 --- a/lib/syntax_tree/language_server/inlay_hints.rb +++ b/lib/syntax_tree/language_server/inlay_hints.rb @@ -69,11 +69,10 @@ def visit_assign(node) # def visit_binary(node) case stack[-2] - in Assign | OpAssign + when Assign, OpAssign parentheses(node.location) - in Binary[operator: operator] if operator != node.operator - parentheses(node.location) - else + when Binary + parentheses(node.location) if stack[-2].operator != node.operator end super @@ -91,9 +90,8 @@ def visit_binary(node) # def visit_if_op(node) case stack[-2] - in Assign | Binary | IfOp | OpAssign + when Assign, Binary, IfOp, OpAssign parentheses(node.location) - else end super From 331172450c064ccf2bfd80e1bc1936d237e97fd0 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 28 Oct 2022 11:20:02 -0400 Subject: [PATCH 171/536] Bump to v4.3.0 --- CHANGELOG.md | 15 ++++++++++++++- Gemfile.lock | 2 +- lib/syntax_tree/version.rb | 2 +- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bbaf044e..45a06c13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,18 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +## [4.3.0] - 2022-10-28 + +### Added + +- [#183](https://github.com/ruby-syntax-tree/syntax_tree/pull/183) - Support TruffleRuby by eliminating internal pattern matching in some places and stopping some tests from running in other places. +- [#184](https://github.com/ruby-syntax-tree/syntax_tree/pull/184) - Remove internal pattern matching entirely. + +### Changed + +- [#183](https://github.com/ruby-syntax-tree/syntax_tree/pull/183) - Pattern matching works against dynamic symbols now. +- [#184](https://github.com/ruby-syntax-tree/syntax_tree/pull/184) - Exit with the correct exit status within the rake tasks. + ## [4.2.0] - 2022-10-25 ### Added @@ -414,7 +426,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - 🎉 Initial release! 🎉 -[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v4.2.0...HEAD +[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v4.3.0...HEAD +[4.3.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v4.2.0...v4.3.0 [4.2.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v4.1.0...v4.2.0 [4.1.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v4.0.2...v4.1.0 [4.0.2]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v4.0.1...v4.0.2 diff --git a/Gemfile.lock b/Gemfile.lock index 339de160..25f461c6 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - syntax_tree (4.2.0) + syntax_tree (4.3.0) prettier_print (>= 1.0.2) GEM diff --git a/lib/syntax_tree/version.rb b/lib/syntax_tree/version.rb index 0b68a850..a12c472d 100644 --- a/lib/syntax_tree/version.rb +++ b/lib/syntax_tree/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SyntaxTree - VERSION = "4.2.0" + VERSION = "4.3.0" end From 178267d30031a8632fe6ff761a197166dd7ed1c4 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Sun, 30 Oct 2022 22:27:20 -0400 Subject: [PATCH 172/536] Fix up linguist percentages --- .gitattributes | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..b24bb2da --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +bin/* linguist-language=Ruby From d15c56f3a1134dac554a1493a18057441e5255c0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Nov 2022 17:39:03 +0000 Subject: [PATCH 173/536] Bump rubocop from 1.37.1 to 1.38.0 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.37.1 to 1.38.0. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.37.1...v1.38.0) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 25f461c6..1c1a127c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -19,7 +19,7 @@ GEM rake (13.0.6) regexp_parser (2.6.0) rexml (3.2.5) - rubocop (1.37.1) + rubocop (1.38.0) json (~> 2.3) parallel (~> 1.10) parser (>= 3.1.2.1) From 5afee6ba9266742c2c90153ffd98bdf7642dd894 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 2 Nov 2022 10:36:10 -0400 Subject: [PATCH 174/536] Remove comments from initializers --- lib/syntax_tree/node.rb | 522 +++++++++++++++++++------------------- lib/syntax_tree/parser.rb | 28 +- 2 files changed, 276 insertions(+), 274 deletions(-) diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index aa133b7f..a1e5c0ae 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -146,11 +146,11 @@ class BEGINBlock < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(lbrace:, statements:, location:, comments: []) + def initialize(lbrace:, statements:, location:) @lbrace = lbrace @statements = statements @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -199,10 +199,10 @@ class CHAR < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, location:, comments: []) + def initialize(value:, location:) @value = value @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -249,11 +249,11 @@ class ENDBlock < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(lbrace:, statements:, location:, comments: []) + def initialize(lbrace:, statements:, location:) @lbrace = lbrace @statements = statements @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -305,10 +305,10 @@ class EndContent < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, location:, comments: []) + def initialize(value:, location:) @value = value @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -392,11 +392,11 @@ def format(q) # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(left:, right:, location:, comments: []) + def initialize(left:, right:, location:) @left = left @right = right @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -453,11 +453,11 @@ class ARef < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(collection:, index:, location:, comments: []) + def initialize(collection:, index:, location:) @collection = collection @index = index @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -514,11 +514,11 @@ class ARefField < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(collection:, index:, location:, comments: []) + def initialize(collection:, index:, location:) @collection = collection @index = index @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -577,10 +577,10 @@ class ArgParen < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(arguments:, location:, comments: []) + def initialize(arguments:, location:) @arguments = arguments @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -650,10 +650,10 @@ class Args < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(parts:, location:, comments: []) + def initialize(parts:, location:) @parts = parts @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -686,10 +686,10 @@ class ArgBlock < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, location:, comments: []) + def initialize(value:, location:) @value = value @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -723,10 +723,10 @@ class ArgStar < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, location:, comments: []) + def initialize(value:, location:) @value = value @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -773,10 +773,10 @@ class ArgsForward < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, location:, comments: []) + def initialize(value:, location:) @value = value @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -955,11 +955,11 @@ def format(q) # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(lbracket:, contents:, location:, comments: []) + def initialize(lbracket:, contents:, location:) @lbracket = lbracket @contents = contents @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -1132,7 +1132,7 @@ def initialize( @rest = rest @posts = posts @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -1209,11 +1209,11 @@ class Assign < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(target:, value:, location:, comments: []) + def initialize(target:, value:, location:) @target = target @value = value @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -1271,11 +1271,11 @@ class Assoc < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(key:, value:, location:, comments: []) + def initialize(key:, value:, location:) @key = key @value = value @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -1330,10 +1330,10 @@ class AssocSplat < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, location:, comments: []) + def initialize(value:, location:) @value = value @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -1368,10 +1368,10 @@ class Backref < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, location:, comments: []) + def initialize(value:, location:) @value = value @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -1403,10 +1403,10 @@ class Backtick < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, location:, comments: []) + def initialize(value:, location:) @value = value @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -1515,10 +1515,10 @@ class BareAssocHash < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(assocs:, location:, comments: []) + def initialize(assocs:, location:) @assocs = assocs @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -1557,10 +1557,10 @@ class Begin < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(bodystmt:, location:, comments: []) + def initialize(bodystmt:, location:) @bodystmt = bodystmt @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -1605,10 +1605,10 @@ class PinnedBegin < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(statement:, location:, comments: []) + def initialize(statement:, location:) @statement = statement @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -1677,12 +1677,12 @@ def name # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(left:, operator:, right:, location:, comments: []) + def initialize(left:, operator:, right:, location:) @left = left @operator = operator @right = right @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -1745,11 +1745,11 @@ class BlockVar < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(params:, locals:, location:, comments: []) + def initialize(params:, locals:, location:) @params = params @locals = locals @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -1803,10 +1803,10 @@ class BlockArg < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(name:, location:, comments: []) + def initialize(name:, location:) @name = name @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -1866,7 +1866,7 @@ def initialize( @else_clause = else_clause @ensure_clause = ensure_clause @location = location - @comments = comments + @comments = [] end def bind(start_char, start_column, end_char, end_column) @@ -2141,12 +2141,12 @@ class BraceBlock < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(lbrace:, block_var:, statements:, location:, comments: []) + def initialize(lbrace:, block_var:, statements:, location:) @lbrace = lbrace @block_var = block_var @statements = statements @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -2371,10 +2371,10 @@ class Break < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(arguments:, location:, comments: []) + def initialize(arguments:, location:) @arguments = arguments @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -2672,7 +2672,7 @@ def initialize( @message = message @arguments = arguments @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -2782,12 +2782,12 @@ class Case < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(keyword:, value:, consequent:, location:, comments: []) + def initialize(keyword:, value:, consequent:, location:) @keyword = keyword @value = value @consequent = consequent @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -2847,12 +2847,12 @@ class RAssign < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, operator:, pattern:, location:, comments: []) + def initialize(value:, operator:, pattern:, location:) @value = value @operator = operator @pattern = pattern @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -2943,12 +2943,12 @@ class ClassDeclaration < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(constant:, superclass:, bodystmt:, location:, comments: []) + def initialize(constant:, superclass:, bodystmt:, location:) @constant = constant @superclass = superclass @bodystmt = bodystmt @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -3049,11 +3049,11 @@ class Command < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(message:, arguments:, location:, comments: []) + def initialize(message:, arguments:, location:) @message = message @arguments = arguments @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -3151,7 +3151,7 @@ def initialize( @message = message @arguments = arguments @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -3323,10 +3323,10 @@ class Const < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, location:, comments: []) + def initialize(value:, location:) @value = value @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -3364,11 +3364,11 @@ class ConstPathField < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(parent:, constant:, location:, comments: []) + def initialize(parent:, constant:, location:) @parent = parent @constant = constant @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -3411,11 +3411,11 @@ class ConstPathRef < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(parent:, constant:, location:, comments: []) + def initialize(parent:, constant:, location:) @parent = parent @constant = constant @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -3457,10 +3457,10 @@ class ConstRef < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(constant:, location:, comments: []) + def initialize(constant:, location:) @constant = constant @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -3493,10 +3493,10 @@ class CVar < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, location:, comments: []) + def initialize(value:, location:) @value = value @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -3535,12 +3535,12 @@ class Def < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(name:, params:, bodystmt:, location:, comments: []) + def initialize(name:, params:, bodystmt:, location:) @name = name @params = params @bodystmt = bodystmt @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -3625,7 +3625,7 @@ def initialize( @paren = paren @statement = statement @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -3690,10 +3690,10 @@ class Defined < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, location:, comments: []) + def initialize(value:, location:) @value = value @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -3761,7 +3761,7 @@ def initialize( @params = params @bodystmt = bodystmt @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -3831,12 +3831,12 @@ class DoBlock < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(keyword:, block_var:, bodystmt:, location:, comments: []) + def initialize(keyword:, block_var:, bodystmt:, location:) @keyword = keyword @block_var = block_var @bodystmt = bodystmt @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -3915,11 +3915,11 @@ class Dot2 < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(left:, right:, location:, comments: []) + def initialize(left:, right:, location:) @left = left @right = right @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -3963,11 +3963,11 @@ class Dot3 < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(left:, right:, location:, comments: []) + def initialize(left:, right:, location:) @left = left @right = right @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -4050,11 +4050,11 @@ class DynaSymbol < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(parts:, quote:, location:, comments: []) + def initialize(parts:, quote:, location:) @parts = parts @quote = quote @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -4161,11 +4161,11 @@ class Else < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(keyword:, statements:, location:, comments: []) + def initialize(keyword:, statements:, location:) @keyword = keyword @statements = statements @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -4231,7 +4231,7 @@ def initialize( @statements = statements @consequent = consequent @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -4435,11 +4435,11 @@ class Ensure < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(keyword:, statements:, location:, comments: []) + def initialize(keyword:, statements:, location:) @keyword = keyword @statements = statements @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -4490,10 +4490,10 @@ class ExcessedComma < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, location:, comments: []) + def initialize(value:, location:) @value = value @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -4532,11 +4532,11 @@ class FCall < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, arguments:, location:, comments: []) + def initialize(value:, arguments:, location:) @value = value @arguments = arguments @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -4591,12 +4591,12 @@ class Field < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(parent:, operator:, name:, location:, comments: []) + def initialize(parent:, operator:, name:, location:) @parent = parent @operator = operator @name = name @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -4639,10 +4639,10 @@ class FloatLiteral < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, location:, comments: []) + def initialize(value:, location:) @value = value @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -4688,13 +4688,13 @@ class FndPtn < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(constant:, left:, values:, right:, location:, comments: []) + def initialize(constant:, left:, values:, right:, location:) @constant = constant @left = left @values = values @right = right @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -4763,12 +4763,12 @@ class For < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(index:, collection:, statements:, location:, comments: []) + def initialize(index:, collection:, statements:, location:) @index = index @collection = collection @statements = statements @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -4822,10 +4822,10 @@ class GVar < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, location:, comments: []) + def initialize(value:, location:) @value = value @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -4887,11 +4887,11 @@ def format(q) # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(lbrace:, assocs:, location:, comments: []) + def initialize(lbrace:, assocs:, location:) @lbrace = lbrace @assocs = assocs @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -4987,7 +4987,7 @@ def initialize( @dedent = dedent @parts = parts @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -5064,10 +5064,10 @@ class HeredocBeg < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, location:, comments: []) + def initialize(value:, location:) @value = value @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -5103,10 +5103,10 @@ class HeredocEnd < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, location:, comments: []) + def initialize(value:, location:) @value = value @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -5195,12 +5195,12 @@ def format(q) # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(constant:, keywords:, keyword_rest:, location:, comments: []) + def initialize(constant:, keywords:, keyword_rest:, location:) @constant = constant @keywords = keywords @keyword_rest = keyword_rest @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -5305,10 +5305,10 @@ class Ident < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, location:, comments: []) + def initialize(value:, location:) @value = value @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -5574,7 +5574,7 @@ def initialize( @statements = statements @consequent = consequent @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -5619,12 +5619,12 @@ class IfOp < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(predicate:, truthy:, falsy:, location:, comments: []) + def initialize(predicate:, truthy:, falsy:, location:) @predicate = predicate @truthy = truthy @falsy = falsy @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -5761,11 +5761,11 @@ class IfMod < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(statement:, predicate:, location:, comments: []) + def initialize(statement:, predicate:, location:) @statement = statement @predicate = predicate @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -5803,10 +5803,10 @@ class Imaginary < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, location:, comments: []) + def initialize(value:, location:) @value = value @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -5848,12 +5848,12 @@ class In < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(pattern:, statements:, consequent:, location:, comments: []) + def initialize(pattern:, statements:, consequent:, location:) @pattern = pattern @statements = statements @consequent = consequent @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -5909,10 +5909,10 @@ class Int < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, location:, comments: []) + def initialize(value:, location:) @value = value @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -5953,10 +5953,10 @@ class IVar < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, location:, comments: []) + def initialize(value:, location:) @value = value @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -6001,11 +6001,11 @@ class Kw < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, location:, comments: []) + def initialize(value:, location:) @value = value @name = value.to_sym @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -6039,10 +6039,10 @@ class KwRestParam < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(name:, location:, comments: []) + def initialize(name:, location:) @name = name @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -6085,10 +6085,10 @@ class Label < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, location:, comments: []) + def initialize(value:, location:) @value = value @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -6155,11 +6155,11 @@ class Lambda < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(params:, statements:, location:, comments: []) + def initialize(params:, statements:, location:) @params = params @statements = statements @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -6263,11 +6263,11 @@ class LambdaVar < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(params:, locals:, location:, comments: []) + def initialize(params:, locals:, location:) @params = params @locals = locals @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -6306,10 +6306,10 @@ class LBrace < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, location:, comments: []) + def initialize(value:, location:) @value = value @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -6339,10 +6339,10 @@ class LBracket < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, location:, comments: []) + def initialize(value:, location:) @value = value @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -6372,10 +6372,10 @@ class LParen < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, location:, comments: []) + def initialize(value:, location:) @value = value @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -6421,11 +6421,11 @@ class MAssign < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(target:, value:, location:, comments: []) + def initialize(target:, value:, location:) @target = target @value = value @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -6468,11 +6468,11 @@ class MethodAddBlock < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(call:, block:, location:, comments: []) + def initialize(call:, block:, location:) @call = call @block = block @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -6529,11 +6529,11 @@ class MLHS < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(parts:, comma: false, location:, comments: []) + def initialize(parts:, comma: false, location:) @parts = parts @comma = comma @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -6573,11 +6573,11 @@ class MLHSParen < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(contents:, comma: false, location:, comments: []) + def initialize(contents:, comma: false, location:) @contents = contents @comma = comma @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -6631,11 +6631,11 @@ class ModuleDeclaration < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(constant:, bodystmt:, location:, comments: []) + def initialize(constant:, bodystmt:, location:) @constant = constant @bodystmt = bodystmt @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -6701,10 +6701,10 @@ class MRHS < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(parts:, location:, comments: []) + def initialize(parts:, location:) @parts = parts @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -6750,10 +6750,10 @@ class Next < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(arguments:, location:, comments: []) + def initialize(arguments:, location:) @arguments = arguments @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -6790,11 +6790,11 @@ class Op < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, location:, comments: []) + def initialize(value:, location:) @value = value @name = value.to_sym @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -6835,12 +6835,12 @@ class OpAssign < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(target:, operator:, value:, location:, comments: []) + def initialize(target:, operator:, value:, location:) @target = target @operator = operator @value = value @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -7081,7 +7081,7 @@ def initialize( @keyword_rest = keyword_rest @block = block @location = location - @comments = comments + @comments = [] end # Params nodes are the most complicated in the tree. Occasionally you want @@ -7187,11 +7187,11 @@ class Paren < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(lparen:, contents:, location:, comments: []) + def initialize(lparen:, contents:, location:) @lparen = lparen @contents = contents @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -7239,10 +7239,10 @@ class Period < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, location:, comments: []) + def initialize(value:, location:) @value = value @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -7272,10 +7272,10 @@ class Program < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(statements:, location:, comments: []) + def initialize(statements:, location:) @statements = statements @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -7316,11 +7316,11 @@ class QSymbols < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(beginning:, elements:, location:, comments: []) + def initialize(beginning:, elements:, location:) @beginning = beginning @elements = elements @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -7410,11 +7410,11 @@ class QWords < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(beginning:, elements:, location:, comments: []) + def initialize(beginning:, elements:, location:) @beginning = beginning @elements = elements @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -7501,10 +7501,10 @@ class RationalLiteral < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, location:, comments: []) + def initialize(value:, location:) @value = value @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -7587,10 +7587,10 @@ class Redo < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, location:, comments: []) + def initialize(value:, location:) @value = value @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -7732,12 +7732,12 @@ class RegexpLiteral < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(beginning:, ending:, parts:, location:, comments: []) + def initialize(beginning:, ending:, parts:, location:) @beginning = beginning @ending = ending @parts = parts @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -7843,11 +7843,11 @@ class RescueEx < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(exceptions:, variable:, location:, comments: []) + def initialize(exceptions:, variable:, location:) @exceptions = exceptions @variable = variable @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -7919,7 +7919,7 @@ def initialize( @statements = statements @consequent = consequent @location = location - @comments = comments + @comments = [] end def bind_end(end_char, end_column) @@ -8004,11 +8004,11 @@ class RescueMod < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(statement:, value:, location:, comments: []) + def initialize(statement:, value:, location:) @statement = statement @value = value @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -8061,10 +8061,10 @@ class RestParam < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(name:, location:, comments: []) + def initialize(name:, location:) @name = name @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -8098,10 +8098,10 @@ class Retry < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, location:, comments: []) + def initialize(value:, location:) @value = value @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -8134,10 +8134,10 @@ class Return < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(arguments:, location:, comments: []) + def initialize(arguments:, location:) @arguments = arguments @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -8170,10 +8170,10 @@ class Return0 < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, location:, comments: []) + def initialize(value:, location:) @value = value @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -8237,11 +8237,11 @@ class SClass < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(target:, bodystmt:, location:, comments: []) + def initialize(target:, bodystmt:, location:) @target = target @bodystmt = bodystmt @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -8294,11 +8294,11 @@ class Statements < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(parser, body:, location:, comments: []) + def initialize(parser, body:, location:) @parser = parser @body = body @location = location - @comments = comments + @comments = [] end def bind(start_char, start_column, end_char, end_column) @@ -8498,11 +8498,11 @@ class StringConcat < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(left:, right:, location:, comments: []) + def initialize(left:, right:, location:) @left = left @right = right @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -8544,10 +8544,10 @@ class StringDVar < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(variable:, location:, comments: []) + def initialize(variable:, location:) @variable = variable @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -8584,10 +8584,10 @@ class StringEmbExpr < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(statements:, location:, comments: []) + def initialize(statements:, location:) @statements = statements @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -8646,11 +8646,11 @@ class StringLiteral < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(parts:, quote:, location:, comments: []) + def initialize(parts:, quote:, location:) @parts = parts @quote = quote @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -8721,10 +8721,10 @@ class Super < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(arguments:, location:, comments: []) + def initialize(arguments:, location:) @arguments = arguments @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -8838,10 +8838,10 @@ class SymbolLiteral < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, location:, comments: []) + def initialize(value:, location:) @value = value @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -8878,11 +8878,11 @@ class Symbols < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(beginning:, elements:, location:, comments: []) + def initialize(beginning:, elements:, location:) @beginning = beginning @elements = elements @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -9031,10 +9031,10 @@ class TopConstField < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(constant:, location:, comments: []) + def initialize(constant:, location:) @constant = constant @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -9069,10 +9069,10 @@ class TopConstRef < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(constant:, location:, comments: []) + def initialize(constant:, location:) @constant = constant @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -9144,10 +9144,10 @@ class TStringContent < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, location:, comments: []) + def initialize(value:, location:) @value = value @location = location - @comments = comments + @comments = [] end def match?(pattern) @@ -9222,11 +9222,11 @@ class Not < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(statement:, parentheses:, location:, comments: []) + def initialize(statement:, parentheses:, location:) @statement = statement @parentheses = parentheses @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -9288,11 +9288,11 @@ class Unary < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(operator:, statement:, location:, comments: []) + def initialize(operator:, statement:, location:) @operator = operator @statement = statement @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -9355,10 +9355,10 @@ def format(q) # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(symbols:, location:, comments: []) + def initialize(symbols:, location:) @symbols = symbols @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -9417,7 +9417,7 @@ def initialize( @statements = statements @consequent = consequent @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -9459,11 +9459,11 @@ class UnlessMod < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(statement:, predicate:, location:, comments: []) + def initialize(statement:, predicate:, location:) @statement = statement @predicate = predicate @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -9556,11 +9556,11 @@ class Until < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(predicate:, statements:, location:, comments: []) + def initialize(predicate:, statements:, location:) @predicate = predicate @statements = statements @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -9612,11 +9612,11 @@ class UntilMod < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(statement:, predicate:, location:, comments: []) + def initialize(statement:, predicate:, location:) @statement = statement @predicate = predicate @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -9678,11 +9678,11 @@ class VarAlias < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(left:, right:, location:, comments: []) + def initialize(left:, right:, location:) @left = left @right = right @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -9722,10 +9722,10 @@ class VarField < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, location:, comments: []) + def initialize(value:, location:) @value = value @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -9766,10 +9766,10 @@ class VarRef < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, location:, comments: []) + def initialize(value:, location:) @value = value @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -9830,10 +9830,10 @@ class PinnedVarRef < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, location:, comments: []) + def initialize(value:, location:) @value = value @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -9870,10 +9870,10 @@ class VCall < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, location:, comments: []) + def initialize(value:, location:) @value = value @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -9910,9 +9910,9 @@ class VoidStmt < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(location:, comments: []) + def initialize(location:) @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -9963,7 +9963,7 @@ def initialize( @statements = statements @consequent = consequent @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -10055,11 +10055,11 @@ class While < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(predicate:, statements:, location:, comments: []) + def initialize(predicate:, statements:, location:) @predicate = predicate @statements = statements @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -10111,11 +10111,11 @@ class WhileMod < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(statement:, predicate:, location:, comments: []) + def initialize(statement:, predicate:, location:) @statement = statement @predicate = predicate @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -10177,10 +10177,10 @@ class Word < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(parts:, location:, comments: []) + def initialize(parts:, location:) @parts = parts @location = location - @comments = comments + @comments = [] end def match?(pattern) @@ -10220,11 +10220,11 @@ class Words < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(beginning:, elements:, location:, comments: []) + def initialize(beginning:, elements:, location:) @beginning = beginning @elements = elements @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -10342,10 +10342,10 @@ class XStringLiteral < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(parts:, location:, comments: []) + def initialize(parts:, location:) @parts = parts @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -10380,10 +10380,10 @@ class Yield < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(arguments:, location:, comments: []) + def initialize(arguments:, location:) @arguments = arguments @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -10430,10 +10430,10 @@ class Yield0 < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, location:, comments: []) + def initialize(value:, location:) @value = value @location = location - @comments = comments + @comments = [] end def accept(visitor) @@ -10466,10 +10466,10 @@ class ZSuper < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, location:, comments: []) + def initialize(value:, location:) @value = value @location = location - @comments = comments + @comments = [] end def accept(visitor) diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index 61a7ca57..abd9bd60 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -2104,17 +2104,20 @@ def on_lambda(params, statements) location = params.contents.location location = location.to(locals.last.location) if locals.any? - Paren.new( - lparen: params.lparen, - contents: - LambdaVar.new( - params: params.contents, - locals: locals, - location: location - ), - location: params.location, - comments: params.comments - ) + node = + Paren.new( + lparen: params.lparen, + contents: + LambdaVar.new( + params: params.contents, + locals: locals, + location: location + ), + location: params.location + ) + + node.comments.concat(params.comments) + node when Params # In this case we've gotten to the <3.2 plain set of parameters. In # this case there cannot be lambda locals, so we will wrap the @@ -2199,8 +2202,7 @@ def lambda_locals(source) on_comma: :item, on_rparen: :final }, - final: { - } + final: {} } tokens[(index + 1)..].each_with_object([]) do |token, locals| From 38a1b036593e1a220a6666817d3c655b432f4656 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 2 Nov 2022 10:46:48 -0400 Subject: [PATCH 175/536] Fold IfMod into If and UnlessMod into Unless --- CHANGELOG.md | 5 + lib/syntax_tree/node.rb | 232 ++++++----------------- lib/syntax_tree/parser.rb | 14 +- lib/syntax_tree/visitor.rb | 6 - lib/syntax_tree/visitor/field_visitor.rb | 16 -- test/node_test.rb | 6 +- 6 files changed, 77 insertions(+), 202 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 45a06c13..5d944f7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +### Changed + +- Nodes no longer have a `comments:` keyword on their initializers. By default, they initialize to an empty array. If you were previously passing comments into the initializer, you should now create the node first, then call `node.comments.concat` to add your comments. +- `IfMod` and `UnlessMod` are no longer nodes. Instead, they have been folded into `If` and `Unless`, respectively. The `If` and `Unless` nodes now have a `modifier?` method to tell you if they were original found in the modifier form. + ## [4.3.0] - 2022-10-28 ### Added diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index a1e5c0ae..2f23e9a1 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -2071,8 +2071,7 @@ def forced_brace_bounds?(q) when Paren, Statements # If we hit certain breakpoints then we know we're safe. return false - when If, IfMod, IfOp, Unless, UnlessMod, While, WhileMod, Until, - UntilMod + when If, IfOp, Unless, While, WhileMod, Until, UntilMod return true if parent.predicate == previous end @@ -3884,7 +3883,7 @@ def format(q) q.format(left) if left case q.parent - when If, IfMod, Unless, UnlessMod + when If, Unless q.text(" #{operator} ") else q.text(operator) @@ -5398,10 +5397,10 @@ def call(q, node) # and default instead to breaking them into multiple lines. def ternaryable?(statement) case statement - when Alias, Assign, Break, Command, CommandCall, Heredoc, If, IfMod, - IfOp, Lambda, MAssign, Next, OpAssign, RescueMod, Return, Return0, - Super, Undef, Unless, UnlessMod, Until, UntilMod, VarAlias, - VoidStmt, While, WhileMod, Yield, Yield0, ZSuper + when Alias, Assign, Break, Command, CommandCall, Heredoc, If, IfOp, + Lambda, MAssign, Next, OpAssign, RescueMod, Return, Return0, Super, + Undef, Unless, Until, UntilMod, VarAlias, VoidStmt, While, + WhileMod, Yield, Yield0, ZSuper # This is a list of nodes that should not be allowed to be a part of a # ternary clause. false @@ -5432,41 +5431,60 @@ def initialize(keyword, node) end def format(q) - # If we can transform this node into a ternary, then we're going to print - # a special version that uses the ternary operator if it fits on one line. - if Ternaryable.call(q, node) - format_ternary(q) - return - end + if node.modifier? + statement = node.statements.body[0] - # If the predicate of the conditional contains an assignment (in which - # case we can't know for certain that that assignment doesn't impact the - # statements inside the conditional) then we can't use the modifier form - # and we must use the block form. - if ContainsAssignment.call(node.predicate) - format_break(q, force: true) - return - end - - if node.consequent || node.statements.empty? || contains_conditional? - q.group { format_break(q, force: true) } + if ContainsAssignment.call(statement) || q.parent.is_a?(In) + q.group { format_flat(q) } + else + q.group { q.if_break { format_break(q, force: false) }.if_flat { format_flat(q) } } + end else - q.group do - q - .if_break { format_break(q, force: false) } - .if_flat do - Parentheses.flat(q) do - q.format(node.statements) - q.text(" #{keyword} ") - q.format(node.predicate) + # If we can transform this node into a ternary, then we're going to + # print a special version that uses the ternary operator if it fits on + # one line. + if Ternaryable.call(q, node) + format_ternary(q) + return + end + + # If the predicate of the conditional contains an assignment (in which + # case we can't know for certain that that assignment doesn't impact the + # statements inside the conditional) then we can't use the modifier form + # and we must use the block form. + if ContainsAssignment.call(node.predicate) + format_break(q, force: true) + return + end + + if node.consequent || node.statements.empty? || contains_conditional? + q.group { format_break(q, force: true) } + else + q.group do + q + .if_break { format_break(q, force: false) } + .if_flat do + Parentheses.flat(q) do + q.format(node.statements) + q.text(" #{keyword} ") + q.format(node.predicate) + end end - end + end end end end private + def format_flat(q) + Parentheses.flat(q) do + q.format(node.statements.body[0]) + q.text(" #{keyword} ") + q.format(node.predicate) + end + end + def format_break(q, force:) q.text("#{keyword} ") q.nest(keyword.length + 1) { q.format(node.predicate) } @@ -5537,7 +5555,7 @@ def contains_conditional? return false if statements.length != 1 case statements.first - when If, IfMod, IfOp, Unless, UnlessMod + when If, IfOp, Unless true else false @@ -5600,6 +5618,11 @@ def deconstruct_keys(_keys) def format(q) ConditionalFormatter.new("if", self).format(q) end + + # Checks if the node was originally found in the modifier form. + def modifier? + predicate.location.start_char > statements.location.start_char + end end # IfOp represents a ternary clause. @@ -5649,10 +5672,9 @@ def deconstruct_keys(_keys) def format(q) force_flat = [ - Alias, Assign, Break, Command, CommandCall, Heredoc, If, IfMod, IfOp, - Lambda, MAssign, Next, OpAssign, RescueMod, Return, Return0, Super, - Undef, Unless, UnlessMod, UntilMod, VarAlias, VoidStmt, WhileMod, Yield, - Yield0, ZSuper + Alias, Assign, Break, Command, CommandCall, Heredoc, If, IfOp, Lambda, + MAssign, Next, OpAssign, RescueMod, Return, Return0, Super, Undef, + Unless, UntilMod, VarAlias, VoidStmt, WhileMod, Yield, Yield0, ZSuper ] if q.parent.is_a?(Paren) || force_flat.include?(truthy.class) || @@ -5704,94 +5726,6 @@ def format_flat(q) end end - # Formats an IfMod or UnlessMod node. - class ConditionalModFormatter - # [String] the keyword associated with this conditional - attr_reader :keyword - - # [IfMod | UnlessMod] the node that is being formatted - attr_reader :node - - def initialize(keyword, node) - @keyword = keyword - @node = node - end - - def format(q) - if ContainsAssignment.call(node.statement) || q.parent.is_a?(In) - q.group { format_flat(q) } - else - q.group { q.if_break { format_break(q) }.if_flat { format_flat(q) } } - end - end - - private - - def format_break(q) - q.text("#{keyword} ") - q.nest(keyword.length + 1) { q.format(node.predicate) } - q.indent do - q.breakable_space - q.format(node.statement) - end - q.breakable_space - q.text("end") - end - - def format_flat(q) - Parentheses.flat(q) do - q.format(node.statement) - q.text(" #{keyword} ") - q.format(node.predicate) - end - end - end - - # IfMod represents the modifier form of an +if+ statement. - # - # expression if predicate - # - class IfMod < Node - # [untyped] the expression to be executed - attr_reader :statement - - # [untyped] the expression to be checked - attr_reader :predicate - - # [Array[ Comment | EmbDoc ]] the comments attached to this node - attr_reader :comments - - def initialize(statement:, predicate:, location:) - @statement = statement - @predicate = predicate - @location = location - @comments = [] - end - - def accept(visitor) - visitor.visit_if_mod(self) - end - - def child_nodes - [statement, predicate] - end - - alias deconstruct child_nodes - - def deconstruct_keys(_keys) - { - statement: statement, - predicate: predicate, - location: location, - comments: comments - } - end - - def format(q) - ConditionalModFormatter.new("if", self).format(q) - end - end - # Imaginary represents an imaginary number literal. # # 1i @@ -9443,50 +9377,10 @@ def deconstruct_keys(_keys) def format(q) ConditionalFormatter.new("unless", self).format(q) end - end - - # UnlessMod represents the modifier form of an +unless+ statement. - # - # expression unless predicate - # - class UnlessMod < Node - # [untyped] the expression to be executed - attr_reader :statement - - # [untyped] the expression to be checked - attr_reader :predicate - - # [Array[ Comment | EmbDoc ]] the comments attached to this node - attr_reader :comments - - def initialize(statement:, predicate:, location:) - @statement = statement - @predicate = predicate - @location = location - @comments = [] - end - - def accept(visitor) - visitor.visit_unless_mod(self) - end - - def child_nodes - [statement, predicate] - end - - alias deconstruct child_nodes - - def deconstruct_keys(_keys) - { - statement: statement, - predicate: predicate, - location: location, - comments: comments - } - end - def format(q) - ConditionalModFormatter.new("unless", self).format(q) + # Checks if the node was originally found in the modifier form. + def modifier? + predicate.location.start_char > statements.location.start_char end end diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index abd9bd60..c4d6f8e9 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -1911,13 +1911,14 @@ def on_ifop(predicate, truthy, falsy) end # :call-seq: - # on_if_mod: (untyped predicate, untyped statement) -> IfMod + # on_if_mod: (untyped predicate, untyped statement) -> If def on_if_mod(predicate, statement) consume_keyword(:if) - IfMod.new( - statement: statement, + If.new( predicate: predicate, + statements: Statements.new(self, body: [statement], location: statement.location), + consequent: nil, location: statement.location.to(predicate.location) ) end @@ -3586,13 +3587,14 @@ def on_unless(predicate, statements, consequent) end # :call-seq: - # on_unless_mod: (untyped predicate, untyped statement) -> UnlessMod + # on_unless_mod: (untyped predicate, untyped statement) -> Unless def on_unless_mod(predicate, statement) consume_keyword(:unless) - UnlessMod.new( - statement: statement, + Unless.new( predicate: predicate, + statements: Statements.new(self, body: [statement], location: statement.location), + consequent: nil, location: statement.location.to(predicate.location) ) end diff --git a/lib/syntax_tree/visitor.rb b/lib/syntax_tree/visitor.rb index e3b52077..0ea83e7b 100644 --- a/lib/syntax_tree/visitor.rb +++ b/lib/syntax_tree/visitor.rb @@ -206,9 +206,6 @@ class Visitor < BasicVisitor # Visit an If node. alias visit_if visit_child_nodes - # Visit an IfMod node. - alias visit_if_mod visit_child_nodes - # Visit an IfOp node. alias visit_if_op visit_child_nodes @@ -431,9 +428,6 @@ class Visitor < BasicVisitor # Visit an Unless node. alias visit_unless visit_child_nodes - # Visit an UnlessMod node. - alias visit_unless_mod visit_child_nodes - # Visit an Until node. alias visit_until visit_child_nodes diff --git a/lib/syntax_tree/visitor/field_visitor.rb b/lib/syntax_tree/visitor/field_visitor.rb index 6c5c6139..aa1b80ab 100644 --- a/lib/syntax_tree/visitor/field_visitor.rb +++ b/lib/syntax_tree/visitor/field_visitor.rb @@ -523,14 +523,6 @@ def visit_if(node) end end - def visit_if_mod(node) - node(node, "if_mod") do - field("statement", node.statement) - field("predicate", node.predicate) - comments(node) - end - end - def visit_if_op(node) node(node, "if_op") do field("predicate", node.predicate) @@ -982,14 +974,6 @@ def visit_unless(node) end end - def visit_unless_mod(node) - node(node, "unless_mod") do - field("statement", node.statement) - field("predicate", node.predicate) - comments(node) - end - end - def visit_until(node) node(node, "until") do field("predicate", node.predicate) diff --git a/test/node_test.rb b/test/node_test.rb index ce26f9ea..8400fa7c 100644 --- a/test/node_test.rb +++ b/test/node_test.rb @@ -583,7 +583,7 @@ def test_if_op end def test_if_mod - assert_node(IfMod, "expression if predicate") + assert_node(If, "expression if predicate") end def test_imaginary @@ -926,10 +926,6 @@ def test_unless assert_node(Unless, "unless value then else end") end - def test_unless_mod - assert_node(UnlessMod, "expression unless predicate") - end - def test_until assert_node(Until, "until value do end") end From b97b9cb03f65a6156657b19b656124a79b276863 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 2 Nov 2022 11:02:50 -0400 Subject: [PATCH 176/536] Fold UntilMod and WhileMod --- CHANGELOG.md | 1 + lib/syntax_tree/node.rb | 226 +++++------------------ lib/syntax_tree/parser.rb | 12 +- lib/syntax_tree/visitor.rb | 6 - lib/syntax_tree/visitor/field_visitor.rb | 16 -- test/node_test.rb | 4 +- 6 files changed, 60 insertions(+), 205 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d944f7a..faed7811 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - Nodes no longer have a `comments:` keyword on their initializers. By default, they initialize to an empty array. If you were previously passing comments into the initializer, you should now create the node first, then call `node.comments.concat` to add your comments. - `IfMod` and `UnlessMod` are no longer nodes. Instead, they have been folded into `If` and `Unless`, respectively. The `If` and `Unless` nodes now have a `modifier?` method to tell you if they were original found in the modifier form. +- `WhileMod` and `UntilMod` are no longer nodes. Instead, they have been folded into `While` and `Until`, respectively. The `While` and `Until` nodes now have a `modifier?` method to tell you if they were originally found in the modifier form. ## [4.3.0] - 2022-10-28 diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 2f23e9a1..71d5e0d8 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -2071,7 +2071,7 @@ def forced_brace_bounds?(q) when Paren, Statements # If we hit certain breakpoints then we know we're safe. return false - when If, IfOp, Unless, While, WhileMod, Until, UntilMod + when If, IfOp, Unless, While, Until return true if parent.predicate == previous end @@ -5399,8 +5399,8 @@ def ternaryable?(statement) case statement when Alias, Assign, Break, Command, CommandCall, Heredoc, If, IfOp, Lambda, MAssign, Next, OpAssign, RescueMod, Return, Return0, Super, - Undef, Unless, Until, UntilMod, VarAlias, VoidStmt, While, - WhileMod, Yield, Yield0, ZSuper + Undef, Unless, Until, VarAlias, VoidStmt, While, Yield, Yield0, + ZSuper # This is a list of nodes that should not be allowed to be a part of a # ternary clause. false @@ -5674,7 +5674,7 @@ def format(q) force_flat = [ Alias, Assign, Break, Command, CommandCall, Heredoc, If, IfOp, Lambda, MAssign, Next, OpAssign, RescueMod, Return, Return0, Super, Undef, - Unless, UntilMod, VarAlias, VoidStmt, WhileMod, Yield, Yield0, ZSuper + Unless, VarAlias, VoidStmt, Yield, Yield0, ZSuper ] if q.parent.is_a?(Paren) || force_flat.include?(truthy.class) || @@ -9384,40 +9384,60 @@ def modifier? end end - # Formats an Until, UntilMod, While, or WhileMod node. + # Formats an Until or While node. class LoopFormatter # [String] the name of the keyword used for this loop attr_reader :keyword - # [Until | UntilMod | While | WhileMod] the node that is being formatted + # [Until | While] the node that is being formatted attr_reader :node - # [untyped] the statements associated with the node - attr_reader :statements - - def initialize(keyword, node, statements) + def initialize(keyword, node) @keyword = keyword @node = node - @statements = statements end def format(q) - if ContainsAssignment.call(node.predicate) + # If we're in the modifier form and we're modifying a `begin`, then this + # is a special case where we need to explicitly use the modifier form + # because otherwise the semantic meaning changes. This looks like: + # + # begin + # foo + # end while bar + # + # Also, if the statement of the modifier includes an assignment, then we + # can't know for certain that it won't impact the predicate, so we need to + # force it to stay as it is. This looks like: + # + # foo = bar while foo + # + if node.modifier? && (statement = node.statements.body.first) && (statement.is_a?(Begin) || ContainsAssignment.call(statement)) + q.format(statement) + q.text(" #{keyword} ") + q.format(node.predicate) + elsif node.statements.empty? + q.group do + q.text("#{keyword} ") + q.nest(keyword.length + 1) { q.format(node.predicate) } + q.breakable_force + q.text("end") + end + elsif ContainsAssignment.call(node.predicate) format_break(q) q.break_parent - return - end - - q.group do - q - .if_break { format_break(q) } - .if_flat do - Parentheses.flat(q) do - q.format(statements) - q.text(" #{keyword} ") - q.format(node.predicate) + else + q.group do + q + .if_break { format_break(q) } + .if_flat do + Parentheses.flat(q) do + q.format(node.statements) + q.text(" #{keyword} ") + q.format(node.predicate) + end end - end + end end end @@ -9428,7 +9448,7 @@ def format_break(q) q.nest(keyword.length + 1) { q.format(node.predicate) } q.indent do q.breakable_empty - q.format(statements) + q.format(node.statements) end q.breakable_empty q.text("end") @@ -9477,83 +9497,11 @@ def deconstruct_keys(_keys) end def format(q) - if statements.empty? - keyword = "until " - - q.group do - q.text(keyword) - q.nest(keyword.length) { q.format(predicate) } - q.breakable_force - q.text("end") - end - else - LoopFormatter.new("until", self, statements).format(q) - end + LoopFormatter.new("until", self).format(q) end - end - # UntilMod represents the modifier form of a +until+ loop. - # - # expression until predicate - # - class UntilMod < Node - # [untyped] the expression to be executed - attr_reader :statement - - # [untyped] the expression to be checked - attr_reader :predicate - - # [Array[ Comment | EmbDoc ]] the comments attached to this node - attr_reader :comments - - def initialize(statement:, predicate:, location:) - @statement = statement - @predicate = predicate - @location = location - @comments = [] - end - - def accept(visitor) - visitor.visit_until_mod(self) - end - - def child_nodes - [statement, predicate] - end - - alias deconstruct child_nodes - - def deconstruct_keys(_keys) - { - statement: statement, - predicate: predicate, - location: location, - comments: comments - } - end - - def format(q) - # If we're in the modifier form and we're modifying a `begin`, then this - # is a special case where we need to explicitly use the modifier form - # because otherwise the semantic meaning changes. This looks like: - # - # begin - # foo - # end until bar - # - # Also, if the statement of the modifier includes an assignment, then we - # can't know for certain that it won't impact the predicate, so we need to - # force it to stay as it is. This looks like: - # - # foo = bar until foo - # - if statement.is_a?(Begin) || ContainsAssignment.call(statement) - q.format(statement) - q.text(" until ") - q.format(predicate) - else - LoopFormatter.new("until", self, statement).format(q) - end + def modifier? + predicate.location.start_char > statements.location.start_char end end @@ -9976,83 +9924,11 @@ def deconstruct_keys(_keys) end def format(q) - if statements.empty? - keyword = "while " - - q.group do - q.text(keyword) - q.nest(keyword.length) { q.format(predicate) } - q.breakable_force - q.text("end") - end - else - LoopFormatter.new("while", self, statements).format(q) - end - end - end - - # WhileMod represents the modifier form of a +while+ loop. - # - # expression while predicate - # - class WhileMod < Node - # [untyped] the expression to be executed - attr_reader :statement - - # [untyped] the expression to be checked - attr_reader :predicate - - # [Array[ Comment | EmbDoc ]] the comments attached to this node - attr_reader :comments - - def initialize(statement:, predicate:, location:) - @statement = statement - @predicate = predicate - @location = location - @comments = [] - end - - def accept(visitor) - visitor.visit_while_mod(self) - end - - def child_nodes - [statement, predicate] - end - - alias deconstruct child_nodes - - def deconstruct_keys(_keys) - { - statement: statement, - predicate: predicate, - location: location, - comments: comments - } + LoopFormatter.new("while", self).format(q) end - def format(q) - # If we're in the modifier form and we're modifying a `begin`, then this - # is a special case where we need to explicitly use the modifier form - # because otherwise the semantic meaning changes. This looks like: - # - # begin - # foo - # end while bar - # - # Also, if the statement of the modifier includes an assignment, then we - # can't know for certain that it won't impact the predicate, so we need to - # force it to stay as it is. This looks like: - # - # foo = bar while foo - # - if statement.is_a?(Begin) || ContainsAssignment.call(statement) - q.format(statement) - q.text(" while ") - q.format(predicate) - else - LoopFormatter.new("while", self, statement).format(q) - end + def modifier? + predicate.location.start_char > statements.location.start_char end end diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index c4d6f8e9..2b5231d2 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -3630,13 +3630,13 @@ def on_until(predicate, statements) end # :call-seq: - # on_until_mod: (untyped predicate, untyped statement) -> UntilMod + # on_until_mod: (untyped predicate, untyped statement) -> Until def on_until_mod(predicate, statement) consume_keyword(:until) - UntilMod.new( - statement: statement, + Until.new( predicate: predicate, + statements: Statements.new(self, body: [statement], location: statement.location), location: statement.location.to(predicate.location) ) end @@ -3756,13 +3756,13 @@ def on_while(predicate, statements) end # :call-seq: - # on_while_mod: (untyped predicate, untyped statement) -> WhileMod + # on_while_mod: (untyped predicate, untyped statement) -> While def on_while_mod(predicate, statement) consume_keyword(:while) - WhileMod.new( - statement: statement, + While.new( predicate: predicate, + statements: Statements.new(self, body: [statement], location: statement.location), location: statement.location.to(predicate.location) ) end diff --git a/lib/syntax_tree/visitor.rb b/lib/syntax_tree/visitor.rb index 0ea83e7b..efbd47c0 100644 --- a/lib/syntax_tree/visitor.rb +++ b/lib/syntax_tree/visitor.rb @@ -431,9 +431,6 @@ class Visitor < BasicVisitor # Visit an Until node. alias visit_until visit_child_nodes - # Visit an UntilMod node. - alias visit_until_mod visit_child_nodes - # Visit a VarAlias node. alias visit_var_alias visit_child_nodes @@ -455,9 +452,6 @@ class Visitor < BasicVisitor # Visit a While node. alias visit_while visit_child_nodes - # Visit a WhileMod node. - alias visit_while_mod visit_child_nodes - # Visit a Word node. alias visit_word visit_child_nodes diff --git a/lib/syntax_tree/visitor/field_visitor.rb b/lib/syntax_tree/visitor/field_visitor.rb index aa1b80ab..853fe4c7 100644 --- a/lib/syntax_tree/visitor/field_visitor.rb +++ b/lib/syntax_tree/visitor/field_visitor.rb @@ -982,14 +982,6 @@ def visit_until(node) end end - def visit_until_mod(node) - node(node, "until_mod") do - field("statement", node.statement) - field("predicate", node.predicate) - comments(node) - end - end - def visit_var_alias(node) node(node, "var_alias") do field("left", node.left) @@ -1040,14 +1032,6 @@ def visit_while(node) end end - def visit_while_mod(node) - node(node, "while_mod") do - field("statement", node.statement) - field("predicate", node.predicate) - comments(node) - end - end - def visit_word(node) node(node, "word") do list("parts", node.parts) diff --git a/test/node_test.rb b/test/node_test.rb index 8400fa7c..1419e151 100644 --- a/test/node_test.rb +++ b/test/node_test.rb @@ -931,7 +931,7 @@ def test_until end def test_until_mod - assert_node(UntilMod, "expression until predicate") + assert_node(Until, "expression until predicate") end def test_var_alias @@ -981,7 +981,7 @@ def test_while end def test_while_mod - assert_node(WhileMod, "expression while predicate") + assert_node(While, "expression while predicate") end def test_word From 672abb2d6c86f1a0ac2a8b04743dfdda0bd3b7ad Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 2 Nov 2022 11:06:58 -0400 Subject: [PATCH 177/536] Fold VarAlias into Alias --- CHANGELOG.md | 1 + lib/syntax_tree/node.rb | 61 ++++-------------------- lib/syntax_tree/parser.rb | 4 +- lib/syntax_tree/visitor.rb | 3 -- lib/syntax_tree/visitor/field_visitor.rb | 8 ---- test/node_test.rb | 2 +- 6 files changed, 13 insertions(+), 66 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index faed7811..e04aaa18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - Nodes no longer have a `comments:` keyword on their initializers. By default, they initialize to an empty array. If you were previously passing comments into the initializer, you should now create the node first, then call `node.comments.concat` to add your comments. - `IfMod` and `UnlessMod` are no longer nodes. Instead, they have been folded into `If` and `Unless`, respectively. The `If` and `Unless` nodes now have a `modifier?` method to tell you if they were original found in the modifier form. - `WhileMod` and `UntilMod` are no longer nodes. Instead, they have been folded into `While` and `Until`, respectively. The `While` and `Until` nodes now have a `modifier?` method to tell you if they were originally found in the modifier form. +- `VarAlias` is no longer a node. Instead it has been folded into the `Alias` node. The `Alias` node now has a `var_alias?` method to tell you if it is aliasing a global variable. ## [4.3.0] - 2022-10-28 diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 71d5e0d8..1c69026b 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -359,7 +359,7 @@ class Alias < Node # Formats an argument to the alias keyword. For symbol literals it uses the # value of the symbol directly to look like bare words. class AliasArgumentFormatter - # [DynaSymbol | SymbolLiteral] the argument being passed to alias + # [Backref | DynaSymbol | GVar | SymbolLiteral] the argument being passed to alias attr_reader :argument def initialize(argument) @@ -383,10 +383,10 @@ def format(q) end end - # [DynaSymbol | SymbolLiteral] the new name of the method + # [DynaSymbol | GVar | SymbolLiteral] the new name of the method attr_reader :left - # [DynaSymbol | SymbolLiteral] the old name of the method + # [Backref | DynaSymbol | GVar | SymbolLiteral] the old name of the method attr_reader :right # [Array[ Comment | EmbDoc ]] the comments attached to this node @@ -428,6 +428,10 @@ def format(q) end end end + + def var_alias? + left.is_a?(GVar) + end end # ARef represents when you're pulling a value out of a collection at a @@ -5399,8 +5403,7 @@ def ternaryable?(statement) case statement when Alias, Assign, Break, Command, CommandCall, Heredoc, If, IfOp, Lambda, MAssign, Next, OpAssign, RescueMod, Return, Return0, Super, - Undef, Unless, Until, VarAlias, VoidStmt, While, Yield, Yield0, - ZSuper + Undef, Unless, Until, VoidStmt, While, Yield, Yield0, ZSuper # This is a list of nodes that should not be allowed to be a part of a # ternary clause. false @@ -5674,7 +5677,7 @@ def format(q) force_flat = [ Alias, Assign, Break, Command, CommandCall, Heredoc, If, IfOp, Lambda, MAssign, Next, OpAssign, RescueMod, Return, Return0, Super, Undef, - Unless, VarAlias, VoidStmt, Yield, Yield0, ZSuper + Unless, VoidStmt, Yield, Yield0, ZSuper ] if q.parent.is_a?(Paren) || force_flat.include?(truthy.class) || @@ -9505,52 +9508,6 @@ def modifier? end end - # VarAlias represents when you're using the +alias+ keyword with global - # variable arguments. - # - # alias $new $old - # - class VarAlias < Node - # [GVar] the new alias of the variable - attr_reader :left - - # [Backref | GVar] the current name of the variable to be aliased - attr_reader :right - - # [Array[ Comment | EmbDoc ]] the comments attached to this node - attr_reader :comments - - def initialize(left:, right:, location:) - @left = left - @right = right - @location = location - @comments = [] - end - - def accept(visitor) - visitor.visit_var_alias(self) - end - - def child_nodes - [left, right] - end - - alias deconstruct child_nodes - - def deconstruct_keys(_keys) - { left: left, right: right, location: location, comments: comments } - end - - def format(q) - keyword = "alias " - - q.text(keyword) - q.format(left) - q.text(" ") - q.format(right) - end - end - # VarField represents a variable that is being assigned a value. As such, it # is always a child of an assignment type node. # diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index 2b5231d2..a28bb296 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -3642,11 +3642,11 @@ def on_until_mod(predicate, statement) end # :call-seq: - # on_var_alias: (GVar left, (Backref | GVar) right) -> VarAlias + # on_var_alias: (GVar left, (Backref | GVar) right) -> Alias def on_var_alias(left, right) keyword = consume_keyword(:alias) - VarAlias.new( + Alias.new( left: left, right: right, location: keyword.location.to(right.location) diff --git a/lib/syntax_tree/visitor.rb b/lib/syntax_tree/visitor.rb index efbd47c0..06fa7e17 100644 --- a/lib/syntax_tree/visitor.rb +++ b/lib/syntax_tree/visitor.rb @@ -431,9 +431,6 @@ class Visitor < BasicVisitor # Visit an Until node. alias visit_until visit_child_nodes - # Visit a VarAlias node. - alias visit_var_alias visit_child_nodes - # Visit a VarField node. alias visit_var_field visit_child_nodes diff --git a/lib/syntax_tree/visitor/field_visitor.rb b/lib/syntax_tree/visitor/field_visitor.rb index 853fe4c7..c5b00c50 100644 --- a/lib/syntax_tree/visitor/field_visitor.rb +++ b/lib/syntax_tree/visitor/field_visitor.rb @@ -982,14 +982,6 @@ def visit_until(node) end end - def visit_var_alias(node) - node(node, "var_alias") do - field("left", node.left) - field("right", node.right) - comments(node) - end - end - def visit_var_field(node) node(node, "var_field") do field("value", node.value) diff --git a/test/node_test.rb b/test/node_test.rb index 1419e151..05ae2254 100644 --- a/test/node_test.rb +++ b/test/node_test.rb @@ -935,7 +935,7 @@ def test_until_mod end def test_var_alias - assert_node(VarAlias, "alias $new $old") + assert_node(Alias, "alias $new $old") end def test_var_field From 7f04f53fd2c0a7f24c404dc10b9b0302adf01f18 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 2 Nov 2022 11:10:01 -0400 Subject: [PATCH 178/536] Fold Yield0 into Yield --- CHANGELOG.md | 1 + lib/syntax_tree/node.rb | 47 ++++-------------------- lib/syntax_tree/parser.rb | 4 +- lib/syntax_tree/visitor.rb | 3 -- lib/syntax_tree/visitor/field_visitor.rb | 4 -- test/node_test.rb | 2 +- 6 files changed, 12 insertions(+), 49 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e04aaa18..148da6f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - `IfMod` and `UnlessMod` are no longer nodes. Instead, they have been folded into `If` and `Unless`, respectively. The `If` and `Unless` nodes now have a `modifier?` method to tell you if they were original found in the modifier form. - `WhileMod` and `UntilMod` are no longer nodes. Instead, they have been folded into `While` and `Until`, respectively. The `While` and `Until` nodes now have a `modifier?` method to tell you if they were originally found in the modifier form. - `VarAlias` is no longer a node. Instead it has been folded into the `Alias` node. The `Alias` node now has a `var_alias?` method to tell you if it is aliasing a global variable. +- `Yield0` is no longer a node. Instead if has been folded into the `Yield` node. The `Yield` node can now have its `arguments` field be `nil`. ## [4.3.0] - 2022-10-28 diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 1c69026b..7d4cb414 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -5403,7 +5403,7 @@ def ternaryable?(statement) case statement when Alias, Assign, Break, Command, CommandCall, Heredoc, If, IfOp, Lambda, MAssign, Next, OpAssign, RescueMod, Return, Return0, Super, - Undef, Unless, Until, VoidStmt, While, Yield, Yield0, ZSuper + Undef, Unless, Until, VoidStmt, While, Yield, ZSuper # This is a list of nodes that should not be allowed to be a part of a # ternary clause. false @@ -5677,7 +5677,7 @@ def format(q) force_flat = [ Alias, Assign, Break, Command, CommandCall, Heredoc, If, IfOp, Lambda, MAssign, Next, OpAssign, RescueMod, Return, Return0, Super, Undef, - Unless, VoidStmt, Yield, Yield0, ZSuper + Unless, VoidStmt, Yield, ZSuper ] if q.parent.is_a?(Paren) || force_flat.include?(truthy.class) || @@ -10101,7 +10101,7 @@ def format(q) # yield value # class Yield < Node - # [Args | Paren] the arguments passed to the yield + # [nil | Args | Paren] the arguments passed to the yield attr_reader :arguments # [Array[ Comment | EmbDoc ]] the comments attached to this node @@ -10128,6 +10128,11 @@ def deconstruct_keys(_keys) end def format(q) + if arguments.nil? + q.text("yield") + return + end + q.group do q.text("yield") @@ -10146,42 +10151,6 @@ def format(q) end end - # Yield0 represents the bare +yield+ keyword with no arguments. - # - # yield - # - class Yield0 < Node - # [String] the value of the keyword - attr_reader :value - - # [Array[ Comment | EmbDoc ]] the comments attached to this node - attr_reader :comments - - def initialize(value:, location:) - @value = value - @location = location - @comments = [] - end - - def accept(visitor) - visitor.visit_yield0(self) - end - - def child_nodes - [] - end - - alias deconstruct child_nodes - - def deconstruct_keys(_keys) - { value: value, location: location, comments: comments } - end - - def format(q) - q.text(value) - end - end - # ZSuper represents the bare +super+ keyword with no arguments. # # super diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index a28bb296..56cba022 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -3896,11 +3896,11 @@ def on_yield(arguments) end # :call-seq: - # on_yield0: () -> Yield0 + # on_yield0: () -> Yield def on_yield0 keyword = consume_keyword(:yield) - Yield0.new(value: keyword.value, location: keyword.location) + Yield.new(arguments: nil, location: keyword.location) end # :call-seq: diff --git a/lib/syntax_tree/visitor.rb b/lib/syntax_tree/visitor.rb index 06fa7e17..a41a3ed7 100644 --- a/lib/syntax_tree/visitor.rb +++ b/lib/syntax_tree/visitor.rb @@ -467,9 +467,6 @@ class Visitor < BasicVisitor # Visit a Yield node. alias visit_yield visit_child_nodes - # Visit a Yield0 node. - alias visit_yield0 visit_child_nodes - # Visit a ZSuper node. alias visit_zsuper visit_child_nodes diff --git a/lib/syntax_tree/visitor/field_visitor.rb b/lib/syntax_tree/visitor/field_visitor.rb index c5b00c50..52dca6f9 100644 --- a/lib/syntax_tree/visitor/field_visitor.rb +++ b/lib/syntax_tree/visitor/field_visitor.rb @@ -1060,10 +1060,6 @@ def visit_yield(node) end end - def visit_yield0(node) - visit_token(node, "yield0") - end - def visit_zsuper(node) visit_token(node, "zsuper") end diff --git a/test/node_test.rb b/test/node_test.rb index 05ae2254..608c7fbe 100644 --- a/test/node_test.rb +++ b/test/node_test.rb @@ -1013,7 +1013,7 @@ def test_yield end def test_yield0 - assert_node(Yield0, "yield") + assert_node(Yield, "yield") end def test_zsuper From 5f64ba7c0b38bfe7e01df913a011c56ec95d0c4a Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 2 Nov 2022 11:19:51 -0400 Subject: [PATCH 179/536] Fold FCall into Call --- CHANGELOG.md | 1 + lib/syntax_tree/node.rb | 101 ++++++++++--------------------------- lib/syntax_tree/parser.rb | 28 +++++----- lib/syntax_tree/visitor.rb | 3 -- test/node_test.rb | 2 +- 5 files changed, 41 insertions(+), 94 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 148da6f8..fbae9d38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - `WhileMod` and `UntilMod` are no longer nodes. Instead, they have been folded into `While` and `Until`, respectively. The `While` and `Until` nodes now have a `modifier?` method to tell you if they were originally found in the modifier form. - `VarAlias` is no longer a node. Instead it has been folded into the `Alias` node. The `Alias` node now has a `var_alias?` method to tell you if it is aliasing a global variable. - `Yield0` is no longer a node. Instead if has been folded into the `Yield` node. The `Yield` node can now have its `arguments` field be `nil`. +- `FCall` is no longer a node. Instead it has been folded into the `Call` node. The `Call` node can now have its `receiver` and `operator` fields be `nil`. ## [4.3.0] - 2022-10-28 diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 7d4cb414..0277d8b1 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -2484,8 +2484,7 @@ def format(q) # nodes. parent = parents[3] if parent.is_a?(DoBlock) - if parent.is_a?(MethodAddBlock) && parent.call.is_a?(FCall) && - parent.call.value.value == "sig" + if parent.is_a?(MethodAddBlock) && parent.call.is_a?(Call) && parent.call.message.value == "sig" threshold = 2 end end @@ -2647,10 +2646,10 @@ def format_child( # receiver.message # class Call < Node - # [untyped] the receiver of the method call + # [nil | untyped] the receiver of the method call attr_reader :receiver - # [:"::" | Op | Period] the operator being used to send the message + # [nil | :"::" | Op | Period] the operator being used to send the message attr_reader :operator # [:call | Backtick | Const | Ident | Op] the message being sent @@ -2705,23 +2704,35 @@ def deconstruct_keys(_keys) end def format(q) - # If we're at the top of a call chain, then we're going to do some - # specialized printing in case we can print it nicely. We _only_ do this - # at the top of the chain to avoid weird recursion issues. - if CallChainFormatter.chained?(receiver) && - !CallChainFormatter.chained?(q.parent) - q.group do - q - .if_break { CallChainFormatter.new(self).format(q) } - .if_flat { format_contents(q) } + if receiver + # If we're at the top of a call chain, then we're going to do some + # specialized printing in case we can print it nicely. We _only_ do this + # at the top of the chain to avoid weird recursion issues. + if CallChainFormatter.chained?(receiver) && !CallChainFormatter.chained?(q.parent) + q.group do + q + .if_break { CallChainFormatter.new(self).format(q) } + .if_flat { format_contents(q) } + end + else + format_contents(q) end else - format_contents(q) + q.format(message) + + if arguments.is_a?(ArgParen) && arguments.arguments.nil? && !message.is_a?(Const) + # If you're using an explicit set of parentheses on something that looks + # like a constant, then we need to match that in order to maintain valid + # Ruby. For example, you could do something like Foo(), on which we + # would need to keep the parentheses to make it look like a method call. + else + q.format(arguments) + end end end # Print out the arguments to this call. If there are no arguments, then do - #nothing. + # nothing. def format_arguments(q) case arguments when ArgParen @@ -4518,64 +4529,6 @@ def format(q) end end - # FCall represents the piece of a method call that comes before any arguments - # (i.e., just the name of the method). It is used in places where the parser - # is sure that it is a method call and not potentially a local variable. - # - # method(argument) - # - # In the above example, it's referring to the +method+ segment. - class FCall < Node - # [Const | Ident] the name of the method - attr_reader :value - - # [nil | ArgParen | Args] the arguments to the method call - attr_reader :arguments - - # [Array[ Comment | EmbDoc ]] the comments attached to this node - attr_reader :comments - - def initialize(value:, arguments:, location:) - @value = value - @arguments = arguments - @location = location - @comments = [] - end - - def accept(visitor) - visitor.visit_fcall(self) - end - - def child_nodes - [value, arguments] - end - - alias deconstruct child_nodes - - def deconstruct_keys(_keys) - { - value: value, - arguments: arguments, - location: location, - comments: comments - } - end - - def format(q) - q.format(value) - - if arguments.is_a?(ArgParen) && arguments.arguments.nil? && - !value.is_a?(Const) - # If you're using an explicit set of parentheses on something that looks - # like a constant, then we need to match that in order to maintain valid - # Ruby. For example, you could do something like Foo(), on which we - # would need to keep the parentheses to make it look like a method call. - else - q.format(arguments) - end - end - end - # Field is always the child of an assignment. It represents assigning to a # “field” on an object. # @@ -6396,7 +6349,7 @@ def format(q) # method {} # class MethodAddBlock < Node - # [Call | Command | CommandCall | FCall] the method call + # [Call | Command | CommandCall] the method call attr_reader :call # [BraceBlock | DoBlock] the block being sent with the method call diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index 56cba022..7fa02c67 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -1608,9 +1608,9 @@ def on_excessed_comma(*) end # :call-seq: - # on_fcall: ((Const | Ident) value) -> FCall + # on_fcall: ((Const | Ident) value) -> Call def on_fcall(value) - FCall.new(value: value, arguments: nil, location: value.location) + Call.new(receiver: nil, operator: nil, message: value, arguments: nil, location: value.location) end # :call-seq: @@ -2305,29 +2305,25 @@ def on_massign(target, value) # :call-seq: # on_method_add_arg: ( - # (Call | FCall) call, + # Call call, # (ArgParen | Args) arguments - # ) -> Call | FCall + # ) -> Call def on_method_add_arg(call, arguments) location = call.location location = location.to(arguments.location) if arguments.is_a?(ArgParen) - if call.is_a?(FCall) - FCall.new(value: call.value, arguments: arguments, location: location) - else - Call.new( - receiver: call.receiver, - operator: call.operator, - message: call.message, - arguments: arguments, - location: location - ) - end + Call.new( + receiver: call.receiver, + operator: call.operator, + message: call.message, + arguments: arguments, + location: location + ) end # :call-seq: # on_method_add_block: ( - # (Call | Command | CommandCall | FCall) call, + # (Call | Command | CommandCall) call, # (BraceBlock | DoBlock) block # ) -> MethodAddBlock def on_method_add_block(call, block) diff --git a/lib/syntax_tree/visitor.rb b/lib/syntax_tree/visitor.rb index a41a3ed7..3b652f89 100644 --- a/lib/syntax_tree/visitor.rb +++ b/lib/syntax_tree/visitor.rb @@ -167,9 +167,6 @@ class Visitor < BasicVisitor # Visit an ExcessedComma node. alias visit_excessed_comma visit_child_nodes - # Visit a FCall node. - alias visit_fcall visit_child_nodes - # Visit a Field node. alias visit_field visit_child_nodes diff --git a/test/node_test.rb b/test/node_test.rb index 608c7fbe..69c147c3 100644 --- a/test/node_test.rb +++ b/test/node_test.rb @@ -487,7 +487,7 @@ def test_excessed_comma end def test_fcall - assert_node(FCall, "method(argument)") + assert_node(Call, "method(argument)") end def test_field From 087350aa095d04b9595b761a7eaaa2d7349300e3 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 2 Nov 2022 11:32:38 -0400 Subject: [PATCH 180/536] Fold Dot2 and Dot3 into RangeLiteral --- CHANGELOG.md | 11 +-- lib/syntax_tree/node.rb | 103 +++++------------------ lib/syntax_tree/parser.rb | 10 ++- lib/syntax_tree/visitor.rb | 9 +- lib/syntax_tree/visitor/field_visitor.rb | 25 ++---- test/node_test.rb | 4 +- 6 files changed, 45 insertions(+), 117 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fbae9d38..25bb7891 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,11 +9,12 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ### Changed - Nodes no longer have a `comments:` keyword on their initializers. By default, they initialize to an empty array. If you were previously passing comments into the initializer, you should now create the node first, then call `node.comments.concat` to add your comments. -- `IfMod` and `UnlessMod` are no longer nodes. Instead, they have been folded into `If` and `Unless`, respectively. The `If` and `Unless` nodes now have a `modifier?` method to tell you if they were original found in the modifier form. -- `WhileMod` and `UntilMod` are no longer nodes. Instead, they have been folded into `While` and `Until`, respectively. The `While` and `Until` nodes now have a `modifier?` method to tell you if they were originally found in the modifier form. -- `VarAlias` is no longer a node. Instead it has been folded into the `Alias` node. The `Alias` node now has a `var_alias?` method to tell you if it is aliasing a global variable. -- `Yield0` is no longer a node. Instead if has been folded into the `Yield` node. The `Yield` node can now have its `arguments` field be `nil`. -- `FCall` is no longer a node. Instead it has been folded into the `Call` node. The `Call` node can now have its `receiver` and `operator` fields be `nil`. +- A lot of nodes have been folded into other nodes to make it easier to interact with the AST. This means that a lot of visit methods have been removed from the visitor and a lot of class definitions are no longer present. This also means that the nodes that received more function now have additional methods or fields to be able to differentiate them. Note that none of these changes have resulted in different formatting. The changes are listed below: + - `IfMod`, `UnlessMod`, `WhileMod`, `UntilMod` have been folded into `If`, `Unless`, `While`, and `Until`. Each of the nodes now have a `modifier?` method to tell if it was originally in the modifier form. Consequently, the `visit_if_mod`, `visit_unless_mod`, `visit_while_mod`, and `visit_until_mod` methods have been removed from the visitor. + - `VarAlias` is no longer a node. Instead it has been folded into the `Alias` node. The `Alias` node now has a `var_alias?` method to tell you if it is aliasing a global variable. Consequently, the `visit_var_alias` method has been removed from the visitor interface. If you were previously using this method, you should now use `visit_alias` instead. + - `Yield0` is no longer a node. Instead if has been folded into the `Yield` node. The `Yield` node can now have its `arguments` field be `nil`. Consequently, the `visit_yield0` method has been removed from the visitor interface. If you were previously using this method, you should now use `visit_yield` instead. + - `FCall` is no longer a node. Instead it has been folded into the `Call` node. The `Call` node can now have its `receiver` and `operator` fields be `nil`. Consequently, the `visit_fcall` method has been removed from the visitor interface. If you were previously using this method, you should now use `visit_call` instead. + - `Dot2` and `Dot3` are no longer nodes. Instead they have become a single new `RangeLiteral` node. This node looks the same as `Dot2` and `Dot3`, except that it additionally has an `operator` field that contains the operator that created the node. Consequently, the `visit_dot2` and `visit_dot3` methods have been removed from the visitor interface. If you were previously using these methods, you should now use `visit_range_literal` instead. ## [4.3.0] - 2022-10-28 diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 0277d8b1..8f3892e1 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -3878,38 +3878,8 @@ def format(q) end end - # Responsible for formatting Dot2 and Dot3 nodes. - class DotFormatter - # [String] the operator to display - attr_reader :operator - - # [Dot2 | Dot3] the node that is being formatter - attr_reader :node - - def initialize(operator, node) - @operator = operator - @node = node - end - - def format(q) - left = node.left - right = node.right - - q.format(left) if left - - case q.parent - when If, Unless - q.text(" #{operator} ") - else - q.text(operator) - end - - q.format(right) if right - end - end - - # Dot2 represents using the .. operator between two expressions. Usually this - # is to create a range object. + # RangeLiteral represents using the .. or the ... operator between two + # expressions. Usually this is to create a range object. # # 1..2 # @@ -3919,25 +3889,29 @@ def format(q) # end # # One of the sides of the expression may be nil, but not both. - class Dot2 < Node + class RangeLiteral < Node # [nil | untyped] the left side of the expression attr_reader :left + # [Op] the operator used for this range + attr_reader :operator + # [nil | untyped] the right side of the expression attr_reader :right # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(left:, right:, location:) + def initialize(left:, operator:, right:, location:) @left = left + @operator = operator @right = right @location = location @comments = [] end def accept(visitor) - visitor.visit_dot2(self) + visitor.visit_range_literal(self) end def child_nodes @@ -3947,59 +3921,20 @@ def child_nodes alias deconstruct child_nodes def deconstruct_keys(_keys) - { left: left, right: right, location: location, comments: comments } + { left: left, operator: operator, right: right, location: location, comments: comments } end def format(q) - DotFormatter.new("..", self).format(q) - end - end - - # Dot3 represents using the ... operator between two expressions. Usually this - # is to create a range object. It's effectively the same event as the Dot2 - # node but with this operator you're asking Ruby to omit the final value. - # - # 1...2 - # - # Like Dot2 it can also be used to create a flip-flop. - # - # if value == 5 ... value == 10 - # end - # - # One of the sides of the expression may be nil, but not both. - class Dot3 < Node - # [nil | untyped] the left side of the expression - attr_reader :left - - # [nil | untyped] the right side of the expression - attr_reader :right - - # [Array[ Comment | EmbDoc ]] the comments attached to this node - attr_reader :comments - - def initialize(left:, right:, location:) - @left = left - @right = right - @location = location - @comments = [] - end - - def accept(visitor) - visitor.visit_dot3(self) - end - - def child_nodes - [left, right] - end - - alias deconstruct child_nodes + q.format(left) if left - def deconstruct_keys(_keys) - { left: left, right: right, location: location, comments: comments } - end + case q.parent + when If, Unless + q.text(" #{operator.value} ") + else + q.text(operator.value) + end - def format(q) - DotFormatter.new("...", self).format(q) + q.format(right) if right end end @@ -9771,7 +9706,7 @@ def format(q) # last argument to the predicate is and endless range, then you are # forced to use the "then" keyword to make it parse properly. last = arguments.parts.last - if (last.is_a?(Dot2) || last.is_a?(Dot3)) && !last.right + if last.is_a?(RangeLiteral) && !last.right q.text(" then") end end diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index 7fa02c67..4bc0bb6f 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -1357,30 +1357,32 @@ def on_do_block(block_var, bodystmt) end # :call-seq: - # on_dot2: ((nil | untyped) left, (nil | untyped) right) -> Dot2 + # on_dot2: ((nil | untyped) left, (nil | untyped) right) -> RangeLiteral def on_dot2(left, right) operator = consume_operator(:"..") beginning = left || operator ending = right || operator - Dot2.new( + RangeLiteral.new( left: left, + operator: operator, right: right, location: beginning.location.to(ending.location) ) end # :call-seq: - # on_dot3: ((nil | untyped) left, (nil | untyped) right) -> Dot3 + # on_dot3: ((nil | untyped) left, (nil | untyped) right) -> RangeLiteral def on_dot3(left, right) operator = consume_operator(:"...") beginning = left || operator ending = right || operator - Dot3.new( + RangeLiteral.new( left: left, + operator: operator, right: right, location: beginning.location.to(ending.location) ) diff --git a/lib/syntax_tree/visitor.rb b/lib/syntax_tree/visitor.rb index 3b652f89..ad4757d7 100644 --- a/lib/syntax_tree/visitor.rb +++ b/lib/syntax_tree/visitor.rb @@ -131,12 +131,6 @@ class Visitor < BasicVisitor # Visit a DoBlock node. alias visit_do_block visit_child_nodes - # Visit a Dot2 node. - alias visit_dot2 visit_child_nodes - - # Visit a Dot3 node. - alias visit_dot3 visit_child_nodes - # Visit a DynaSymbol node. alias visit_dyna_symbol visit_child_nodes @@ -305,6 +299,9 @@ class Visitor < BasicVisitor # Visit a QWordsBeg node. alias visit_qwords_beg visit_child_nodes + # Visit a RangeLiteral node + alias visit_range_literal visit_child_nodes + # Visit a RAssign node. alias visit_rassign visit_child_nodes diff --git a/lib/syntax_tree/visitor/field_visitor.rb b/lib/syntax_tree/visitor/field_visitor.rb index 52dca6f9..ac7b3603 100644 --- a/lib/syntax_tree/visitor/field_visitor.rb +++ b/lib/syntax_tree/visitor/field_visitor.rb @@ -362,22 +362,6 @@ def visit_do_block(node) end end - def visit_dot2(node) - node(node, "dot2") do - field("left", node.left) if node.left - field("right", node.right) if node.right - comments(node) - end - end - - def visit_dot3(node) - node(node, "dot3") do - field("left", node.left) if node.left - field("right", node.right) if node.right - comments(node) - end - end - def visit_dyna_symbol(node) node(node, "dyna_symbol") do list("parts", node.parts) @@ -739,6 +723,15 @@ def visit_qwords_beg(node) node(node, "qwords_beg") { field("value", node.value) } end + def visit_range_literal(node) + node(node, "range_literal") do + field("left", node.left) if node.left + field("operator", node.operator) + field("right", node.right) if node.right + comments(node) + end + end + def visit_rassign(node) node(node, "rassign") do field("value", node.value) diff --git a/test/node_test.rb b/test/node_test.rb index 69c147c3..d27c0d5f 100644 --- a/test/node_test.rb +++ b/test/node_test.rb @@ -414,11 +414,11 @@ def test_do_block end def test_dot2 - assert_node(Dot2, "1..3") + assert_node(RangeLiteral, "1..3") end def test_dot3 - assert_node(Dot3, "1...3") + assert_node(RangeLiteral, "1...3") end def test_dyna_symbol From a99d08124ade363c0b9371897918e0ded74f42c3 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 2 Nov 2022 12:02:42 -0400 Subject: [PATCH 181/536] Fold DefEndless into Def and Defs nodes --- CHANGELOG.md | 1 + lib/syntax_tree/node.rb | 171 +++++++------------- lib/syntax_tree/parser.rb | 16 +- lib/syntax_tree/visitor.rb | 3 - lib/syntax_tree/visitor/field_visitor.rb | 14 -- lib/syntax_tree/visitor/with_environment.rb | 4 - test/fixtures/def_endless.rb | 4 - test/node_test.rb | 4 +- 8 files changed, 70 insertions(+), 147 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 25bb7891..67b4e150 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - `Yield0` is no longer a node. Instead if has been folded into the `Yield` node. The `Yield` node can now have its `arguments` field be `nil`. Consequently, the `visit_yield0` method has been removed from the visitor interface. If you were previously using this method, you should now use `visit_yield` instead. - `FCall` is no longer a node. Instead it has been folded into the `Call` node. The `Call` node can now have its `receiver` and `operator` fields be `nil`. Consequently, the `visit_fcall` method has been removed from the visitor interface. If you were previously using this method, you should now use `visit_call` instead. - `Dot2` and `Dot3` are no longer nodes. Instead they have become a single new `RangeLiteral` node. This node looks the same as `Dot2` and `Dot3`, except that it additionally has an `operator` field that contains the operator that created the node. Consequently, the `visit_dot2` and `visit_dot3` methods have been removed from the visitor interface. If you were previously using these methods, you should now use `visit_range_literal` instead. + - `DefEndless` has been folded into the `Def` and `Defs` nodes. ## [4.3.0] - 2022-10-28 diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 8f3892e1..ebfcc336 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -3108,7 +3108,7 @@ def align(q, node, &block) part = parts.first case part - when Def, Defs, DefEndless + when Def, Defs q.text(" ") yield when IfOp @@ -3540,10 +3540,10 @@ class Def < Node # [Backtick | Const | Ident | Kw | Op] the name of the method attr_reader :name - # [Params | Paren] the parameter declaration for the method + # [nil | Params | Paren] the parameter declaration for the method attr_reader :params - # [BodyStmt] the expressions to be executed by the method + # [BodyStmt | untyped] the expressions to be executed by the method attr_reader :bodystmt # [Array[ Comment | EmbDoc ]] the comments attached to this node @@ -3583,112 +3583,41 @@ def format(q) q.text("def ") q.format(name) - if !params.is_a?(Params) || !params.empty? || params.comments.any? + case params + when Paren q.format(params) + when Params + q.format(params) if !params.empty? || params.comments.any? end end - unless bodystmt.empty? - q.indent do - q.breakable_force - q.format(bodystmt) + if endless? + q.text(" =") + q.group do + q.indent do + q.breakable_space + q.format(bodystmt) + end + end + else + unless bodystmt.empty? + q.indent do + q.breakable_force + q.format(bodystmt) + end end - end - q.breakable_force - q.text("end") + q.breakable_force + q.text("end") + end end end - end - - # DefEndless represents defining a single-line method since Ruby 3.0+. - # - # def method = result - # - class DefEndless < Node - # [untyped] the target where the method is being defined - attr_reader :target - - # [Op | Period] the operator being used to declare the method - attr_reader :operator - - # [Backtick | Const | Ident | Kw | Op] the name of the method - attr_reader :name - - # [nil | Params | Paren] the parameter declaration for the method - attr_reader :paren - - # [untyped] the expression to be executed by the method - attr_reader :statement - - # [Array[ Comment | EmbDoc ]] the comments attached to this node - attr_reader :comments - - def initialize( - target:, - operator:, - name:, - paren:, - statement:, - location:, - comments: [] - ) - @target = target - @operator = operator - @name = name - @paren = paren - @statement = statement - @location = location - @comments = [] - end - - def accept(visitor) - visitor.visit_def_endless(self) - end - def child_nodes - [target, operator, name, paren, statement] - end - - alias deconstruct child_nodes - - def deconstruct_keys(_keys) - { - target: target, - operator: operator, - name: name, - paren: paren, - statement: statement, - location: location, - comments: comments - } - end - - def format(q) - q.group do - q.text("def ") - - if target - q.format(target) - q.format(CallOperatorFormatter.new(operator), stackable: false) - end - - q.format(name) - - if paren - params = paren - params = params.contents if params.is_a?(Paren) - q.format(paren) unless params.empty? - end - - q.text(" =") - q.group do - q.indent do - q.breakable_space - q.format(statement) - end - end - end + # Returns true if the method was found in the source in the "endless" form, + # i.e. where the method body is defined using the `=` operator after the + # method name and parameters. + def endless? + !bodystmt.is_a?(BodyStmt) end end @@ -3751,10 +3680,10 @@ class Defs < Node # [Backtick | Const | Ident | Kw | Op] the name of the method attr_reader :name - # [Params | Paren] the parameter declaration for the method + # [nil | Params | Paren] the parameter declaration for the method attr_reader :params - # [BodyStmt] the expressions to be executed by the method + # [BodyStmt | untyped] the expressions to be executed by the method attr_reader :bodystmt # [Array[ Comment | EmbDoc ]] the comments attached to this node @@ -3808,22 +3737,42 @@ def format(q) q.format(CallOperatorFormatter.new(operator), stackable: false) q.format(name) - if !params.is_a?(Params) || !params.empty? || params.comments.any? + case params + when Paren q.format(params) + when Params + q.format(params) if !params.empty? || params.comments.any? end end - unless bodystmt.empty? - q.indent do - q.breakable_force - q.format(bodystmt) + if endless? + q.text(" =") + q.group do + q.indent do + q.breakable_space + q.format(bodystmt) + end + end + else + unless bodystmt.empty? + q.indent do + q.breakable_force + q.format(bodystmt) + end end - end - q.breakable_force - q.text("end") + q.breakable_force + q.text("end") + end end end + + # Returns true if the method was found in the source in the "endless" form, + # i.e. where the method body is defined using the `=` operator after the + # method name and parameters. + def endless? + !bodystmt.is_a?(BodyStmt) + end end # DoBlock represents passing a block to a method call using the +do+ and +end+ @@ -6971,7 +6920,7 @@ def format(q) end case q.parent - when Def, Defs, DefEndless + when Def, Defs q.nest(0) do q.text("(") q.group do diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index 4bc0bb6f..508348d8 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -1181,7 +1181,7 @@ def on_cvar(value) # (Backtick | Const | Ident | Kw | Op) name, # (nil | Params | Paren) params, # untyped bodystmt - # ) -> Def | DefEndless + # ) -> Def def on_def(name, params, bodystmt) # Make sure to delete this token in case you're defining something like # def class which would lead to this being a kw and causing all kinds of @@ -1234,12 +1234,10 @@ def on_def(name, params, bodystmt) # the statements list. Before, it was just the individual statement. statement = bodystmt.is_a?(BodyStmt) ? bodystmt.statements : bodystmt - DefEndless.new( - target: nil, - operator: nil, + Def.new( name: name, - paren: params, - statement: statement, + params: params, + bodystmt: statement, location: beginning.location.to(bodystmt.location) ) end @@ -1322,12 +1320,12 @@ def on_defs(target, operator, name, params, bodystmt) # the statements list. Before, it was just the individual statement. statement = bodystmt.is_a?(BodyStmt) ? bodystmt.statements : bodystmt - DefEndless.new( + Defs.new( target: target, operator: operator, name: name, - paren: params, - statement: statement, + params: params, + bodystmt: statement, location: beginning.location.to(bodystmt.location) ) end diff --git a/lib/syntax_tree/visitor.rb b/lib/syntax_tree/visitor.rb index ad4757d7..3bf3c72d 100644 --- a/lib/syntax_tree/visitor.rb +++ b/lib/syntax_tree/visitor.rb @@ -119,9 +119,6 @@ class Visitor < BasicVisitor # Visit a Def node. alias visit_def visit_child_nodes - # Visit a DefEndless node. - alias visit_def_endless visit_child_nodes - # Visit a Defined node. alias visit_defined visit_child_nodes diff --git a/lib/syntax_tree/visitor/field_visitor.rb b/lib/syntax_tree/visitor/field_visitor.rb index ac7b3603..14ecd2fc 100644 --- a/lib/syntax_tree/visitor/field_visitor.rb +++ b/lib/syntax_tree/visitor/field_visitor.rb @@ -322,20 +322,6 @@ def visit_def(node) end end - def visit_def_endless(node) - node(node, "def_endless") do - if node.target - field("target", node.target) - field("operator", node.operator) - end - - field("name", node.name) - field("paren", node.paren) if node.paren - field("statement", node.statement) - comments(node) - end - end - def visit_defined(node) node(node, "defined") do field("value", node.value) diff --git a/lib/syntax_tree/visitor/with_environment.rb b/lib/syntax_tree/visitor/with_environment.rb index 043cbd4c..006f7b09 100644 --- a/lib/syntax_tree/visitor/with_environment.rb +++ b/lib/syntax_tree/visitor/with_environment.rb @@ -60,10 +60,6 @@ def visit_defs(node) with_new_environment { super } end - def visit_def_endless(node) - with_new_environment { super } - end - # Visit for keeping track of local arguments, such as method and block # arguments def visit_params(node) diff --git a/test/fixtures/def_endless.rb b/test/fixtures/def_endless.rb index 15ea518b..4595fba9 100644 --- a/test/fixtures/def_endless.rb +++ b/test/fixtures/def_endless.rb @@ -4,8 +4,6 @@ def foo = bar def foo(bar) = baz % def foo() = bar -- -def foo = bar % # >= 3.1.0 def foo = bar baz % # >= 3.1.0 @@ -14,8 +12,6 @@ def self.foo = bar def self.foo(bar) = baz % # >= 3.1.0 def self.foo() = bar -- -def self.foo = bar % # >= 3.1.0 def self.foo = bar baz % diff --git a/test/node_test.rb b/test/node_test.rb index d27c0d5f..49f6f921 100644 --- a/test/node_test.rb +++ b/test/node_test.rb @@ -379,13 +379,13 @@ def method guard_version("3.0.0") do def test_def_endless - assert_node(DefEndless, "def method = result") + assert_node(Def, "def method = result") end end guard_version("3.1.0") do def test_def_endless_command - assert_node(DefEndless, "def method = result argument") + assert_node(Def, "def method = result argument") end end From 4a9a7c6a8b55764676f47b2ddcd10e614b4aff8c Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 2 Nov 2022 12:07:46 -0400 Subject: [PATCH 182/536] Fold Def and DefEndless --- CHANGELOG.md | 2 +- lib/syntax_tree/node.rb | 135 +++----------------- lib/syntax_tree/parser.rb | 10 +- lib/syntax_tree/visitor.rb | 3 - lib/syntax_tree/visitor/field_visitor.rb | 13 +- lib/syntax_tree/visitor/with_environment.rb | 4 - test/node_test.rb | 4 +- 7 files changed, 33 insertions(+), 138 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 67b4e150..10c33420 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - `Yield0` is no longer a node. Instead if has been folded into the `Yield` node. The `Yield` node can now have its `arguments` field be `nil`. Consequently, the `visit_yield0` method has been removed from the visitor interface. If you were previously using this method, you should now use `visit_yield` instead. - `FCall` is no longer a node. Instead it has been folded into the `Call` node. The `Call` node can now have its `receiver` and `operator` fields be `nil`. Consequently, the `visit_fcall` method has been removed from the visitor interface. If you were previously using this method, you should now use `visit_call` instead. - `Dot2` and `Dot3` are no longer nodes. Instead they have become a single new `RangeLiteral` node. This node looks the same as `Dot2` and `Dot3`, except that it additionally has an `operator` field that contains the operator that created the node. Consequently, the `visit_dot2` and `visit_dot3` methods have been removed from the visitor interface. If you were previously using these methods, you should now use `visit_range_literal` instead. - - `DefEndless` has been folded into the `Def` and `Defs` nodes. + - `DefEndless` and `Defs` have both been folded into the `Def` node. The `Def` node now has the `target` and `operator` fields which originally came from `Defs` which can both be `nil`. It also now has an `endless?` method on it to tell if the original node was found in the endless form. Finally the `bodystmt` field can now either be a `BodyStmt` as it was or any other kind of node since that was the body of the `DefEndless` node. The `visit_defs` and `visit_def_endless` methods on the visitor have therefore been removed. ## [4.3.0] - 2022-10-28 diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index ebfcc336..b1080f1f 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -3108,7 +3108,7 @@ def align(q, node, &block) part = parts.first case part - when Def, Defs + when Def q.text(" ") yield when IfOp @@ -3535,8 +3535,15 @@ def format(q) # Def represents defining a regular method on the current self object. # # def method(param) result end + # def object.method(param) result end # class Def < Node + # [nil | untyped] the target where the method is being defined + attr_reader :target + + # [nil | Op | Period] the operator being used to declare the method + attr_reader :operator + # [Backtick | Const | Ident | Kw | Op] the name of the method attr_reader :name @@ -3549,7 +3556,9 @@ class Def < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(name:, params:, bodystmt:, location:) + def initialize(target:, operator:, name:, params:, bodystmt:, location:) + @target = target + @operator = operator @name = name @params = params @bodystmt = bodystmt @@ -3562,13 +3571,15 @@ def accept(visitor) end def child_nodes - [name, params, bodystmt] + [target, operator, name, params, bodystmt] end alias deconstruct child_nodes def deconstruct_keys(_keys) { + target: target, + operator: operator, name: name, params: params, bodystmt: bodystmt, @@ -3581,6 +3592,12 @@ def format(q) q.group do q.group do q.text("def ") + + if target + q.format(target) + q.format(CallOperatorFormatter.new(operator), stackable: false) + end + q.format(name) case params @@ -3666,115 +3683,6 @@ def format(q) end end - # Defs represents defining a singleton method on an object. - # - # def object.method(param) result end - # - class Defs < Node - # [untyped] the target where the method is being defined - attr_reader :target - - # [Op | Period] the operator being used to declare the method - attr_reader :operator - - # [Backtick | Const | Ident | Kw | Op] the name of the method - attr_reader :name - - # [nil | Params | Paren] the parameter declaration for the method - attr_reader :params - - # [BodyStmt | untyped] the expressions to be executed by the method - attr_reader :bodystmt - - # [Array[ Comment | EmbDoc ]] the comments attached to this node - attr_reader :comments - - def initialize( - target:, - operator:, - name:, - params:, - bodystmt:, - location:, - comments: [] - ) - @target = target - @operator = operator - @name = name - @params = params - @bodystmt = bodystmt - @location = location - @comments = [] - end - - def accept(visitor) - visitor.visit_defs(self) - end - - def child_nodes - [target, operator, name, params, bodystmt] - end - - alias deconstruct child_nodes - - def deconstruct_keys(_keys) - { - target: target, - operator: operator, - name: name, - params: params, - bodystmt: bodystmt, - location: location, - comments: comments - } - end - - def format(q) - q.group do - q.group do - q.text("def ") - q.format(target) - q.format(CallOperatorFormatter.new(operator), stackable: false) - q.format(name) - - case params - when Paren - q.format(params) - when Params - q.format(params) if !params.empty? || params.comments.any? - end - end - - if endless? - q.text(" =") - q.group do - q.indent do - q.breakable_space - q.format(bodystmt) - end - end - else - unless bodystmt.empty? - q.indent do - q.breakable_force - q.format(bodystmt) - end - end - - q.breakable_force - q.text("end") - end - end - end - - # Returns true if the method was found in the source in the "endless" form, - # i.e. where the method body is defined using the `=` operator after the - # method name and parameters. - def endless? - !bodystmt.is_a?(BodyStmt) - end - end - # DoBlock represents passing a block to a method call using the +do+ and +end+ # keywords. # @@ -6919,8 +6827,7 @@ def format(q) return end - case q.parent - when Def, Defs + if q.parent.is_a?(Def) q.nest(0) do q.text("(") q.group do diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index 508348d8..797f69d2 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -1224,6 +1224,8 @@ def on_def(name, params, bodystmt) ) Def.new( + target: nil, + operator: nil, name: name, params: params, bodystmt: bodystmt, @@ -1235,6 +1237,8 @@ def on_def(name, params, bodystmt) statement = bodystmt.is_a?(BodyStmt) ? bodystmt.statements : bodystmt Def.new( + target: nil, + operator: nil, name: name, params: params, bodystmt: statement, @@ -1268,7 +1272,7 @@ def on_defined(value) # (Backtick | Const | Ident | Kw | Op) name, # (Params | Paren) params, # BodyStmt bodystmt - # ) -> Defs + # ) -> Def def on_defs(target, operator, name, params, bodystmt) # Make sure to delete this token in case you're defining something # like def class which would lead to this being a kw and causing all kinds @@ -1307,7 +1311,7 @@ def on_defs(target, operator, name, params, bodystmt) ending.location.start_column ) - Defs.new( + Def.new( target: target, operator: operator, name: name, @@ -1320,7 +1324,7 @@ def on_defs(target, operator, name, params, bodystmt) # the statements list. Before, it was just the individual statement. statement = bodystmt.is_a?(BodyStmt) ? bodystmt.statements : bodystmt - Defs.new( + Def.new( target: target, operator: operator, name: name, diff --git a/lib/syntax_tree/visitor.rb b/lib/syntax_tree/visitor.rb index 3bf3c72d..9748d0e1 100644 --- a/lib/syntax_tree/visitor.rb +++ b/lib/syntax_tree/visitor.rb @@ -122,9 +122,6 @@ class Visitor < BasicVisitor # Visit a Defined node. alias visit_defined visit_child_nodes - # Visit a Defs node. - alias visit_defs visit_child_nodes - # Visit a DoBlock node. alias visit_do_block visit_child_nodes diff --git a/lib/syntax_tree/visitor/field_visitor.rb b/lib/syntax_tree/visitor/field_visitor.rb index 14ecd2fc..0fc8c908 100644 --- a/lib/syntax_tree/visitor/field_visitor.rb +++ b/lib/syntax_tree/visitor/field_visitor.rb @@ -315,6 +315,8 @@ def visit_cvar(node) def visit_def(node) node(node, "def") do + field("target", node.target) + field("operator", node.operator) field("name", node.name) field("params", node.params) field("bodystmt", node.bodystmt) @@ -329,17 +331,6 @@ def visit_defined(node) end end - def visit_defs(node) - node(node, "defs") do - field("target", node.target) - field("operator", node.operator) - field("name", node.name) - field("params", node.params) - field("bodystmt", node.bodystmt) - comments(node) - end - end - def visit_do_block(node) node(node, "do_block") do field("block_var", node.block_var) if node.block_var diff --git a/lib/syntax_tree/visitor/with_environment.rb b/lib/syntax_tree/visitor/with_environment.rb index 006f7b09..59033d50 100644 --- a/lib/syntax_tree/visitor/with_environment.rb +++ b/lib/syntax_tree/visitor/with_environment.rb @@ -56,10 +56,6 @@ def visit_def(node) with_new_environment { super } end - def visit_defs(node) - with_new_environment { super } - end - # Visit for keeping track of local arguments, such as method and block # arguments def visit_params(node) diff --git a/test/node_test.rb b/test/node_test.rb index 49f6f921..8bb28131 100644 --- a/test/node_test.rb +++ b/test/node_test.rb @@ -394,7 +394,7 @@ def test_defined end def test_defs - assert_node(Defs, "def object.method(param) result end") + assert_node(Def, "def object.method(param) result end") end def test_defs_paramless @@ -403,7 +403,7 @@ def object.method end SOURCE - assert_node(Defs, source) + assert_node(Def, source) end def test_do_block From 3445e6435639914c73309d412ad639c11a165763 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 2 Nov 2022 12:24:42 -0400 Subject: [PATCH 183/536] Fold DoBlock and BraceBlock into Block --- CHANGELOG.md | 1 + lib/syntax_tree/node.rb | 396 ++++++++++------------- lib/syntax_tree/parser.rb | 16 +- lib/syntax_tree/visitor.rb | 9 +- lib/syntax_tree/visitor/field_visitor.rb | 24 +- test/node_test.rb | 6 +- 6 files changed, 186 insertions(+), 266 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 10c33420..92f03f01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - `FCall` is no longer a node. Instead it has been folded into the `Call` node. The `Call` node can now have its `receiver` and `operator` fields be `nil`. Consequently, the `visit_fcall` method has been removed from the visitor interface. If you were previously using this method, you should now use `visit_call` instead. - `Dot2` and `Dot3` are no longer nodes. Instead they have become a single new `RangeLiteral` node. This node looks the same as `Dot2` and `Dot3`, except that it additionally has an `operator` field that contains the operator that created the node. Consequently, the `visit_dot2` and `visit_dot3` methods have been removed from the visitor interface. If you were previously using these methods, you should now use `visit_range_literal` instead. - `DefEndless` and `Defs` have both been folded into the `Def` node. The `Def` node now has the `target` and `operator` fields which originally came from `Defs` which can both be `nil`. It also now has an `endless?` method on it to tell if the original node was found in the endless form. Finally the `bodystmt` field can now either be a `BodyStmt` as it was or any other kind of node since that was the body of the `DefEndless` node. The `visit_defs` and `visit_def_endless` methods on the visitor have therefore been removed. + - `DoBlock` and `BraceBlock` have now been folded into a `Block` node. The `Block` node now has a `keywords?` method on it that returns true if the block was constructed with the `do`..`end` keywords. The `visit_do_block` and `visit_brace_block` methods on the visitor have therefore been removed and replaced with the `visit_block` method. ## [4.3.0] - 2022-10-28 diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index b1080f1f..0ab33b83 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -1961,222 +1961,6 @@ def format(q) end end - # Responsible for formatting either a BraceBlock or a DoBlock. - class BlockFormatter - # Formats the opening brace or keyword of a block. - class BlockOpenFormatter - # [String] the actual output that should be printed - attr_reader :text - - # [LBrace | Keyword] the node that is being represented - attr_reader :node - - def initialize(text, node) - @text = text - @node = node - end - - def comments - node.comments - end - - def format(q) - q.text(text) - end - end - - # [BraceBlock | DoBlock] the block node to be formatted - attr_reader :node - - # [LBrace | Keyword] the node that opens the block - attr_reader :block_open - - # [String] the string that closes the block - attr_reader :block_close - - # [BodyStmt | Statements] the statements inside the block - attr_reader :statements - - def initialize(node, block_open, block_close, statements) - @node = node - @block_open = block_open - @block_close = block_close - @statements = statements - end - - def format(q) - # If this is nested anywhere inside of a Command or CommandCall node, then - # we can't change which operators we're using for the bounds of the block. - break_opening, break_closing, flat_opening, flat_closing = - if unchangeable_bounds?(q) - [block_open.value, block_close, block_open.value, block_close] - elsif forced_do_end_bounds?(q) - %w[do end do end] - elsif forced_brace_bounds?(q) - %w[{ } { }] - else - %w[do end { }] - end - - # If the receiver of this block a Command or CommandCall node, then there - # are no parentheses around the arguments to that command, so we need to - # break the block. - case q.parent.call - when Command, CommandCall - q.break_parent - format_break(q, break_opening, break_closing) - return - end - - q.group do - q - .if_break { format_break(q, break_opening, break_closing) } - .if_flat { format_flat(q, flat_opening, flat_closing) } - end - end - - private - - # If this is nested anywhere inside certain nodes, then we can't change - # which operators/keywords we're using for the bounds of the block. - def unchangeable_bounds?(q) - q.parents.any? do |parent| - # If we hit a statements, then we're safe to use whatever since we - # know for certain we're going to get split over multiple lines - # anyway. - case parent - when Statements, ArgParen - break false - when Command, CommandCall - true - else - false - end - end - end - - # If we're a sibling of a control-flow keyword, then we're going to have to - # use the do..end bounds. - def forced_do_end_bounds?(q) - case q.parent.call - when Break, Next, Return, Super - true - else - false - end - end - - # If we're the predicate of a loop or conditional, then we're going to have - # to go with the {..} bounds. - def forced_brace_bounds?(q) - previous = nil - q.parents.any? do |parent| - case parent - when Paren, Statements - # If we hit certain breakpoints then we know we're safe. - return false - when If, IfOp, Unless, While, Until - return true if parent.predicate == previous - end - - previous = parent - false - end - end - - def format_break(q, opening, closing) - q.text(" ") - q.format(BlockOpenFormatter.new(opening, block_open), stackable: false) - - if node.block_var - q.text(" ") - q.format(node.block_var) - end - - unless statements.empty? - q.indent do - q.breakable_space - q.format(statements) - end - end - - q.breakable_space - q.text(closing) - end - - def format_flat(q, opening, closing) - q.text(" ") - q.format(BlockOpenFormatter.new(opening, block_open), stackable: false) - - if node.block_var - q.breakable_space - q.format(node.block_var) - q.breakable_space - end - - if statements.empty? - q.text(" ") if opening == "do" - else - q.breakable_space unless node.block_var - q.format(statements) - q.breakable_space - end - - q.text(closing) - end - end - - # BraceBlock represents passing a block to a method call using the { } - # operators. - # - # method { |variable| variable + 1 } - # - class BraceBlock < Node - # [LBrace] the left brace that opens this block - attr_reader :lbrace - - # [nil | BlockVar] the optional set of parameters to the block - attr_reader :block_var - - # [Statements] the list of expressions to evaluate within the block - attr_reader :statements - - # [Array[ Comment | EmbDoc ]] the comments attached to this node - attr_reader :comments - - def initialize(lbrace:, block_var:, statements:, location:) - @lbrace = lbrace - @block_var = block_var - @statements = statements - @location = location - @comments = [] - end - - def accept(visitor) - visitor.visit_brace_block(self) - end - - def child_nodes - [lbrace, block_var, statements] - end - - alias deconstruct child_nodes - - def deconstruct_keys(_keys) - { - lbrace: lbrace, - block_var: block_var, - statements: statements, - location: location, - comments: comments - } - end - - def format(q) - BlockFormatter.new(self, lbrace, "}", statements).format(q) - end - end - # Formats either a Break, Next, or Return node. class FlowControlFormatter # [String] the keyword to print @@ -2479,10 +2263,10 @@ def format(q) # https://github.com/prettier/plugin-ruby/issues/863. parents = q.parents.take(4) if (parent = parents[2]) - # If we're at a do_block, then we want to go one more level up. This is - # because do blocks have BodyStmt nodes instead of just Statements - # nodes. - parent = parents[3] if parent.is_a?(DoBlock) + # If we're at a block with the `do` keywords, then we want to go one + # more level up. This is because do blocks have BodyStmt nodes instead + # of just Statements nodes. + parent = parents[3] if parent.is_a?(Block) && parent.keywords? if parent.is_a?(MethodAddBlock) && parent.call.is_a?(Call) && parent.call.message.value == "sig" threshold = 2 @@ -3683,27 +3467,51 @@ def format(q) end end - # DoBlock represents passing a block to a method call using the +do+ and +end+ - # keywords. + # Block represents passing a block to a method call using the +do+ and +end+ + # keywords or the +{+ and +}+ operators. # # method do |value| # end # - class DoBlock < Node - # [Kw] the do keyword that opens this block - attr_reader :keyword + # method { |value| } + # + class Block < Node + # Formats the opening brace or keyword of a block. + class BlockOpenFormatter + # [String] the actual output that should be printed + attr_reader :text + + # [LBrace | Keyword] the node that is being represented + attr_reader :node + + def initialize(text, node) + @text = text + @node = node + end + + def comments + node.comments + end + + def format(q) + q.text(text) + end + end + + # [LBrace | Kw] the left brace or the do keyword that opens this block + attr_reader :opening # [nil | BlockVar] the optional variable declaration within this block attr_reader :block_var - # [BodyStmt] the expressions to be executed within this block + # [BodyStmt | Statements] the expressions to be executed within this block attr_reader :bodystmt # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(keyword:, block_var:, bodystmt:, location:) - @keyword = keyword + def initialize(opening:, block_var:, bodystmt:, location:) + @opening = opening @block_var = block_var @bodystmt = bodystmt @location = location @@ -3711,18 +3519,18 @@ def initialize(keyword:, block_var:, bodystmt:, location:) end def accept(visitor) - visitor.visit_do_block(self) + visitor.visit_block(self) end def child_nodes - [keyword, block_var, bodystmt] + [opening, block_var, bodystmt] end alias deconstruct child_nodes def deconstruct_keys(_keys) { - keyword: keyword, + opening: opening, block_var: block_var, bodystmt: bodystmt, location: location, @@ -3731,7 +3539,129 @@ def deconstruct_keys(_keys) end def format(q) - BlockFormatter.new(self, keyword, "end", bodystmt).format(q) + # If this is nested anywhere inside of a Command or CommandCall node, then + # we can't change which operators we're using for the bounds of the block. + break_opening, break_closing, flat_opening, flat_closing = + if unchangeable_bounds?(q) + block_close = keywords? ? "end" : "}" + [opening.value, block_close, opening.value, block_close] + elsif forced_do_end_bounds?(q) + %w[do end do end] + elsif forced_brace_bounds?(q) + %w[{ } { }] + else + %w[do end { }] + end + + # If the receiver of this block a Command or CommandCall node, then there + # are no parentheses around the arguments to that command, so we need to + # break the block. + case q.parent.call + when Command, CommandCall + q.break_parent + format_break(q, break_opening, break_closing) + return + end + + q.group do + q + .if_break { format_break(q, break_opening, break_closing) } + .if_flat { format_flat(q, flat_opening, flat_closing) } + end + end + + def keywords? + opening.is_a?(Kw) + end + + private + + # If this is nested anywhere inside certain nodes, then we can't change + # which operators/keywords we're using for the bounds of the block. + def unchangeable_bounds?(q) + q.parents.any? do |parent| + # If we hit a statements, then we're safe to use whatever since we + # know for certain we're going to get split over multiple lines + # anyway. + case parent + when Statements, ArgParen + break false + when Command, CommandCall + true + else + false + end + end + end + + # If we're a sibling of a control-flow keyword, then we're going to have to + # use the do..end bounds. + def forced_do_end_bounds?(q) + case q.parent.call + when Break, Next, Return, Super + true + else + false + end + end + + # If we're the predicate of a loop or conditional, then we're going to have + # to go with the {..} bounds. + def forced_brace_bounds?(q) + previous = nil + q.parents.any? do |parent| + case parent + when Paren, Statements + # If we hit certain breakpoints then we know we're safe. + return false + when If, IfOp, Unless, While, Until + return true if parent.predicate == previous + end + + previous = parent + false + end + end + + def format_break(q, break_opening, break_closing) + q.text(" ") + q.format(BlockOpenFormatter.new(break_opening, opening), stackable: false) + + if block_var + q.text(" ") + q.format(block_var) + end + + unless bodystmt.empty? + q.indent do + q.breakable_space + q.format(bodystmt) + end + end + + q.breakable_space + q.text(break_closing) + end + + def format_flat(q, flat_opening, flat_closing) + q.text(" ") + q.format(BlockOpenFormatter.new(flat_opening, opening), stackable: false) + + if block_var + q.breakable_space + q.format(block_var) + q.breakable_space + end + + if bodystmt.empty? + q.text(" ") if flat_opening == "do" + else + q.breakable_space unless block_var + q.format(bodystmt) + q.breakable_space + end + + q.text(flat_closing) end end @@ -6144,7 +6074,7 @@ class MethodAddBlock < Node # [Call | Command | CommandCall] the method call attr_reader :call - # [BraceBlock | DoBlock] the block being sent with the method call + # [Block] the block being sent with the method call attr_reader :block # [Array[ Comment | EmbDoc ]] the comments attached to this node diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index 797f69d2..c7ac5e74 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -920,7 +920,7 @@ def on_bodystmt(statements, rescue_clause, else_clause, ensure_clause) # on_brace_block: ( # (nil | BlockVar) block_var, # Statements statements - # ) -> BraceBlock + # ) -> Block def on_brace_block(block_var, statements) lbrace = consume_token(LBrace) rbrace = consume_token(RBrace) @@ -947,10 +947,10 @@ def on_brace_block(block_var, statements) end_column: rbrace.location.end_column ) - BraceBlock.new( - lbrace: lbrace, + Block.new( + opening: lbrace, block_var: block_var, - statements: statements, + bodystmt: statements, location: location ) end @@ -1336,7 +1336,7 @@ def on_defs(target, operator, name, params, bodystmt) end # :call-seq: - # on_do_block: (BlockVar block_var, BodyStmt bodystmt) -> DoBlock + # on_do_block: (BlockVar block_var, BodyStmt bodystmt) -> Block def on_do_block(block_var, bodystmt) beginning = consume_keyword(:do) ending = consume_keyword(:end) @@ -1350,8 +1350,8 @@ def on_do_block(block_var, bodystmt) ending.location.start_column ) - DoBlock.new( - keyword: beginning, + Block.new( + opening: beginning, block_var: block_var, bodystmt: bodystmt, location: beginning.location.to(ending.location) @@ -2328,7 +2328,7 @@ def on_method_add_arg(call, arguments) # :call-seq: # on_method_add_block: ( # (Call | Command | CommandCall) call, - # (BraceBlock | DoBlock) block + # Block block # ) -> MethodAddBlock def on_method_add_block(call, block) MethodAddBlock.new( diff --git a/lib/syntax_tree/visitor.rb b/lib/syntax_tree/visitor.rb index 9748d0e1..2ee0fc15 100644 --- a/lib/syntax_tree/visitor.rb +++ b/lib/syntax_tree/visitor.rb @@ -62,6 +62,9 @@ class Visitor < BasicVisitor # Visit a Binary node. alias visit_binary visit_child_nodes + # Visit a Block node. + alias visit_block visit_child_nodes + # Visit a BlockArg node. alias visit_blockarg visit_child_nodes @@ -71,9 +74,6 @@ class Visitor < BasicVisitor # Visit a BodyStmt node. alias visit_bodystmt visit_child_nodes - # Visit a BraceBlock node. - alias visit_brace_block visit_child_nodes - # Visit a Break node. alias visit_break visit_child_nodes @@ -122,9 +122,6 @@ class Visitor < BasicVisitor # Visit a Defined node. alias visit_defined visit_child_nodes - # Visit a DoBlock node. - alias visit_do_block visit_child_nodes - # Visit a DynaSymbol node. alias visit_dyna_symbol visit_child_nodes diff --git a/lib/syntax_tree/visitor/field_visitor.rb b/lib/syntax_tree/visitor/field_visitor.rb index 0fc8c908..e1e75474 100644 --- a/lib/syntax_tree/visitor/field_visitor.rb +++ b/lib/syntax_tree/visitor/field_visitor.rb @@ -184,6 +184,14 @@ def visit_binary(node) end end + def visit_block(node) + node(node, "block") do + field("block_var", node.block_var) if node.block_var + field("bodystmt", node.bodystmt) + comments(node) + end + end + def visit_blockarg(node) node(node, "blockarg") do field("name", node.name) if node.name @@ -209,14 +217,6 @@ def visit_bodystmt(node) end end - def visit_brace_block(node) - node(node, "brace_block") do - field("block_var", node.block_var) if node.block_var - field("statements", node.statements) - comments(node) - end - end - def visit_break(node) node(node, "break") do field("arguments", node.arguments) @@ -331,14 +331,6 @@ def visit_defined(node) end end - def visit_do_block(node) - node(node, "do_block") do - field("block_var", node.block_var) if node.block_var - field("bodystmt", node.bodystmt) - comments(node) - end - end - def visit_dyna_symbol(node) node(node, "dyna_symbol") do list("parts", node.parts) diff --git a/test/node_test.rb b/test/node_test.rb index 8bb28131..05618552 100644 --- a/test/node_test.rb +++ b/test/node_test.rb @@ -276,7 +276,7 @@ def test_brace_block source = "method { |variable| variable + 1 }" at = location(chars: 7..34) - assert_node(BraceBlock, source, at: at, &:block) + assert_node(Block, source, at: at, &:block) end def test_break @@ -410,7 +410,7 @@ def test_do_block source = "method do |variable| variable + 1 end" at = location(chars: 7..37) - assert_node(DoBlock, source, at: at, &:block) + assert_node(Block, source, at: at, &:block) end def test_dot2 @@ -647,7 +647,7 @@ def test_lbrace source = "method {}" at = location(chars: 7..8) - assert_node(LBrace, source, at: at) { |node| node.block.lbrace } + assert_node(LBrace, source, at: at) { |node| node.block.opening } end def test_lparen From 0059a828241892c84c83196c758da92a4e6bfdbf Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 2 Nov 2022 18:16:15 -0400 Subject: [PATCH 184/536] Remove the "value" field from ZSuper --- CHANGELOG.md | 1 + lib/syntax_tree/node.rb | 10 +++------- lib/syntax_tree/parser.rb | 2 +- lib/syntax_tree/visitor/field_visitor.rb | 4 +++- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 92f03f01..989c0657 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - `Dot2` and `Dot3` are no longer nodes. Instead they have become a single new `RangeLiteral` node. This node looks the same as `Dot2` and `Dot3`, except that it additionally has an `operator` field that contains the operator that created the node. Consequently, the `visit_dot2` and `visit_dot3` methods have been removed from the visitor interface. If you were previously using these methods, you should now use `visit_range_literal` instead. - `DefEndless` and `Defs` have both been folded into the `Def` node. The `Def` node now has the `target` and `operator` fields which originally came from `Defs` which can both be `nil`. It also now has an `endless?` method on it to tell if the original node was found in the endless form. Finally the `bodystmt` field can now either be a `BodyStmt` as it was or any other kind of node since that was the body of the `DefEndless` node. The `visit_defs` and `visit_def_endless` methods on the visitor have therefore been removed. - `DoBlock` and `BraceBlock` have now been folded into a `Block` node. The `Block` node now has a `keywords?` method on it that returns true if the block was constructed with the `do`..`end` keywords. The `visit_do_block` and `visit_brace_block` methods on the visitor have therefore been removed and replaced with the `visit_block` method. +- The `ZSuper` node no longer has a `value` field associated with it (which was always a "super" string literal). ## [4.3.0] - 2022-10-28 diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 0ab33b83..f6baef2f 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -9830,14 +9830,10 @@ def format(q) # super # class ZSuper < Node - # [String] the value of the keyword - attr_reader :value - # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, location:) - @value = value + def initialize(location:) @location = location @comments = [] end @@ -9853,11 +9849,11 @@ def child_nodes alias deconstruct child_nodes def deconstruct_keys(_keys) - { value: value, location: location, comments: comments } + { location: location, comments: comments } end def format(q) - q.text(value) + q.text("super") end end end diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index c7ac5e74..32b4562a 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -3908,7 +3908,7 @@ def on_yield0 def on_zsuper keyword = consume_keyword(:super) - ZSuper.new(value: keyword.value, location: keyword.location) + ZSuper.new(location: keyword.location) end end end diff --git a/lib/syntax_tree/visitor/field_visitor.rb b/lib/syntax_tree/visitor/field_visitor.rb index e1e75474..4fab1b6c 100644 --- a/lib/syntax_tree/visitor/field_visitor.rb +++ b/lib/syntax_tree/visitor/field_visitor.rb @@ -1023,7 +1023,9 @@ def visit_yield(node) end def visit_zsuper(node) - visit_token(node, "zsuper") + node(node, "zsuper") do + comments(node) + end end def visit___end__(node) From 0bdddcccd08827032434fd699391416d547fc944 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 2 Nov 2022 18:21:25 -0400 Subject: [PATCH 185/536] Fold Return0 into Return --- CHANGELOG.md | 1 + lib/syntax_tree/node.rb | 53 ++++++------------------ lib/syntax_tree/parser.rb | 4 +- lib/syntax_tree/visitor.rb | 3 -- lib/syntax_tree/visitor/field_visitor.rb | 4 -- test/node_test.rb | 2 +- 6 files changed, 16 insertions(+), 51 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 989c0657..2d1fb994 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - `Dot2` and `Dot3` are no longer nodes. Instead they have become a single new `RangeLiteral` node. This node looks the same as `Dot2` and `Dot3`, except that it additionally has an `operator` field that contains the operator that created the node. Consequently, the `visit_dot2` and `visit_dot3` methods have been removed from the visitor interface. If you were previously using these methods, you should now use `visit_range_literal` instead. - `DefEndless` and `Defs` have both been folded into the `Def` node. The `Def` node now has the `target` and `operator` fields which originally came from `Defs` which can both be `nil`. It also now has an `endless?` method on it to tell if the original node was found in the endless form. Finally the `bodystmt` field can now either be a `BodyStmt` as it was or any other kind of node since that was the body of the `DefEndless` node. The `visit_defs` and `visit_def_endless` methods on the visitor have therefore been removed. - `DoBlock` and `BraceBlock` have now been folded into a `Block` node. The `Block` node now has a `keywords?` method on it that returns true if the block was constructed with the `do`..`end` keywords. The `visit_do_block` and `visit_brace_block` methods on the visitor have therefore been removed and replaced with the `visit_block` method. + - `Return0` is no longer a node. Instead if has been folded into the `Return` node. The `Return` node can now have its `arguments` field be `nil`. Consequently, the `visit_return0` method has been removed from the visitor interface. If you were previously using this method, you should now use `visit_return` instead. - The `ZSuper` node no longer has a `value` field associated with it (which was always a "super" string literal). ## [4.3.0] - 2022-10-28 diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index f6baef2f..31e27da1 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -1975,6 +1975,13 @@ def initialize(keyword, node) end def format(q) + # If there are no arguments associated with this flow control, then we can + # safely just print the keyword and return. + if node.arguments.nil? + q.text(keyword) + return + end + q.group do q.text(keyword) @@ -5077,8 +5084,8 @@ def call(q, node) def ternaryable?(statement) case statement when Alias, Assign, Break, Command, CommandCall, Heredoc, If, IfOp, - Lambda, MAssign, Next, OpAssign, RescueMod, Return, Return0, Super, - Undef, Unless, Until, VoidStmt, While, Yield, ZSuper + Lambda, MAssign, Next, OpAssign, RescueMod, Return, Super, Undef, + Unless, Until, VoidStmt, While, Yield, ZSuper # This is a list of nodes that should not be allowed to be a part of a # ternary clause. false @@ -5351,8 +5358,8 @@ def deconstruct_keys(_keys) def format(q) force_flat = [ Alias, Assign, Break, Command, CommandCall, Heredoc, If, IfOp, Lambda, - MAssign, Next, OpAssign, RescueMod, Return, Return0, Super, Undef, - Unless, VoidStmt, Yield, ZSuper + MAssign, Next, OpAssign, RescueMod, Return, Super, Undef, Unless, + VoidStmt, Yield, ZSuper ] if q.parent.is_a?(Paren) || force_flat.include?(truthy.class) || @@ -7739,7 +7746,7 @@ def format(q) # return value # class Return < Node - # [Args] the arguments being passed to the keyword + # [nil | Args] the arguments being passed to the keyword attr_reader :arguments # [Array[ Comment | EmbDoc ]] the comments attached to this node @@ -7770,42 +7777,6 @@ def format(q) end end - # Return0 represents the bare +return+ keyword with no arguments. - # - # return - # - class Return0 < Node - # [String] the value of the keyword - attr_reader :value - - # [Array[ Comment | EmbDoc ]] the comments attached to this node - attr_reader :comments - - def initialize(value:, location:) - @value = value - @location = location - @comments = [] - end - - def accept(visitor) - visitor.visit_return0(self) - end - - def child_nodes - [] - end - - alias deconstruct child_nodes - - def deconstruct_keys(_keys) - { value: value, location: location, comments: comments } - end - - def format(q) - q.text(value) - end - end - # RParen represents the use of a right parenthesis, i.e., +)+. class RParen < Node # [String] the parenthesis diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index 32b4562a..9fe750d7 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -3084,11 +3084,11 @@ def on_return(arguments) end # :call-seq: - # on_return0: () -> Return0 + # on_return0: () -> Return def on_return0 keyword = consume_keyword(:return) - Return0.new(value: keyword.value, location: keyword.location) + Return.new(arguments: nil, location: keyword.location) end # :call-seq: diff --git a/lib/syntax_tree/visitor.rb b/lib/syntax_tree/visitor.rb index 2ee0fc15..57aca619 100644 --- a/lib/syntax_tree/visitor.rb +++ b/lib/syntax_tree/visitor.rb @@ -338,9 +338,6 @@ class Visitor < BasicVisitor # Visit a Return node. alias visit_return visit_child_nodes - # Visit a Return0 node. - alias visit_return0 visit_child_nodes - # Visit a RParen node. alias visit_rparen visit_child_nodes diff --git a/lib/syntax_tree/visitor/field_visitor.rb b/lib/syntax_tree/visitor/field_visitor.rb index 4fab1b6c..150a4e97 100644 --- a/lib/syntax_tree/visitor/field_visitor.rb +++ b/lib/syntax_tree/visitor/field_visitor.rb @@ -789,10 +789,6 @@ def visit_return(node) end end - def visit_return0(node) - visit_token(node, "return0") - end - def visit_rparen(node) node(node, "rparen") { field("value", node.value) } end diff --git a/test/node_test.rb b/test/node_test.rb index 05618552..cbfc6173 100644 --- a/test/node_test.rb +++ b/test/node_test.rb @@ -841,7 +841,7 @@ def test_return end def test_return0 - assert_node(Return0, "return") + assert_node(Return, "return") end def test_sclass From 1a006d47262494ed7f4b7a29ab7667d63b9e3fa2 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 2 Nov 2022 18:24:12 -0400 Subject: [PATCH 186/536] Drop the value attribute from Redo and Retry --- CHANGELOG.md | 2 +- lib/syntax_tree/node.rb | 20 ++++++-------------- lib/syntax_tree/parser.rb | 4 ++-- lib/syntax_tree/visitor/field_visitor.rb | 6 ++++-- 4 files changed, 13 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d1fb994..d3c3e915 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - `DefEndless` and `Defs` have both been folded into the `Def` node. The `Def` node now has the `target` and `operator` fields which originally came from `Defs` which can both be `nil`. It also now has an `endless?` method on it to tell if the original node was found in the endless form. Finally the `bodystmt` field can now either be a `BodyStmt` as it was or any other kind of node since that was the body of the `DefEndless` node. The `visit_defs` and `visit_def_endless` methods on the visitor have therefore been removed. - `DoBlock` and `BraceBlock` have now been folded into a `Block` node. The `Block` node now has a `keywords?` method on it that returns true if the block was constructed with the `do`..`end` keywords. The `visit_do_block` and `visit_brace_block` methods on the visitor have therefore been removed and replaced with the `visit_block` method. - `Return0` is no longer a node. Instead if has been folded into the `Return` node. The `Return` node can now have its `arguments` field be `nil`. Consequently, the `visit_return0` method has been removed from the visitor interface. If you were previously using this method, you should now use `visit_return` instead. -- The `ZSuper` node no longer has a `value` field associated with it (which was always a "super" string literal). +- The `Redo`, `Retry`, and `ZSuper` nodes no longer have `value` fields associated with them (which were always string literals corresponding to the keyword being used). ## [4.3.0] - 2022-10-28 diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 31e27da1..f93f4935 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -7199,14 +7199,10 @@ def deconstruct_keys(_keys) # redo # class Redo < Node - # [String] the value of the keyword - attr_reader :value - # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, location:) - @value = value + def initialize(location:) @location = location @comments = [] end @@ -7222,11 +7218,11 @@ def child_nodes alias deconstruct child_nodes def deconstruct_keys(_keys) - { value: value, location: location, comments: comments } + { location: location, comments: comments } end def format(q) - q.text(value) + q.text("redo") end end @@ -7710,14 +7706,10 @@ def format(q) # retry # class Retry < Node - # [String] the value of the keyword - attr_reader :value - # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, location:) - @value = value + def initialize(location:) @location = location @comments = [] end @@ -7733,11 +7725,11 @@ def child_nodes alias deconstruct child_nodes def deconstruct_keys(_keys) - { value: value, location: location, comments: comments } + { location: location, comments: comments } end def format(q) - q.text(value) + q.text("retry") end end diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index 9fe750d7..ef0b77a0 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -2903,7 +2903,7 @@ def on_rbracket(value) def on_redo keyword = consume_keyword(:redo) - Redo.new(value: keyword.value, location: keyword.location) + Redo.new(location: keyword.location) end # :call-seq: @@ -3069,7 +3069,7 @@ def on_rest_param(name) def on_retry keyword = consume_keyword(:retry) - Retry.new(value: keyword.value, location: keyword.location) + Retry.new(location: keyword.location) end # :call-seq: diff --git a/lib/syntax_tree/visitor/field_visitor.rb b/lib/syntax_tree/visitor/field_visitor.rb index 150a4e97..d32dc877 100644 --- a/lib/syntax_tree/visitor/field_visitor.rb +++ b/lib/syntax_tree/visitor/field_visitor.rb @@ -723,7 +723,7 @@ def visit_rbracket(node) end def visit_redo(node) - visit_token(node, "redo") + node(node, "redo") { comments(node) } end def visit_regexp_beg(node) @@ -779,7 +779,9 @@ def visit_rest_param(node) end def visit_retry(node) - visit_token(node, "retry") + node(node, "retry") do + comments(node) + end end def visit_return(node) From 73e59c7760809af285eb231772a94be32e6768f7 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 2 Nov 2022 18:25:55 -0400 Subject: [PATCH 187/536] Remove ArgsForward value --- CHANGELOG.md | 2 +- lib/syntax_tree/node.rb | 10 +++------- lib/syntax_tree/parser.rb | 2 +- lib/syntax_tree/visitor/field_visitor.rb | 2 +- 4 files changed, 6 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d3c3e915..a2fae5b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - `DefEndless` and `Defs` have both been folded into the `Def` node. The `Def` node now has the `target` and `operator` fields which originally came from `Defs` which can both be `nil`. It also now has an `endless?` method on it to tell if the original node was found in the endless form. Finally the `bodystmt` field can now either be a `BodyStmt` as it was or any other kind of node since that was the body of the `DefEndless` node. The `visit_defs` and `visit_def_endless` methods on the visitor have therefore been removed. - `DoBlock` and `BraceBlock` have now been folded into a `Block` node. The `Block` node now has a `keywords?` method on it that returns true if the block was constructed with the `do`..`end` keywords. The `visit_do_block` and `visit_brace_block` methods on the visitor have therefore been removed and replaced with the `visit_block` method. - `Return0` is no longer a node. Instead if has been folded into the `Return` node. The `Return` node can now have its `arguments` field be `nil`. Consequently, the `visit_return0` method has been removed from the visitor interface. If you were previously using this method, you should now use `visit_return` instead. -- The `Redo`, `Retry`, and `ZSuper` nodes no longer have `value` fields associated with them (which were always string literals corresponding to the keyword being used). +- The `ArgsForward`, `Redo`, `Retry`, and `ZSuper` nodes no longer have `value` fields associated with them (which were always string literals corresponding to the keyword being used). ## [4.3.0] - 2022-10-28 diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index f93f4935..b1ddcc43 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -771,14 +771,10 @@ def format(q) # The ArgsForward node appears in both the caller (the request method calls) # and the callee (the get and post definitions). class ArgsForward < Node - # [String] the value of the operator - attr_reader :value - # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(value:, location:) - @value = value + def initialize(location:) @location = location @comments = [] end @@ -794,11 +790,11 @@ def child_nodes alias deconstruct child_nodes def deconstruct_keys(_keys) - { value: value, location: location, comments: comments } + { location: location, comments: comments } end def format(q) - q.text(value) + q.text("...") end end diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index ef0b77a0..324dfb47 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -575,7 +575,7 @@ def on_args_add_star(arguments, argument) def on_args_forward op = consume_operator(:"...") - ArgsForward.new(value: op.value, location: op.location) + ArgsForward.new(location: op.location) end # :call-seq: diff --git a/lib/syntax_tree/visitor/field_visitor.rb b/lib/syntax_tree/visitor/field_visitor.rb index d32dc877..c8014106 100644 --- a/lib/syntax_tree/visitor/field_visitor.rb +++ b/lib/syntax_tree/visitor/field_visitor.rb @@ -103,7 +103,7 @@ def visit_args(node) end def visit_args_forward(node) - visit_token(node, "args_forward") + node(node, "args_forward") { comments(node) } end def visit_array(node) From 321c5b9ba56f021abfe098ab126fc925c1b769e4 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 2 Nov 2022 18:30:51 -0400 Subject: [PATCH 188/536] Move block down to CommandCall --- CHANGELOG.md | 1 + lib/syntax_tree/node.rb | 12 ++++++++++-- lib/syntax_tree/parser.rb | 27 ++++++++++++++++++++++----- 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a2fae5b7..435c4f54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - `DoBlock` and `BraceBlock` have now been folded into a `Block` node. The `Block` node now has a `keywords?` method on it that returns true if the block was constructed with the `do`..`end` keywords. The `visit_do_block` and `visit_brace_block` methods on the visitor have therefore been removed and replaced with the `visit_block` method. - `Return0` is no longer a node. Instead if has been folded into the `Return` node. The `Return` node can now have its `arguments` field be `nil`. Consequently, the `visit_return0` method has been removed from the visitor interface. If you were previously using this method, you should now use `visit_return` instead. - The `ArgsForward`, `Redo`, `Retry`, and `ZSuper` nodes no longer have `value` fields associated with them (which were always string literals corresponding to the keyword being used). +- `CommandCall` now has a `block` attribute on it. This attribute is used in the place where you would previously have a `MethodAddBlock` structure. Where before the `MethodAddBlock` would have the `CommandCall` and `Block` as its two children, you now just have one `CommandCall` node with the `block` attribute set to the `Block` node. ## [4.3.0] - 2022-10-28 diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index b1ddcc43..9cd2dba6 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -2936,6 +2936,9 @@ class CommandCall < Node # [nil | Args] the arguments going along with the message attr_reader :arguments + # [nil | Block] the block associated with this method call + attr_reader :block + # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments @@ -2944,6 +2947,7 @@ def initialize( operator:, message:, arguments:, + block:, location:, comments: [] ) @@ -2951,6 +2955,7 @@ def initialize( @operator = operator @message = message @arguments = arguments + @block = block @location = location @comments = [] end @@ -2971,6 +2976,7 @@ def deconstruct_keys(_keys) operator: operator, message: message, arguments: arguments, + block: block, location: location, comments: comments } @@ -3011,6 +3017,8 @@ def format(q) end end end + + q.format(block) if block end private @@ -3559,8 +3567,8 @@ def format(q) # If the receiver of this block a Command or CommandCall node, then there # are no parentheses around the arguments to that command, so we need to # break the block. - case q.parent.call - when Command, CommandCall + parent = q.parent + if (parent.is_a?(MethodAddBlock) && parent.call.is_a?(Command)) || parent.is_a?(CommandCall) q.break_parent format_break(q, break_opening, break_closing) return diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index 324dfb47..257a25c2 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -1095,6 +1095,7 @@ def on_command_call(receiver, operator, message, arguments) operator: operator, message: message, arguments: arguments, + block: nil, location: receiver.location.to(ending.location) ) end @@ -2331,11 +2332,27 @@ def on_method_add_arg(call, arguments) # Block block # ) -> MethodAddBlock def on_method_add_block(call, block) - MethodAddBlock.new( - call: call, - block: block, - location: call.location.to(block.location) - ) + case call + when CommandCall + node = + CommandCall.new( + receiver: call.receiver, + operator: call.operator, + message: call.message, + arguments: call.arguments, + block: block, + location: call.location.to(block.location) + ) + + node.comments.concat(call.comments) + node + else + MethodAddBlock.new( + call: call, + block: block, + location: call.location.to(block.location) + ) + end end # :call-seq: From e0ce4ffbcdad0c8dddb8bcf25a807ebc08c64f3d Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 2 Nov 2022 18:34:39 -0400 Subject: [PATCH 189/536] Move block down to the command node --- CHANGELOG.md | 2 +- lib/syntax_tree/node.rb | 17 ++++++++++++----- lib/syntax_tree/parser.rb | 12 ++++++++++++ 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 435c4f54..642e7866 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - `DoBlock` and `BraceBlock` have now been folded into a `Block` node. The `Block` node now has a `keywords?` method on it that returns true if the block was constructed with the `do`..`end` keywords. The `visit_do_block` and `visit_brace_block` methods on the visitor have therefore been removed and replaced with the `visit_block` method. - `Return0` is no longer a node. Instead if has been folded into the `Return` node. The `Return` node can now have its `arguments` field be `nil`. Consequently, the `visit_return0` method has been removed from the visitor interface. If you were previously using this method, you should now use `visit_return` instead. - The `ArgsForward`, `Redo`, `Retry`, and `ZSuper` nodes no longer have `value` fields associated with them (which were always string literals corresponding to the keyword being used). -- `CommandCall` now has a `block` attribute on it. This attribute is used in the place where you would previously have a `MethodAddBlock` structure. Where before the `MethodAddBlock` would have the `CommandCall` and `Block` as its two children, you now just have one `CommandCall` node with the `block` attribute set to the `Block` node. +- The `Command` and `CommandCall` nodes now has `block` attributes on them. These attributes are used in the place where you would previously have had a `MethodAddBlock` structure. Where before the `MethodAddBlock` would have the command and block as its two children, you now just have one command node with the `block` attribute set to the `Block` node. ## [4.3.0] - 2022-10-28 diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 9cd2dba6..e966e32f 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -2847,12 +2847,16 @@ class Command < Node # [Args] the arguments being sent with the message attr_reader :arguments + # [nil | Block] the optional block being passed to the method + attr_reader :block + # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(message:, arguments:, location:) + def initialize(message:, arguments:, block:, location:) @message = message @arguments = arguments + @block = block @location = location @comments = [] end @@ -2862,7 +2866,7 @@ def accept(visitor) end def child_nodes - [message, arguments] + [message, arguments, block] end alias deconstruct child_nodes @@ -2871,6 +2875,7 @@ def deconstruct_keys(_keys) { message: message, arguments: arguments, + block: block, location: location, comments: comments } @@ -2881,6 +2886,8 @@ def format(q) q.format(message) align(q, self) { q.format(arguments) } end + + q.format(block) if block end private @@ -2965,7 +2972,7 @@ def accept(visitor) end def child_nodes - [receiver, message, arguments] + [receiver, message, arguments, block] end alias deconstruct child_nodes @@ -3567,8 +3574,8 @@ def format(q) # If the receiver of this block a Command or CommandCall node, then there # are no parentheses around the arguments to that command, so we need to # break the block. - parent = q.parent - if (parent.is_a?(MethodAddBlock) && parent.call.is_a?(Command)) || parent.is_a?(CommandCall) + case q.parent + when Command, CommandCall q.break_parent format_break(q, break_opening, break_closing) return diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index 257a25c2..9344653b 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -1076,6 +1076,7 @@ def on_command(message, arguments) Command.new( message: message, arguments: arguments, + block: nil, location: message.location.to(arguments.location) ) end @@ -2333,6 +2334,17 @@ def on_method_add_arg(call, arguments) # ) -> MethodAddBlock def on_method_add_block(call, block) case call + when Command + node = + Command.new( + message: call.message, + arguments: call.arguments, + block: block, + location: call.location.to(block.location) + ) + + node.comments.concat(call.comments) + node when CommandCall node = CommandCall.new( From 9bbf6cf5e65718fbe777bb3bde0f3929bfefa04f Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Tue, 8 Nov 2022 11:55:30 +0100 Subject: [PATCH 190/536] Unskip test_multiple_inline_scripts on TruffleRuby * StringIO is thread-safe now so this test should pass reliably. --- test/cli_test.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/test/cli_test.rb b/test/cli_test.rb index c00fb338..b4ef0afc 100644 --- a/test/cli_test.rb +++ b/test/cli_test.rb @@ -148,7 +148,6 @@ def test_inline_script end def test_multiple_inline_scripts - skip if RUBY_ENGINE == "truffleruby" # Relies on a thread-safe StringIO stdio, = capture_io { SyntaxTree::CLI.run(%w[format -e 1+1 -e 2+2]) } assert_equal(["1 + 1", "2 + 2"], stdio.split("\n").sort) end From b8bebe0aee7335e06e6f6271dd07408abb0b269b Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 2 Nov 2022 18:40:40 -0400 Subject: [PATCH 191/536] Fix up call chaining with new folded nodes --- lib/syntax_tree/node.rb | 71 ++++++++++++++++-------- lib/syntax_tree/parser.rb | 23 ++++++-- lib/syntax_tree/visitor/field_visitor.rb | 8 +-- 3 files changed, 68 insertions(+), 34 deletions(-) diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index e966e32f..97bec379 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -359,7 +359,8 @@ class Alias < Node # Formats an argument to the alias keyword. For symbol literals it uses the # value of the symbol directly to look like bare words. class AliasArgumentFormatter - # [Backref | DynaSymbol | GVar | SymbolLiteral] the argument being passed to alias + # [Backref | DynaSymbol | GVar | SymbolLiteral] the argument being passed + # to alias attr_reader :argument def initialize(argument) @@ -2245,14 +2246,26 @@ def format(q) when Call case (receiver = child.receiver) when Call - children << receiver + if receiver.receiver.nil? + break + else + children << receiver + end when MethodAddBlock - receiver.call.is_a?(Call) ? children << receiver : break + if receiver.call.is_a?(Call) && !receiver.call.receiver.nil? + children << receiver + else + break + end else break end when MethodAddBlock - child.call.is_a?(Call) ? children << child.call : break + if child.call.is_a?(Call) && !child.call.receiver.nil? + children << child.call + else + break + end else break end @@ -2271,7 +2284,8 @@ def format(q) # of just Statements nodes. parent = parents[3] if parent.is_a?(Block) && parent.keywords? - if parent.is_a?(MethodAddBlock) && parent.call.is_a?(Call) && parent.call.message.value == "sig" + if parent.is_a?(MethodAddBlock) && parent.call.is_a?(Call) && + parent.call.message.value == "sig" threshold = 2 end end @@ -2300,7 +2314,7 @@ def format_chain(q, children) # formatter so it's as if we had descending normally into them. This is # necessary so they can check their parents as normal. q.stack.concat(children) - q.format(children.last.receiver) + q.format(children.last.receiver) if children.last.receiver q.group do if attach_directly?(children.last) @@ -2343,7 +2357,8 @@ def format_chain(q, children) # If the parent call node has a comment on the message then we need # to print the operator trailing in order to keep it working. last_child = children.last - if last_child.is_a?(Call) && last_child.message.comments.any? + if last_child.is_a?(Call) && last_child.message.comments.any? && + last_child.operator q.format(CallOperatorFormatter.new(last_child.operator)) skip_operator = true else @@ -2372,9 +2387,9 @@ def self.chained?(node) case node when Call - true + !node.receiver.nil? when MethodAddBlock - node.call.is_a?(Call) + node.call.is_a?(Call) && !node.call.receiver.nil? else false end @@ -2405,7 +2420,7 @@ def format_child( case child when Call q.group do - unless skip_operator + if !skip_operator && child.operator q.format(CallOperatorFormatter.new(child.operator)) end q.format(child.message) if child.message != :call @@ -2495,7 +2510,8 @@ def format(q) # If we're at the top of a call chain, then we're going to do some # specialized printing in case we can print it nicely. We _only_ do this # at the top of the chain to avoid weird recursion issues. - if CallChainFormatter.chained?(receiver) && !CallChainFormatter.chained?(q.parent) + if CallChainFormatter.chained?(receiver) && + !CallChainFormatter.chained?(q.parent) q.group do q .if_break { CallChainFormatter.new(self).format(q) } @@ -2507,11 +2523,13 @@ def format(q) else q.format(message) - if arguments.is_a?(ArgParen) && arguments.arguments.nil? && !message.is_a?(Const) - # If you're using an explicit set of parentheses on something that looks - # like a constant, then we need to match that in order to maintain valid - # Ruby. For example, you could do something like Foo(), on which we - # would need to keep the parentheses to make it look like a method call. + if arguments.is_a?(ArgParen) && arguments.arguments.nil? && + !message.is_a?(Const) + # If you're using an explicit set of parentheses on something that + # looks like a constant, then we need to match that in order to + # maintain valid Ruby. For example, you could do something like Foo(), + # on which we would need to keep the parentheses to make it look like + # a method call. else q.format(arguments) end @@ -3726,7 +3744,13 @@ def child_nodes alias deconstruct child_nodes def deconstruct_keys(_keys) - { left: left, operator: operator, right: right, location: location, comments: comments } + { + left: left, + operator: operator, + right: right, + location: location, + comments: comments + } end def format(q) @@ -5133,7 +5157,11 @@ def format(q) if ContainsAssignment.call(statement) || q.parent.is_a?(In) q.group { format_flat(q) } else - q.group { q.if_break { format_break(q, force: false) }.if_flat { format_flat(q) } } + q.group do + q + .if_break { format_break(q, force: false) } + .if_flat { format_flat(q) } + end end else # If we can transform this node into a ternary, then we're going to @@ -9063,7 +9091,8 @@ def format(q) # # foo = bar while foo # - if node.modifier? && (statement = node.statements.body.first) && (statement.is_a?(Begin) || ContainsAssignment.call(statement)) + if node.modifier? && (statement = node.statements.body.first) && + (statement.is_a?(Begin) || ContainsAssignment.call(statement)) q.format(statement) q.text(" #{keyword} ") q.format(node.predicate) @@ -9466,9 +9495,7 @@ def format(q) # last argument to the predicate is and endless range, then you are # forced to use the "then" keyword to make it parse properly. last = arguments.parts.last - if last.is_a?(RangeLiteral) && !last.right - q.text(" then") - end + q.text(" then") if last.is_a?(RangeLiteral) && !last.right end end diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index 9344653b..cd14672e 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -1616,7 +1616,13 @@ def on_excessed_comma(*) # :call-seq: # on_fcall: ((Const | Ident) value) -> Call def on_fcall(value) - Call.new(receiver: nil, operator: nil, message: value, arguments: nil, location: value.location) + Call.new( + receiver: nil, + operator: nil, + message: value, + arguments: nil, + location: value.location + ) end # :call-seq: @@ -1923,7 +1929,8 @@ def on_if_mod(predicate, statement) If.new( predicate: predicate, - statements: Statements.new(self, body: [statement], location: statement.location), + statements: + Statements.new(self, body: [statement], location: statement.location), consequent: nil, location: statement.location.to(predicate.location) ) @@ -2209,7 +2216,8 @@ def lambda_locals(source) on_comma: :item, on_rparen: :final }, - final: {} + final: { + } } tokens[(index + 1)..].each_with_object([]) do |token, locals| @@ -3622,7 +3630,8 @@ def on_unless_mod(predicate, statement) Unless.new( predicate: predicate, - statements: Statements.new(self, body: [statement], location: statement.location), + statements: + Statements.new(self, body: [statement], location: statement.location), consequent: nil, location: statement.location.to(predicate.location) ) @@ -3665,7 +3674,8 @@ def on_until_mod(predicate, statement) Until.new( predicate: predicate, - statements: Statements.new(self, body: [statement], location: statement.location), + statements: + Statements.new(self, body: [statement], location: statement.location), location: statement.location.to(predicate.location) ) end @@ -3791,7 +3801,8 @@ def on_while_mod(predicate, statement) While.new( predicate: predicate, - statements: Statements.new(self, body: [statement], location: statement.location), + statements: + Statements.new(self, body: [statement], location: statement.location), location: statement.location.to(predicate.location) ) end diff --git a/lib/syntax_tree/visitor/field_visitor.rb b/lib/syntax_tree/visitor/field_visitor.rb index c8014106..01c7de4e 100644 --- a/lib/syntax_tree/visitor/field_visitor.rb +++ b/lib/syntax_tree/visitor/field_visitor.rb @@ -779,9 +779,7 @@ def visit_rest_param(node) end def visit_retry(node) - node(node, "retry") do - comments(node) - end + node(node, "retry") { comments(node) } end def visit_return(node) @@ -1021,9 +1019,7 @@ def visit_yield(node) end def visit_zsuper(node) - node(node, "zsuper") do - comments(node) - end + node(node, "zsuper") { comments(node) } end def visit___end__(node) From 686f938b66a6033349d5bd88b94366335be66691 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 8 Nov 2022 10:45:34 -0500 Subject: [PATCH 192/536] Create #copy API for mutating nodes --- lib/syntax_tree/node.rb | 1437 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 1437 insertions(+) diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 97bec379..2882687c 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -161,6 +161,15 @@ def child_nodes [lbrace, statements] end + def copy(lbrace: nil, statements: nil, location: nil, comments: nil) + BEGINBlock.new( + lbrace: lbrace || self.lbrace, + statements: statements || self.statements, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -213,6 +222,14 @@ def child_nodes [] end + def copy(value: nil, location: nil, comments: nil) + CHAR.new( + value: value || self.value, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -264,6 +281,15 @@ def child_nodes [lbrace, statements] end + def copy(lbrace: nil, statements: nil, location: nil, comments: nil) + ENDBlock.new( + lbrace: lbrace || self.lbrace, + statements: statements || self.statements, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -319,6 +345,14 @@ def child_nodes [] end + def copy(value: nil, location: nil, comments: nil) + EndContent.new( + value: value || self.value, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -408,6 +442,15 @@ def child_nodes [left, right] end + def copy(left: nil, right: nil, location: nil, comments: nil) + Alias.new( + left: left || self.left, + right: right || self.right, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -473,6 +516,15 @@ def child_nodes [collection, index] end + def copy(collection: nil, index: nil, location: nil, comments: nil) + ARef.new( + collection: collection || self.collection, + index: index || self.index, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -534,6 +586,15 @@ def child_nodes [collection, index] end + def copy(collection: nil, index: nil, location: nil, comments: nil) + ARefField.new( + collection: collection || self.collection, + index: index || self.index, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -596,6 +657,14 @@ def child_nodes [arguments] end + def copy(arguments: nil, location: nil, comments: nil) + ArgParen.new( + arguments: arguments || self.arguments, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -669,6 +738,14 @@ def child_nodes parts end + def copy(parts: nil, location: nil, comments: nil) + Args.new( + parts: parts || self.parts, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -705,6 +782,14 @@ def child_nodes [value] end + def copy(value: nil, location: nil, comments: nil) + ArgBlock.new( + value: value || self.value, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -742,6 +827,14 @@ def child_nodes [value] end + def copy(value: nil, location: nil, comments: nil) + ArgStar.new( + value: value || self.value, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -788,6 +881,13 @@ def child_nodes [] end + def copy(location: nil, comments: nil) + ArgsForward.new( + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -971,6 +1071,15 @@ def child_nodes [lbracket, contents] end + def copy(lbracket: nil, contents: nil, location: nil, comments: nil) + ArrayLiteral.new( + lbracket: lbracket || self.lbracket, + contents: contents || self.contents, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -1144,6 +1253,24 @@ def child_nodes [constant, *requireds, rest, *posts] end + def copy( + constant: nil, + requireds: nil, + rest: nil, + posts: nil, + location: nil, + comments: nil + ) + AryPtn.new( + constant: constant || self.constant, + requireds: requireds || self.requireds, + rest: rest || self.rest, + posts: posts || self.posts, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -1225,6 +1352,15 @@ def child_nodes [target, value] end + def copy(target: nil, value: nil, location: nil, comments: nil) + Assign.new( + target: target || self.target, + value: value || self.value, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -1287,6 +1423,15 @@ def child_nodes [key, value] end + def copy(key: nil, value: nil, location: nil, comments: nil) + Assoc.new( + key: key || self.key, + value: value || self.value, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -1345,6 +1490,14 @@ def child_nodes [value] end + def copy(value: nil, location: nil, comments: nil) + AssocSplat.new( + value: value || self.value, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -1383,6 +1536,14 @@ def child_nodes [] end + def copy(value: nil, location: nil, comments: nil) + Backref.new( + value: value || self.value, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -1418,6 +1579,14 @@ def child_nodes [] end + def copy(value: nil, location: nil, comments: nil) + Backtick.new( + value: value || self.value, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -1530,6 +1699,14 @@ def child_nodes assocs end + def copy(assocs: nil, location: nil, comments: nil) + BareAssocHash.new( + assocs: assocs || self.assocs, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -1572,6 +1749,14 @@ def child_nodes [bodystmt] end + def copy(bodystmt: nil, location: nil, comments: nil) + Begin.new( + bodystmt: bodystmt || self.bodystmt, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -1620,6 +1805,14 @@ def child_nodes [statement] end + def copy(statement: nil, location: nil, comments: nil) + PinnedBegin.new( + statement: statement || self.statement, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -1694,6 +1887,16 @@ def child_nodes [left, right] end + def copy(left: nil, operator: nil, right: nil, location: nil, comments: nil) + Binary.new( + left: left || self.left, + operator: operator || self.operator, + right: right || self.right, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -1761,6 +1964,15 @@ def child_nodes [params, *locals] end + def copy(params: nil, locals: nil, location: nil, comments: nil) + BlockVar.new( + params: params || self.params, + locals: locals || self.locals, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -1818,6 +2030,14 @@ def child_nodes [name] end + def copy(name: nil, location: nil, comments: nil) + BlockArg.new( + name: name || self.name, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -1912,6 +2132,24 @@ def child_nodes [statements, rescue_clause, else_keyword, else_clause, ensure_clause] end + def copy( + statements: nil, + rescue_clause: nil, + else_clause: nil, + ensure_clause: nil, + location: nil, + comments: nil + ) + BodyStmt.new( + statements: statements || self.statements, + rescue_clause: rescue_clause || self.rescue_clause, + else_clause: else_clause || self.else_clause, + ensure_clause: ensure_clause || self.ensure_clause, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -2176,6 +2414,14 @@ def child_nodes [arguments] end + def copy(arguments: nil, location: nil, comments: nil) + Break.new( + arguments: arguments || self.arguments, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -2492,6 +2738,24 @@ def child_nodes ] end + def copy( + receiver: nil, + operator: nil, + message: nil, + arguments: nil, + location: nil, + comments: nil + ) + Call.new( + receiver: receiver || self.receiver, + operator: operator || self.operator, + message: message || self.message, + arguments: arguments || self.arguments, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -2617,6 +2881,22 @@ def child_nodes [keyword, value, consequent] end + def copy( + keyword: nil, + value: nil, + consequent: nil, + location: nil, + comments: nil + ) + Case.new( + keyword: keyword || self.keyword, + value: value || self.value, + consequent: consequent || self.consequent, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -2682,6 +2962,22 @@ def child_nodes [value, operator, pattern] end + def copy( + value: nil, + operator: nil, + pattern: nil, + location: nil, + comments: nil + ) + RAssign.new( + value: value || self.value, + operator: operator || self.operator, + pattern: pattern || self.pattern, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -2778,6 +3074,22 @@ def child_nodes [constant, superclass, bodystmt] end + def copy( + constant: nil, + superclass: nil, + bodystmt: nil, + location: nil, + comments: nil + ) + ClassDeclaration.new( + constant: constant || self.constant, + superclass: superclass || self.superclass, + bodystmt: bodystmt || self.bodystmt, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -2845,6 +3157,10 @@ def child_nodes [] end + def copy(value: nil, location: nil) + Comma.new(value: value || self.value, location: location || self.location) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -2887,6 +3203,22 @@ def child_nodes [message, arguments, block] end + def copy( + message: nil, + arguments: nil, + block: nil, + location: nil, + comments: nil + ) + Command.new( + message: message || self.message, + arguments: arguments || self.arguments, + block: block || self.block, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -2993,6 +3325,26 @@ def child_nodes [receiver, message, arguments, block] end + def copy( + receiver: nil, + operator: nil, + message: nil, + arguments: nil, + block: nil, + location: nil, + comments: nil + ) + CommandCall.new( + receiver: receiver || self.receiver, + operator: operator || self.operator, + message: message || self.message, + arguments: arguments || self.arguments, + block: block || self.block, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -3125,6 +3477,14 @@ def child_nodes [] end + def copy(value: nil, inline: nil, location: nil) + Comment.new( + value: value || self.value, + inline: inline || self.inline, + location: location || self.location + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -3171,6 +3531,14 @@ def child_nodes [] end + def copy(value: nil, location: nil, comments: nil) + Const.new( + value: value || self.value, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -3213,6 +3581,15 @@ def child_nodes [parent, constant] end + def copy(parent: nil, constant: nil, location: nil, comments: nil) + ConstPathField.new( + parent: parent || self.parent, + constant: constant || self.constant, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -3260,6 +3637,15 @@ def child_nodes [parent, constant] end + def copy(parent: nil, constant: nil, location: nil, comments: nil) + ConstPathRef.new( + parent: parent || self.parent, + constant: constant || self.constant, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -3305,6 +3691,14 @@ def child_nodes [constant] end + def copy(constant: nil, location: nil, comments: nil) + ConstRef.new( + constant: constant || self.constant, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -3341,6 +3735,14 @@ def child_nodes [] end + def copy(value: nil, location: nil, comments: nil) + CVar.new( + value: value || self.value, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -3394,6 +3796,26 @@ def child_nodes [target, operator, name, params, bodystmt] end + def copy( + target: nil, + operator: nil, + name: nil, + params: nil, + bodystmt: nil, + location: nil, + comments: nil + ) + Def.new( + target: target || self.target, + operator: operator || self.operator, + name: name || self.name, + params: params || self.params, + bodystmt: bodystmt || self.bodystmt, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -3484,6 +3906,14 @@ def child_nodes [value] end + def copy(value: nil, location: nil, comments: nil) + Defined.new( + value: value || self.value, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -3562,6 +3992,22 @@ def child_nodes [opening, block_var, bodystmt] end + def copy( + opening: nil, + block_var: nil, + bodystmt: nil, + location: nil, + comments: nil + ) + Block.new( + opening: opening || self.opening, + block_var: block_var || self.block_var, + bodystmt: bodystmt || self.bodystmt, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -3741,6 +4187,16 @@ def child_nodes [left, right] end + def copy(left: nil, operator: nil, right: nil, location: nil, comments: nil) + RangeLiteral.new( + left: left || self.left, + operator: operator || self.operator, + right: right || self.right, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -3843,6 +4299,15 @@ def child_nodes parts end + def copy(parts: nil, quote: nil, location: nil, comments: nil) + DynaSymbol.new( + parts: parts || self.parts, + quote: quote || self.quote, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -3954,6 +4419,15 @@ def child_nodes [keyword, statements] end + def copy(keyword: nil, statements: nil, location: nil, comments: nil) + Else.new( + keyword: keyword || self.keyword, + statements: statements || self.statements, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -4020,6 +4494,22 @@ def child_nodes [predicate, statements, consequent] end + def copy( + predicate: nil, + statements: nil, + consequent: nil, + location: nil, + comments: nil + ) + Elsif.new( + predicate: predicate || self.predicate, + statements: statements || self.statements, + consequent: consequent || self.consequent, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -4092,6 +4582,13 @@ def child_nodes [] end + def copy(value: nil, location: nil) + EmbDoc.new( + value: value || self.value, + location: location || self.location + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -4127,6 +4624,13 @@ def child_nodes [] end + def copy(value: nil, location: nil) + EmbExprBeg.new( + value: value || self.value, + location: location || self.location + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -4157,6 +4661,13 @@ def child_nodes [] end + def copy(value: nil, location: nil) + EmbExprEnd.new( + value: value || self.value, + location: location || self.location + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -4189,6 +4700,13 @@ def child_nodes [] end + def copy(value: nil, location: nil) + EmbVar.new( + value: value || self.value, + location: location || self.location + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -4228,6 +4746,15 @@ def child_nodes [keyword, statements] end + def copy(keyword: nil, statements: nil, location: nil, comments: nil) + Ensure.new( + keyword: keyword || self.keyword, + statements: statements || self.statements, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -4282,6 +4809,14 @@ def child_nodes [] end + def copy(value: nil, location: nil, comments: nil) + ExcessedComma.new( + value: value || self.value, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -4327,6 +4862,22 @@ def child_nodes [parent, (operator if operator != :"::"), name] end + def copy( + parent: nil, + operator: nil, + name: nil, + location: nil, + comments: nil + ) + Field.new( + parent: parent || self.parent, + operator: operator || self.operator, + name: name || self.name, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -4373,6 +4924,14 @@ def child_nodes [] end + def copy(value: nil, location: nil, comments: nil) + FloatLiteral.new( + value: value || self.value, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -4425,6 +4984,24 @@ def child_nodes [constant, left, *values, right] end + def copy( + constant: nil, + left: nil, + values: nil, + right: nil, + location: nil, + comments: nil + ) + FndPtn.new( + constant: constant || self.constant, + left: left || self.left, + values: values || self.values, + right: right || self.right, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -4499,6 +5076,22 @@ def child_nodes [index, collection, statements] end + def copy( + index: nil, + collection: nil, + statements: nil, + location: nil, + comments: nil + ) + For.new( + index: index || self.index, + collection: collection || self.collection, + statements: statements || self.statements, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -4556,6 +5149,14 @@ def child_nodes [] end + def copy(value: nil, location: nil, comments: nil) + GVar.new( + value: value || self.value, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -4622,6 +5223,15 @@ def child_nodes [lbrace] + assocs end + def copy(lbrace: nil, assocs: nil, location: nil, comments: nil) + HashLiteral.new( + lbrace: lbrace || self.lbrace, + assocs: assocs || self.assocs, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -4718,6 +5328,22 @@ def child_nodes [beginning, *parts, ending] end + def copy( + beginning: nil, + location: nil, + ending: nil, + parts: nil, + comments: nil + ) + Heredoc.new( + beginning: beginning || self.beginning, + location: location || self.location, + ending: ending || self.ending, + parts: parts || self.parts, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -4798,6 +5424,14 @@ def child_nodes [] end + def copy(value: nil, location: nil, comments: nil) + HeredocBeg.new( + value: value || self.value, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -4837,6 +5471,14 @@ def child_nodes [] end + def copy(value: nil, location: nil, comments: nil) + HeredocEnd.new( + value: value || self.value, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -4931,6 +5573,22 @@ def child_nodes [constant, *keywords.flatten(1), keyword_rest] end + def copy( + constant: nil, + keywords: nil, + keyword_rest: nil, + location: nil, + comments: nil + ) + HshPtn.new( + constant: constant || self.constant, + keywords: keywords || self.keywords, + keyword_rest: keyword_rest || self.keyword_rest, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -5039,6 +5697,14 @@ def child_nodes [] end + def copy(value: nil, location: nil, comments: nil) + Ident.new( + value: value || self.value, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -5327,6 +5993,22 @@ def child_nodes [predicate, statements, consequent] end + def copy( + predicate: nil, + statements: nil, + consequent: nil, + location: nil, + comments: nil + ) + If.new( + predicate: predicate || self.predicate, + statements: statements || self.statements, + consequent: consequent || self.consequent, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -5382,6 +6064,22 @@ def child_nodes [predicate, truthy, falsy] end + def copy( + predicate: nil, + truthy: nil, + falsy: nil, + location: nil, + comments: nil + ) + IfOp.new( + predicate: predicate || self.predicate, + truthy: truthy || self.truthy, + falsy: falsy || self.falsy, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -5475,6 +6173,14 @@ def child_nodes [] end + def copy(value: nil, location: nil, comments: nil) + Imaginary.new( + value: value || self.value, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -5522,6 +6228,22 @@ def child_nodes [pattern, statements, consequent] end + def copy( + pattern: nil, + statements: nil, + consequent: nil, + location: nil, + comments: nil + ) + In.new( + pattern: pattern || self.pattern, + statements: statements || self.statements, + consequent: consequent || self.consequent, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -5581,6 +6303,14 @@ def child_nodes [] end + def copy(value: nil, location: nil, comments: nil) + Int.new( + value: value || self.value, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -5625,6 +6355,14 @@ def child_nodes [] end + def copy(value: nil, location: nil, comments: nil) + IVar.new( + value: value || self.value, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -5674,6 +6412,14 @@ def child_nodes [] end + def copy(value: nil, location: nil, comments: nil) + Kw.new( + value: value || self.value, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -5711,6 +6457,14 @@ def child_nodes [name] end + def copy(name: nil, location: nil, comments: nil) + KwRestParam.new( + name: name || self.name, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -5757,6 +6511,14 @@ def child_nodes [] end + def copy(value: nil, location: nil, comments: nil) + Label.new( + value: value || self.value, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -5792,6 +6554,13 @@ def child_nodes [] end + def copy(value: nil, location: nil) + LabelEnd.new( + value: value || self.value, + location: location || self.location + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -5828,6 +6597,15 @@ def child_nodes [params, statements] end + def copy(params: nil, statements: nil, location: nil, comments: nil) + Lambda.new( + params: params || self.params, + statements: statements || self.statements, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -5936,6 +6714,15 @@ def child_nodes [params, *locals] end + def copy(params: nil, locals: nil, location: nil, comments: nil) + LambdaVar.new( + params: params || self.params, + locals: locals || self.locals, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -5978,6 +6765,14 @@ def child_nodes [] end + def copy(value: nil, location: nil, comments: nil) + LBrace.new( + value: value || self.value, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -6011,6 +6806,14 @@ def child_nodes [] end + def copy(value: nil, location: nil, comments: nil) + LBracket.new( + value: value || self.value, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -6044,6 +6847,14 @@ def child_nodes [] end + def copy(value: nil, location: nil, comments: nil) + LParen.new( + value: value || self.value, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -6094,6 +6905,15 @@ def child_nodes [target, value] end + def copy(target: nil, value: nil, location: nil, comments: nil) + MAssign.new( + target: target || self.target, + value: value || self.value, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -6141,6 +6961,15 @@ def child_nodes [call, block] end + def copy(call: nil, block: nil, location: nil, comments: nil) + MethodAddBlock.new( + call: call || self.call, + block: block || self.block, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -6202,6 +7031,15 @@ def child_nodes parts end + def copy(parts: nil, location: nil, comma: nil, comments: nil) + MLHS.new( + parts: parts || self.parts, + location: location || self.location, + comma: comma || self.comma, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -6246,6 +7084,14 @@ def child_nodes [contents] end + def copy(contents: nil, location: nil, comments: nil) + MLHSParen.new( + contents: contents || self.contents, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -6304,6 +7150,15 @@ def child_nodes [constant, bodystmt] end + def copy(constant: nil, bodystmt: nil, location: nil, comments: nil) + ModuleDeclaration.new( + constant: constant || self.constant, + bodystmt: bodystmt || self.bodystmt, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -6373,6 +7228,14 @@ def child_nodes parts end + def copy(parts: nil, location: nil, comments: nil) + MRHS.new( + parts: parts || self.parts, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -6422,6 +7285,14 @@ def child_nodes [arguments] end + def copy(arguments: nil, location: nil, comments: nil) + Next.new( + arguments: arguments || self.arguments, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -6463,6 +7334,14 @@ def child_nodes [] end + def copy(value: nil, location: nil, comments: nil) + Op.new( + value: value || self.value, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -6509,6 +7388,22 @@ def child_nodes [target, operator, value] end + def copy( + target: nil, + operator: nil, + value: nil, + location: nil, + comments: nil + ) + OpAssign.new( + target: target || self.target, + operator: operator || self.operator, + value: value || self.value, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -6767,6 +7662,30 @@ def child_nodes ] end + def copy( + location: nil, + requireds: nil, + optionals: nil, + rest: nil, + posts: nil, + keywords: nil, + keyword_rest: nil, + block: nil, + comments: nil + ) + Params.new( + location: location || self.location, + requireds: requireds || self.requireds, + optionals: optionals || self.optionals, + rest: rest || self.rest, + posts: posts || self.posts, + keywords: keywords || self.keywords, + keyword_rest: keyword_rest || self.keyword_rest, + block: block || self.block, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -6859,6 +7778,15 @@ def child_nodes [lparen, contents] end + def copy(lparen: nil, contents: nil, location: nil, comments: nil) + Paren.new( + lparen: lparen || self.lparen, + contents: contents || self.contents, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -6910,6 +7838,14 @@ def child_nodes [] end + def copy(value: nil, location: nil, comments: nil) + Period.new( + value: value || self.value, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -6943,6 +7879,14 @@ def child_nodes [statements] end + def copy(statements: nil, location: nil, comments: nil) + Program.new( + statements: statements || self.statements, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -6988,6 +7932,15 @@ def child_nodes [] end + def copy(beginning: nil, elements: nil, location: nil, comments: nil) + QSymbols.new( + beginning: beginning || self.beginning, + elements: elements || self.elements, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -7046,6 +7999,13 @@ def child_nodes [] end + def copy(value: nil, location: nil) + QSymbolsBeg.new( + value: value || self.value, + location: location || self.location + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -7082,6 +8042,15 @@ def child_nodes [] end + def copy(beginning: nil, elements: nil, location: nil, comments: nil) + QWords.new( + beginning: beginning || self.beginning, + elements: elements || self.elements, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -7140,6 +8109,13 @@ def child_nodes [] end + def copy(value: nil, location: nil) + QWordsBeg.new( + value: value || self.value, + location: location || self.location + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -7172,6 +8148,14 @@ def child_nodes [] end + def copy(value: nil, location: nil, comments: nil) + RationalLiteral.new( + value: value || self.value, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -7201,6 +8185,13 @@ def child_nodes [] end + def copy(value: nil, location: nil) + RBrace.new( + value: value || self.value, + location: location || self.location + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -7226,6 +8217,13 @@ def child_nodes [] end + def copy(value: nil, location: nil) + RBracket.new( + value: value || self.value, + location: location || self.location + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -7254,6 +8252,13 @@ def child_nodes [] end + def copy(location: nil, comments: nil) + Redo.new( + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -7293,6 +8298,14 @@ def child_nodes parts end + def copy(beginning: nil, parts: nil, location: nil) + RegexpContent.new( + beginning: beginning || self.beginning, + parts: parts || self.parts, + location: location || self.location + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -7326,6 +8339,13 @@ def child_nodes [] end + def copy(value: nil, location: nil) + RegexpBeg.new( + value: value || self.value, + location: location || self.location + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -7360,6 +8380,13 @@ def child_nodes [] end + def copy(value: nil, location: nil) + RegexpEnd.new( + value: value || self.value, + location: location || self.location + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -7401,6 +8428,24 @@ def child_nodes parts end + def copy( + beginning: nil, + ending: nil, + options: nil, + parts: nil, + location: nil, + comments: nil + ) + RegexpLiteral.new( + beginning: beginning || self.beginning, + ending: ending || self.ending, + options: options || self.options, + parts: parts || self.parts, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -7511,6 +8556,15 @@ def child_nodes [*exceptions, variable] end + def copy(exceptions: nil, variable: nil, location: nil, comments: nil) + RescueEx.new( + exceptions: exceptions || self.exceptions, + variable: variable || self.variable, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -7605,6 +8659,24 @@ def child_nodes [keyword, exception, statements, consequent] end + def copy( + keyword: nil, + exception: nil, + statements: nil, + consequent: nil, + location: nil, + comments: nil + ) + Rescue.new( + keyword: keyword || self.keyword, + exception: exception || self.exception, + statements: statements || self.statements, + consequent: consequent || self.consequent, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -7672,6 +8744,15 @@ def child_nodes [statement, value] end + def copy(statement: nil, value: nil, location: nil, comments: nil) + RescueMod.new( + statement: statement || self.statement, + value: value || self.value, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -7728,6 +8809,14 @@ def child_nodes [name] end + def copy(name: nil, location: nil, comments: nil) + RestParam.new( + name: name || self.name, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -7761,6 +8850,13 @@ def child_nodes [] end + def copy(location: nil, comments: nil) + Retry.new( + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -7797,6 +8893,14 @@ def child_nodes [arguments] end + def copy(arguments: nil, location: nil, comments: nil) + Return.new( + arguments: arguments || self.arguments, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -7826,6 +8930,13 @@ def child_nodes [] end + def copy(value: nil, location: nil) + RParen.new( + value: value || self.value, + location: location || self.location + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -7865,6 +8976,15 @@ def child_nodes [target, bodystmt] end + def copy(target: nil, bodystmt: nil, location: nil, comments: nil) + SClass.new( + target: target || self.target, + bodystmt: bodystmt || self.bodystmt, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -7969,6 +9089,15 @@ def child_nodes body end + def copy(parser: nil, body: nil, location: nil, comments: nil) + Statements.new( + parser: parser || self.parser, + body: body || self.body, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -8088,6 +9217,13 @@ def child_nodes parts end + def copy(parts: nil, location: nil) + StringContent.new( + parts: parts || self.parts, + location: location || self.location + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -8126,6 +9262,15 @@ def child_nodes [left, right] end + def copy(left: nil, right: nil, location: nil, comments: nil) + StringConcat.new( + left: left || self.left, + right: right || self.right, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -8171,6 +9316,14 @@ def child_nodes [variable] end + def copy(variable: nil, location: nil, comments: nil) + StringDVar.new( + variable: variable || self.variable, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -8211,6 +9364,14 @@ def child_nodes [statements] end + def copy(statements: nil, location: nil, comments: nil) + StringEmbExpr.new( + statements: statements || self.statements, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -8274,6 +9435,15 @@ def child_nodes parts end + def copy(parts: nil, quote: nil, location: nil, comments: nil) + StringLiteral.new( + parts: parts || self.parts, + quote: quote || self.quote, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -8348,6 +9518,14 @@ def child_nodes [arguments] end + def copy(arguments: nil, location: nil, comments: nil) + Super.new( + arguments: arguments || self.arguments, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -8401,6 +9579,13 @@ def child_nodes [] end + def copy(value: nil, location: nil) + SymBeg.new( + value: value || self.value, + location: location || self.location + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -8431,6 +9616,13 @@ def child_nodes [] end + def copy(value: nil, location: nil) + SymbolContent.new( + value: value || self.value, + location: location || self.location + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -8465,6 +9657,14 @@ def child_nodes [value] end + def copy(value: nil, location: nil, comments: nil) + SymbolLiteral.new( + value: value || self.value, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -8506,6 +9706,15 @@ def child_nodes [] end + def copy(beginning: nil, elements: nil, location: nil, comments: nil) + Symbols.new( + beginning: beginning || self.beginning, + elements: elements || self.elements, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -8565,6 +9774,13 @@ def child_nodes [] end + def copy(value: nil, location: nil) + SymbolsBeg.new( + value: value || self.value, + location: location || self.location + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -8594,6 +9810,13 @@ def child_nodes [] end + def copy(value: nil, location: nil) + TLambda.new( + value: value || self.value, + location: location || self.location + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -8624,6 +9847,13 @@ def child_nodes [] end + def copy(value: nil, location: nil) + TLamBeg.new( + value: value || self.value, + location: location || self.location + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -8658,6 +9888,14 @@ def child_nodes [constant] end + def copy(constant: nil, location: nil, comments: nil) + TopConstField.new( + constant: constant || self.constant, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -8696,6 +9934,14 @@ def child_nodes [constant] end + def copy(constant: nil, location: nil, comments: nil) + TopConstRef.new( + constant: constant || self.constant, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -8735,6 +9981,13 @@ def child_nodes [] end + def copy(value: nil, location: nil) + TStringBeg.new( + value: value || self.value, + location: location || self.location + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -8775,6 +10028,14 @@ def child_nodes [] end + def copy(value: nil, location: nil, comments: nil) + TStringContent.new( + value: value || self.value, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -8813,6 +10074,13 @@ def child_nodes [] end + def copy(value: nil, location: nil) + TStringEnd.new( + value: value || self.value, + location: location || self.location + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -8850,6 +10118,15 @@ def child_nodes [statement] end + def copy(statement: nil, parentheses: nil, location: nil, comments: nil) + Not.new( + statement: statement || self.statement, + parentheses: parentheses || self.parentheses, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -8916,6 +10193,15 @@ def child_nodes [statement] end + def copy(operator: nil, statement: nil, location: nil, comments: nil) + Unary.new( + operator: operator || self.operator, + statement: statement || self.statement, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -8982,6 +10268,14 @@ def child_nodes symbols end + def copy(symbols: nil, location: nil, comments: nil) + Undef.new( + symbols: symbols || self.symbols, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -9041,6 +10335,22 @@ def child_nodes [predicate, statements, consequent] end + def copy( + predicate: nil, + statements: nil, + consequent: nil, + location: nil, + comments: nil + ) + Unless.new( + predicate: predicate || self.predicate, + statements: statements || self.statements, + consequent: consequent || self.consequent, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -9165,6 +10475,15 @@ def child_nodes [predicate, statements] end + def copy(predicate: nil, statements: nil, location: nil, comments: nil) + Until.new( + predicate: predicate || self.predicate, + statements: statements || self.statements, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -9212,6 +10531,14 @@ def child_nodes [value] end + def copy(value: nil, location: nil, comments: nil) + VarField.new( + value: value || self.value, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -9256,6 +10583,14 @@ def child_nodes [value] end + def copy(value: nil, location: nil, comments: nil) + VarRef.new( + value: value || self.value, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -9320,6 +10655,14 @@ def child_nodes [value] end + def copy(value: nil, location: nil, comments: nil) + PinnedVarRef.new( + value: value || self.value, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -9360,6 +10703,14 @@ def child_nodes [value] end + def copy(value: nil, location: nil, comments: nil) + VCall.new( + value: value || self.value, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -9399,6 +10750,13 @@ def child_nodes [] end + def copy(location: nil, comments: nil) + VoidStmt.new( + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -9450,6 +10808,22 @@ def child_nodes [arguments, statements, consequent] end + def copy( + arguments: nil, + statements: nil, + consequent: nil, + location: nil, + comments: nil + ) + When.new( + arguments: arguments || self.arguments, + statements: statements || self.statements, + consequent: consequent || self.consequent, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -9544,6 +10918,15 @@ def child_nodes [predicate, statements] end + def copy(predicate: nil, statements: nil, location: nil, comments: nil) + While.new( + predicate: predicate || self.predicate, + statements: statements || self.statements, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -9597,6 +10980,14 @@ def child_nodes parts end + def copy(parts: nil, location: nil, comments: nil) + Word.new( + parts: parts || self.parts, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -9637,6 +11028,15 @@ def child_nodes [] end + def copy(beginning: nil, elements: nil, location: nil, comments: nil) + Words.new( + beginning: beginning || self.beginning, + elements: elements || self.elements, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -9696,6 +11096,13 @@ def child_nodes [] end + def copy(value: nil, location: nil) + WordsBeg.new( + value: value || self.value, + location: location || self.location + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -9725,6 +11132,13 @@ def child_nodes parts end + def copy(parts: nil, location: nil) + XString.new( + parts: parts || self.parts, + location: location || self.location + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -9758,6 +11172,14 @@ def child_nodes parts end + def copy(parts: nil, location: nil, comments: nil) + XStringLiteral.new( + parts: parts || self.parts, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -9796,6 +11218,14 @@ def child_nodes [arguments] end + def copy(arguments: nil, location: nil, comments: nil) + Yield.new( + arguments: arguments || self.arguments, + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) @@ -9847,6 +11277,13 @@ def child_nodes [] end + def copy(location: nil, comments: nil) + ZSuper.new( + location: location || self.location, + comments: comments || self.comments + ) + end + alias deconstruct child_nodes def deconstruct_keys(_keys) From 1ee58b2fe410fe9ebce7bb6ada60a818796bd2f7 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 8 Nov 2022 11:37:44 -0500 Subject: [PATCH 193/536] Create mutation visitor --- lib/syntax_tree.rb | 1 + lib/syntax_tree/node.rb | 811 ++++++------------ lib/syntax_tree/visitor/mutation_visitor.rb | 887 ++++++++++++++++++++ test/test_helper.rb | 3 + 4 files changed, 1149 insertions(+), 553 deletions(-) create mode 100644 lib/syntax_tree/visitor/mutation_visitor.rb diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index df2f43a9..5808cd15 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -16,6 +16,7 @@ require_relative "syntax_tree/visitor/field_visitor" require_relative "syntax_tree/visitor/json_visitor" require_relative "syntax_tree/visitor/match_visitor" +require_relative "syntax_tree/visitor/mutation_visitor" require_relative "syntax_tree/visitor/pretty_print_visitor" require_relative "syntax_tree/visitor/environment" require_relative "syntax_tree/visitor/with_environment" diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 2882687c..fb4bab21 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -161,12 +161,11 @@ def child_nodes [lbrace, statements] end - def copy(lbrace: nil, statements: nil, location: nil, comments: nil) + def copy(lbrace: nil, statements: nil, location: nil) BEGINBlock.new( lbrace: lbrace || self.lbrace, statements: statements || self.statements, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -222,12 +221,8 @@ def child_nodes [] end - def copy(value: nil, location: nil, comments: nil) - CHAR.new( - value: value || self.value, - location: location || self.location, - comments: comments || self.comments - ) + def copy(value: nil, location: nil) + CHAR.new(value: value || self.value, location: location || self.location) end alias deconstruct child_nodes @@ -281,12 +276,11 @@ def child_nodes [lbrace, statements] end - def copy(lbrace: nil, statements: nil, location: nil, comments: nil) + def copy(lbrace: nil, statements: nil, location: nil) ENDBlock.new( lbrace: lbrace || self.lbrace, statements: statements || self.statements, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -345,11 +339,10 @@ def child_nodes [] end - def copy(value: nil, location: nil, comments: nil) + def copy(value: nil, location: nil) EndContent.new( value: value || self.value, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -442,12 +435,11 @@ def child_nodes [left, right] end - def copy(left: nil, right: nil, location: nil, comments: nil) + def copy(left: nil, right: nil, location: nil) Alias.new( left: left || self.left, right: right || self.right, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -516,12 +508,11 @@ def child_nodes [collection, index] end - def copy(collection: nil, index: nil, location: nil, comments: nil) + def copy(collection: nil, index: nil, location: nil) ARef.new( collection: collection || self.collection, index: index || self.index, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -586,12 +577,11 @@ def child_nodes [collection, index] end - def copy(collection: nil, index: nil, location: nil, comments: nil) + def copy(collection: nil, index: nil, location: nil) ARefField.new( collection: collection || self.collection, index: index || self.index, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -657,11 +647,10 @@ def child_nodes [arguments] end - def copy(arguments: nil, location: nil, comments: nil) + def copy(arguments: nil, location: nil) ArgParen.new( arguments: arguments || self.arguments, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -738,12 +727,8 @@ def child_nodes parts end - def copy(parts: nil, location: nil, comments: nil) - Args.new( - parts: parts || self.parts, - location: location || self.location, - comments: comments || self.comments - ) + def copy(parts: nil, location: nil) + Args.new(parts: parts || self.parts, location: location || self.location) end alias deconstruct child_nodes @@ -782,11 +767,10 @@ def child_nodes [value] end - def copy(value: nil, location: nil, comments: nil) + def copy(value: nil, location: nil) ArgBlock.new( value: value || self.value, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -827,11 +811,10 @@ def child_nodes [value] end - def copy(value: nil, location: nil, comments: nil) + def copy(value: nil, location: nil) ArgStar.new( value: value || self.value, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -881,11 +864,8 @@ def child_nodes [] end - def copy(location: nil, comments: nil) - ArgsForward.new( - location: location || self.location, - comments: comments || self.comments - ) + def copy(location: nil) + ArgsForward.new(location: location || self.location) end alias deconstruct child_nodes @@ -1071,12 +1051,11 @@ def child_nodes [lbracket, contents] end - def copy(lbracket: nil, contents: nil, location: nil, comments: nil) + def copy(lbracket: nil, contents: nil, location: nil) ArrayLiteral.new( lbracket: lbracket || self.lbracket, contents: contents || self.contents, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -1258,16 +1237,14 @@ def copy( requireds: nil, rest: nil, posts: nil, - location: nil, - comments: nil + location: nil ) AryPtn.new( constant: constant || self.constant, requireds: requireds || self.requireds, rest: rest || self.rest, posts: posts || self.posts, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -1352,12 +1329,11 @@ def child_nodes [target, value] end - def copy(target: nil, value: nil, location: nil, comments: nil) + def copy(target: nil, value: nil, location: nil) Assign.new( target: target || self.target, value: value || self.value, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -1397,7 +1373,7 @@ def skip_indent? # # { key1: value1, key2: value2 } # - # In the above example, the would be two AssocNew nodes. + # In the above example, the would be two Assoc nodes. class Assoc < Node # [untyped] the key of this pair attr_reader :key @@ -1423,12 +1399,11 @@ def child_nodes [key, value] end - def copy(key: nil, value: nil, location: nil, comments: nil) + def copy(key: nil, value: nil, location: nil) Assoc.new( key: key || self.key, value: value || self.value, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -1490,11 +1465,10 @@ def child_nodes [value] end - def copy(value: nil, location: nil, comments: nil) + def copy(value: nil, location: nil) AssocSplat.new( value: value || self.value, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -1536,11 +1510,10 @@ def child_nodes [] end - def copy(value: nil, location: nil, comments: nil) + def copy(value: nil, location: nil) Backref.new( value: value || self.value, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -1579,11 +1552,10 @@ def child_nodes [] end - def copy(value: nil, location: nil, comments: nil) + def copy(value: nil, location: nil) Backtick.new( value: value || self.value, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -1699,11 +1671,10 @@ def child_nodes assocs end - def copy(assocs: nil, location: nil, comments: nil) + def copy(assocs: nil, location: nil) BareAssocHash.new( assocs: assocs || self.assocs, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -1749,11 +1720,10 @@ def child_nodes [bodystmt] end - def copy(bodystmt: nil, location: nil, comments: nil) + def copy(bodystmt: nil, location: nil) Begin.new( bodystmt: bodystmt || self.bodystmt, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -1805,11 +1775,10 @@ def child_nodes [statement] end - def copy(statement: nil, location: nil, comments: nil) + def copy(statement: nil, location: nil) PinnedBegin.new( statement: statement || self.statement, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -1887,13 +1856,12 @@ def child_nodes [left, right] end - def copy(left: nil, operator: nil, right: nil, location: nil, comments: nil) + def copy(left: nil, operator: nil, right: nil, location: nil) Binary.new( left: left || self.left, operator: operator || self.operator, right: right || self.right, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -1964,12 +1932,11 @@ def child_nodes [params, *locals] end - def copy(params: nil, locals: nil, location: nil, comments: nil) + def copy(params: nil, locals: nil, location: nil) BlockVar.new( params: params || self.params, locals: locals || self.locals, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -2030,12 +1997,8 @@ def child_nodes [name] end - def copy(name: nil, location: nil, comments: nil) - BlockArg.new( - name: name || self.name, - location: location || self.location, - comments: comments || self.comments - ) + def copy(name: nil, location: nil) + BlockArg.new(name: name || self.name, location: location || self.location) end alias deconstruct child_nodes @@ -2135,18 +2098,18 @@ def child_nodes def copy( statements: nil, rescue_clause: nil, + else_keyword: nil, else_clause: nil, ensure_clause: nil, - location: nil, - comments: nil + location: nil ) BodyStmt.new( statements: statements || self.statements, rescue_clause: rescue_clause || self.rescue_clause, + else_keyword: else_keyword || self.else_keyword, else_clause: else_clause || self.else_clause, ensure_clause: ensure_clause || self.ensure_clause, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -2156,6 +2119,7 @@ def deconstruct_keys(_keys) { statements: statements, rescue_clause: rescue_clause, + else_keyword: else_keyword, else_clause: else_clause, ensure_clause: ensure_clause, location: location, @@ -2414,11 +2378,10 @@ def child_nodes [arguments] end - def copy(arguments: nil, location: nil, comments: nil) + def copy(arguments: nil, location: nil) Break.new( arguments: arguments || self.arguments, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -2743,16 +2706,14 @@ def copy( operator: nil, message: nil, arguments: nil, - location: nil, - comments: nil + location: nil ) Call.new( receiver: receiver || self.receiver, operator: operator || self.operator, message: message || self.message, arguments: arguments || self.arguments, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -2881,19 +2842,12 @@ def child_nodes [keyword, value, consequent] end - def copy( - keyword: nil, - value: nil, - consequent: nil, - location: nil, - comments: nil - ) + def copy(keyword: nil, value: nil, consequent: nil, location: nil) Case.new( keyword: keyword || self.keyword, value: value || self.value, consequent: consequent || self.consequent, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -2962,19 +2916,12 @@ def child_nodes [value, operator, pattern] end - def copy( - value: nil, - operator: nil, - pattern: nil, - location: nil, - comments: nil - ) + def copy(value: nil, operator: nil, pattern: nil, location: nil) RAssign.new( value: value || self.value, operator: operator || self.operator, pattern: pattern || self.pattern, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -3074,19 +3021,12 @@ def child_nodes [constant, superclass, bodystmt] end - def copy( - constant: nil, - superclass: nil, - bodystmt: nil, - location: nil, - comments: nil - ) + def copy(constant: nil, superclass: nil, bodystmt: nil, location: nil) ClassDeclaration.new( constant: constant || self.constant, superclass: superclass || self.superclass, bodystmt: bodystmt || self.bodystmt, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -3203,19 +3143,12 @@ def child_nodes [message, arguments, block] end - def copy( - message: nil, - arguments: nil, - block: nil, - location: nil, - comments: nil - ) + def copy(message: nil, arguments: nil, block: nil, location: nil) Command.new( message: message || self.message, arguments: arguments || self.arguments, block: block || self.block, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -3331,8 +3264,7 @@ def copy( message: nil, arguments: nil, block: nil, - location: nil, - comments: nil + location: nil ) CommandCall.new( receiver: receiver || self.receiver, @@ -3340,8 +3272,7 @@ def copy( message: message || self.message, arguments: arguments || self.arguments, block: block || self.block, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -3531,12 +3462,8 @@ def child_nodes [] end - def copy(value: nil, location: nil, comments: nil) - Const.new( - value: value || self.value, - location: location || self.location, - comments: comments || self.comments - ) + def copy(value: nil, location: nil) + Const.new(value: value || self.value, location: location || self.location) end alias deconstruct child_nodes @@ -3581,12 +3508,11 @@ def child_nodes [parent, constant] end - def copy(parent: nil, constant: nil, location: nil, comments: nil) + def copy(parent: nil, constant: nil, location: nil) ConstPathField.new( parent: parent || self.parent, constant: constant || self.constant, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -3637,12 +3563,11 @@ def child_nodes [parent, constant] end - def copy(parent: nil, constant: nil, location: nil, comments: nil) + def copy(parent: nil, constant: nil, location: nil) ConstPathRef.new( parent: parent || self.parent, constant: constant || self.constant, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -3691,11 +3616,10 @@ def child_nodes [constant] end - def copy(constant: nil, location: nil, comments: nil) + def copy(constant: nil, location: nil) ConstRef.new( constant: constant || self.constant, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -3735,12 +3659,8 @@ def child_nodes [] end - def copy(value: nil, location: nil, comments: nil) - CVar.new( - value: value || self.value, - location: location || self.location, - comments: comments || self.comments - ) + def copy(value: nil, location: nil) + CVar.new(value: value || self.value, location: location || self.location) end alias deconstruct child_nodes @@ -3802,8 +3722,7 @@ def copy( name: nil, params: nil, bodystmt: nil, - location: nil, - comments: nil + location: nil ) Def.new( target: target || self.target, @@ -3811,8 +3730,7 @@ def copy( name: name || self.name, params: params || self.params, bodystmt: bodystmt || self.bodystmt, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -3906,11 +3824,10 @@ def child_nodes [value] end - def copy(value: nil, location: nil, comments: nil) + def copy(value: nil, location: nil) Defined.new( value: value || self.value, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -3992,19 +3909,12 @@ def child_nodes [opening, block_var, bodystmt] end - def copy( - opening: nil, - block_var: nil, - bodystmt: nil, - location: nil, - comments: nil - ) + def copy(opening: nil, block_var: nil, bodystmt: nil, location: nil) Block.new( opening: opening || self.opening, block_var: block_var || self.block_var, bodystmt: bodystmt || self.bodystmt, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -4187,13 +4097,12 @@ def child_nodes [left, right] end - def copy(left: nil, operator: nil, right: nil, location: nil, comments: nil) + def copy(left: nil, operator: nil, right: nil, location: nil) RangeLiteral.new( left: left || self.left, operator: operator || self.operator, right: right || self.right, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -4299,12 +4208,11 @@ def child_nodes parts end - def copy(parts: nil, quote: nil, location: nil, comments: nil) + def copy(parts: nil, quote: nil, location: nil) DynaSymbol.new( parts: parts || self.parts, quote: quote || self.quote, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -4419,12 +4327,11 @@ def child_nodes [keyword, statements] end - def copy(keyword: nil, statements: nil, location: nil, comments: nil) + def copy(keyword: nil, statements: nil, location: nil) Else.new( keyword: keyword || self.keyword, statements: statements || self.statements, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -4494,19 +4401,12 @@ def child_nodes [predicate, statements, consequent] end - def copy( - predicate: nil, - statements: nil, - consequent: nil, - location: nil, - comments: nil - ) + def copy(predicate: nil, statements: nil, consequent: nil, location: nil) Elsif.new( predicate: predicate || self.predicate, statements: statements || self.statements, consequent: consequent || self.consequent, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -4746,12 +4646,11 @@ def child_nodes [keyword, statements] end - def copy(keyword: nil, statements: nil, location: nil, comments: nil) + def copy(keyword: nil, statements: nil, location: nil) Ensure.new( keyword: keyword || self.keyword, statements: statements || self.statements, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -4809,11 +4708,10 @@ def child_nodes [] end - def copy(value: nil, location: nil, comments: nil) + def copy(value: nil, location: nil) ExcessedComma.new( value: value || self.value, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -4862,19 +4760,12 @@ def child_nodes [parent, (operator if operator != :"::"), name] end - def copy( - parent: nil, - operator: nil, - name: nil, - location: nil, - comments: nil - ) + def copy(parent: nil, operator: nil, name: nil, location: nil) Field.new( parent: parent || self.parent, operator: operator || self.operator, name: name || self.name, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -4924,11 +4815,10 @@ def child_nodes [] end - def copy(value: nil, location: nil, comments: nil) + def copy(value: nil, location: nil) FloatLiteral.new( value: value || self.value, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -4984,21 +4874,13 @@ def child_nodes [constant, left, *values, right] end - def copy( - constant: nil, - left: nil, - values: nil, - right: nil, - location: nil, - comments: nil - ) + def copy(constant: nil, left: nil, values: nil, right: nil, location: nil) FndPtn.new( constant: constant || self.constant, left: left || self.left, values: values || self.values, right: right || self.right, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -5076,19 +4958,12 @@ def child_nodes [index, collection, statements] end - def copy( - index: nil, - collection: nil, - statements: nil, - location: nil, - comments: nil - ) + def copy(index: nil, collection: nil, statements: nil, location: nil) For.new( index: index || self.index, collection: collection || self.collection, statements: statements || self.statements, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -5149,12 +5024,8 @@ def child_nodes [] end - def copy(value: nil, location: nil, comments: nil) - GVar.new( - value: value || self.value, - location: location || self.location, - comments: comments || self.comments - ) + def copy(value: nil, location: nil) + GVar.new(value: value || self.value, location: location || self.location) end alias deconstruct child_nodes @@ -5202,7 +5073,7 @@ def format(q) # [LBrace] the left brace that opens this hash attr_reader :lbrace - # [Array[ AssocNew | AssocSplat ]] the optional contents of the hash + # [Array[ Assoc | AssocSplat ]] the optional contents of the hash attr_reader :assocs # [Array[ Comment | EmbDoc ]] the comments attached to this node @@ -5223,12 +5094,11 @@ def child_nodes [lbrace] + assocs end - def copy(lbrace: nil, assocs: nil, location: nil, comments: nil) + def copy(lbrace: nil, assocs: nil, location: nil) HashLiteral.new( lbrace: lbrace || self.lbrace, assocs: assocs || self.assocs, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -5328,19 +5198,12 @@ def child_nodes [beginning, *parts, ending] end - def copy( - beginning: nil, - location: nil, - ending: nil, - parts: nil, - comments: nil - ) + def copy(beginning: nil, location: nil, ending: nil, parts: nil) Heredoc.new( beginning: beginning || self.beginning, location: location || self.location, ending: ending || self.ending, - parts: parts || self.parts, - comments: comments || self.comments + parts: parts || self.parts ) end @@ -5424,11 +5287,10 @@ def child_nodes [] end - def copy(value: nil, location: nil, comments: nil) + def copy(value: nil, location: nil) HeredocBeg.new( value: value || self.value, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -5471,11 +5333,10 @@ def child_nodes [] end - def copy(value: nil, location: nil, comments: nil) + def copy(value: nil, location: nil) HeredocEnd.new( value: value || self.value, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -5573,19 +5434,12 @@ def child_nodes [constant, *keywords.flatten(1), keyword_rest] end - def copy( - constant: nil, - keywords: nil, - keyword_rest: nil, - location: nil, - comments: nil - ) + def copy(constant: nil, keywords: nil, keyword_rest: nil, location: nil) HshPtn.new( constant: constant || self.constant, keywords: keywords || self.keywords, keyword_rest: keyword_rest || self.keyword_rest, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -5697,12 +5551,8 @@ def child_nodes [] end - def copy(value: nil, location: nil, comments: nil) - Ident.new( - value: value || self.value, - location: location || self.location, - comments: comments || self.comments - ) + def copy(value: nil, location: nil) + Ident.new(value: value || self.value, location: location || self.location) end alias deconstruct child_nodes @@ -5965,7 +5815,7 @@ class If < Node # [Statements] the expressions to be executed attr_reader :statements - # [nil, Elsif, Else] the next clause in the chain + # [nil | Elsif | Else] the next clause in the chain attr_reader :consequent # [Array[ Comment | EmbDoc ]] the comments attached to this node @@ -5993,19 +5843,12 @@ def child_nodes [predicate, statements, consequent] end - def copy( - predicate: nil, - statements: nil, - consequent: nil, - location: nil, - comments: nil - ) + def copy(predicate: nil, statements: nil, consequent: nil, location: nil) If.new( predicate: predicate || self.predicate, statements: statements || self.statements, consequent: consequent || self.consequent, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -6064,19 +5907,12 @@ def child_nodes [predicate, truthy, falsy] end - def copy( - predicate: nil, - truthy: nil, - falsy: nil, - location: nil, - comments: nil - ) + def copy(predicate: nil, truthy: nil, falsy: nil, location: nil) IfOp.new( predicate: predicate || self.predicate, truthy: truthy || self.truthy, falsy: falsy || self.falsy, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -6173,11 +6009,10 @@ def child_nodes [] end - def copy(value: nil, location: nil, comments: nil) + def copy(value: nil, location: nil) Imaginary.new( value: value || self.value, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -6228,19 +6063,12 @@ def child_nodes [pattern, statements, consequent] end - def copy( - pattern: nil, - statements: nil, - consequent: nil, - location: nil, - comments: nil - ) + def copy(pattern: nil, statements: nil, consequent: nil, location: nil) In.new( pattern: pattern || self.pattern, statements: statements || self.statements, consequent: consequent || self.consequent, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -6303,12 +6131,8 @@ def child_nodes [] end - def copy(value: nil, location: nil, comments: nil) - Int.new( - value: value || self.value, - location: location || self.location, - comments: comments || self.comments - ) + def copy(value: nil, location: nil) + Int.new(value: value || self.value, location: location || self.location) end alias deconstruct child_nodes @@ -6355,12 +6179,8 @@ def child_nodes [] end - def copy(value: nil, location: nil, comments: nil) - IVar.new( - value: value || self.value, - location: location || self.location, - comments: comments || self.comments - ) + def copy(value: nil, location: nil) + IVar.new(value: value || self.value, location: location || self.location) end alias deconstruct child_nodes @@ -6412,12 +6232,8 @@ def child_nodes [] end - def copy(value: nil, location: nil, comments: nil) - Kw.new( - value: value || self.value, - location: location || self.location, - comments: comments || self.comments - ) + def copy(value: nil, location: nil) + Kw.new(value: value || self.value, location: location || self.location) end alias deconstruct child_nodes @@ -6457,11 +6273,10 @@ def child_nodes [name] end - def copy(name: nil, location: nil, comments: nil) + def copy(name: nil, location: nil) KwRestParam.new( name: name || self.name, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -6511,12 +6326,8 @@ def child_nodes [] end - def copy(value: nil, location: nil, comments: nil) - Label.new( - value: value || self.value, - location: location || self.location, - comments: comments || self.comments - ) + def copy(value: nil, location: nil) + Label.new(value: value || self.value, location: location || self.location) end alias deconstruct child_nodes @@ -6597,12 +6408,11 @@ def child_nodes [params, statements] end - def copy(params: nil, statements: nil, location: nil, comments: nil) + def copy(params: nil, statements: nil, location: nil) Lambda.new( params: params || self.params, statements: statements || self.statements, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -6714,12 +6524,11 @@ def child_nodes [params, *locals] end - def copy(params: nil, locals: nil, location: nil, comments: nil) + def copy(params: nil, locals: nil, location: nil) LambdaVar.new( params: params || self.params, locals: locals || self.locals, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -6765,11 +6574,10 @@ def child_nodes [] end - def copy(value: nil, location: nil, comments: nil) + def copy(value: nil, location: nil) LBrace.new( value: value || self.value, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -6806,11 +6614,10 @@ def child_nodes [] end - def copy(value: nil, location: nil, comments: nil) + def copy(value: nil, location: nil) LBracket.new( value: value || self.value, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -6847,11 +6654,10 @@ def child_nodes [] end - def copy(value: nil, location: nil, comments: nil) + def copy(value: nil, location: nil) LParen.new( value: value || self.value, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -6905,12 +6711,11 @@ def child_nodes [target, value] end - def copy(target: nil, value: nil, location: nil, comments: nil) + def copy(target: nil, value: nil, location: nil) MAssign.new( target: target || self.target, value: value || self.value, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -6961,12 +6766,11 @@ def child_nodes [call, block] end - def copy(call: nil, block: nil, location: nil, comments: nil) + def copy(call: nil, block: nil, location: nil) MethodAddBlock.new( call: call || self.call, block: block || self.block, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -7031,12 +6835,11 @@ def child_nodes parts end - def copy(parts: nil, location: nil, comma: nil, comments: nil) + def copy(parts: nil, location: nil, comma: nil) MLHS.new( parts: parts || self.parts, location: location || self.location, - comma: comma || self.comma, - comments: comments || self.comments + comma: comma || self.comma ) end @@ -7084,11 +6887,10 @@ def child_nodes [contents] end - def copy(contents: nil, location: nil, comments: nil) + def copy(contents: nil, location: nil) MLHSParen.new( contents: contents || self.contents, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -7150,12 +6952,11 @@ def child_nodes [constant, bodystmt] end - def copy(constant: nil, bodystmt: nil, location: nil, comments: nil) + def copy(constant: nil, bodystmt: nil, location: nil) ModuleDeclaration.new( constant: constant || self.constant, bodystmt: bodystmt || self.bodystmt, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -7228,12 +7029,8 @@ def child_nodes parts end - def copy(parts: nil, location: nil, comments: nil) - MRHS.new( - parts: parts || self.parts, - location: location || self.location, - comments: comments || self.comments - ) + def copy(parts: nil, location: nil) + MRHS.new(parts: parts || self.parts, location: location || self.location) end alias deconstruct child_nodes @@ -7285,11 +7082,10 @@ def child_nodes [arguments] end - def copy(arguments: nil, location: nil, comments: nil) + def copy(arguments: nil, location: nil) Next.new( arguments: arguments || self.arguments, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -7334,12 +7130,8 @@ def child_nodes [] end - def copy(value: nil, location: nil, comments: nil) - Op.new( - value: value || self.value, - location: location || self.location, - comments: comments || self.comments - ) + def copy(value: nil, location: nil) + Op.new(value: value || self.value, location: location || self.location) end alias deconstruct child_nodes @@ -7388,19 +7180,12 @@ def child_nodes [target, operator, value] end - def copy( - target: nil, - operator: nil, - value: nil, - location: nil, - comments: nil - ) + def copy(target: nil, operator: nil, value: nil, location: nil) OpAssign.new( target: target || self.target, operator: operator || self.operator, value: value || self.value, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -7670,8 +7455,7 @@ def copy( posts: nil, keywords: nil, keyword_rest: nil, - block: nil, - comments: nil + block: nil ) Params.new( location: location || self.location, @@ -7681,8 +7465,7 @@ def copy( posts: posts || self.posts, keywords: keywords || self.keywords, keyword_rest: keyword_rest || self.keyword_rest, - block: block || self.block, - comments: comments || self.comments + block: block || self.block ) end @@ -7778,12 +7561,11 @@ def child_nodes [lparen, contents] end - def copy(lparen: nil, contents: nil, location: nil, comments: nil) + def copy(lparen: nil, contents: nil, location: nil) Paren.new( lparen: lparen || self.lparen, contents: contents || self.contents, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -7838,11 +7620,10 @@ def child_nodes [] end - def copy(value: nil, location: nil, comments: nil) + def copy(value: nil, location: nil) Period.new( value: value || self.value, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -7879,11 +7660,10 @@ def child_nodes [statements] end - def copy(statements: nil, location: nil, comments: nil) + def copy(statements: nil, location: nil) Program.new( statements: statements || self.statements, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -7932,12 +7712,11 @@ def child_nodes [] end - def copy(beginning: nil, elements: nil, location: nil, comments: nil) + def copy(beginning: nil, elements: nil, location: nil) QSymbols.new( beginning: beginning || self.beginning, elements: elements || self.elements, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -8042,12 +7821,11 @@ def child_nodes [] end - def copy(beginning: nil, elements: nil, location: nil, comments: nil) + def copy(beginning: nil, elements: nil, location: nil) QWords.new( beginning: beginning || self.beginning, elements: elements || self.elements, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -8148,11 +7926,10 @@ def child_nodes [] end - def copy(value: nil, location: nil, comments: nil) + def copy(value: nil, location: nil) RationalLiteral.new( value: value || self.value, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -8252,11 +8029,8 @@ def child_nodes [] end - def copy(location: nil, comments: nil) - Redo.new( - location: location || self.location, - comments: comments || self.comments - ) + def copy(location: nil) + Redo.new(location: location || self.location) end alias deconstruct child_nodes @@ -8428,21 +8202,12 @@ def child_nodes parts end - def copy( - beginning: nil, - ending: nil, - options: nil, - parts: nil, - location: nil, - comments: nil - ) + def copy(beginning: nil, ending: nil, parts: nil, location: nil) RegexpLiteral.new( beginning: beginning || self.beginning, ending: ending || self.ending, - options: options || self.options, parts: parts || self.parts, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -8556,12 +8321,11 @@ def child_nodes [*exceptions, variable] end - def copy(exceptions: nil, variable: nil, location: nil, comments: nil) + def copy(exceptions: nil, variable: nil, location: nil) RescueEx.new( exceptions: exceptions || self.exceptions, variable: variable || self.variable, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -8664,16 +8428,14 @@ def copy( exception: nil, statements: nil, consequent: nil, - location: nil, - comments: nil + location: nil ) Rescue.new( keyword: keyword || self.keyword, exception: exception || self.exception, statements: statements || self.statements, consequent: consequent || self.consequent, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -8744,12 +8506,11 @@ def child_nodes [statement, value] end - def copy(statement: nil, value: nil, location: nil, comments: nil) + def copy(statement: nil, value: nil, location: nil) RescueMod.new( statement: statement || self.statement, value: value || self.value, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -8809,11 +8570,10 @@ def child_nodes [name] end - def copy(name: nil, location: nil, comments: nil) + def copy(name: nil, location: nil) RestParam.new( name: name || self.name, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -8850,11 +8610,8 @@ def child_nodes [] end - def copy(location: nil, comments: nil) - Retry.new( - location: location || self.location, - comments: comments || self.comments - ) + def copy(location: nil) + Retry.new(location: location || self.location) end alias deconstruct child_nodes @@ -8893,11 +8650,10 @@ def child_nodes [arguments] end - def copy(arguments: nil, location: nil, comments: nil) + def copy(arguments: nil, location: nil) Return.new( arguments: arguments || self.arguments, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -8976,12 +8732,11 @@ def child_nodes [target, bodystmt] end - def copy(target: nil, bodystmt: nil, location: nil, comments: nil) + def copy(target: nil, bodystmt: nil, location: nil) SClass.new( target: target || self.target, bodystmt: bodystmt || self.bodystmt, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -9089,12 +8844,11 @@ def child_nodes body end - def copy(parser: nil, body: nil, location: nil, comments: nil) + def copy(body: nil, location: nil) Statements.new( - parser: parser || self.parser, + parser, body: body || self.body, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -9262,12 +9016,11 @@ def child_nodes [left, right] end - def copy(left: nil, right: nil, location: nil, comments: nil) + def copy(left: nil, right: nil, location: nil) StringConcat.new( left: left || self.left, right: right || self.right, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -9316,11 +9069,10 @@ def child_nodes [variable] end - def copy(variable: nil, location: nil, comments: nil) + def copy(variable: nil, location: nil) StringDVar.new( variable: variable || self.variable, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -9364,11 +9116,10 @@ def child_nodes [statements] end - def copy(statements: nil, location: nil, comments: nil) + def copy(statements: nil, location: nil) StringEmbExpr.new( statements: statements || self.statements, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -9435,12 +9186,11 @@ def child_nodes parts end - def copy(parts: nil, quote: nil, location: nil, comments: nil) + def copy(parts: nil, quote: nil, location: nil) StringLiteral.new( parts: parts || self.parts, quote: quote || self.quote, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -9518,11 +9268,10 @@ def child_nodes [arguments] end - def copy(arguments: nil, location: nil, comments: nil) + def copy(arguments: nil, location: nil) Super.new( arguments: arguments || self.arguments, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -9657,11 +9406,10 @@ def child_nodes [value] end - def copy(value: nil, location: nil, comments: nil) + def copy(value: nil, location: nil) SymbolLiteral.new( value: value || self.value, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -9706,12 +9454,11 @@ def child_nodes [] end - def copy(beginning: nil, elements: nil, location: nil, comments: nil) + def copy(beginning: nil, elements: nil, location: nil) Symbols.new( beginning: beginning || self.beginning, elements: elements || self.elements, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -9888,11 +9635,10 @@ def child_nodes [constant] end - def copy(constant: nil, location: nil, comments: nil) + def copy(constant: nil, location: nil) TopConstField.new( constant: constant || self.constant, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -9934,11 +9680,10 @@ def child_nodes [constant] end - def copy(constant: nil, location: nil, comments: nil) + def copy(constant: nil, location: nil) TopConstRef.new( constant: constant || self.constant, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -10028,11 +9773,10 @@ def child_nodes [] end - def copy(value: nil, location: nil, comments: nil) + def copy(value: nil, location: nil) TStringContent.new( value: value || self.value, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -10118,12 +9862,11 @@ def child_nodes [statement] end - def copy(statement: nil, parentheses: nil, location: nil, comments: nil) + def copy(statement: nil, parentheses: nil, location: nil) Not.new( statement: statement || self.statement, parentheses: parentheses || self.parentheses, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -10193,12 +9936,11 @@ def child_nodes [statement] end - def copy(operator: nil, statement: nil, location: nil, comments: nil) + def copy(operator: nil, statement: nil, location: nil) Unary.new( operator: operator || self.operator, statement: statement || self.statement, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -10268,11 +10010,10 @@ def child_nodes symbols end - def copy(symbols: nil, location: nil, comments: nil) + def copy(symbols: nil, location: nil) Undef.new( symbols: symbols || self.symbols, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -10307,7 +10048,7 @@ class Unless < Node # [Statements] the expressions to be executed attr_reader :statements - # [nil, Elsif, Else] the next clause in the chain + # [nil | Elsif | Else] the next clause in the chain attr_reader :consequent # [Array[ Comment | EmbDoc ]] the comments attached to this node @@ -10335,19 +10076,12 @@ def child_nodes [predicate, statements, consequent] end - def copy( - predicate: nil, - statements: nil, - consequent: nil, - location: nil, - comments: nil - ) + def copy(predicate: nil, statements: nil, consequent: nil, location: nil) Unless.new( predicate: predicate || self.predicate, statements: statements || self.statements, consequent: consequent || self.consequent, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -10475,12 +10209,11 @@ def child_nodes [predicate, statements] end - def copy(predicate: nil, statements: nil, location: nil, comments: nil) + def copy(predicate: nil, statements: nil, location: nil) Until.new( predicate: predicate || self.predicate, statements: statements || self.statements, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -10531,11 +10264,10 @@ def child_nodes [value] end - def copy(value: nil, location: nil, comments: nil) + def copy(value: nil, location: nil) VarField.new( value: value || self.value, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -10583,11 +10315,10 @@ def child_nodes [value] end - def copy(value: nil, location: nil, comments: nil) + def copy(value: nil, location: nil) VarRef.new( value: value || self.value, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -10655,11 +10386,10 @@ def child_nodes [value] end - def copy(value: nil, location: nil, comments: nil) + def copy(value: nil, location: nil) PinnedVarRef.new( value: value || self.value, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -10703,12 +10433,8 @@ def child_nodes [value] end - def copy(value: nil, location: nil, comments: nil) - VCall.new( - value: value || self.value, - location: location || self.location, - comments: comments || self.comments - ) + def copy(value: nil, location: nil) + VCall.new(value: value || self.value, location: location || self.location) end alias deconstruct child_nodes @@ -10750,11 +10476,8 @@ def child_nodes [] end - def copy(location: nil, comments: nil) - VoidStmt.new( - location: location || self.location, - comments: comments || self.comments - ) + def copy(location: nil) + VoidStmt.new(location: location || self.location) end alias deconstruct child_nodes @@ -10808,19 +10531,12 @@ def child_nodes [arguments, statements, consequent] end - def copy( - arguments: nil, - statements: nil, - consequent: nil, - location: nil, - comments: nil - ) + def copy(arguments: nil, statements: nil, consequent: nil, location: nil) When.new( arguments: arguments || self.arguments, statements: statements || self.statements, consequent: consequent || self.consequent, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -10918,12 +10634,11 @@ def child_nodes [predicate, statements] end - def copy(predicate: nil, statements: nil, location: nil, comments: nil) + def copy(predicate: nil, statements: nil, location: nil) While.new( predicate: predicate || self.predicate, statements: statements || self.statements, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -10980,12 +10695,8 @@ def child_nodes parts end - def copy(parts: nil, location: nil, comments: nil) - Word.new( - parts: parts || self.parts, - location: location || self.location, - comments: comments || self.comments - ) + def copy(parts: nil, location: nil) + Word.new(parts: parts || self.parts, location: location || self.location) end alias deconstruct child_nodes @@ -11028,12 +10739,11 @@ def child_nodes [] end - def copy(beginning: nil, elements: nil, location: nil, comments: nil) + def copy(beginning: nil, elements: nil, location: nil) Words.new( beginning: beginning || self.beginning, elements: elements || self.elements, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -11172,11 +10882,10 @@ def child_nodes parts end - def copy(parts: nil, location: nil, comments: nil) + def copy(parts: nil, location: nil) XStringLiteral.new( parts: parts || self.parts, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -11218,11 +10927,10 @@ def child_nodes [arguments] end - def copy(arguments: nil, location: nil, comments: nil) + def copy(arguments: nil, location: nil) Yield.new( arguments: arguments || self.arguments, - location: location || self.location, - comments: comments || self.comments + location: location || self.location ) end @@ -11277,11 +10985,8 @@ def child_nodes [] end - def copy(location: nil, comments: nil) - ZSuper.new( - location: location || self.location, - comments: comments || self.comments - ) + def copy(location: nil) + ZSuper.new(location: location || self.location) end alias deconstruct child_nodes diff --git a/lib/syntax_tree/visitor/mutation_visitor.rb b/lib/syntax_tree/visitor/mutation_visitor.rb new file mode 100644 index 00000000..c2bdc7fc --- /dev/null +++ b/lib/syntax_tree/visitor/mutation_visitor.rb @@ -0,0 +1,887 @@ +# frozen_string_literal: true + +module SyntaxTree + class Visitor + # This visitor walks through the tree and copies each node as it is being + # visited. This is useful for mutating the tree before it is formatted. + class MutationVisitor < Visitor + # Visit a BEGINBlock node. + def visit_BEGIN(node) + node.copy( + lbrace: visit(node.lbrace), + statements: visit(node.statements) + ) + end + + # Visit a CHAR node. + def visit_CHAR(node) + node.copy + end + + # Visit a ENDBlock node. + def visit_END(node) + node.copy( + lbrace: visit(node.lbrace), + statements: visit(node.statements) + ) + end + + # Visit a EndContent node. + def visit___end__(node) + node.copy + end + + # Visit a Alias node. + def visit_alias(node) + node.copy(left: visit(node.left), right: visit(node.right)) + end + + # Visit a ARef node. + def visit_aref(node) + node.copy(index: visit(node.index)) + end + + # Visit a ARefField node. + def visit_aref_field(node) + node.copy(index: visit(node.index)) + end + + # Visit a ArgParen node. + def visit_arg_paren(node) + node.copy(arguments: visit(node.arguments)) + end + + # Visit a Args node. + def visit_args(node) + node.copy(parts: visit_all(node.parts)) + end + + # Visit a ArgBlock node. + def visit_arg_block(node) + node.copy(value: visit(node.value)) + end + + # Visit a ArgStar node. + def visit_arg_star(node) + node.copy(value: visit(node.value)) + end + + # Visit a ArgsForward node. + def visit_args_forward(node) + node.copy + end + + # Visit a ArrayLiteral node. + def visit_array(node) + node.copy( + lbracket: visit(node.lbracket), + contents: visit(node.contents) + ) + end + + # Visit a AryPtn node. + def visit_aryptn(node) + node.copy( + constant: visit(node.constant), + requireds: visit_all(node.requireds), + rest: visit(node.rest), + posts: visit_all(node.posts) + ) + end + + # Visit a Assign node. + def visit_assign(node) + node.copy(target: visit(node.target)) + end + + # Visit a Assoc node. + def visit_assoc(node) + node.copy + end + + # Visit a AssocSplat node. + def visit_assoc_splat(node) + node.copy + end + + # Visit a Backref node. + def visit_backref(node) + node.copy + end + + # Visit a Backtick node. + def visit_backtick(node) + node.copy + end + + # Visit a BareAssocHash node. + def visit_bare_assoc_hash(node) + node.copy(assocs: visit_all(node.assocs)) + end + + # Visit a Begin node. + def visit_begin(node) + node.copy(bodystmt: visit(node.bodystmt)) + end + + # Visit a PinnedBegin node. + def visit_pinned_begin(node) + node.copy + end + + # Visit a Binary node. + def visit_binary(node) + node.copy + end + + # Visit a BlockVar node. + def visit_block_var(node) + node.copy(params: visit(node.params), locals: visit_all(node.locals)) + end + + # Visit a BlockArg node. + def visit_blockarg(node) + node.copy(name: visit(node.name)) + end + + # Visit a BodyStmt node. + def visit_bodystmt(node) + node.copy( + statements: visit(node.statements), + rescue_clause: visit(node.rescue_clause), + else_clause: visit(node.else_clause), + ensure_clause: visit(node.ensure_clause) + ) + end + + # Visit a Break node. + def visit_break(node) + node.copy(arguments: visit(node.arguments)) + end + + # Visit a Call node. + def visit_call(node) + node.copy( + receiver: visit(node.receiver), + operator: node.operator == :"::" ? :"::" : visit(node.operator), + message: node.message == :call ? :call : visit(node.message), + arguments: visit(node.arguments) + ) + end + + # Visit a Case node. + def visit_case(node) + node.copy( + keyword: visit(node.keyword), + value: visit(node.value), + consequent: visit(node.consequent) + ) + end + + # Visit a RAssign node. + def visit_rassign(node) + node.copy(operator: visit(node.operator)) + end + + # Visit a ClassDeclaration node. + def visit_class(node) + node.copy( + constant: visit(node.constant), + superclass: visit(node.superclass), + bodystmt: visit(node.bodystmt) + ) + end + + # Visit a Comma node. + def visit_comma(node) + node.copy + end + + # Visit a Command node. + def visit_command(node) + node.copy( + message: visit(node.message), + arguments: visit(node.arguments), + block: visit(node.block) + ) + end + + # Visit a CommandCall node. + def visit_command_call(node) + node.copy( + operator: node.operator == :"::" ? :"::" : visit(node.operator), + message: visit(node.message), + arguments: visit(node.arguments), + block: visit(node.block) + ) + end + + # Visit a Comment node. + def visit_comment(node) + node.copy + end + + # Visit a Const node. + def visit_const(node) + node.copy + end + + # Visit a ConstPathField node. + def visit_const_path_field(node) + node.copy(constant: visit(node.constant)) + end + + # Visit a ConstPathRef node. + def visit_const_path_ref(node) + node.copy(constant: visit(node.constant)) + end + + # Visit a ConstRef node. + def visit_const_ref(node) + node.copy(constant: visit(node.constant)) + end + + # Visit a CVar node. + def visit_cvar(node) + node.copy + end + + # Visit a Def node. + def visit_def(node) + node.copy( + target: visit(node.target), + operator: visit(node.operator), + name: visit(node.name), + params: visit(node.params), + bodystmt: visit(node.bodystmt) + ) + end + + # Visit a Defined node. + def visit_defined(node) + node.copy + end + + # Visit a Block node. + def visit_block(node) + node.copy( + opening: visit(node.opening), + block_var: visit(node.block_var), + bodystmt: visit(node.bodystmt) + ) + end + + # Visit a RangeLiteral node. + def visit_range_literal(node) + node.copy( + left: visit(node.left), + operator: visit(node.operator), + right: visit(node.right) + ) + end + + # Visit a DynaSymbol node. + def visit_dyna_symbol(node) + node.copy(parts: visit_all(node.parts)) + end + + # Visit a Else node. + def visit_else(node) + node.copy( + keyword: visit(node.keyword), + statements: visit(node.statements) + ) + end + + # Visit a Elsif node. + def visit_elsif(node) + node.copy( + statements: visit(node.statements), + consequent: visit(node.consequent) + ) + end + + # Visit a EmbDoc node. + def visit_embdoc(node) + node.copy + end + + # Visit a EmbExprBeg node. + def visit_embexpr_beg(node) + node.copy + end + + # Visit a EmbExprEnd node. + def visit_embexpr_end(node) + node.copy + end + + # Visit a EmbVar node. + def visit_embvar(node) + node.copy + end + + # Visit a Ensure node. + def visit_ensure(node) + node.copy( + keyword: visit(node.keyword), + statements: visit(node.statements) + ) + end + + # Visit a ExcessedComma node. + def visit_excessed_comma(node) + node.copy + end + + # Visit a Field node. + def visit_field(node) + node.copy( + operator: node.operator == :"::" ? :"::" : visit(node.operator), + name: visit(node.name) + ) + end + + # Visit a FloatLiteral node. + def visit_float(node) + node.copy + end + + # Visit a FndPtn node. + def visit_fndptn(node) + node.copy( + constant: visit(node.constant), + left: visit(node.left), + values: visit_all(node.values), + right: visit(node.right) + ) + end + + # Visit a For node. + def visit_for(node) + node.copy(index: visit(node.index), statements: visit(node.statements)) + end + + # Visit a GVar node. + def visit_gvar(node) + node.copy + end + + # Visit a HashLiteral node. + def visit_hash(node) + node.copy(lbrace: visit(node.lbrace), assocs: visit_all(node.assocs)) + end + + # Visit a Heredoc node. + def visit_heredoc(node) + node.copy( + beginning: visit(node.beginning), + ending: visit(node.ending), + parts: visit_all(node.parts) + ) + end + + # Visit a HeredocBeg node. + def visit_heredoc_beg(node) + node.copy + end + + # Visit a HeredocEnd node. + def visit_heredoc_end(node) + node.copy + end + + # Visit a HshPtn node. + def visit_hshptn(node) + node.copy( + constant: visit(node.constant), + keywords: + node.keywords.map { |label, value| [visit(label), visit(value)] }, + keyword_rest: visit(node.keyword_rest) + ) + end + + # Visit a Ident node. + def visit_ident(node) + node.copy + end + + # Visit a If node. + def visit_if(node) + node.copy( + statements: visit(node.statements), + consequent: visit(node.consequent) + ) + end + + # Visit a IfOp node. + def visit_if_op(node) + node.copy + end + + # Visit a Imaginary node. + def visit_imaginary(node) + node.copy + end + + # Visit a In node. + def visit_in(node) + node.copy( + statements: visit(node.statements), + consequent: visit(node.consequent) + ) + end + + # Visit a Int node. + def visit_int(node) + node.copy + end + + # Visit a IVar node. + def visit_ivar(node) + node.copy + end + + # Visit a Kw node. + def visit_kw(node) + node.copy + end + + # Visit a KwRestParam node. + def visit_kwrest_param(node) + node.copy(name: visit(node.name)) + end + + # Visit a Label node. + def visit_label(node) + node.copy + end + + # Visit a LabelEnd node. + def visit_label_end(node) + node.copy + end + + # Visit a Lambda node. + def visit_lambda(node) + node.copy( + params: visit(node.params), + statements: visit(node.statements) + ) + end + + # Visit a LambdaVar node. + def visit_lambda_var(node) + node.copy(params: visit(node.params), locals: visit_all(node.locals)) + end + + # Visit a LBrace node. + def visit_lbrace(node) + node.copy + end + + # Visit a LBracket node. + def visit_lbracket(node) + node.copy + end + + # Visit a LParen node. + def visit_lparen(node) + node.copy + end + + # Visit a MAssign node. + def visit_massign(node) + node.copy(target: visit(node.target)) + end + + # Visit a MethodAddBlock node. + def visit_method_add_block(node) + node.copy(call: visit(node.call), block: visit(node.block)) + end + + # Visit a MLHS node. + def visit_mlhs(node) + node.copy(parts: visit_all(node.parts)) + end + + # Visit a MLHSParen node. + def visit_mlhs_paren(node) + node.copy(contents: visit(node.contents)) + end + + # Visit a ModuleDeclaration node. + def visit_module(node) + node.copy( + constant: visit(node.constant), + bodystmt: visit(node.bodystmt) + ) + end + + # Visit a MRHS node. + def visit_mrhs(node) + node.copy(parts: visit_all(node.parts)) + end + + # Visit a Next node. + def visit_next(node) + node.copy(arguments: visit(node.arguments)) + end + + # Visit a Op node. + def visit_op(node) + node.copy + end + + # Visit a OpAssign node. + def visit_opassign(node) + node.copy(target: visit(node.target), operator: visit(node.operator)) + end + + # Visit a Params node. + def visit_params(node) + node.copy( + requireds: visit_all(node.requireds), + optionals: + node.optionals.map { |ident, value| [visit(ident), visit(value)] }, + rest: visit(node.rest), + posts: visit_all(node.posts), + keywords: + node.keywords.map { |ident, value| [visit(ident), visit(value)] }, + keyword_rest: + node.keyword_rest == :nil ? :nil : visit(node.keyword_rest), + block: visit(node.block) + ) + end + + # Visit a Paren node. + def visit_paren(node) + node.copy(lparen: visit(node.lparen), contents: visit(node.contents)) + end + + # Visit a Period node. + def visit_period(node) + node.copy + end + + # Visit a Program node. + def visit_program(node) + node.copy(statements: visit(node.statements)) + end + + # Visit a QSymbols node. + def visit_qsymbols(node) + node.copy( + beginning: visit(node.beginning), + elements: visit_all(node.elements) + ) + end + + # Visit a QSymbolsBeg node. + def visit_qsymbols_beg(node) + node.copy + end + + # Visit a QWords node. + def visit_qwords(node) + node.copy( + beginning: visit(node.beginning), + elements: visit_all(node.elements) + ) + end + + # Visit a QWordsBeg node. + def visit_qwords_beg(node) + node.copy + end + + # Visit a RationalLiteral node. + def visit_rational(node) + node.copy + end + + # Visit a RBrace node. + def visit_rbrace(node) + node.copy + end + + # Visit a RBracket node. + def visit_rbracket(node) + node.copy + end + + # Visit a Redo node. + def visit_redo(node) + node.copy + end + + # Visit a RegexpContent node. + def visit_regexp_content(node) + node.copy(parts: visit_all(node.parts)) + end + + # Visit a RegexpBeg node. + def visit_regexp_beg(node) + node.copy + end + + # Visit a RegexpEnd node. + def visit_regexp_end(node) + node.copy + end + + # Visit a RegexpLiteral node. + def visit_regexp_literal(node) + node.copy(parts: visit_all(node.parts)) + end + + # Visit a RescueEx node. + def visit_rescue_ex(node) + node.copy(variable: visit(node.variable)) + end + + # Visit a Rescue node. + def visit_rescue(node) + node.copy( + keyword: visit(node.keyword), + exception: visit(node.exception), + statements: visit(node.statements), + consequent: visit(node.consequent) + ) + end + + # Visit a RescueMod node. + def visit_rescue_mod(node) + node.copy + end + + # Visit a RestParam node. + def visit_rest_param(node) + node.copy(name: visit(node.name)) + end + + # Visit a Retry node. + def visit_retry(node) + node.copy + end + + # Visit a Return node. + def visit_return(node) + node.copy(arguments: visit(node.arguments)) + end + + # Visit a RParen node. + def visit_rparen(node) + node.copy + end + + # Visit a SClass node. + def visit_sclass(node) + node.copy(bodystmt: visit(node.bodystmt)) + end + + # Visit a Statements node. + def visit_statements(node) + node.copy(body: visit_all(node.body)) + end + + # Visit a StringContent node. + def visit_string_content(node) + node.copy(parts: visit_all(node.parts)) + end + + # Visit a StringConcat node. + def visit_string_concat(node) + node.copy(left: visit(node.left), right: visit(node.right)) + end + + # Visit a StringDVar node. + def visit_string_dvar(node) + node.copy(variable: visit(node.variable)) + end + + # Visit a StringEmbExpr node. + def visit_string_embexpr(node) + node.copy(statements: visit(node.statements)) + end + + # Visit a StringLiteral node. + def visit_string_literal(node) + node.copy(parts: visit_all(node.parts)) + end + + # Visit a Super node. + def visit_super(node) + node.copy(arguments: visit(node.arguments)) + end + + # Visit a SymBeg node. + def visit_symbeg(node) + node.copy + end + + # Visit a SymbolContent node. + def visit_symbol_content(node) + node.copy(value: visit(node.value)) + end + + # Visit a SymbolLiteral node. + def visit_symbol_literal(node) + node.copy(value: visit(node.value)) + end + + # Visit a Symbols node. + def visit_symbols(node) + node.copy( + beginning: visit(node.beginning), + elements: visit_all(node.elements) + ) + end + + # Visit a SymbolsBeg node. + def visit_symbols_beg(node) + node.copy + end + + # Visit a TLambda node. + def visit_tlambda(node) + node.copy + end + + # Visit a TLamBeg node. + def visit_tlambeg(node) + node.copy + end + + # Visit a TopConstField node. + def visit_top_const_field(node) + node.copy(constant: visit(node.constant)) + end + + # Visit a TopConstRef node. + def visit_top_const_ref(node) + node.copy(constant: visit(node.constant)) + end + + # Visit a TStringBeg node. + def visit_tstring_beg(node) + node.copy + end + + # Visit a TStringContent node. + def visit_tstring_content(node) + node.copy + end + + # Visit a TStringEnd node. + def visit_tstring_end(node) + node.copy + end + + # Visit a Not node. + def visit_not(node) + node.copy(statement: visit(node.statement)) + end + + # Visit a Unary node. + def visit_unary(node) + node.copy + end + + # Visit a Undef node. + def visit_undef(node) + node.copy(symbols: visit_all(node.symbols)) + end + + # Visit a Unless node. + def visit_unless(node) + node.copy( + statements: visit(node.statements), + consequent: visit(node.consequent) + ) + end + + # Visit a Until node. + def visit_until(node) + node.copy(statements: visit(node.statements)) + end + + # Visit a VarField node. + def visit_var_field(node) + node.copy(value: visit(node.value)) + end + + # Visit a VarRef node. + def visit_var_ref(node) + node.copy(value: visit(node.value)) + end + + # Visit a PinnedVarRef node. + def visit_pinned_var_ref(node) + node.copy(value: visit(node.value)) + end + + # Visit a VCall node. + def visit_vcall(node) + node.copy(value: visit(node.value)) + end + + # Visit a VoidStmt node. + def visit_void_stmt(node) + node.copy + end + + # Visit a When node. + def visit_when(node) + node.copy( + arguments: visit(node.arguments), + statements: visit(node.statements), + consequent: visit(node.consequent) + ) + end + + # Visit a While node. + def visit_while(node) + node.copy(statements: visit(node.statements)) + end + + # Visit a Word node. + def visit_word(node) + node.copy(parts: visit_all(node.parts)) + end + + # Visit a Words node. + def visit_words(node) + node.copy( + beginning: visit(node.beginning), + elements: visit_all(node.elements) + ) + end + + # Visit a WordsBeg node. + def visit_words_beg(node) + node.copy + end + + # Visit a XString node. + def visit_xstring(node) + node.copy(parts: visit_all(node.parts)) + end + + # Visit a XStringLiteral node. + def visit_xstring_literal(node) + node.copy(parts: visit_all(node.parts)) + end + + # Visit a Yield node. + def visit_yield(node) + node.copy(arguments: visit(node.arguments)) + end + + # Visit a ZSuper node. + def visit_zsuper(node) + node.copy + end + end + end +end diff --git a/test/test_helper.rb b/test/test_helper.rb index c46022ae..21bb75e3 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -61,6 +61,9 @@ def assert_syntax_tree(node) refute_includes(pretty, "#<") assert_includes(pretty, type) + # Assert that we can get back a new tree by using the mutation visitor. + node.accept(Visitor::MutationVisitor.new) + # Serialize the node to JSON, parse it back out, and assert that we have # found the expected type. json = node.to_json From bab6417a328b93112ae8edbbda7d255a1ac6d959 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 8 Nov 2022 12:25:04 -0500 Subject: [PATCH 194/536] Add the === operator to check for matching --- lib/syntax_tree/node.rb | 693 ++++++++++++++++++++++++++++++++++++++++ test/test_helper.rb | 2 +- 2 files changed, 694 insertions(+), 1 deletion(-) diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index fb4bab21..c9789e1e 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -127,6 +127,18 @@ def construct_keys end end + # When we're implementing the === operator for a node, we oftentimes need to + # compare two arrays. We want to skip over the === definition of array and use + # our own here, so we do that using this module. + module ArrayMatch + def self.call(left, right) + left.length === right.length && + left + .zip(right) + .all? { |left_value, right_value| left_value === right_value } + end + end + # BEGINBlock represents the use of the +BEGIN+ keyword, which hooks into the # lifecycle of the interpreter. Whatever is inside the block will get executed # when the program starts. @@ -192,6 +204,11 @@ def format(q) q.text("}") end end + + def ===(other) + other.is_a?(BEGINBlock) && lbrace === other.lbrace && + statements === other.statements + end end # CHAR irepresents a single codepoint in the script encoding. @@ -240,6 +257,10 @@ def format(q) q.text(q.quote) end end + + def ===(other) + other.is_a?(CHAR) && value === other.value + end end # ENDBlock represents the use of the +END+ keyword, which hooks into the @@ -307,6 +328,11 @@ def format(q) q.text("}") end end + + def ===(other) + other.is_a?(ENDBlock) && lbrace === other.lbrace && + statements === other.statements + end end # EndContent represents the use of __END__ syntax, which allows individual @@ -369,6 +395,10 @@ def format(q) q.breakable_return if value.end_with?("\n") end + + def ===(other) + other.is_a?(EndContent) && value === other.value + end end # Alias represents the use of the +alias+ keyword with regular arguments (not @@ -465,6 +495,10 @@ def format(q) end end + def ===(other) + other.is_a?(Alias) && left === other.left && right === other.right + end + def var_alias? left.is_a?(GVar) end @@ -543,6 +577,11 @@ def format(q) q.text("]") end end + + def ===(other) + other.is_a?(ARef) && collection === other.collection && + index === other.index + end end # ARefField represents assigning values into collections at specific indices. @@ -612,6 +651,11 @@ def format(q) q.text("]") end end + + def ===(other) + other.is_a?(ARefField) && collection === other.collection && + index === other.index + end end # ArgParen represents wrapping arguments to a method inside a set of @@ -678,6 +722,10 @@ def format(q) q.text(")") end + def ===(other) + other.is_a?(ArgParen) && arguments === other.arguments + end + private def trailing_comma? @@ -740,6 +788,10 @@ def deconstruct_keys(_keys) def format(q) q.seplist(parts) { |part| q.format(part) } end + + def ===(other) + other.is_a?(Args) && ArrayMatch.call(parts, other.parts) + end end # ArgBlock represents using a block operator on an expression. @@ -784,6 +836,10 @@ def format(q) q.text("&") q.format(value) if value end + + def ===(other) + other.is_a?(ArgBlock) && value === other.value + end end # Star represents using a splat operator on an expression. @@ -828,6 +884,10 @@ def format(q) q.text("*") q.format(value) if value end + + def ===(other) + other.is_a?(ArgStar) && value === other.value + end end # ArgsForward represents forwarding all kinds of arguments onto another method @@ -877,6 +937,10 @@ def deconstruct_keys(_keys) def format(q) q.text("...") end + + def ===(other) + other.is_a?(ArgsForward) + end end # ArrayLiteral represents an array literal, which can optionally contain @@ -1107,6 +1171,11 @@ def format(q) end end + def ===(other) + other.is_a?(ArrayLiteral) && lbracket === other.lbracket && + contents === other.contents + end + private def qwords? @@ -1278,6 +1347,12 @@ def format(q) q.text("]") end end + + def ===(other) + other.is_a?(AryPtn) && constant === other.constant && + ArrayMatch.call(requireds, other.requireds) && rest === other.rest && + ArrayMatch.call(posts, other.posts) + end end # Determins if the following value should be indented or not. @@ -1360,6 +1435,10 @@ def format(q) end end + def ===(other) + other.is_a?(Assign) && target === other.target && value === other.value + end + private def skip_indent? @@ -1421,6 +1500,10 @@ def format(q) end end + def ===(other) + other.is_a?(Assoc) && key === other.key && value === other.value + end + private def format_contents(q) @@ -1482,6 +1565,10 @@ def format(q) q.text("**") q.format(value) end + + def ===(other) + other.is_a?(AssocSplat) && value === other.value + end end # Backref represents a global variable referencing a matched value. It comes @@ -1526,6 +1613,10 @@ def deconstruct_keys(_keys) def format(q) q.text(value) end + + def ===(other) + other.is_a?(Backref) && value === other.value + end end # Backtick represents the use of the ` operator. It's usually found being used @@ -1568,6 +1659,10 @@ def deconstruct_keys(_keys) def format(q) q.text(value) end + + def ===(other) + other.is_a?(Backtick) && value === other.value + end end # This module is responsible for formatting the assocs contained within a @@ -1688,6 +1783,10 @@ def format(q) q.seplist(assocs) { |assoc| q.format(assoc) } end + def ===(other) + other.is_a?(BareAssocHash) && ArrayMatch.call(assocs, other.assocs) + end + def format_key(q, key) (@key_formatter ||= HashKeyFormatter.for(self)).format_key(q, key) end @@ -1746,6 +1845,10 @@ def format(q) q.breakable_force q.text("end") end + + def ===(other) + other.is_a?(Begin) && bodystmt === other.bodystmt + end end # PinnedBegin represents a pinning a nested statement within pattern matching. @@ -1801,6 +1904,10 @@ def format(q) end end end + + def ===(other) + other.is_a?(PinnedBegin) && statement === other.statement + end end # Binary represents any expression that involves two sub-expressions with an @@ -1898,6 +2005,11 @@ def format(q) end end end + + def ===(other) + other.is_a?(Binary) && left === other.left && + operator === other.operator && right === other.right + end end # BlockVar represents the parameters being declared for a block. Effectively @@ -1970,6 +2082,11 @@ def format(q) end q.text("|") end + + def ===(other) + other.is_a?(BlockVar) && params === other.params && + ArrayMatch.call(locals, other.locals) + end end # BlockArg represents declaring a block parameter on a method definition. @@ -2011,6 +2128,10 @@ def format(q) q.text("&") q.format(name) if name end + + def ===(other) + other.is_a?(BlockArg) && name === other.name + end end # bodystmt can't actually determine its bounds appropriately because it @@ -2158,6 +2279,14 @@ def format(q) end end end + + def ===(other) + other.is_a?(BodyStmt) && statements === other.statements && + rescue_clause === other.rescue_clause && + else_keyword === other.else_keyword && + else_clause === other.else_clause && + ensure_clause === other.ensure_clause + end end # Formats either a Break, Next, or Return node. @@ -2394,6 +2523,10 @@ def deconstruct_keys(_keys) def format(q) FlowControlFormatter.new("break", self).format(q) end + + def ===(other) + other.is_a?(Break) && arguments === other.arguments + end end # Wraps a call operator (which can be a string literal :: or an Op node or a @@ -2761,6 +2894,12 @@ def format(q) end end + def ===(other) + other.is_a?(Call) && receiver === other.receiver && + operator === other.operator && message === other.message && + arguments === other.arguments + end + # Print out the arguments to this call. If there are no arguments, then do # nothing. def format_arguments(q) @@ -2879,6 +3018,11 @@ def format(q) q.text("end") end end + + def ===(other) + other.is_a?(Case) && keyword === other.keyword && value === other.value && + consequent === other.consequent + end end # RAssign represents a single-line pattern match. @@ -2957,6 +3101,11 @@ def format(q) end end end + + def ===(other) + other.is_a?(RAssign) && value === other.value && + operator === other.operator && pattern === other.pattern + end end # Class represents defining a class using the +class+ keyword. @@ -3064,6 +3213,11 @@ def format(q) end end + def ===(other) + other.is_a?(ClassDeclaration) && constant === other.constant && + superclass === other.superclass && bodystmt === other.bodystmt + end + private def format_declaration(q) @@ -3106,6 +3260,10 @@ def copy(value: nil, location: nil) def deconstruct_keys(_keys) { value: value, location: location } end + + def ===(other) + other.is_a?(Comma) && value === other.value + end end # Command represents a method call with arguments and no parentheses. Note @@ -3173,6 +3331,11 @@ def format(q) q.format(block) if block end + def ===(other) + other.is_a?(Command) && message === other.message && + arguments === other.arguments && block === other.block + end + private def align(q, node, &block) @@ -3329,6 +3492,12 @@ def format(q) q.format(block) if block end + def ===(other) + other.is_a?(CommandCall) && receiver === other.receiver && + operator === other.operator && message === other.message && + arguments === other.arguments && block === other.block + end + private def argument_alignment(q, doc) @@ -3425,6 +3594,10 @@ def deconstruct_keys(_keys) def format(q) q.text(value) end + + def ===(other) + other.is_a?(Comment) && value === other.value && inline === other.inline + end end # Const represents a literal value that _looks_ like a constant. This could @@ -3475,6 +3648,10 @@ def deconstruct_keys(_keys) def format(q) q.text(value) end + + def ===(other) + other.is_a?(Const) && value === other.value + end end # ConstPathField represents the child node of some kind of assignment. It @@ -3532,6 +3709,11 @@ def format(q) q.text("::") q.format(constant) end + + def ===(other) + other.is_a?(ConstPathField) && parent === other.parent && + constant === other.constant + end end # ConstPathRef represents referencing a constant by a path. @@ -3587,6 +3769,11 @@ def format(q) q.text("::") q.format(constant) end + + def ===(other) + other.is_a?(ConstPathRef) && parent === other.parent && + constant === other.constant + end end # ConstRef represents the name of the constant being used in a class or module @@ -3632,6 +3819,10 @@ def deconstruct_keys(_keys) def format(q) q.format(constant) end + + def ===(other) + other.is_a?(ConstRef) && constant === other.constant + end end # CVar represents the use of a class variable. @@ -3672,6 +3863,10 @@ def deconstruct_keys(_keys) def format(q) q.text(value) end + + def ===(other) + other.is_a?(CVar) && value === other.value + end end # Def represents defining a regular method on the current self object. @@ -3790,6 +3985,12 @@ def format(q) end end + def ===(other) + other.is_a?(Def) && target === other.target && + operator === other.operator && name === other.name && + params === other.params && bodystmt === other.bodystmt + end + # Returns true if the method was found in the source in the "endless" form, # i.e. where the method body is defined using the `=` operator after the # method name and parameters. @@ -3848,6 +4049,10 @@ def format(q) end q.text(")") end + + def ===(other) + other.is_a?(Defined) && value === other.value + end end # Block represents passing a block to a method call using the +do+ and +end+ @@ -3962,6 +4167,11 @@ def format(q) end end + def ===(other) + other.is_a?(Block) && opening === other.opening && + block_var === other.block_var && bodystmt === other.bodystmt + end + def keywords? opening.is_a?(Kw) end @@ -4130,6 +4340,11 @@ def format(q) q.format(right) if right end + + def ===(other) + other.is_a?(RangeLiteral) && left === other.left && + operator === other.operator && right === other.right + end end # Responsible for providing information about quotes to be used for strings @@ -4251,6 +4466,11 @@ def format(q) q.text(closing_quote) end + def ===(other) + other.is_a?(DynaSymbol) && ArrayMatch.call(parts, other.parts) && + quote === other.quote + end + private # Here we determine the quotes to use for a dynamic symbol. It's bound by a @@ -4358,6 +4578,11 @@ def format(q) end end end + + def ===(other) + other.is_a?(Else) && keyword === other.keyword && + statements === other.statements + end end # Elsif represents another clause in an +if+ or +unless+ chain. @@ -4444,6 +4669,11 @@ def format(q) end end end + + def ===(other) + other.is_a?(Elsif) && predicate === other.predicate && + statements === other.statements && consequent === other.consequent + end end # EmbDoc represents a multi-line comment. @@ -4499,6 +4729,10 @@ def format(q) q.trim q.text(value) end + + def ===(other) + other.is_a?(EmbDoc) && value === other.value + end end # EmbExprBeg represents the beginning token for using interpolation inside of @@ -4536,6 +4770,10 @@ def copy(value: nil, location: nil) def deconstruct_keys(_keys) { value: value, location: location } end + + def ===(other) + other.is_a?(EmbExprBeg) && value === other.value + end end # EmbExprEnd represents the ending token for using interpolation inside of a @@ -4573,6 +4811,10 @@ def copy(value: nil, location: nil) def deconstruct_keys(_keys) { value: value, location: location } end + + def ===(other) + other.is_a?(EmbExprEnd) && value === other.value + end end # EmbVar represents the use of shorthand interpolation for an instance, class, @@ -4612,6 +4854,10 @@ def copy(value: nil, location: nil) def deconstruct_keys(_keys) { value: value, location: location } end + + def ===(other) + other.is_a?(EmbVar) && value === other.value + end end # Ensure represents the use of the +ensure+ keyword and its subsequent @@ -4675,6 +4921,11 @@ def format(q) end end end + + def ===(other) + other.is_a?(Ensure) && keyword === other.keyword && + statements === other.statements + end end # ExcessedComma represents a trailing comma in a list of block parameters. It @@ -4724,6 +4975,10 @@ def deconstruct_keys(_keys) def format(q) q.text(value) end + + def ===(other) + other.is_a?(ExcessedComma) && value === other.value + end end # Field is always the child of an assignment. It represents assigning to a @@ -4788,6 +5043,11 @@ def format(q) q.format(name) end end + + def ===(other) + other.is_a?(Field) && parent === other.parent && + operator === other.operator && name === other.name + end end # FloatLiteral represents a floating point number literal. @@ -4831,6 +5091,10 @@ def deconstruct_keys(_keys) def format(q) q.text(value) end + + def ===(other) + other.is_a?(FloatLiteral) && value === other.value + end end # FndPtn represents matching against a pattern where you find a pattern in an @@ -4921,6 +5185,12 @@ def format(q) q.text("]") end end + + def ===(other) + other.is_a?(FndPtn) && constant === other.constant && + left === other.left && ArrayMatch.call(values, other.values) && + right === other.right + end end # For represents using a +for+ loop. @@ -4997,6 +5267,11 @@ def format(q) q.text("end") end end + + def ===(other) + other.is_a?(For) && index === other.index && + collection === other.collection && statements === other.statements + end end # GVar represents a global variable literal. @@ -5037,6 +5312,10 @@ def deconstruct_keys(_keys) def format(q) q.text(value) end + + def ===(other) + other.is_a?(GVar) && value === other.value + end end # HashLiteral represents a hash literal. @@ -5116,6 +5395,11 @@ def format(q) end end + def ===(other) + other.is_a?(HashLiteral) && lbrace === other.lbrace && + ArrayMatch.call(assocs, other.assocs) + end + def format_key(q, key) (@key_formatter ||= HashKeyFormatter.for(self)).format_key(q, key) end @@ -5257,6 +5541,11 @@ def format(q) end end end + + def ===(other) + other.is_a?(Heredoc) && beginning === other.beginning && + ending === other.ending && ArrayMatch.call(parts, other.parts) + end end # HeredocBeg represents the beginning declaration of a heredoc. @@ -5303,6 +5592,10 @@ def deconstruct_keys(_keys) def format(q) q.text(value) end + + def ===(other) + other.is_a?(HeredocBeg) && value === other.value + end end # HeredocEnd represents the closing declaration of a heredoc. @@ -5349,6 +5642,10 @@ def deconstruct_keys(_keys) def format(q) q.text(value) end + + def ===(other) + other.is_a?(HeredocEnd) && value === other.value + end end # HshPtn represents matching against a hash pattern using the Ruby 2.7+ @@ -5507,6 +5804,15 @@ def format(q) end end + def ===(other) + other.is_a?(HshPtn) && constant === other.constant && + keywords.length == other.keywords.length && + keywords + .zip(other.keywords) + .all? { |left, right| ArrayMatch.call(left, right) } && + keyword_rest === other.keyword_rest + end + private def format_contents(q, parts, nested) @@ -5564,6 +5870,10 @@ def deconstruct_keys(_keys) def format(q) q.text(value) end + + def ===(other) + other.is_a?(Ident) && value === other.value + end end # If the predicate of a conditional or loop contains an assignment (in which @@ -5868,6 +6178,11 @@ def format(q) ConditionalFormatter.new("if", self).format(q) end + def ===(other) + other.is_a?(If) && predicate === other.predicate && + statements === other.statements && consequent === other.consequent + end + # Checks if the node was originally found in the modifier form. def modifier? predicate.location.start_char > statements.location.start_char @@ -5944,6 +6259,11 @@ def format(q) q.group { q.if_break { format_break(q) }.if_flat { format_flat(q) } } end + def ===(other) + other.is_a?(IfOp) && predicate === other.predicate && + truthy === other.truthy && falsy === other.falsy + end + private def format_break(q) @@ -6025,6 +6345,10 @@ def deconstruct_keys(_keys) def format(q) q.text(value) end + + def ===(other) + other.is_a?(Imaginary) && value === other.value + end end # In represents using the +in+ keyword within the Ruby 2.7+ pattern matching @@ -6104,6 +6428,11 @@ def format(q) end end end + + def ===(other) + other.is_a?(In) && pattern === other.pattern && + statements === other.statements && consequent === other.consequent + end end # Int represents an integer number literal. @@ -6152,6 +6481,10 @@ def format(q) q.text(value) end end + + def ===(other) + other.is_a?(Int) && value === other.value + end end # IVar represents an instance variable literal. @@ -6192,6 +6525,10 @@ def deconstruct_keys(_keys) def format(q) q.text(value) end + + def ===(other) + other.is_a?(IVar) && value === other.value + end end # Kw represents the use of a keyword. It can be almost anywhere in the syntax @@ -6245,6 +6582,10 @@ def deconstruct_keys(_keys) def format(q) q.text(value) end + + def ===(other) + other.is_a?(Kw) && value === other.value + end end # KwRestParam represents defining a parameter in a method definition that @@ -6290,6 +6631,10 @@ def format(q) q.text("**") q.format(name) if name end + + def ===(other) + other.is_a?(KwRestParam) && name === other.name + end end # Label represents the use of an identifier to associate with an object. You @@ -6339,6 +6684,10 @@ def deconstruct_keys(_keys) def format(q) q.text(value) end + + def ===(other) + other.is_a?(Label) && value === other.value + end end # LabelEnd represents the end of a dynamic symbol. @@ -6377,6 +6726,10 @@ def copy(value: nil, location: nil) def deconstruct_keys(_keys) { value: value, location: location } end + + def ===(other) + other.is_a?(LabelEnd) && value === other.value + end end # Lambda represents using a lambda literal (not the lambda method call). @@ -6489,6 +6842,11 @@ def format(q) end end end + + def ===(other) + other.is_a?(Lambda) && params === other.params && + statements === other.statements + end end # LambdaVar represents the parameters being declared for a lambda. Effectively @@ -6550,6 +6908,11 @@ def format(q) q.seplist(locals, BlockVar::SEPARATOR) { |local| q.format(local) } end end + + def ===(other) + other.is_a?(LambdaVar) && params === other.params && + ArrayMatch.call(locals, other.locals) + end end # LBrace represents the use of a left brace, i.e., {. @@ -6590,6 +6953,10 @@ def deconstruct_keys(_keys) def format(q) q.text(value) end + + def ===(other) + other.is_a?(LBrace) && value === other.value + end end # LBracket represents the use of a left bracket, i.e., [. @@ -6630,6 +6997,10 @@ def deconstruct_keys(_keys) def format(q) q.text(value) end + + def ===(other) + other.is_a?(LBracket) && value === other.value + end end # LParen represents the use of a left parenthesis, i.e., (. @@ -6670,6 +7041,10 @@ def deconstruct_keys(_keys) def format(q) q.text(value) end + + def ===(other) + other.is_a?(LParen) && value === other.value + end end # MAssign is a parent node of any kind of multiple assignment. This includes @@ -6735,6 +7110,10 @@ def format(q) end end end + + def ===(other) + other.is_a?(MAssign) && target === other.target && value === other.value + end end # MethodAddBlock represents a method call with a block argument. @@ -6796,6 +7175,11 @@ def format(q) end end + def ===(other) + other.is_a?(MethodAddBlock) && call === other.call && + block === other.block + end + def format_contents(q) q.format(call) q.format(block) @@ -6853,6 +7237,11 @@ def format(q) q.seplist(parts) { |part| q.format(part) } q.text(",") if comma end + + def ===(other) + other.is_a?(MLHS) && ArrayMatch.call(parts, other.parts) && + comma === other.comma + end end # MLHSParen represents parentheses being used to destruct values in a multiple @@ -6920,6 +7309,10 @@ def format(q) q.text(")") end end + + def ===(other) + other.is_a?(MLHSParen) && contents === other.contents + end end # ModuleDeclaration represents defining a module using the +module+ keyword. @@ -6993,6 +7386,11 @@ def format(q) end end + def ===(other) + other.is_a?(ModuleDeclaration) && constant === other.constant && + bodystmt === other.bodystmt + end + private def format_declaration(q) @@ -7042,6 +7440,10 @@ def deconstruct_keys(_keys) def format(q) q.seplist(parts) { |part| q.format(part) } end + + def ===(other) + other.is_a?(MRHS) && ArrayMatch.call(parts, other.parts) + end end # Next represents using the +next+ keyword. @@ -7098,6 +7500,10 @@ def deconstruct_keys(_keys) def format(q) FlowControlFormatter.new("next", self).format(q) end + + def ===(other) + other.is_a?(Next) && arguments === other.arguments + end end # Op represents an operator literal in the source. @@ -7143,6 +7549,10 @@ def deconstruct_keys(_keys) def format(q) q.text(value) end + + def ===(other) + other.is_a?(Op) && value === other.value + end end # OpAssign represents assigning a value to a variable or constant using an @@ -7219,6 +7629,11 @@ def format(q) end end + def ===(other) + other.is_a?(OpAssign) && target === other.target && + operator === other.operator && value === other.value + end + private def skip_indent? @@ -7522,6 +7937,20 @@ def format(q) end end + def ===(other) + other.is_a?(Params) && ArrayMatch.call(requireds, other.requireds) && + optionals.length == other.optionals.length && + optionals + .zip(other.optionals) + .all? { |left, right| ArrayMatch.call(left, right) } && + rest === other.rest && ArrayMatch.call(posts, other.posts) && + keywords.length == other.keywords.length && + keywords + .zip(other.keywords) + .all? { |left, right| ArrayMatch.call(left, right) } && + keyword_rest === other.keyword_rest && block === other.block + end + private def format_contents(q, parts) @@ -7595,6 +8024,11 @@ def format(q) q.text(")") end end + + def ===(other) + other.is_a?(Paren) && lparen === other.lparen && + contents === other.contents + end end # Period represents the use of the +.+ operator. It is usually found in method @@ -7636,6 +8070,10 @@ def deconstruct_keys(_keys) def format(q) q.text(value) end + + def ===(other) + other.is_a?(Period) && value === other.value + end end # Program represents the overall syntax tree. @@ -7681,6 +8119,10 @@ def format(q) # replicate the text exactly so we will just let it be. q.breakable_force unless statements.body.last.is_a?(EndContent) end + + def ===(other) + other.is_a?(Program) && statements === other.statements + end end # QSymbols represents a symbol literal array without interpolation. @@ -7752,6 +8194,11 @@ def format(q) end q.text(closing) end + + def ===(other) + other.is_a?(QSymbols) && beginning === other.beginning && + ArrayMatch.call(elements, other.elements) + end end # QSymbolsBeg represents the beginning of a symbol literal array. @@ -7790,6 +8237,10 @@ def copy(value: nil, location: nil) def deconstruct_keys(_keys) { value: value, location: location } end + + def ===(other) + other.is_a?(QSymbolsBeg) && value === other.value + end end # QWords represents a string literal array without interpolation. @@ -7861,6 +8312,11 @@ def format(q) end q.text(closing) end + + def ===(other) + other.is_a?(QWords) && beginning === other.beginning && + ArrayMatch.call(elements, other.elements) + end end # QWordsBeg represents the beginning of a string literal array. @@ -7899,6 +8355,10 @@ def copy(value: nil, location: nil) def deconstruct_keys(_keys) { value: value, location: location } end + + def ===(other) + other.is_a?(QWordsBeg) && value === other.value + end end # RationalLiteral represents the use of a rational number literal. @@ -7942,6 +8402,10 @@ def deconstruct_keys(_keys) def format(q) q.text(value) end + + def ===(other) + other.is_a?(RationalLiteral) && value === other.value + end end # RBrace represents the use of a right brace, i.e., +++. @@ -7974,6 +8438,10 @@ def copy(value: nil, location: nil) def deconstruct_keys(_keys) { value: value, location: location } end + + def ===(other) + other.is_a?(RBrace) && value === other.value + end end # RBracket represents the use of a right bracket, i.e., +]+. @@ -8006,6 +8474,10 @@ def copy(value: nil, location: nil) def deconstruct_keys(_keys) { value: value, location: location } end + + def ===(other) + other.is_a?(RBracket) && value === other.value + end end # Redo represents the use of the +redo+ keyword. @@ -8042,6 +8514,10 @@ def deconstruct_keys(_keys) def format(q) q.text("redo") end + + def ===(other) + other.is_a?(Redo) + end end # RegexpContent represents the body of a regular expression. @@ -8085,6 +8561,11 @@ def copy(beginning: nil, parts: nil, location: nil) def deconstruct_keys(_keys) { beginning: beginning, parts: parts, location: location } end + + def ===(other) + other.is_a?(RegexpContent) && beginning === other.beginning && + parts === other.parts + end end # RegexpBeg represents the start of a regular expression literal. @@ -8125,6 +8606,10 @@ def copy(value: nil, location: nil) def deconstruct_keys(_keys) { value: value, location: location } end + + def ===(other) + other.is_a?(RegexpBeg) && value === other.value + end end # RegexpEnd represents the end of a regular expression literal. @@ -8166,6 +8651,10 @@ def copy(value: nil, location: nil) def deconstruct_keys(_keys) { value: value, location: location } end + + def ===(other) + other.is_a?(RegexpEnd) && value === other.value + end end # RegexpLiteral represents a regular expression literal. @@ -8264,6 +8753,12 @@ def format(q) end end + def ===(other) + other.is_a?(RegexpLiteral) && beginning === other.beginning && + ending === other.ending && options === other.options && + ArrayMatch.call(parts, other.parts) + end + def options ending[1..] end @@ -8353,6 +8848,11 @@ def format(q) end end end + + def ===(other) + other.is_a?(RescueEx) && exceptions === other.exceptions && + variable === other.variable + end end # Rescue represents the use of the rescue keyword inside of a BodyStmt node. @@ -8475,6 +8975,12 @@ def format(q) end end end + + def ===(other) + other.is_a?(Rescue) && keyword === other.keyword && + exception === other.exception && statements === other.statements && + consequent === other.consequent + end end # RescueMod represents the use of the modifier form of a +rescue+ clause. @@ -8542,6 +9048,11 @@ def format(q) end q.text("end") end + + def ===(other) + other.is_a?(RescueMod) && statement === other.statement && + value === other.value + end end # RestParam represents defining a parameter in a method definition that @@ -8587,6 +9098,10 @@ def format(q) q.text("*") q.format(name) if name end + + def ===(other) + other.is_a?(RestParam) && name === other.name + end end # Retry represents the use of the +retry+ keyword. @@ -8623,6 +9138,10 @@ def deconstruct_keys(_keys) def format(q) q.text("retry") end + + def ===(other) + other.is_a?(Retry) + end end # Return represents using the +return+ keyword with arguments. @@ -8666,6 +9185,10 @@ def deconstruct_keys(_keys) def format(q) FlowControlFormatter.new("return", self).format(q) end + + def ===(other) + other.is_a?(Return) && arguments === other.arguments + end end # RParen represents the use of a right parenthesis, i.e., +)+. @@ -8698,6 +9221,10 @@ def copy(value: nil, location: nil) def deconstruct_keys(_keys) { value: value, location: location } end + + def ===(other) + other.is_a?(RParen) && value === other.value + end end # SClass represents a block of statements that should be evaluated within the @@ -8763,6 +9290,11 @@ def format(q) end q.text("end") end + + def ===(other) + other.is_a?(SClass) && target === other.target && + bodystmt === other.bodystmt + end end # Everything that has a block of code inside of it has a list of statements. @@ -8906,6 +9438,10 @@ def format(q) end end + def ===(other) + other.is_a?(Statements) && ArrayMatch.call(body, other.body) + end + private # As efficiently as possible, gather up all of the comments that have been @@ -8983,6 +9519,10 @@ def copy(parts: nil, location: nil) def deconstruct_keys(_keys) { parts: parts, location: location } end + + def ===(other) + other.is_a?(StringContent) && ArrayMatch.call(parts, other.parts) + end end # StringConcat represents concatenating two strings together using a backward @@ -9040,6 +9580,10 @@ def format(q) end end end + + def ===(other) + other.is_a?(StringConcat) && left === other.left && right === other.right + end end # StringDVar represents shorthand interpolation of a variable into a string. @@ -9087,6 +9631,10 @@ def format(q) q.format(variable) q.text("}") end + + def ===(other) + other.is_a?(StringDVar) && variable === other.variable + end end # StringEmbExpr represents interpolated content. It can be contained within a @@ -9154,6 +9702,10 @@ def format(q) end end end + + def ===(other) + other.is_a?(StringEmbExpr) && statements === other.statements + end end # StringLiteral represents a string literal. @@ -9240,6 +9792,11 @@ def format(q) end q.text(closing_quote) end + + def ===(other) + other.is_a?(StringLiteral) && ArrayMatch.call(parts, other.parts) && + quote === other.quote + end end # Super represents using the +super+ keyword with arguments. It can optionally @@ -9293,6 +9850,10 @@ def format(q) end end end + + def ===(other) + other.is_a?(Super) && arguments === other.arguments + end end # SymBeg represents the beginning of a symbol literal. @@ -9340,6 +9901,10 @@ def copy(value: nil, location: nil) def deconstruct_keys(_keys) { value: value, location: location } end + + def ===(other) + other.is_a?(SymBeg) && value === other.value + end end # SymbolContent represents symbol contents and is always the child of a @@ -9377,6 +9942,10 @@ def copy(value: nil, location: nil) def deconstruct_keys(_keys) { value: value, location: location } end + + def ===(other) + other.is_a?(SymbolContent) && value === other.value + end end # SymbolLiteral represents a symbol in the system with no interpolation @@ -9423,6 +9992,10 @@ def format(q) q.text(":") q.format(value) end + + def ===(other) + other.is_a?(SymbolLiteral) && value === other.value + end end # Symbols represents a symbol array literal with interpolation. @@ -9494,6 +10067,11 @@ def format(q) end q.text(closing) end + + def ===(other) + other.is_a?(Symbols) && beginning === other.beginning && + ArrayMatch.call(elements, other.elements) + end end # SymbolsBeg represents the start of a symbol array literal with @@ -9533,6 +10111,10 @@ def copy(value: nil, location: nil) def deconstruct_keys(_keys) { value: value, location: location } end + + def ===(other) + other.is_a?(SymbolsBeg) && value === other.value + end end # TLambda represents the beginning of a lambda literal. @@ -9569,6 +10151,10 @@ def copy(value: nil, location: nil) def deconstruct_keys(_keys) { value: value, location: location } end + + def ===(other) + other.is_a?(TLambda) && value === other.value + end end # TLamBeg represents the beginning of the body of a lambda literal using @@ -9606,6 +10192,10 @@ def copy(value: nil, location: nil) def deconstruct_keys(_keys) { value: value, location: location } end + + def ===(other) + other.is_a?(TLamBeg) && value === other.value + end end # TopConstField is always the child node of some kind of assignment. It @@ -9652,6 +10242,10 @@ def format(q) q.text("::") q.format(constant) end + + def ===(other) + other.is_a?(TopConstField) && constant === other.constant + end end # TopConstRef is very similar to TopConstField except that it is not involved @@ -9697,6 +10291,10 @@ def format(q) q.text("::") q.format(constant) end + + def ===(other) + other.is_a?(TopConstRef) && constant === other.constant + end end # TStringBeg represents the beginning of a string literal. @@ -9738,6 +10336,10 @@ def copy(value: nil, location: nil) def deconstruct_keys(_keys) { value: value, location: location } end + + def ===(other) + other.is_a?(TStringBeg) && value === other.value + end end # TStringContent represents plain characters inside of an entity that accepts @@ -9789,6 +10391,10 @@ def deconstruct_keys(_keys) def format(q) q.text(value) end + + def ===(other) + other.is_a?(TStringContent) && value === other.value + end end # TStringEnd represents the end of a string literal. @@ -9830,6 +10436,10 @@ def copy(value: nil, location: nil) def deconstruct_keys(_keys) { value: value, location: location } end + + def ===(other) + other.is_a?(TStringEnd) && value === other.value + end end # Not represents the unary +not+ method being called on an expression. @@ -9904,6 +10514,11 @@ def format(q) end end end + + def ===(other) + other.is_a?(Not) && statement === other.statement && + parentheses === other.parentheses + end end # Unary represents a unary method being called on an expression, as in +!+ or @@ -9959,6 +10574,11 @@ def format(q) q.text(operator) q.format(statement) end + + def ===(other) + other.is_a?(Unary) && operator === other.operator && + statement === other.statement + end end # Undef represents the use of the +undef+ keyword. @@ -10034,6 +10654,10 @@ def format(q) end end end + + def ===(other) + other.is_a?(Undef) && ArrayMatch.call(symbols, other.symbols) + end end # Unless represents the first clause in an +unless+ chain. @@ -10101,6 +10725,11 @@ def format(q) ConditionalFormatter.new("unless", self).format(q) end + def ===(other) + other.is_a?(Unless) && predicate === other.predicate && + statements === other.statements && consequent === other.consequent + end + # Checks if the node was originally found in the modifier form. def modifier? predicate.location.start_char > statements.location.start_char @@ -10232,6 +10861,11 @@ def format(q) LoopFormatter.new("until", self).format(q) end + def ===(other) + other.is_a?(Until) && predicate === other.predicate && + statements === other.statements + end + def modifier? predicate.location.start_char > statements.location.start_char end @@ -10284,6 +10918,10 @@ def format(q) q.format(value) end end + + def ===(other) + other.is_a?(VarField) && value === other.value + end end # VarRef represents a variable reference. @@ -10332,6 +10970,10 @@ def format(q) q.format(value) end + def ===(other) + other.is_a?(VarRef) && value === other.value + end + # Oh man I hate this so much. Basically, ripper doesn't provide enough # functionality to actually know where pins are within an expression. So we # have to walk the tree ourselves and insert more information. In doing so, @@ -10405,6 +11047,10 @@ def format(q) q.format(value) end end + + def ===(other) + other.is_a?(PinnedVarRef) && value === other.value + end end # VCall represent any plain named object with Ruby that could be either a @@ -10447,6 +11093,10 @@ def format(q) q.format(value) end + def ===(other) + other.is_a?(VCall) && value === other.value + end + def access_control? @access_control ||= %w[private protected public].include?(value.value) end @@ -10488,6 +11138,10 @@ def deconstruct_keys(_keys) def format(q) end + + def ===(other) + other.is_a?(VoidStmt) + end end # When represents a +when+ clause in a +case+ chain. @@ -10602,6 +11256,11 @@ def format(q) end end end + + def ===(other) + other.is_a?(When) && arguments === other.arguments && + statements === other.statements && consequent === other.consequent + end end # While represents a +while+ loop. @@ -10657,6 +11316,11 @@ def format(q) LoopFormatter.new("while", self).format(q) end + def ===(other) + other.is_a?(While) && predicate === other.predicate && + statements === other.statements + end + def modifier? predicate.location.start_char > statements.location.start_char end @@ -10708,6 +11372,10 @@ def deconstruct_keys(_keys) def format(q) q.format_each(parts) end + + def ===(other) + other.is_a?(Word) && ArrayMatch.call(parts, other.parts) + end end # Words represents a string literal array with interpolation. @@ -10779,6 +11447,11 @@ def format(q) end q.text(closing) end + + def ===(other) + other.is_a?(Words) && beginning === other.beginning && + ArrayMatch.call(elements, other.elements) + end end # WordsBeg represents the beginning of a string literal array with @@ -10818,6 +11491,10 @@ def copy(value: nil, location: nil) def deconstruct_keys(_keys) { value: value, location: location } end + + def ===(other) + other.is_a?(WordsBeg) && value === other.value + end end # XString represents the contents of an XStringLiteral. @@ -10854,6 +11531,10 @@ def copy(parts: nil, location: nil) def deconstruct_keys(_keys) { parts: parts, location: location } end + + def ===(other) + other.is_a?(XString) && ArrayMatch.call(parts, other.parts) + end end # XStringLiteral represents a string that gets executed. @@ -10900,6 +11581,10 @@ def format(q) q.format_each(parts) q.text("`") end + + def ===(other) + other.is_a?(XStringLiteral) && ArrayMatch.call(parts, other.parts) + end end # Yield represents using the +yield+ keyword with arguments. @@ -10962,6 +11647,10 @@ def format(q) end end end + + def ===(other) + other.is_a?(Yield) && arguments === other.arguments + end end # ZSuper represents the bare +super+ keyword with no arguments. @@ -10998,5 +11687,9 @@ def deconstruct_keys(_keys) def format(q) q.text("super") end + + def ===(other) + other.is_a?(ZSuper) + end end end diff --git a/test/test_helper.rb b/test/test_helper.rb index 21bb75e3..1683e7cf 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -62,7 +62,7 @@ def assert_syntax_tree(node) assert_includes(pretty, type) # Assert that we can get back a new tree by using the mutation visitor. - node.accept(Visitor::MutationVisitor.new) + assert_operator node, :===, node.accept(Visitor::MutationVisitor.new) # Serialize the node to JSON, parse it back out, and assert that we have # found the expected type. From 6c41e756ede60302fd83c119df2ed0d3a2b62da5 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 8 Nov 2022 12:35:30 -0500 Subject: [PATCH 195/536] Track parent context in the mutation visitor --- lib/syntax_tree/visitor/mutation_visitor.rb | 28 ++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/lib/syntax_tree/visitor/mutation_visitor.rb b/lib/syntax_tree/visitor/mutation_visitor.rb index c2bdc7fc..8b7f187b 100644 --- a/lib/syntax_tree/visitor/mutation_visitor.rb +++ b/lib/syntax_tree/visitor/mutation_visitor.rb @@ -4,7 +4,33 @@ module SyntaxTree class Visitor # This visitor walks through the tree and copies each node as it is being # visited. This is useful for mutating the tree before it is formatted. - class MutationVisitor < Visitor + class MutationVisitor < BasicVisitor + # Here we maintain a stack of parent nodes so that it's easy to reflect on + # the context of a given node while mutating it. + attr_reader :stack + + def initialize + @stack = [] + end + + # This is the main entrypoint that's going to be called when we're + # recursing down through the tree. + def visit(node) + return unless node + + stack << node + result = node.accept(self) + + stack.pop + result + end + + # This is a small helper to visit an array of nodes and return the result + # of visiting them all. + def visit_all(nodes) + nodes.map { |node| visit(node) } + end + # Visit a BEGINBlock node. def visit_BEGIN(node) node.copy( From 3086f41642a3f66aa7bed39a2b8471113e387cec Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 8 Nov 2022 13:01:42 -0500 Subject: [PATCH 196/536] Ensure comments are copied over correctly for copy method --- lib/syntax_tree/node.rb | 1665 ++++++++++++------- lib/syntax_tree/visitor/mutation_visitor.rb | 2 +- 2 files changed, 1071 insertions(+), 596 deletions(-) diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index c9789e1e..219c0de1 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -83,6 +83,17 @@ def self.fixed(line:, char:, column:) end_column: column ) end + + def self.default + new( + start_line: 1, + start_char: 0, + start_column: 0, + end_line: 1, + end_char: 0, + end_column: 0 + ) + end end # This is the parent node of all of the syntax tree nodes. It's pretty much @@ -174,11 +185,15 @@ def child_nodes end def copy(lbrace: nil, statements: nil, location: nil) - BEGINBlock.new( - lbrace: lbrace || self.lbrace, - statements: statements || self.statements, - location: location || self.location - ) + node = + BEGINBlock.new( + lbrace: lbrace || self.lbrace, + statements: statements || self.statements, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -239,7 +254,14 @@ def child_nodes end def copy(value: nil, location: nil) - CHAR.new(value: value || self.value, location: location || self.location) + node = + CHAR.new( + value: value || self.value, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -298,11 +320,15 @@ def child_nodes end def copy(lbrace: nil, statements: nil, location: nil) - ENDBlock.new( - lbrace: lbrace || self.lbrace, - statements: statements || self.statements, - location: location || self.location - ) + node = + ENDBlock.new( + lbrace: lbrace || self.lbrace, + statements: statements || self.statements, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -366,10 +392,14 @@ def child_nodes end def copy(value: nil, location: nil) - EndContent.new( - value: value || self.value, - location: location || self.location - ) + node = + EndContent.new( + value: value || self.value, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -466,11 +496,15 @@ def child_nodes end def copy(left: nil, right: nil, location: nil) - Alias.new( - left: left || self.left, - right: right || self.right, - location: location || self.location - ) + node = + Alias.new( + left: left || self.left, + right: right || self.right, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -543,11 +577,15 @@ def child_nodes end def copy(collection: nil, index: nil, location: nil) - ARef.new( - collection: collection || self.collection, - index: index || self.index, - location: location || self.location - ) + node = + ARef.new( + collection: collection || self.collection, + index: index || self.index, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -617,11 +655,15 @@ def child_nodes end def copy(collection: nil, index: nil, location: nil) - ARefField.new( - collection: collection || self.collection, - index: index || self.index, - location: location || self.location - ) + node = + ARefField.new( + collection: collection || self.collection, + index: index || self.index, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -692,10 +734,14 @@ def child_nodes end def copy(arguments: nil, location: nil) - ArgParen.new( - arguments: arguments || self.arguments, - location: location || self.location - ) + node = + ArgParen.new( + arguments: arguments || self.arguments, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -776,7 +822,14 @@ def child_nodes end def copy(parts: nil, location: nil) - Args.new(parts: parts || self.parts, location: location || self.location) + node = + Args.new( + parts: parts || self.parts, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -820,10 +873,14 @@ def child_nodes end def copy(value: nil, location: nil) - ArgBlock.new( - value: value || self.value, - location: location || self.location - ) + node = + ArgBlock.new( + value: value || self.value, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -868,10 +925,14 @@ def child_nodes end def copy(value: nil, location: nil) - ArgStar.new( - value: value || self.value, - location: location || self.location - ) + node = + ArgStar.new( + value: value || self.value, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -925,7 +986,10 @@ def child_nodes end def copy(location: nil) - ArgsForward.new(location: location || self.location) + node = ArgsForward.new(location: location || self.location) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -1116,11 +1180,15 @@ def child_nodes end def copy(lbracket: nil, contents: nil, location: nil) - ArrayLiteral.new( - lbracket: lbracket || self.lbracket, - contents: contents || self.contents, - location: location || self.location - ) + node = + ArrayLiteral.new( + lbracket: lbracket || self.lbracket, + contents: contents || self.contents, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -1277,14 +1345,7 @@ def format(q) # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize( - constant:, - requireds:, - rest:, - posts:, - location:, - comments: [] - ) + def initialize(constant:, requireds:, rest:, posts:, location:) @constant = constant @requireds = requireds @rest = rest @@ -1308,13 +1369,17 @@ def copy( posts: nil, location: nil ) - AryPtn.new( - constant: constant || self.constant, - requireds: requireds || self.requireds, - rest: rest || self.rest, - posts: posts || self.posts, - location: location || self.location - ) + node = + AryPtn.new( + constant: constant || self.constant, + requireds: requireds || self.requireds, + rest: rest || self.rest, + posts: posts || self.posts, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -1405,11 +1470,15 @@ def child_nodes end def copy(target: nil, value: nil, location: nil) - Assign.new( - target: target || self.target, - value: value || self.value, - location: location || self.location - ) + node = + Assign.new( + target: target || self.target, + value: value || self.value, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -1479,11 +1548,15 @@ def child_nodes end def copy(key: nil, value: nil, location: nil) - Assoc.new( - key: key || self.key, - value: value || self.value, - location: location || self.location - ) + node = + Assoc.new( + key: key || self.key, + value: value || self.value, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -1549,10 +1622,14 @@ def child_nodes end def copy(value: nil, location: nil) - AssocSplat.new( - value: value || self.value, - location: location || self.location - ) + node = + AssocSplat.new( + value: value || self.value, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -1598,10 +1675,14 @@ def child_nodes end def copy(value: nil, location: nil) - Backref.new( - value: value || self.value, - location: location || self.location - ) + node = + Backref.new( + value: value || self.value, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -1644,10 +1725,14 @@ def child_nodes end def copy(value: nil, location: nil) - Backtick.new( - value: value || self.value, - location: location || self.location - ) + node = + Backtick.new( + value: value || self.value, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -1767,10 +1852,14 @@ def child_nodes end def copy(assocs: nil, location: nil) - BareAssocHash.new( - assocs: assocs || self.assocs, - location: location || self.location - ) + node = + BareAssocHash.new( + assocs: assocs || self.assocs, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -1820,10 +1909,14 @@ def child_nodes end def copy(bodystmt: nil, location: nil) - Begin.new( - bodystmt: bodystmt || self.bodystmt, - location: location || self.location - ) + node = + Begin.new( + bodystmt: bodystmt || self.bodystmt, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -1879,10 +1972,14 @@ def child_nodes end def copy(statement: nil, location: nil) - PinnedBegin.new( - statement: statement || self.statement, - location: location || self.location - ) + node = + PinnedBegin.new( + statement: statement || self.statement, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -1964,12 +2061,16 @@ def child_nodes end def copy(left: nil, operator: nil, right: nil, location: nil) - Binary.new( - left: left || self.left, - operator: operator || self.operator, - right: right || self.right, - location: location || self.location - ) + node = + Binary.new( + left: left || self.left, + operator: operator || self.operator, + right: right || self.right, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -2045,11 +2146,15 @@ def child_nodes end def copy(params: nil, locals: nil, location: nil) - BlockVar.new( - params: params || self.params, - locals: locals || self.locals, - location: location || self.location - ) + node = + BlockVar.new( + params: params || self.params, + locals: locals || self.locals, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -2115,7 +2220,14 @@ def child_nodes end def copy(name: nil, location: nil) - BlockArg.new(name: name || self.name, location: location || self.location) + node = + BlockArg.new( + name: name || self.name, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -2162,8 +2274,7 @@ def initialize( else_keyword:, else_clause:, ensure_clause:, - location:, - comments: [] + location: ) @statements = statements @rescue_clause = rescue_clause @@ -2224,14 +2335,18 @@ def copy( ensure_clause: nil, location: nil ) - BodyStmt.new( - statements: statements || self.statements, - rescue_clause: rescue_clause || self.rescue_clause, - else_keyword: else_keyword || self.else_keyword, - else_clause: else_clause || self.else_clause, - ensure_clause: ensure_clause || self.ensure_clause, - location: location || self.location - ) + node = + BodyStmt.new( + statements: statements || self.statements, + rescue_clause: rescue_clause || self.rescue_clause, + else_keyword: else_keyword || self.else_keyword, + else_clause: else_clause || self.else_clause, + ensure_clause: ensure_clause || self.ensure_clause, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -2508,10 +2623,14 @@ def child_nodes end def copy(arguments: nil, location: nil) - Break.new( - arguments: arguments || self.arguments, - location: location || self.location - ) + node = + Break.new( + arguments: arguments || self.arguments, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -2805,14 +2924,7 @@ class Call < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize( - receiver:, - operator:, - message:, - arguments:, - location:, - comments: [] - ) + def initialize(receiver:, operator:, message:, arguments:, location:) @receiver = receiver @operator = operator @message = message @@ -2841,13 +2953,17 @@ def copy( arguments: nil, location: nil ) - Call.new( - receiver: receiver || self.receiver, - operator: operator || self.operator, - message: message || self.message, - arguments: arguments || self.arguments, - location: location || self.location - ) + node = + Call.new( + receiver: receiver || self.receiver, + operator: operator || self.operator, + message: message || self.message, + arguments: arguments || self.arguments, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -2982,12 +3098,16 @@ def child_nodes end def copy(keyword: nil, value: nil, consequent: nil, location: nil) - Case.new( - keyword: keyword || self.keyword, - value: value || self.value, - consequent: consequent || self.consequent, - location: location || self.location - ) + node = + Case.new( + keyword: keyword || self.keyword, + value: value || self.value, + consequent: consequent || self.consequent, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -3061,12 +3181,16 @@ def child_nodes end def copy(value: nil, operator: nil, pattern: nil, location: nil) - RAssign.new( - value: value || self.value, - operator: operator || self.operator, - pattern: pattern || self.pattern, - location: location || self.location - ) + node = + RAssign.new( + value: value || self.value, + operator: operator || self.operator, + pattern: pattern || self.pattern, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -3171,12 +3295,16 @@ def child_nodes end def copy(constant: nil, superclass: nil, bodystmt: nil, location: nil) - ClassDeclaration.new( - constant: constant || self.constant, - superclass: superclass || self.superclass, - bodystmt: bodystmt || self.bodystmt, - location: location || self.location - ) + node = + ClassDeclaration.new( + constant: constant || self.constant, + superclass: superclass || self.superclass, + bodystmt: bodystmt || self.bodystmt, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -3302,12 +3430,16 @@ def child_nodes end def copy(message: nil, arguments: nil, block: nil, location: nil) - Command.new( - message: message || self.message, - arguments: arguments || self.arguments, - block: block || self.block, - location: location || self.location - ) + node = + Command.new( + message: message || self.message, + arguments: arguments || self.arguments, + block: block || self.block, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -3401,8 +3533,7 @@ def initialize( message:, arguments:, block:, - location:, - comments: [] + location: ) @receiver = receiver @operator = operator @@ -3429,14 +3560,18 @@ def copy( block: nil, location: nil ) - CommandCall.new( - receiver: receiver || self.receiver, - operator: operator || self.operator, - message: message || self.message, - arguments: arguments || self.arguments, - block: block || self.block, - location: location || self.location - ) + node = + CommandCall.new( + receiver: receiver || self.receiver, + operator: operator || self.operator, + message: message || self.message, + arguments: arguments || self.arguments, + block: block || self.block, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -3636,7 +3771,14 @@ def child_nodes end def copy(value: nil, location: nil) - Const.new(value: value || self.value, location: location || self.location) + node = + Const.new( + value: value || self.value, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -3686,11 +3828,15 @@ def child_nodes end def copy(parent: nil, constant: nil, location: nil) - ConstPathField.new( - parent: parent || self.parent, - constant: constant || self.constant, - location: location || self.location - ) + node = + ConstPathField.new( + parent: parent || self.parent, + constant: constant || self.constant, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -3746,11 +3892,15 @@ def child_nodes end def copy(parent: nil, constant: nil, location: nil) - ConstPathRef.new( - parent: parent || self.parent, - constant: constant || self.constant, - location: location || self.location - ) + node = + ConstPathRef.new( + parent: parent || self.parent, + constant: constant || self.constant, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -3804,10 +3954,14 @@ def child_nodes end def copy(constant: nil, location: nil) - ConstRef.new( - constant: constant || self.constant, - location: location || self.location - ) + node = + ConstRef.new( + constant: constant || self.constant, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -3851,7 +4005,14 @@ def child_nodes end def copy(value: nil, location: nil) - CVar.new(value: value || self.value, location: location || self.location) + node = + CVar.new( + value: value || self.value, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -3919,14 +4080,18 @@ def copy( bodystmt: nil, location: nil ) - Def.new( - target: target || self.target, - operator: operator || self.operator, - name: name || self.name, - params: params || self.params, - bodystmt: bodystmt || self.bodystmt, - location: location || self.location - ) + node = + Def.new( + target: target || self.target, + operator: operator || self.operator, + name: name || self.name, + params: params || self.params, + bodystmt: bodystmt || self.bodystmt, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -4026,10 +4191,14 @@ def child_nodes end def copy(value: nil, location: nil) - Defined.new( - value: value || self.value, - location: location || self.location - ) + node = + Defined.new( + value: value || self.value, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -4115,12 +4284,16 @@ def child_nodes end def copy(opening: nil, block_var: nil, bodystmt: nil, location: nil) - Block.new( - opening: opening || self.opening, - block_var: block_var || self.block_var, - bodystmt: bodystmt || self.bodystmt, - location: location || self.location - ) + node = + Block.new( + opening: opening || self.opening, + block_var: block_var || self.block_var, + bodystmt: bodystmt || self.bodystmt, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -4308,12 +4481,16 @@ def child_nodes end def copy(left: nil, operator: nil, right: nil, location: nil) - RangeLiteral.new( - left: left || self.left, - operator: operator || self.operator, - right: right || self.right, - location: location || self.location - ) + node = + RangeLiteral.new( + left: left || self.left, + operator: operator || self.operator, + right: right || self.right, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -4424,11 +4601,15 @@ def child_nodes end def copy(parts: nil, quote: nil, location: nil) - DynaSymbol.new( - parts: parts || self.parts, - quote: quote || self.quote, - location: location || self.location - ) + node = + DynaSymbol.new( + parts: parts || self.parts, + quote: quote || self.quote, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -4548,11 +4729,15 @@ def child_nodes end def copy(keyword: nil, statements: nil, location: nil) - Else.new( - keyword: keyword || self.keyword, - statements: statements || self.statements, - location: location || self.location - ) + node = + Else.new( + keyword: keyword || self.keyword, + statements: statements || self.statements, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -4604,13 +4789,7 @@ class Elsif < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize( - predicate:, - statements:, - consequent:, - location:, - comments: [] - ) + def initialize(predicate:, statements:, consequent:, location:) @predicate = predicate @statements = statements @consequent = consequent @@ -4627,12 +4806,16 @@ def child_nodes end def copy(predicate: nil, statements: nil, consequent: nil, location: nil) - Elsif.new( - predicate: predicate || self.predicate, - statements: statements || self.statements, - consequent: consequent || self.consequent, - location: location || self.location - ) + node = + Elsif.new( + predicate: predicate || self.predicate, + statements: statements || self.statements, + consequent: consequent || self.consequent, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -4893,11 +5076,15 @@ def child_nodes end def copy(keyword: nil, statements: nil, location: nil) - Ensure.new( - keyword: keyword || self.keyword, - statements: statements || self.statements, - location: location || self.location - ) + node = + Ensure.new( + keyword: keyword || self.keyword, + statements: statements || self.statements, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -4960,10 +5147,14 @@ def child_nodes end def copy(value: nil, location: nil) - ExcessedComma.new( - value: value || self.value, - location: location || self.location - ) + node = + ExcessedComma.new( + value: value || self.value, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -5016,12 +5207,16 @@ def child_nodes end def copy(parent: nil, operator: nil, name: nil, location: nil) - Field.new( - parent: parent || self.parent, - operator: operator || self.operator, - name: name || self.name, - location: location || self.location - ) + node = + Field.new( + parent: parent || self.parent, + operator: operator || self.operator, + name: name || self.name, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -5076,10 +5271,14 @@ def child_nodes end def copy(value: nil, location: nil) - FloatLiteral.new( - value: value || self.value, - location: location || self.location - ) + node = + FloatLiteral.new( + value: value || self.value, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -5139,13 +5338,17 @@ def child_nodes end def copy(constant: nil, left: nil, values: nil, right: nil, location: nil) - FndPtn.new( - constant: constant || self.constant, - left: left || self.left, - values: values || self.values, - right: right || self.right, - location: location || self.location - ) + node = + FndPtn.new( + constant: constant || self.constant, + left: left || self.left, + values: values || self.values, + right: right || self.right, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -5229,12 +5432,16 @@ def child_nodes end def copy(index: nil, collection: nil, statements: nil, location: nil) - For.new( - index: index || self.index, - collection: collection || self.collection, - statements: statements || self.statements, - location: location || self.location - ) + node = + For.new( + index: index || self.index, + collection: collection || self.collection, + statements: statements || self.statements, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -5300,7 +5507,14 @@ def child_nodes end def copy(value: nil, location: nil) - GVar.new(value: value || self.value, location: location || self.location) + node = + GVar.new( + value: value || self.value, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -5374,11 +5588,15 @@ def child_nodes end def copy(lbrace: nil, assocs: nil, location: nil) - HashLiteral.new( - lbrace: lbrace || self.lbrace, - assocs: assocs || self.assocs, - location: location || self.location - ) + node = + HashLiteral.new( + lbrace: lbrace || self.lbrace, + assocs: assocs || self.assocs, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -5458,14 +5676,7 @@ class Heredoc < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize( - beginning:, - ending: nil, - dedent: 0, - parts: [], - location:, - comments: [] - ) + def initialize(beginning:, ending: nil, dedent: 0, parts: [], location:) @beginning = beginning @ending = ending @dedent = dedent @@ -5483,12 +5694,16 @@ def child_nodes end def copy(beginning: nil, location: nil, ending: nil, parts: nil) - Heredoc.new( - beginning: beginning || self.beginning, - location: location || self.location, - ending: ending || self.ending, - parts: parts || self.parts - ) + node = + Heredoc.new( + beginning: beginning || self.beginning, + location: location || self.location, + ending: ending || self.ending, + parts: parts || self.parts + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -5577,10 +5792,14 @@ def child_nodes end def copy(value: nil, location: nil) - HeredocBeg.new( - value: value || self.value, - location: location || self.location - ) + node = + HeredocBeg.new( + value: value || self.value, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -5627,10 +5846,14 @@ def child_nodes end def copy(value: nil, location: nil) - HeredocEnd.new( - value: value || self.value, - location: location || self.location - ) + node = + HeredocEnd.new( + value: value || self.value, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -5732,12 +5955,16 @@ def child_nodes end def copy(constant: nil, keywords: nil, keyword_rest: nil, location: nil) - HshPtn.new( - constant: constant || self.constant, - keywords: keywords || self.keywords, - keyword_rest: keyword_rest || self.keyword_rest, - location: location || self.location - ) + node = + HshPtn.new( + constant: constant || self.constant, + keywords: keywords || self.keywords, + keyword_rest: keyword_rest || self.keyword_rest, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -5858,7 +6085,14 @@ def child_nodes end def copy(value: nil, location: nil) - Ident.new(value: value || self.value, location: location || self.location) + node = + Ident.new( + value: value || self.value, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -6131,13 +6365,7 @@ class If < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize( - predicate:, - statements:, - consequent:, - location:, - comments: [] - ) + def initialize(predicate:, statements:, consequent:, location:) @predicate = predicate @statements = statements @consequent = consequent @@ -6154,12 +6382,16 @@ def child_nodes end def copy(predicate: nil, statements: nil, consequent: nil, location: nil) - If.new( - predicate: predicate || self.predicate, - statements: statements || self.statements, - consequent: consequent || self.consequent, - location: location || self.location - ) + node = + If.new( + predicate: predicate || self.predicate, + statements: statements || self.statements, + consequent: consequent || self.consequent, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -6223,12 +6455,16 @@ def child_nodes end def copy(predicate: nil, truthy: nil, falsy: nil, location: nil) - IfOp.new( - predicate: predicate || self.predicate, - truthy: truthy || self.truthy, - falsy: falsy || self.falsy, - location: location || self.location - ) + node = + IfOp.new( + predicate: predicate || self.predicate, + truthy: truthy || self.truthy, + falsy: falsy || self.falsy, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -6330,10 +6566,14 @@ def child_nodes end def copy(value: nil, location: nil) - Imaginary.new( - value: value || self.value, - location: location || self.location - ) + node = + Imaginary.new( + value: value || self.value, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -6388,12 +6628,16 @@ def child_nodes end def copy(pattern: nil, statements: nil, consequent: nil, location: nil) - In.new( - pattern: pattern || self.pattern, - statements: statements || self.statements, - consequent: consequent || self.consequent, - location: location || self.location - ) + node = + In.new( + pattern: pattern || self.pattern, + statements: statements || self.statements, + consequent: consequent || self.consequent, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -6461,7 +6705,11 @@ def child_nodes end def copy(value: nil, location: nil) - Int.new(value: value || self.value, location: location || self.location) + node = + Int.new(value: value || self.value, location: location || self.location) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -6513,7 +6761,14 @@ def child_nodes end def copy(value: nil, location: nil) - IVar.new(value: value || self.value, location: location || self.location) + node = + IVar.new( + value: value || self.value, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -6570,7 +6825,11 @@ def child_nodes end def copy(value: nil, location: nil) - Kw.new(value: value || self.value, location: location || self.location) + node = + Kw.new(value: value || self.value, location: location || self.location) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -6615,10 +6874,14 @@ def child_nodes end def copy(name: nil, location: nil) - KwRestParam.new( - name: name || self.name, - location: location || self.location - ) + node = + KwRestParam.new( + name: name || self.name, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -6672,7 +6935,14 @@ def child_nodes end def copy(value: nil, location: nil) - Label.new(value: value || self.value, location: location || self.location) + node = + Label.new( + value: value || self.value, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -6762,11 +7032,15 @@ def child_nodes end def copy(params: nil, statements: nil, location: nil) - Lambda.new( - params: params || self.params, - statements: statements || self.statements, - location: location || self.location - ) + node = + Lambda.new( + params: params || self.params, + statements: statements || self.statements, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -6883,11 +7157,15 @@ def child_nodes end def copy(params: nil, locals: nil, location: nil) - LambdaVar.new( - params: params || self.params, - locals: locals || self.locals, - location: location || self.location - ) + node = + LambdaVar.new( + params: params || self.params, + locals: locals || self.locals, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -6938,10 +7216,14 @@ def child_nodes end def copy(value: nil, location: nil) - LBrace.new( - value: value || self.value, - location: location || self.location - ) + node = + LBrace.new( + value: value || self.value, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -6982,10 +7264,14 @@ def child_nodes end def copy(value: nil, location: nil) - LBracket.new( - value: value || self.value, - location: location || self.location - ) + node = + LBracket.new( + value: value || self.value, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -7026,10 +7312,14 @@ def child_nodes end def copy(value: nil, location: nil) - LParen.new( - value: value || self.value, - location: location || self.location - ) + node = + LParen.new( + value: value || self.value, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -7087,11 +7377,15 @@ def child_nodes end def copy(target: nil, value: nil, location: nil) - MAssign.new( - target: target || self.target, - value: value || self.value, - location: location || self.location - ) + node = + MAssign.new( + target: target || self.target, + value: value || self.value, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -7146,11 +7440,15 @@ def child_nodes end def copy(call: nil, block: nil, location: nil) - MethodAddBlock.new( - call: call || self.call, - block: block || self.block, - location: location || self.location - ) + node = + MethodAddBlock.new( + call: call || self.call, + block: block || self.block, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -7220,11 +7518,15 @@ def child_nodes end def copy(parts: nil, location: nil, comma: nil) - MLHS.new( - parts: parts || self.parts, - location: location || self.location, - comma: comma || self.comma - ) + node = + MLHS.new( + parts: parts || self.parts, + location: location || self.location, + comma: comma || self.comma + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -7277,10 +7579,14 @@ def child_nodes end def copy(contents: nil, location: nil) - MLHSParen.new( - contents: contents || self.contents, - location: location || self.location - ) + node = + MLHSParen.new( + contents: contents || self.contents, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -7346,11 +7652,15 @@ def child_nodes end def copy(constant: nil, bodystmt: nil, location: nil) - ModuleDeclaration.new( - constant: constant || self.constant, - bodystmt: bodystmt || self.bodystmt, - location: location || self.location - ) + node = + ModuleDeclaration.new( + constant: constant || self.constant, + bodystmt: bodystmt || self.bodystmt, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -7428,7 +7738,14 @@ def child_nodes end def copy(parts: nil, location: nil) - MRHS.new(parts: parts || self.parts, location: location || self.location) + node = + MRHS.new( + parts: parts || self.parts, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -7485,10 +7802,14 @@ def child_nodes end def copy(arguments: nil, location: nil) - Next.new( - arguments: arguments || self.arguments, - location: location || self.location - ) + node = + Next.new( + arguments: arguments || self.arguments, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -7537,7 +7858,11 @@ def child_nodes end def copy(value: nil, location: nil) - Op.new(value: value || self.value, location: location || self.location) + node = + Op.new(value: value || self.value, location: location || self.location) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -7591,12 +7916,16 @@ def child_nodes end def copy(target: nil, operator: nil, value: nil, location: nil) - OpAssign.new( - target: target || self.target, - operator: operator || self.operator, - value: value || self.value, - location: location || self.location - ) + node = + OpAssign.new( + target: target || self.target, + operator: operator || self.operator, + value: value || self.value, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -7823,8 +8152,7 @@ def initialize( keywords: [], keyword_rest: nil, block: nil, - location:, - comments: [] + location: ) @requireds = requireds @optionals = optionals @@ -7872,16 +8200,20 @@ def copy( keyword_rest: nil, block: nil ) - Params.new( - location: location || self.location, - requireds: requireds || self.requireds, - optionals: optionals || self.optionals, - rest: rest || self.rest, - posts: posts || self.posts, - keywords: keywords || self.keywords, - keyword_rest: keyword_rest || self.keyword_rest, - block: block || self.block - ) + node = + Params.new( + location: location || self.location, + requireds: requireds || self.requireds, + optionals: optionals || self.optionals, + rest: rest || self.rest, + posts: posts || self.posts, + keywords: keywords || self.keywords, + keyword_rest: keyword_rest || self.keyword_rest, + block: block || self.block + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -7991,11 +8323,15 @@ def child_nodes end def copy(lparen: nil, contents: nil, location: nil) - Paren.new( - lparen: lparen || self.lparen, - contents: contents || self.contents, - location: location || self.location - ) + node = + Paren.new( + lparen: lparen || self.lparen, + contents: contents || self.contents, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -8055,10 +8391,14 @@ def child_nodes end def copy(value: nil, location: nil) - Period.new( - value: value || self.value, - location: location || self.location - ) + node = + Period.new( + value: value || self.value, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -8099,10 +8439,14 @@ def child_nodes end def copy(statements: nil, location: nil) - Program.new( - statements: statements || self.statements, - location: location || self.location - ) + node = + Program.new( + statements: statements || self.statements, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -8155,11 +8499,15 @@ def child_nodes end def copy(beginning: nil, elements: nil, location: nil) - QSymbols.new( - beginning: beginning || self.beginning, - elements: elements || self.elements, - location: location || self.location - ) + node = + QSymbols.new( + beginning: beginning || self.beginning, + elements: elements || self.elements, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -8387,10 +8735,14 @@ def child_nodes end def copy(value: nil, location: nil) - RationalLiteral.new( - value: value || self.value, - location: location || self.location - ) + node = + RationalLiteral.new( + value: value || self.value, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -8502,7 +8854,10 @@ def child_nodes end def copy(location: nil) - Redo.new(location: location || self.location) + node = Redo.new(location: location || self.location) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -8692,12 +9047,16 @@ def child_nodes end def copy(beginning: nil, ending: nil, parts: nil, location: nil) - RegexpLiteral.new( - beginning: beginning || self.beginning, - ending: ending || self.ending, - parts: parts || self.parts, - location: location || self.location - ) + node = + RegexpLiteral.new( + beginning: beginning || self.beginning, + ending: ending || self.ending, + parts: parts || self.parts, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -8817,11 +9176,15 @@ def child_nodes end def copy(exceptions: nil, variable: nil, location: nil) - RescueEx.new( - exceptions: exceptions || self.exceptions, - variable: variable || self.variable, - location: location || self.location - ) + node = + RescueEx.new( + exceptions: exceptions || self.exceptions, + variable: variable || self.variable, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -8877,14 +9240,7 @@ class Rescue < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize( - keyword:, - exception:, - statements:, - consequent:, - location:, - comments: [] - ) + def initialize(keyword:, exception:, statements:, consequent:, location:) @keyword = keyword @exception = exception @statements = statements @@ -8930,13 +9286,17 @@ def copy( consequent: nil, location: nil ) - Rescue.new( - keyword: keyword || self.keyword, - exception: exception || self.exception, - statements: statements || self.statements, - consequent: consequent || self.consequent, - location: location || self.location - ) + node = + Rescue.new( + keyword: keyword || self.keyword, + exception: exception || self.exception, + statements: statements || self.statements, + consequent: consequent || self.consequent, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -9013,11 +9373,15 @@ def child_nodes end def copy(statement: nil, value: nil, location: nil) - RescueMod.new( - statement: statement || self.statement, - value: value || self.value, - location: location || self.location - ) + node = + RescueMod.new( + statement: statement || self.statement, + value: value || self.value, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -9082,10 +9446,14 @@ def child_nodes end def copy(name: nil, location: nil) - RestParam.new( - name: name || self.name, - location: location || self.location - ) + node = + RestParam.new( + name: name || self.name, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -9126,7 +9494,10 @@ def child_nodes end def copy(location: nil) - Retry.new(location: location || self.location) + node = Retry.new(location: location || self.location) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -9170,10 +9541,14 @@ def child_nodes end def copy(arguments: nil, location: nil) - Return.new( - arguments: arguments || self.arguments, - location: location || self.location - ) + node = + Return.new( + arguments: arguments || self.arguments, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -9260,11 +9635,15 @@ def child_nodes end def copy(target: nil, bodystmt: nil, location: nil) - SClass.new( - target: target || self.target, - bodystmt: bodystmt || self.bodystmt, - location: location || self.location - ) + node = + SClass.new( + target: target || self.target, + bodystmt: bodystmt || self.bodystmt, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -9377,11 +9756,15 @@ def child_nodes end def copy(body: nil, location: nil) - Statements.new( - parser, - body: body || self.body, - location: location || self.location - ) + node = + Statements.new( + parser, + body: body || self.body, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -9557,11 +9940,15 @@ def child_nodes end def copy(left: nil, right: nil, location: nil) - StringConcat.new( - left: left || self.left, - right: right || self.right, - location: location || self.location - ) + node = + StringConcat.new( + left: left || self.left, + right: right || self.right, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -9614,10 +10001,14 @@ def child_nodes end def copy(variable: nil, location: nil) - StringDVar.new( - variable: variable || self.variable, - location: location || self.location - ) + node = + StringDVar.new( + variable: variable || self.variable, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -9665,10 +10056,14 @@ def child_nodes end def copy(statements: nil, location: nil) - StringEmbExpr.new( - statements: statements || self.statements, - location: location || self.location - ) + node = + StringEmbExpr.new( + statements: statements || self.statements, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -9739,11 +10134,15 @@ def child_nodes end def copy(parts: nil, quote: nil, location: nil) - StringLiteral.new( - parts: parts || self.parts, - quote: quote || self.quote, - location: location || self.location - ) + node = + StringLiteral.new( + parts: parts || self.parts, + quote: quote || self.quote, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -9826,10 +10225,14 @@ def child_nodes end def copy(arguments: nil, location: nil) - Super.new( - arguments: arguments || self.arguments, - location: location || self.location - ) + node = + Super.new( + arguments: arguments || self.arguments, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -9976,10 +10379,14 @@ def child_nodes end def copy(value: nil, location: nil) - SymbolLiteral.new( - value: value || self.value, - location: location || self.location - ) + node = + SymbolLiteral.new( + value: value || self.value, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -10226,10 +10633,14 @@ def child_nodes end def copy(constant: nil, location: nil) - TopConstField.new( - constant: constant || self.constant, - location: location || self.location - ) + node = + TopConstField.new( + constant: constant || self.constant, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -10275,10 +10686,14 @@ def child_nodes end def copy(constant: nil, location: nil) - TopConstRef.new( - constant: constant || self.constant, - location: location || self.location - ) + node = + TopConstRef.new( + constant: constant || self.constant, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -10376,10 +10791,14 @@ def child_nodes end def copy(value: nil, location: nil) - TStringContent.new( - value: value || self.value, - location: location || self.location - ) + node = + TStringContent.new( + value: value || self.value, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -10473,11 +10892,15 @@ def child_nodes end def copy(statement: nil, parentheses: nil, location: nil) - Not.new( - statement: statement || self.statement, - parentheses: parentheses || self.parentheses, - location: location || self.location - ) + node = + Not.new( + statement: statement || self.statement, + parentheses: parentheses || self.parentheses, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -10552,11 +10975,15 @@ def child_nodes end def copy(operator: nil, statement: nil, location: nil) - Unary.new( - operator: operator || self.operator, - statement: statement || self.statement, - location: location || self.location - ) + node = + Unary.new( + operator: operator || self.operator, + statement: statement || self.statement, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -10631,10 +11058,14 @@ def child_nodes end def copy(symbols: nil, location: nil) - Undef.new( - symbols: symbols || self.symbols, - location: location || self.location - ) + node = + Undef.new( + symbols: symbols || self.symbols, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -10678,13 +11109,7 @@ class Unless < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize( - predicate:, - statements:, - consequent:, - location:, - comments: [] - ) + def initialize(predicate:, statements:, consequent:, location:) @predicate = predicate @statements = statements @consequent = consequent @@ -10701,12 +11126,16 @@ def child_nodes end def copy(predicate: nil, statements: nil, consequent: nil, location: nil) - Unless.new( - predicate: predicate || self.predicate, - statements: statements || self.statements, - consequent: consequent || self.consequent, - location: location || self.location - ) + node = + Unless.new( + predicate: predicate || self.predicate, + statements: statements || self.statements, + consequent: consequent || self.consequent, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -10839,11 +11268,15 @@ def child_nodes end def copy(predicate: nil, statements: nil, location: nil) - Until.new( - predicate: predicate || self.predicate, - statements: statements || self.statements, - location: location || self.location - ) + node = + Until.new( + predicate: predicate || self.predicate, + statements: statements || self.statements, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -10899,10 +11332,14 @@ def child_nodes end def copy(value: nil, location: nil) - VarField.new( - value: value || self.value, - location: location || self.location - ) + node = + VarField.new( + value: value || self.value, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -10954,10 +11391,14 @@ def child_nodes end def copy(value: nil, location: nil) - VarRef.new( - value: value || self.value, - location: location || self.location - ) + node = + VarRef.new( + value: value || self.value, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -11029,10 +11470,14 @@ def child_nodes end def copy(value: nil, location: nil) - PinnedVarRef.new( - value: value || self.value, - location: location || self.location - ) + node = + PinnedVarRef.new( + value: value || self.value, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -11080,7 +11525,14 @@ def child_nodes end def copy(value: nil, location: nil) - VCall.new(value: value || self.value, location: location || self.location) + node = + VCall.new( + value: value || self.value, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -11127,7 +11579,10 @@ def child_nodes end def copy(location: nil) - VoidStmt.new(location: location || self.location) + node = VoidStmt.new(location: location || self.location) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -11163,13 +11618,7 @@ class When < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize( - arguments:, - statements:, - consequent:, - location:, - comments: [] - ) + def initialize(arguments:, statements:, consequent:, location:) @arguments = arguments @statements = statements @consequent = consequent @@ -11186,12 +11635,16 @@ def child_nodes end def copy(arguments: nil, statements: nil, consequent: nil, location: nil) - When.new( - arguments: arguments || self.arguments, - statements: statements || self.statements, - consequent: consequent || self.consequent, - location: location || self.location - ) + node = + When.new( + arguments: arguments || self.arguments, + statements: statements || self.statements, + consequent: consequent || self.consequent, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -11294,11 +11747,15 @@ def child_nodes end def copy(predicate: nil, statements: nil, location: nil) - While.new( - predicate: predicate || self.predicate, - statements: statements || self.statements, - location: location || self.location - ) + node = + While.new( + predicate: predicate || self.predicate, + statements: statements || self.statements, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -11360,7 +11817,14 @@ def child_nodes end def copy(parts: nil, location: nil) - Word.new(parts: parts || self.parts, location: location || self.location) + node = + Word.new( + parts: parts || self.parts, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -11564,10 +12028,14 @@ def child_nodes end def copy(parts: nil, location: nil) - XStringLiteral.new( - parts: parts || self.parts, - location: location || self.location - ) + node = + XStringLiteral.new( + parts: parts || self.parts, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -11613,10 +12081,14 @@ def child_nodes end def copy(arguments: nil, location: nil) - Yield.new( - arguments: arguments || self.arguments, - location: location || self.location - ) + node = + Yield.new( + arguments: arguments || self.arguments, + location: location || self.location + ) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes @@ -11675,7 +12147,10 @@ def child_nodes end def copy(location: nil) - ZSuper.new(location: location || self.location) + node = ZSuper.new(location: location || self.location) + + node.comments.concat(comments.map(&:copy)) + node end alias deconstruct child_nodes diff --git a/lib/syntax_tree/visitor/mutation_visitor.rb b/lib/syntax_tree/visitor/mutation_visitor.rb index 8b7f187b..6ad7feef 100644 --- a/lib/syntax_tree/visitor/mutation_visitor.rb +++ b/lib/syntax_tree/visitor/mutation_visitor.rb @@ -24,7 +24,7 @@ def visit(node) stack.pop result end - + # This is a small helper to visit an array of nodes and return the result # of visiting them all. def visit_all(nodes) From 648145396c968f300d75eb51b6ae4168220df7f5 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 8 Nov 2022 15:03:16 -0500 Subject: [PATCH 197/536] Final tests, documentation, changelog for mutations --- CHANGELOG.md | 6 ++ README.md | 70 +++++++++++++++++++++ lib/syntax_tree.rb | 7 +++ lib/syntax_tree/node.rb | 30 +++++++++ lib/syntax_tree/visitor/mutation_visitor.rb | 45 ++++++++----- test/mutation_test.rb | 47 ++++++++++++++ 6 files changed, 188 insertions(+), 17 deletions(-) create mode 100644 test/mutation_test.rb diff --git a/CHANGELOG.md b/CHANGELOG.md index 642e7866..d8a848f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +### Added + +- Every node now implements the `#copy(**)` method, which provides a copy of the node with the given attributes replaced. +- Every node now implements the `#===(other)` method, which checks if the given node matches the current node for all attributes except for comments and location. +- There is a new `SyntaxTree::Visitor::MutationVisitor` and its convenience method `SyntaxTree.mutation` which can be used to mutate a syntax tree. For details on how to use this visitor, check the README. + ### Changed - Nodes no longer have a `comments:` keyword on their initializers. By default, they initialize to an empty array. If you were previously passing comments into the initializer, you should now create the node first, then call `node.comments.concat` to add your comments. diff --git a/README.md b/README.md index 368c9361..2bd333ae 100644 --- a/README.md +++ b/README.md @@ -27,17 +27,21 @@ It is built with only standard library dependencies. It additionally ships with - [SyntaxTree.read(filepath)](#syntaxtreereadfilepath) - [SyntaxTree.parse(source)](#syntaxtreeparsesource) - [SyntaxTree.format(source)](#syntaxtreeformatsource) + - [SyntaxTree.mutation(&block)](#syntaxtreemutationblock) - [SyntaxTree.search(source, query, &block)](#syntaxtreesearchsource-query-block) - [Nodes](#nodes) - [child_nodes](#child_nodes) + - [copy(**)](#copy) - [Pattern matching](#pattern-matching) - [pretty_print(q)](#pretty_printq) - [to_json(*opts)](#to_jsonopts) - [format(q)](#formatq) + - [===(other)](#other) - [construct_keys](#construct_keys) - [Visitor](#visitor) - [visit_method](#visit_method) - [BasicVisitor](#basicvisitor) + - [MutationVisitor](#mutationvisitor) - [Language server](#language-server) - [textDocument/formatting](#textdocumentformatting) - [textDocument/inlayHint](#textdocumentinlayhint) @@ -332,6 +336,10 @@ This function takes an input string containing Ruby code and returns the syntax This function takes an input string containing Ruby code, parses it into its underlying syntax tree, and formats it back out to a string. You can optionally pass a second argument to this method as well that is the maximum width to print. It defaults to `80`. +### SyntaxTree.mutation(&block) + +This function yields a new mutation visitor to the block, and then returns the initialized visitor. It's effectively a shortcut for creating a `SyntaxTree::Visitor::MutationVisitor` without having to remember the class name. For more information on that visitor, see the definition below. + ### SyntaxTree.search(source, query, &block) This function takes an input string containing Ruby code, an input string containing a valid Ruby `in` clause expression that can be used to match against nodes in the tree (can be generated using `stree expr`, `stree match`, or `Node#construct_keys`), and a block. Each node that matches the given query will be yielded to the block. The block will receive the node as its only argument. @@ -350,6 +358,20 @@ program.child_nodes.first.child_nodes.first # => (binary (int "1") :+ (int "1")) ``` +### copy + +This method returns a copy of the node, with the given attributes replaced. + +```ruby +program = SyntaxTree.parse("1 + 1") + +binary = program.statements.body.first +# => (binary (int "1") + (int "1")) + +binary.copy(operator: :-) +# => (binary (int "1") - (int "1")) +``` + ### Pattern matching Pattern matching is another way to descend the tree which is more specific than using `child_nodes`. Using Ruby's built-in pattern matching, you can extract the same information but be as specific about your constraints as you like. For example, with minimal constraints: @@ -407,6 +429,18 @@ formatter.output.join # => "1 + 1" ``` +### ===(other) + +Every node responds to `===`, which is used to check if the given other node matches all of the attributes of the current node except for location and comments. For example: + +```ruby +program1 = SyntaxTree.parse("1 + 1") +program2 = SyntaxTree.parse("1 + 1") + +program1 === program2 +# => true +``` + ### construct_keys Every node responds to `construct_keys`, which will return a string that contains a Ruby pattern-matching expression that could be used to match against the current node. It's meant to be used in tooling and through the CLI mostly. @@ -495,6 +529,42 @@ end The visitor defined above will error out unless it's only visiting a `SyntaxTree::Int` node. This is useful in a couple of ways, e.g., if you're trying to define a visitor to handle the whole tree but it's currently a work-in-progress. +### MutationVisitor + +The `MutationVisitor` is a visitor that can be used to mutate the tree. It works by defining a default `visit_*` method that returns a copy of the given node with all of its attributes visited. This new node will replace the old node in the tree. Typically, you use the `#mutate` method on it to define mutations using patterns. For example: + +```ruby +# Create a new visitor +visitor = SyntaxTree::Visitor::MutationVisitor.new + +# Specify that it should mutate If nodes with assignments in their predicates +visitor.mutate("If[predicate: Assign | OpAssign]") do |node| + # Get the existing If's predicate node + predicate = node.predicate + + # Create a new predicate node that wraps the existing predicate node + # in parentheses + predicate = + SyntaxTree::Paren.new( + lparen: SyntaxTree::LParen.default, + contents: predicate, + location: predicate.location + ) + + # Return a copy of this node with the new predicate + node.copy(predicate: predicate) +end + +source = "if a = 1; end" +program = SyntaxTree.parse(source) + +SyntaxTree::Formatter.format(source, program) +# => "if a = 1\nend\n" + +SyntaxTree::Formatter.format(source, program.accept(visitor)) +# => "if (a = 1)\nend\n" +``` + ### WithEnvironment The `WithEnvironment` module can be included in visitors to automatically keep track of local variables and arguments diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index 5808cd15..aff4404c 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -62,6 +62,13 @@ def self.format(source, maxwidth = DEFAULT_PRINT_WIDTH) formatter.output.join end + # A convenience method for creating a new mutation visitor. + def self.mutation + visitor = Visitor::MutationVisitor.new + yield visitor + visitor + end + # Returns the source from the given filepath taking into account any potential # magic encoding comments. def self.read(filepath) diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 219c0de1..639eb7be 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -84,6 +84,9 @@ def self.fixed(line:, char:, column:) ) end + # A convenience method that is typically used when you don't care about the + # location of a node, but need to create a Location instance to pass to a + # constructor. def self.default new( start_line: 1, @@ -7239,6 +7242,15 @@ def format(q) def ===(other) other.is_a?(LBrace) && value === other.value end + + # Because some nodes keep around a { token so that comments can be attached + # to it if they occur in the source, oftentimes an LBrace is a child of + # another node. This means it's required at initialization time. To make it + # easier to create LBrace nodes without any specific value, this method + # provides a default node. + def self.default + new(value: "{", location: Location.default) + end end # LBracket represents the use of a left bracket, i.e., [. @@ -7287,6 +7299,15 @@ def format(q) def ===(other) other.is_a?(LBracket) && value === other.value end + + # Because some nodes keep around a [ token so that comments can be attached + # to it if they occur in the source, oftentimes an LBracket is a child of + # another node. This means it's required at initialization time. To make it + # easier to create LBracket nodes without any specific value, this method + # provides a default node. + def self.default + new(value: "[", location: Location.default) + end end # LParen represents the use of a left parenthesis, i.e., (. @@ -7335,6 +7356,15 @@ def format(q) def ===(other) other.is_a?(LParen) && value === other.value end + + # Because some nodes keep around a ( token so that comments can be attached + # to it if they occur in the source, oftentimes an LParen is a child of + # another node. This means it's required at initialization time. To make it + # easier to create LParen nodes without any specific value, this method + # provides a default node. + def self.default + new(value: "(", location: Location.default) + end end # MAssign is a parent node of any kind of multiple assignment. This includes diff --git a/lib/syntax_tree/visitor/mutation_visitor.rb b/lib/syntax_tree/visitor/mutation_visitor.rb index 6ad7feef..6e7d4ff2 100644 --- a/lib/syntax_tree/visitor/mutation_visitor.rb +++ b/lib/syntax_tree/visitor/mutation_visitor.rb @@ -5,30 +5,33 @@ class Visitor # This visitor walks through the tree and copies each node as it is being # visited. This is useful for mutating the tree before it is formatted. class MutationVisitor < BasicVisitor - # Here we maintain a stack of parent nodes so that it's easy to reflect on - # the context of a given node while mutating it. - attr_reader :stack + attr_reader :mutations def initialize - @stack = [] + @mutations = [] end - # This is the main entrypoint that's going to be called when we're - # recursing down through the tree. + # Create a new mutation based on the given query that will mutate the node + # using the given block. The block should return a new node that will take + # the place of the given node in the tree. These blocks frequently make + # use of the `copy` method on nodes to create a new node with the same + # properties as the original node. + def mutate(query, &block) + mutations << [Pattern.new(query).compile, block] + end + + # This is the base visit method for each node in the tree. It first + # creates a copy of the node using the visit_* methods defined below. Then + # it checks each mutation in sequence and calls it if it finds a match. def visit(node) return unless node - - stack << node result = node.accept(self) - stack.pop - result - end + mutations.each do |(pattern, mutation)| + result = mutation.call(result) if pattern.call(result) + end - # This is a small helper to visit an array of nodes and return the result - # of visiting them all. - def visit_all(nodes) - nodes.map { |node| visit(node) } + result end # Visit a BEGINBlock node. @@ -435,6 +438,7 @@ def visit_ident(node) # Visit a If node. def visit_if(node) node.copy( + predicate: visit(node.predicate), statements: visit(node.statements), consequent: visit(node.consequent) ) @@ -822,6 +826,7 @@ def visit_undef(node) # Visit a Unless node. def visit_unless(node) node.copy( + predicate: visit(node.predicate), statements: visit(node.statements), consequent: visit(node.consequent) ) @@ -829,7 +834,10 @@ def visit_unless(node) # Visit a Until node. def visit_until(node) - node.copy(statements: visit(node.statements)) + node.copy( + predicate: visit(node.predicate), + statements: visit(node.statements) + ) end # Visit a VarField node. @@ -868,7 +876,10 @@ def visit_when(node) # Visit a While node. def visit_while(node) - node.copy(statements: visit(node.statements)) + node.copy( + predicate: visit(node.predicate), + statements: visit(node.statements) + ) end # Visit a Word node. diff --git a/test/mutation_test.rb b/test/mutation_test.rb new file mode 100644 index 00000000..ab607beb --- /dev/null +++ b/test/mutation_test.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +require_relative "test_helper" + +module SyntaxTree + class MutationTest < Minitest::Test + def test_mutates_based_on_patterns + source = <<~RUBY + if a = b + c + end + RUBY + + expected = <<~RUBY + if (a = b) + c + end + RUBY + + program = SyntaxTree.parse(source).accept(build_mutation) + assert_equal(expected, SyntaxTree::Formatter.format(source, program)) + end + + private + + def build_mutation + SyntaxTree.mutation do |mutation| + mutation.mutate("If[predicate: Assign | OpAssign]") do |node| + # Get the existing If's predicate node + predicate = node.predicate + + # Create a new predicate node that wraps the existing predicate node + # in parentheses + predicate = + SyntaxTree::Paren.new( + lparen: SyntaxTree::LParen.default, + contents: predicate, + location: predicate.location + ) + + # Return a copy of this node with the new predicate + node.copy(predicate: predicate) + end + end + end + end +end From 416e48147b4eb52fefb5a42103287f33bdee87b5 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 8 Nov 2022 15:10:03 -0500 Subject: [PATCH 198/536] Cleaner README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2bd333ae..e87ec765 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ It is built with only standard library dependencies. It additionally ships with - [SyntaxTree.search(source, query, &block)](#syntaxtreesearchsource-query-block) - [Nodes](#nodes) - [child_nodes](#child_nodes) - - [copy(**)](#copy) + - [copy(**attrs)](#copyattrs) - [Pattern matching](#pattern-matching) - [pretty_print(q)](#pretty_printq) - [to_json(*opts)](#to_jsonopts) @@ -358,7 +358,7 @@ program.child_nodes.first.child_nodes.first # => (binary (int "1") :+ (int "1")) ``` -### copy +### copy(**attrs) This method returns a copy of the node, with the given attributes replaced. From 52bed533a0b1377565c9a5b10fbe3ce87aff16ce Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 8 Nov 2022 16:41:43 -0500 Subject: [PATCH 199/536] Increase test coverage --- lib/syntax_tree/cli.rb | 12 ++-- lib/syntax_tree/pattern.rb | 6 +- lib/syntax_tree/visitor/environment.rb | 11 ++-- lib/syntax_tree/visitor/field_visitor.rb | 8 --- test/cli_test.rb | 66 +++++++++++++++++++- test/language_server_test.rb | 18 ++++++ test/rake_test.rb | 20 +++++- test/search_test.rb | 79 +++++++++++++++++++++--- test/test_helper.rb | 7 +-- test/visitor_with_environment_test.rb | 40 ++++++++++++ 10 files changed, 233 insertions(+), 34 deletions(-) diff --git a/lib/syntax_tree/cli.rb b/lib/syntax_tree/cli.rb index 62e8ab68..11c93537 100644 --- a/lib/syntax_tree/cli.rb +++ b/lib/syntax_tree/cli.rb @@ -497,10 +497,14 @@ def run(argv) Dir .glob(pattern) .each do |filepath| - if File.readable?(filepath) && - options.ignore_files.none? { File.fnmatch?(_1, filepath) } - queue << FileItem.new(filepath) - end + # Skip past invalid filepaths by default. + next unless File.readable?(filepath) + + # Skip past any ignored filepaths. + next if options.ignore_files.any? { File.fnmatch(_1, filepath) } + + # Otherwise, a new file item for the given filepath to the list. + queue << FileItem.new(filepath) end end diff --git a/lib/syntax_tree/pattern.rb b/lib/syntax_tree/pattern.rb index 439d573f..ca49c6bf 100644 --- a/lib/syntax_tree/pattern.rb +++ b/lib/syntax_tree/pattern.rb @@ -142,11 +142,11 @@ def compile_binary(node) def compile_const(node) value = node.value - if SyntaxTree.const_defined?(value) + if SyntaxTree.const_defined?(value, false) clazz = SyntaxTree.const_get(value) ->(other) { clazz === other } - elsif Object.const_defined?(value) + elsif Object.const_defined?(value, false) clazz = Object.const_get(value) ->(other) { clazz === other } @@ -179,7 +179,7 @@ def compile_dyna_symbol(node) ->(other) { symbol === other } else - compile_error(root) + compile_error(node) end end diff --git a/lib/syntax_tree/visitor/environment.rb b/lib/syntax_tree/visitor/environment.rb index dfcf0a80..b07a5203 100644 --- a/lib/syntax_tree/visitor/environment.rb +++ b/lib/syntax_tree/visitor/environment.rb @@ -4,10 +4,6 @@ module SyntaxTree # The environment class is used to keep track of local variables and arguments # inside a particular scope class Environment - # [Array[Local]] The local variables and arguments defined in this - # environment - attr_reader :locals - # This class tracks the occurrences of a local variable or argument class Local # [Symbol] The type of the local (e.g. :argument, :variable) @@ -38,6 +34,13 @@ def add_usage(location) end end + # [Array[Local]] The local variables and arguments defined in this + # environment + attr_reader :locals + + # [Environment | nil] The parent environment + attr_reader :parent + # initialize: (Environment | nil parent) -> void def initialize(parent = nil) @locals = {} diff --git a/lib/syntax_tree/visitor/field_visitor.rb b/lib/syntax_tree/visitor/field_visitor.rb index 01c7de4e..b56d771c 100644 --- a/lib/syntax_tree/visitor/field_visitor.rb +++ b/lib/syntax_tree/visitor/field_visitor.rb @@ -388,14 +388,6 @@ def visit_excessed_comma(node) visit_token(node, "excessed_comma") end - def visit_fcall(node) - node(node, "fcall") do - field("value", node.value) - field("arguments", node.arguments) if node.arguments - comments(node) - end - end - def visit_field(node) node(node, "field") do field("parent", node.parent) diff --git a/test/cli_test.rb b/test/cli_test.rb index b4ef0afc..9740806d 100644 --- a/test/cli_test.rb +++ b/test/cli_test.rb @@ -41,6 +41,7 @@ def test_ast_ignore def test_ast_syntax_error result = run_cli("ast", contents: "foo\n<>\nbar\n") assert_includes(result.stderr, "syntax error") + refute_equal(0, result.status) end def test_check @@ -51,6 +52,7 @@ def test_check def test_check_unformatted result = run_cli("check", contents: "foo") assert_includes(result.stderr, "expected") + refute_equal(0, result.status) end def test_check_print_width @@ -59,6 +61,17 @@ def test_check_print_width assert_includes(result.stdio, "match") end + def test_check_target_ruby_version + previous = Formatter::OPTIONS[:target_ruby_version] + + begin + result = run_cli("check", "--target-ruby-version=2.6.0") + assert_includes(result.stdio, "match") + ensure + Formatter::OPTIONS[:target_ruby_version] = previous + end + end + def test_debug result = run_cli("debug") assert_includes(result.stdio, "idempotently") @@ -71,6 +84,7 @@ def test_debug_non_idempotent_format SyntaxTree.stub(:format, formatting) do result = run_cli("debug") assert_includes(result.stderr, "idempotently") + refute_equal(0, result.status) end end @@ -84,6 +98,12 @@ def test_expr assert_includes(result.stdio, "SyntaxTree::Ident") end + def test_expr_more_than_one + result = run_cli("expr", contents: "1; 2") + assert_includes(result.stderr, "single expression") + refute_equal(0, result.status) + end + def test_format result = run_cli("format") assert_equal("test\n", result.stdio) @@ -104,6 +124,17 @@ def test_search assert_equal(2, result.stdio.lines.length) end + def test_search_multi_line + result = run_cli("search", "Binary", contents: "1 +\n2") + assert_equal(1, result.stdio.lines.length) + end + + def test_search_invalid + result = run_cli("search", "FooBar") + assert_includes(result.stderr, "unable") + refute_equal(0, result.status) + end + def test_version result = run_cli("version") assert_includes(result.stdio, SyntaxTree::VERSION.to_s) @@ -120,6 +151,29 @@ def test_write def test_write_syntax_tree result = run_cli("write", contents: "<>") assert_includes(result.stderr, "syntax error") + refute_equal(0, result.status) + end + + def test_write_script + args = ["write", "-e", "1 + 2"] + stdout, stderr = capture_io { SyntaxTree::CLI.run(args) } + + assert_includes stdout, "script" + assert_empty stderr + end + + def test_write_stdin + previous = $stdin + $stdin = StringIO.new("1 + 2") + + begin + stdout, stderr = capture_io { SyntaxTree::CLI.run(["write"]) } + + assert_includes stdout, "stdin" + assert_empty stderr + ensure + $stdin = previous + end end def test_help @@ -128,8 +182,10 @@ def test_help end def test_help_default - *, stderr = capture_io { SyntaxTree::CLI.run(["foobar"]) } + status = 0 + *, stderr = capture_io { status = SyntaxTree::CLI.run(["foobar"]) } assert_includes(stderr, "stree help") + refute_equal(0, status) end def test_no_arguments @@ -215,6 +271,7 @@ def test_print_width_args_with_config_file_override result = run_cli("check", "--print-width=82", contents: contents) assert_includes(result.stderr, "expected") + refute_equal(0, result.status) end end @@ -251,7 +308,12 @@ def run_cli(command, *args, contents: :default) status = nil stdio, stderr = capture_io do - status = SyntaxTree::CLI.run([command, *args, tempfile.path]) + status = + begin + SyntaxTree::CLI.run([command, *args, tempfile.path]) + rescue SystemExit => error + error.status + end end Result.new(status: status, stdio: stdio, stderr: stderr) diff --git a/test/language_server_test.rb b/test/language_server_test.rb index 8e1ed9a7..2fe4e60a 100644 --- a/test/language_server_test.rb +++ b/test/language_server_test.rb @@ -159,6 +159,24 @@ def test_inlay_hint assert_equal(3, responses.dig(1, :result).size) end + def test_inlay_hint_invalid + responses = run_server([ + Initialize.new(1), + TextDocumentDidOpen.new("file:///path/to/file.rb", "<>"), + TextDocumentInlayHint.new(2, "file:///path/to/file.rb"), + Shutdown.new(3) + ]) + + shape = LanguageServer::Request[[ + { id: 1, result: { capabilities: Hash } }, + { id: 2, result: :any }, + { id: 3, result: {} } + ]] + + assert_operator(shape, :===, responses) + assert_equal(0, responses.dig(1, :result).size) + end + def test_visualizing responses = run_server([ Initialize.new(1), diff --git a/test/rake_test.rb b/test/rake_test.rb index bd315cc6..d07fc49c 100644 --- a/test/rake_test.rb +++ b/test/rake_test.rb @@ -8,12 +8,28 @@ module Rake class CheckTaskTest < Minitest::Test Invocation = Struct.new(:args) + def test_task_command + assert_raises(NotImplementedError) { Task.new.command } + end + def test_check_task source_files = "{app,config,lib}/**/*.rb" - CheckTask.new { |t| t.source_files = source_files } + + CheckTask.new do |t| + t.source_files = source_files + t.print_width = 100 + t.target_ruby_version = Gem::Version.new("2.6.0") + end + + expected = [ + "check", + "--print-width=100", + "--target-ruby-version=2.6.0", + source_files + ] invocation = invoke("stree:check") - assert_equal(["check", source_files], invocation.args) + assert_equal(expected, invocation.args) end def test_write_task diff --git a/test/search_test.rb b/test/search_test.rb index 314142e3..9f7d89b8 100644 --- a/test/search_test.rb +++ b/test/search_test.rb @@ -4,6 +4,50 @@ module SyntaxTree class SearchTest < Minitest::Test + def test_search_invalid_syntax + assert_raises(Pattern::CompilationError) { search("", "<>") } + end + + def test_search_invalid_constant + assert_raises(Pattern::CompilationError) { search("", "Foo") } + end + + def test_search_invalid_nested_constant + assert_raises(Pattern::CompilationError) { search("", "Foo::Bar") } + end + + def test_search_regexp_with_interpolation + assert_raises(Pattern::CompilationError) { search("", "/\#{foo}/") } + end + + def test_search_string_with_interpolation + assert_raises(Pattern::CompilationError) { search("", '"#{foo}"') } + end + + def test_search_symbol_with_interpolation + assert_raises(Pattern::CompilationError) { search("", ":\"\#{foo}\"") } + end + + def test_search_invalid_node + assert_raises(Pattern::CompilationError) { search("", "Int[^foo]") } + end + + def test_search_self + assert_raises(Pattern::CompilationError) { search("", "self") } + end + + def test_search_array_pattern_no_constant + results = search("1 + 2", "[Int, Int]") + + assert_equal 1, results.length + end + + def test_search_array_pattern + results = search("1 + 2", "Binary[Int, Int]") + + assert_equal 1, results.length + end + def test_search_binary_or results = search("Foo + Bar + 1", "VarRef | Int") @@ -18,12 +62,24 @@ def test_search_const assert_equal %w[Bar Baz Foo], results.map { |node| node.value.value }.sort end + def test_search_object_const + results = search("1 + 2 + 3", "Int[value: String]") + + assert_equal 3, results.length + end + def test_search_syntax_tree_const results = search("Foo + Bar + Baz", "SyntaxTree::VarRef") assert_equal 3, results.length end + def test_search_hash_pattern_no_constant + results = search("Foo + Bar + Baz", "{ value: Const }") + + assert_equal 3, results.length + end + def test_search_hash_pattern_string results = search("Foo + Bar + Baz", "VarRef[value: Const[value: 'Foo']]") @@ -39,13 +95,25 @@ def test_search_hash_pattern_regexp end def test_search_string_empty - results = search("''", "StringLiteral[parts: []]") + results = search("", "''") - assert_equal 1, results.length + assert_empty results end def test_search_symbol_empty - results = search(":''", "DynaSymbol[parts: []]") + results = search("", ":''") + + assert_empty results + end + + def test_search_symbol_plain + results = search("1 + 2", "Binary[operator: :'+']") + + assert_equal 1, results.length + end + + def test_search_symbol + results = search("1 + 2", "Binary[operator: :+]") assert_equal 1, results.length end @@ -53,10 +121,7 @@ def test_search_symbol_empty private def search(source, query) - pattern = Pattern.new(query).compile - program = SyntaxTree.parse(source) - - Search.new(pattern).scan(program).to_a + SyntaxTree.search(source, query).to_a end end end diff --git a/test/test_helper.rb b/test/test_helper.rb index 1683e7cf..b2cd6787 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -2,10 +2,9 @@ require "simplecov" SimpleCov.start do - unless ENV["CI"] - add_filter("accept_methods_test.rb") - add_filter("idempotency_test.rb") - end + add_filter("idempotency_test.rb") unless ENV["CI"] + add_group("lib", "lib") + add_group("test", "test") end $LOAD_PATH.unshift(File.expand_path("../lib", __dir__)) diff --git a/test/visitor_with_environment_test.rb b/test/visitor_with_environment_test.rb index b37bad16..cc4007fe 100644 --- a/test/visitor_with_environment_test.rb +++ b/test/visitor_with_environment_test.rb @@ -615,5 +615,45 @@ def test_double_nested_arguments assert_equal(1, argument.definitions[0].start_line) assert_equal(5, argument.usages[0].start_line) end + + class Resolver < Visitor + include WithEnvironment + + attr_reader :locals + + def initialize + @locals = [] + end + + def visit_assign(node) + level = 0 + environment = current_environment + level += 1 until (environment = environment.parent).nil? + + locals << [node.target.value.value, level] + super + end + end + + def test_class + source = <<~RUBY + module Level0 + level0 = 0 + + module Level1 + level1 = 1 + + class Level2 + level2 = 2 + end + end + end + RUBY + + visitor = Resolver.new + SyntaxTree.parse(source).accept(visitor) + + assert_equal [["level0", 0], ["level1", 1], ["level2", 2]], visitor.locals + end end end From a3c65df07baa42c348ac9a8c17463807b05d7f4e Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 9 Nov 2022 09:46:33 -0500 Subject: [PATCH 200/536] Support formatting from a non-main ractor --- CHANGELOG.md | 1 + Gemfile.lock | 4 +- lib/syntax_tree.rb | 8 ++- lib/syntax_tree/cli.rb | 70 +++++++++++++++------- lib/syntax_tree/formatter.rb | 76 +++++++++++++++++------- lib/syntax_tree/node.rb | 13 ++-- lib/syntax_tree/plugin/single_quotes.rb | 6 +- lib/syntax_tree/plugin/trailing_comma.rb | 6 +- syntax_tree.gemspec | 2 +- test/cli_test.rb | 10 +--- test/plugin/single_quotes_test.rb | 5 +- test/plugin/trailing_comma_test.rb | 5 +- test/ractor_test.rb | 37 ++++++++++++ test/test_helper.rb | 25 +------- 14 files changed, 177 insertions(+), 91 deletions(-) create mode 100644 test/ractor_test.rb diff --git a/CHANGELOG.md b/CHANGELOG.md index d8a848f5..dc5637ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - `Return0` is no longer a node. Instead if has been folded into the `Return` node. The `Return` node can now have its `arguments` field be `nil`. Consequently, the `visit_return0` method has been removed from the visitor interface. If you were previously using this method, you should now use `visit_return` instead. - The `ArgsForward`, `Redo`, `Retry`, and `ZSuper` nodes no longer have `value` fields associated with them (which were always string literals corresponding to the keyword being used). - The `Command` and `CommandCall` nodes now has `block` attributes on them. These attributes are used in the place where you would previously have had a `MethodAddBlock` structure. Where before the `MethodAddBlock` would have the command and block as its two children, you now just have one command node with the `block` attribute set to the `Block` node. +- Previously the formatting options were defined on an unfrozen hash called `SyntaxTree::Formatter::OPTIONS`. It was globally mutable, which made it impossible to reference from within a Ractor. As such, it has now been replaced with `SyntaxTree::Formatter::Options.new` which creates a new options object instance that can be modified without impacting global state. As a part of this change, formatting can now be performed from within a non-main Ractor. In order to check if the `plugin/single_quotes` plugin has been loaded, check if `SyntaxTree::Formatter::SINGLE_QUOTES` is defined. In order to check if the `plugin/trailing_comma` plugin has been loaded, check if `SyntaxTree::Formatter::TRAILING_COMMA` is defined. ## [4.3.0] - 2022-10-28 diff --git a/Gemfile.lock b/Gemfile.lock index 1c1a127c..351c842a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -2,7 +2,7 @@ PATH remote: . specs: syntax_tree (4.3.0) - prettier_print (>= 1.0.2) + prettier_print (>= 1.1.0) GEM remote: https://rubygems.org/ @@ -14,7 +14,7 @@ GEM parallel (1.22.1) parser (3.1.2.1) ast (~> 2.4.1) - prettier_print (1.0.2) + prettier_print (1.1.0) rainbow (3.1.1) rake (13.0.6) regexp_parser (2.6.0) diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index aff4404c..c2cb3484 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -54,8 +54,12 @@ def self.parse(source) end # Parses the given source and returns the formatted source. - def self.format(source, maxwidth = DEFAULT_PRINT_WIDTH) - formatter = Formatter.new(source, [], maxwidth) + def self.format( + source, + maxwidth = DEFAULT_PRINT_WIDTH, + options: Formatter::Options.new + ) + formatter = Formatter.new(source, [], maxwidth, options: options) parse(source).format(formatter) formatter.flush diff --git a/lib/syntax_tree/cli.rb b/lib/syntax_tree/cli.rb index 11c93537..3975df18 100644 --- a/lib/syntax_tree/cli.rb +++ b/lib/syntax_tree/cli.rb @@ -131,9 +131,14 @@ class UnformattedError < StandardError def run(item) source = item.source - if source != item.handler.format(source, options.print_width) - raise UnformattedError - end + formatted = + item.handler.format( + source, + options.print_width, + options: options.formatter_options + ) + + raise UnformattedError if source != formatted rescue StandardError warn("[#{Color.yellow("warn")}] #{item.filepath}") raise @@ -156,13 +161,23 @@ class NonIdempotentFormatError < StandardError def run(item) handler = item.handler - warning = "[#{Color.yellow("warn")}] #{item.filepath}" - formatted = handler.format(item.source, options.print_width) - if formatted != handler.format(formatted, options.print_width) - raise NonIdempotentFormatError - end + formatted = + handler.format( + item.source, + options.print_width, + options: options.formatter_options + ) + + double_formatted = + handler.format( + formatted, + options.print_width, + options: options.formatter_options + ) + + raise NonIdempotentFormatError if formatted != double_formatted rescue StandardError warn(warning) raise @@ -182,7 +197,9 @@ class Doc < Action def run(item) source = item.source - formatter = Formatter.new(source, []) + formatter_options = options.formatter_options + formatter = Formatter.new(source, [], options: formatter_options) + item.handler.parse(source).format(formatter) pp formatter.groups.first end @@ -206,7 +223,14 @@ def run(item) # An action of the CLI that formats the input source and prints it out. class Format < Action def run(item) - puts item.handler.format(item.source, options.print_width) + formatted = + item.handler.format( + item.source, + options.print_width, + options: options.formatter_options + ) + + puts formatted end end @@ -273,7 +297,13 @@ def run(item) start = Time.now source = item.source - formatted = item.handler.format(source, options.print_width) + formatted = + item.handler.format( + source, + options.print_width, + options: options.formatter_options + ) + File.write(filepath, formatted) if item.writable? color = source == formatted ? Color.gray(filepath) : filepath @@ -347,20 +377,16 @@ class Options :plugins, :print_width, :scripts, - :target_ruby_version + :formatter_options - def initialize(print_width: DEFAULT_PRINT_WIDTH) + def initialize @ignore_files = [] @plugins = [] - @print_width = print_width + @print_width = DEFAULT_PRINT_WIDTH @scripts = [] - @target_ruby_version = nil + @formatter_options = Formatter::Options.new end - # TODO: This function causes a couple of side-effects that I really don't - # like to have here. It mutates the global state by requiring the plugins, - # and mutates the global options hash by adding the target ruby version. - # That should be done on a config-by-config basis, not here. def parse(arguments) parser.parse!(arguments) end @@ -404,8 +430,10 @@ def parser # If there is a target ruby version specified on the command line, # parse that out and use it when formatting. opts.on("--target-ruby-version=VERSION") do |version| - @target_ruby_version = Gem::Version.new(version) - Formatter::OPTIONS[:target_ruby_version] = @target_ruby_version + @formatter_options = + Formatter::Options.new( + target_ruby_version: Formatter::SemanticVersion.new(version) + ) end end end diff --git a/lib/syntax_tree/formatter.rb b/lib/syntax_tree/formatter.rb index f878490c..d5d251c6 100644 --- a/lib/syntax_tree/formatter.rb +++ b/lib/syntax_tree/formatter.rb @@ -4,21 +4,63 @@ module SyntaxTree # A slightly enhanced PP that knows how to format recursively including # comments. class Formatter < PrettierPrint + # Unfortunately, Gem::Version.new is not ractor-safe because it performs + # global caching using a class variable. This works around that by just + # setting the instance variables directly. + class SemanticVersion < ::Gem::Version + def initialize(version) + @version = version + @segments = nil + end + end + # We want to minimize as much as possible the number of options that are # available in syntax tree. For the most part, if users want non-default # formatting, they should override the format methods on the specific nodes # themselves. However, because of some history with prettier and the fact # that folks have become entrenched in their ways, we decided to provide a # small amount of configurability. - # - # Note that we're keeping this in a global-ish hash instead of just - # overriding methods on classes so that other plugins can reference this if - # necessary. For example, the RBS plugin references the quote style. - OPTIONS = { - quote: "\"", - trailing_comma: false, - target_ruby_version: Gem::Version.new(RUBY_VERSION) - } + class Options + attr_reader :quote, :trailing_comma, :target_ruby_version + + def initialize( + quote: :default, + trailing_comma: :default, + target_ruby_version: :default + ) + @quote = + if quote == :default + # We ship with a single quotes plugin that will define this + # constant. That constant is responsible for determining the default + # quote style. If it's defined, we default to single quotes, + # otherwise we default to double quotes. + defined?(SINGLE_QUOTES) ? "'" : "\"" + else + quote + end + + @trailing_comma = + if trailing_comma == :default + # We ship with a trailing comma plugin that will define this + # constant. That constant is responsible for determining the default + # trailing comma value. If it's defined, then we default to true. + # Otherwise we default to false. + defined?(TRAILING_COMMA) + else + trailing_comma + end + + @target_ruby_version = + if target_ruby_version == :default + # The default target Ruby version is the current version of Ruby. + # This is really only used for very niche cases, and it shouldn't be + # used by most users. + SemanticVersion.new(RUBY_VERSION) + else + target_ruby_version + end + end + end COMMENT_PRIORITY = 1 HEREDOC_PRIORITY = 2 @@ -30,22 +72,16 @@ class Formatter < PrettierPrint attr_reader :quote, :trailing_comma, :target_ruby_version alias trailing_comma? trailing_comma - def initialize( - source, - *args, - quote: OPTIONS[:quote], - trailing_comma: OPTIONS[:trailing_comma], - target_ruby_version: OPTIONS[:target_ruby_version] - ) + def initialize(source, *args, options: Options.new) super(*args) @source = source @stack = [] - # Memoizing these values per formatter to make access faster. - @quote = quote - @trailing_comma = trailing_comma - @target_ruby_version = target_ruby_version + # Memoizing these values to make access faster. + @quote = options.quote + @trailing_comma = options.trailing_comma + @target_ruby_version = options.target_ruby_version end def self.format(source, node) diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 639eb7be..1ed74e12 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -1026,7 +1026,7 @@ def call(q) end end - BREAKABLE_SPACE_SEPARATOR = BreakableSpaceSeparator.new + BREAKABLE_SPACE_SEPARATOR = BreakableSpaceSeparator.new.freeze # Formats an array of multiple simple string literals into the %w syntax. class QWordsFormatter @@ -1759,7 +1759,7 @@ def ===(other) module HashKeyFormatter # Formats the keys of a hash literal using labels. class Labels - LABEL = /\A[A-Za-z_](\w*[\w!?])?\z/ + LABEL = /\A[A-Za-z_](\w*[\w!?])?\z/.freeze def format_key(q, key) case key @@ -2176,7 +2176,7 @@ def call(q) # We'll keep a single instance of this separator around for all block vars # to cut down on allocations. - SEPARATOR = Separator.new + SEPARATOR = Separator.new.freeze def format(q) q.text("|") @@ -5723,7 +5723,8 @@ def deconstruct_keys(_keys) # This is a very specific behavior where you want to force a newline, but # don't want to force the break parent. - SEPARATOR = PrettierPrint::Breakable.new(" ", 1, indent: false, force: true) + SEPARATOR = + PrettierPrint::Breakable.new(" ", 1, indent: false, force: true).freeze def format(q) q.group do @@ -6025,7 +6026,7 @@ def format(q) format_contents(q, parts, nested) end - if q.target_ruby_version < Gem::Version.new("2.7.3") + if q.target_ruby_version < Formatter::SemanticVersion.new("2.7.3") q.text(" }") else q.breakable_space @@ -11703,7 +11704,7 @@ def call(q) # We're going to keep a single instance of this separator around so we don't # have to allocate a new one every time we format a when clause. - SEPARATOR = Separator.new + SEPARATOR = Separator.new.freeze def format(q) keyword = "when " diff --git a/lib/syntax_tree/plugin/single_quotes.rb b/lib/syntax_tree/plugin/single_quotes.rb index c6e829e0..c7405e2c 100644 --- a/lib/syntax_tree/plugin/single_quotes.rb +++ b/lib/syntax_tree/plugin/single_quotes.rb @@ -1,3 +1,7 @@ # frozen_string_literal: true -SyntaxTree::Formatter::OPTIONS[:quote] = "'" +module SyntaxTree + class Formatter + SINGLE_QUOTES = true + end +end diff --git a/lib/syntax_tree/plugin/trailing_comma.rb b/lib/syntax_tree/plugin/trailing_comma.rb index 878703c3..1ae2b96d 100644 --- a/lib/syntax_tree/plugin/trailing_comma.rb +++ b/lib/syntax_tree/plugin/trailing_comma.rb @@ -1,3 +1,7 @@ # frozen_string_literal: true -SyntaxTree::Formatter::OPTIONS[:trailing_comma] = true +module SyntaxTree + class Formatter + TRAILING_COMMA = true + end +end diff --git a/syntax_tree.gemspec b/syntax_tree.gemspec index c82a8e98..19f4ee97 100644 --- a/syntax_tree.gemspec +++ b/syntax_tree.gemspec @@ -25,7 +25,7 @@ Gem::Specification.new do |spec| spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } spec.require_paths = %w[lib] - spec.add_dependency "prettier_print", ">= 1.0.2" + spec.add_dependency "prettier_print", ">= 1.1.0" spec.add_development_dependency "bundler" spec.add_development_dependency "minitest" diff --git a/test/cli_test.rb b/test/cli_test.rb index 9740806d..7c9e2652 100644 --- a/test/cli_test.rb +++ b/test/cli_test.rb @@ -62,14 +62,8 @@ def test_check_print_width end def test_check_target_ruby_version - previous = Formatter::OPTIONS[:target_ruby_version] - - begin - result = run_cli("check", "--target-ruby-version=2.6.0") - assert_includes(result.stdio, "match") - ensure - Formatter::OPTIONS[:target_ruby_version] = previous - end + result = run_cli("check", "--target-ruby-version=2.6.0") + assert_includes(result.stdio, "match") end def test_debug diff --git a/test/plugin/single_quotes_test.rb b/test/plugin/single_quotes_test.rb index 719f33c1..6ce10448 100644 --- a/test/plugin/single_quotes_test.rb +++ b/test/plugin/single_quotes_test.rb @@ -4,8 +4,6 @@ module SyntaxTree class SingleQuotesTest < Minitest::Test - OPTIONS = Plugin.options("syntax_tree/plugin/single_quotes") - def test_empty_string_literal assert_format("''\n", "\"\"") end @@ -36,7 +34,8 @@ def test_label private def assert_format(expected, source = expected) - formatter = Formatter.new(source, [], **OPTIONS) + options = Formatter::Options.new(quote: "'") + formatter = Formatter.new(source, [], options: options) SyntaxTree.parse(source).format(formatter) formatter.flush diff --git a/test/plugin/trailing_comma_test.rb b/test/plugin/trailing_comma_test.rb index ba9ad846..7f6e49a8 100644 --- a/test/plugin/trailing_comma_test.rb +++ b/test/plugin/trailing_comma_test.rb @@ -4,8 +4,6 @@ module SyntaxTree class TrailingCommaTest < Minitest::Test - OPTIONS = Plugin.options("syntax_tree/plugin/trailing_comma") - def test_arg_paren_flat assert_format("foo(a)\n") end @@ -82,7 +80,8 @@ def test_hash_literal_break private def assert_format(expected, source = expected) - formatter = Formatter.new(source, [], **OPTIONS) + options = Formatter::Options.new(trailing_comma: true) + formatter = Formatter.new(source, [], options: options) SyntaxTree.parse(source).format(formatter) formatter.flush diff --git a/test/ractor_test.rb b/test/ractor_test.rb new file mode 100644 index 00000000..e697b48e --- /dev/null +++ b/test/ractor_test.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +# Don't run this test if we're in a version of Ruby that doesn't have Ractors. +return unless defined?(Ractor) + +# Don't run this version on Ruby 3.0.0. For some reason it just hangs within the +# main Ractor waiting for this children. Not going to investigate it since it's +# already been fixed in 3.1.0. +return if Gem::Version.new(RUBY_VERSION) < Gem::Version.new("3.1.0") + +require_relative "test_helper" + +module SyntaxTree + class RactorTest < Minitest::Test + def test_formatting + ractors = + filepaths.map do |filepath| + # At the moment we have to parse in the main Ractor because Ripper is + # not marked as a Ractor-safe extension. + source = SyntaxTree.read(filepath) + program = SyntaxTree.parse(source) + + Ractor.new(source, program, name: filepath) do |source, program| + SyntaxTree::Formatter.format(source, program) + end + end + + ractors.each(&:take) + end + + private + + def filepaths + Dir.glob(File.expand_path("../lib/syntax_tree/{node,parser}.rb", __dir__)) + end + end +end diff --git a/test/test_helper.rb b/test/test_helper.rb index b2cd6787..77627e26 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -86,26 +86,6 @@ def assert_syntax_tree(node) end end -module SyntaxTree - module Plugin - # A couple of plugins modify the options hash on the formatter. They're - # modeled as files that should be required so that it's simple for the CLI - # and the library to use the same code path. In this case we're going to - # require the file for the plugin but ensure it doesn't make any lasting - # changes. - def self.options(path) - previous_options = SyntaxTree::Formatter::OPTIONS.dup - - begin - require path - SyntaxTree::Formatter::OPTIONS.dup - ensure - SyntaxTree::Formatter::OPTIONS.merge!(previous_options) - end - end - end -end - # There are a bunch of fixtures defined in test/fixtures. They exercise every # possible combination of syntax that leads to variations in the types of nodes. # They are used for testing various parts of Syntax Tree, including formatting, @@ -153,9 +133,8 @@ def self.each_fixture # If there's a comment starting with >= that starts after the % that # delineates the test, then we're going to check if the version # satisfies that constraint. - if comment&.start_with?(">=") && - (ruby_version < Gem::Version.new(comment.split[1])) - next + if comment&.start_with?(">=") + next if ruby_version < Gem::Version.new(comment.split[1]) end name = :"#{fixture}_#{index}" From 8e31ca9e323a05048f4c577335dfdd860c43c576 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 9 Nov 2022 11:22:31 -0500 Subject: [PATCH 201/536] More naming changes --- .rubocop.yml | 2 +- CHANGELOG.md | 16 +-- README.md | 2 +- lib/syntax_tree/node.rb | 136 ++++++++++---------- lib/syntax_tree/parser.rb | 124 ++++++++---------- lib/syntax_tree/visitor.rb | 16 +-- lib/syntax_tree/visitor/field_visitor.rb | 4 +- lib/syntax_tree/visitor/mutation_visitor.rb | 16 +-- test/mutation_test.rb | 2 +- test/node_test.rb | 54 ++++---- 10 files changed, 180 insertions(+), 192 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index 27efc39a..6c9be677 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -7,7 +7,7 @@ AllCops: SuggestExtensions: false TargetRubyVersion: 2.7 Exclude: - - '{bin,coverage,pkg,test/fixtures,vendor,tmp}/**/*' + - '{.git,.github,bin,coverage,pkg,test/fixtures,vendor,tmp}/**/*' - test.rb Layout/LineLength: diff --git a/CHANGELOG.md b/CHANGELOG.md index dc5637ce..2e4783e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,14 +16,14 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - Nodes no longer have a `comments:` keyword on their initializers. By default, they initialize to an empty array. If you were previously passing comments into the initializer, you should now create the node first, then call `node.comments.concat` to add your comments. - A lot of nodes have been folded into other nodes to make it easier to interact with the AST. This means that a lot of visit methods have been removed from the visitor and a lot of class definitions are no longer present. This also means that the nodes that received more function now have additional methods or fields to be able to differentiate them. Note that none of these changes have resulted in different formatting. The changes are listed below: - - `IfMod`, `UnlessMod`, `WhileMod`, `UntilMod` have been folded into `If`, `Unless`, `While`, and `Until`. Each of the nodes now have a `modifier?` method to tell if it was originally in the modifier form. Consequently, the `visit_if_mod`, `visit_unless_mod`, `visit_while_mod`, and `visit_until_mod` methods have been removed from the visitor. - - `VarAlias` is no longer a node. Instead it has been folded into the `Alias` node. The `Alias` node now has a `var_alias?` method to tell you if it is aliasing a global variable. Consequently, the `visit_var_alias` method has been removed from the visitor interface. If you were previously using this method, you should now use `visit_alias` instead. - - `Yield0` is no longer a node. Instead if has been folded into the `Yield` node. The `Yield` node can now have its `arguments` field be `nil`. Consequently, the `visit_yield0` method has been removed from the visitor interface. If you were previously using this method, you should now use `visit_yield` instead. - - `FCall` is no longer a node. Instead it has been folded into the `Call` node. The `Call` node can now have its `receiver` and `operator` fields be `nil`. Consequently, the `visit_fcall` method has been removed from the visitor interface. If you were previously using this method, you should now use `visit_call` instead. - - `Dot2` and `Dot3` are no longer nodes. Instead they have become a single new `RangeLiteral` node. This node looks the same as `Dot2` and `Dot3`, except that it additionally has an `operator` field that contains the operator that created the node. Consequently, the `visit_dot2` and `visit_dot3` methods have been removed from the visitor interface. If you were previously using these methods, you should now use `visit_range_literal` instead. - - `DefEndless` and `Defs` have both been folded into the `Def` node. The `Def` node now has the `target` and `operator` fields which originally came from `Defs` which can both be `nil`. It also now has an `endless?` method on it to tell if the original node was found in the endless form. Finally the `bodystmt` field can now either be a `BodyStmt` as it was or any other kind of node since that was the body of the `DefEndless` node. The `visit_defs` and `visit_def_endless` methods on the visitor have therefore been removed. - - `DoBlock` and `BraceBlock` have now been folded into a `Block` node. The `Block` node now has a `keywords?` method on it that returns true if the block was constructed with the `do`..`end` keywords. The `visit_do_block` and `visit_brace_block` methods on the visitor have therefore been removed and replaced with the `visit_block` method. - - `Return0` is no longer a node. Instead if has been folded into the `Return` node. The `Return` node can now have its `arguments` field be `nil`. Consequently, the `visit_return0` method has been removed from the visitor interface. If you were previously using this method, you should now use `visit_return` instead. + - `IfMod`, `UnlessMod`, `WhileMod`, `UntilMod` have been folded into `IfNode`, `UnlessNode`, `WhileNode`, and `UntilNode`. Each of the nodes now have a `modifier?` method to tell if it was originally in the modifier form. Consequently, the `visit_if_mod`, `visit_unless_mod`, `visit_while_mod`, and `visit_until_mod` methods have been removed from the visitor. + - `VarAlias` is no longer a node, and the `Alias` node has been renamed. They have been folded into the `AliasNode` node. The `AliasNode` node now has a `var_alias?` method to tell you if it is aliasing a global variable. Consequently, the `visit_var_alias` method has been removed from the visitor interface. If you were previously using this method, you should now use `visit_alias` instead. + - `Yield0` is no longer a node, and the `Yield` node has been renamed. They has been folded into the `YieldNode` node. The `YieldNode` node can now have its `arguments` field be `nil`. Consequently, the `visit_yield0` method has been removed from the visitor interface. If you were previously using this method, you should now use `visit_yield` instead. + - `FCall` is no longer a node, and the `Call` node has been renamed. They have been folded into the `CallNode` node. The `CallNode` node can now have its `receiver` and `operator` fields be `nil`. Consequently, the `visit_fcall` method has been removed from the visitor interface. If you were previously using this method, you should now use `visit_call` instead. + - `Dot2` and `Dot3` are no longer nodes. Instead they have become a single new `RangeNode` node. This node looks the same as `Dot2` and `Dot3`, except that it additionally has an `operator` field that contains the operator that created the node. Consequently, the `visit_dot2` and `visit_dot3` methods have been removed from the visitor interface. If you were previously using these methods, you should now use `visit_range` instead. + - `Def`, `DefEndless`, and `Defs` have been folded into the `DefNode` node. The `DefNode` node now has the `target` and `operator` fields which originally came from `Defs` which can both be `nil`. It also now has an `endless?` method on it to tell if the original node was found in the endless form. Finally the `bodystmt` field can now either be a `BodyStmt` as it was or any other kind of node since that was the body of the `DefEndless` node. The `visit_defs` and `visit_def_endless` methods on the visitor have therefore been removed. + - `DoBlock` and `BraceBlock` have now been folded into a `BlockNode` node. The `BlockNode` node now has a `keywords?` method on it that returns true if the block was constructed with the `do`..`end` keywords. The `visit_do_block` and `visit_brace_block` methods on the visitor have therefore been removed and replaced with the `visit_block` method. + - `Return0` is no longer a node, and the `Return` node has been renamed. They have been folded into the `ReturnNode` node. The `ReturnNode` node can now have its `arguments` field be `nil`. Consequently, the `visit_return0` method has been removed from the visitor interface. If you were previously using this method, you should now use `visit_return` instead. - The `ArgsForward`, `Redo`, `Retry`, and `ZSuper` nodes no longer have `value` fields associated with them (which were always string literals corresponding to the keyword being used). - The `Command` and `CommandCall` nodes now has `block` attributes on them. These attributes are used in the place where you would previously have had a `MethodAddBlock` structure. Where before the `MethodAddBlock` would have the command and block as its two children, you now just have one command node with the `block` attribute set to the `Block` node. - Previously the formatting options were defined on an unfrozen hash called `SyntaxTree::Formatter::OPTIONS`. It was globally mutable, which made it impossible to reference from within a Ractor. As such, it has now been replaced with `SyntaxTree::Formatter::Options.new` which creates a new options object instance that can be modified without impacting global state. As a part of this change, formatting can now be performed from within a non-main Ractor. In order to check if the `plugin/single_quotes` plugin has been loaded, check if `SyntaxTree::Formatter::SINGLE_QUOTES` is defined. In order to check if the `plugin/trailing_comma` plugin has been loaded, check if `SyntaxTree::Formatter::TRAILING_COMMA` is defined. diff --git a/README.md b/README.md index e87ec765..050877ee 100644 --- a/README.md +++ b/README.md @@ -538,7 +538,7 @@ The `MutationVisitor` is a visitor that can be used to mutate the tree. It works visitor = SyntaxTree::Visitor::MutationVisitor.new # Specify that it should mutate If nodes with assignments in their predicates -visitor.mutate("If[predicate: Assign | OpAssign]") do |node| +visitor.mutate("IfNode[predicate: Assign | OpAssign]") do |node| # Get the existing If's predicate node predicate = node.predicate diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 1ed74e12..f32789a3 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -445,7 +445,7 @@ def ===(other) # can either provide bare words (like the example above) or you can provide # symbols (note that this includes dynamic symbols like # :"left-#{middle}-right"). - class Alias < Node + class AliasNode < Node # Formats an argument to the alias keyword. For symbol literals it uses the # value of the symbol directly to look like bare words. class AliasArgumentFormatter @@ -500,7 +500,7 @@ def child_nodes def copy(left: nil, right: nil, location: nil) node = - Alias.new( + AliasNode.new( left: left || self.left, right: right || self.right, location: location || self.location @@ -533,7 +533,7 @@ def format(q) end def ===(other) - other.is_a?(Alias) && left === other.left && right === other.right + other.is_a?(AliasNode) && left === other.left && right === other.right end def var_alias? @@ -1430,7 +1430,7 @@ def self.skip_indent?(value) when ArrayLiteral, HashLiteral, Heredoc, Lambda, QSymbols, QWords, Symbols, Words true - when Call + when CallNode skip_indent?(value.receiver) when DynaSymbol value.quote.start_with?("%s") @@ -2707,16 +2707,16 @@ def format(q) # longer at a chainable node. loop do case (child = children.last) - when Call + when CallNode case (receiver = child.receiver) - when Call + when CallNode if receiver.receiver.nil? break else children << receiver end when MethodAddBlock - if receiver.call.is_a?(Call) && !receiver.call.receiver.nil? + if receiver.call.is_a?(CallNode) && !receiver.call.receiver.nil? children << receiver else break @@ -2725,7 +2725,7 @@ def format(q) break end when MethodAddBlock - if child.call.is_a?(Call) && !child.call.receiver.nil? + if child.call.is_a?(CallNode) && !child.call.receiver.nil? children << child.call else break @@ -2746,9 +2746,9 @@ def format(q) # If we're at a block with the `do` keywords, then we want to go one # more level up. This is because do blocks have BodyStmt nodes instead # of just Statements nodes. - parent = parents[3] if parent.is_a?(Block) && parent.keywords? + parent = parents[3] if parent.is_a?(BlockNode) && parent.keywords? - if parent.is_a?(MethodAddBlock) && parent.call.is_a?(Call) && + if parent.is_a?(MethodAddBlock) && parent.call.is_a?(CallNode) && parent.call.message.value == "sig" threshold = 2 end @@ -2772,7 +2772,7 @@ def format_chain(q, children) empty_except_last = children .drop(1) - .all? { |child| child.is_a?(Call) && child.arguments.nil? } + .all? { |child| child.is_a?(CallNode) && child.arguments.nil? } # Here, we're going to add all of the children onto the stack of the # formatter so it's as if we had descending normally into them. This is @@ -2793,8 +2793,8 @@ def format_chain(q, children) skip_operator = false while (child = children.pop) - if child.is_a?(Call) - if child.receiver.is_a?(Call) && + if child.is_a?(CallNode) + if child.receiver.is_a?(CallNode) && (child.receiver.message != :call) && (child.receiver.message.value == "where") && (child.message.value == "not") @@ -2821,7 +2821,7 @@ def format_chain(q, children) # If the parent call node has a comment on the message then we need # to print the operator trailing in order to keep it working. last_child = children.last - if last_child.is_a?(Call) && last_child.message.comments.any? && + if last_child.is_a?(CallNode) && last_child.message.comments.any? && last_child.operator q.format(CallOperatorFormatter.new(last_child.operator)) skip_operator = true @@ -2838,7 +2838,7 @@ def format_chain(q, children) if empty_except_last case node - when Call + when CallNode node.format_arguments(q) when MethodAddBlock q.format(node.block) @@ -2850,10 +2850,10 @@ def self.chained?(node) return false if ENV["STREE_FAST_FORMAT"] case node - when Call + when CallNode !node.receiver.nil? when MethodAddBlock - node.call.is_a?(Call) && !node.call.receiver.nil? + node.call.is_a?(CallNode) && !node.call.receiver.nil? else false end @@ -2866,7 +2866,8 @@ def self.chained?(node) # format it separately here. def attach_directly?(node) case node.receiver - when ArrayLiteral, HashLiteral, Heredoc, If, Unless, XStringLiteral + when ArrayLiteral, HashLiteral, Heredoc, IfNode, UnlessNode, + XStringLiteral true else false @@ -2882,7 +2883,7 @@ def format_child( ) # First, format the actual contents of the child. case child - when Call + when CallNode q.group do if !skip_operator && child.operator q.format(CallOperatorFormatter.new(child.operator)) @@ -2907,11 +2908,11 @@ def format_child( end end - # Call represents a method call. + # CallNode represents a method call. # # receiver.message # - class Call < Node + class CallNode < Node # [nil | untyped] the receiver of the method call attr_reader :receiver @@ -2957,7 +2958,7 @@ def copy( location: nil ) node = - Call.new( + CallNode.new( receiver: receiver || self.receiver, operator: operator || self.operator, message: message || self.message, @@ -3014,7 +3015,7 @@ def format(q) end def ===(other) - other.is_a?(Call) && receiver === other.receiver && + other.is_a?(CallNode) && receiver === other.receiver && operator === other.operator && message === other.message && arguments === other.arguments end @@ -3483,7 +3484,7 @@ def align(q, node, &block) part = parts.first case part - when Def + when DefNode q.text(" ") yield when IfOp @@ -4038,7 +4039,7 @@ def ===(other) # def method(param) result end # def object.method(param) result end # - class Def < Node + class DefNode < Node # [nil | untyped] the target where the method is being defined attr_reader :target @@ -4084,7 +4085,7 @@ def copy( location: nil ) node = - Def.new( + DefNode.new( target: target || self.target, operator: operator || self.operator, name: name || self.name, @@ -4154,7 +4155,7 @@ def format(q) end def ===(other) - other.is_a?(Def) && target === other.target && + other.is_a?(DefNode) && target === other.target && operator === other.operator && name === other.name && params === other.params && bodystmt === other.bodystmt end @@ -4235,7 +4236,7 @@ def ===(other) # # method { |value| } # - class Block < Node + class BlockNode < Node # Formats the opening brace or keyword of a block. class BlockOpenFormatter # [String] the actual output that should be printed @@ -4288,7 +4289,7 @@ def child_nodes def copy(opening: nil, block_var: nil, bodystmt: nil, location: nil) node = - Block.new( + BlockNode.new( opening: opening || self.opening, block_var: block_var || self.block_var, bodystmt: bodystmt || self.bodystmt, @@ -4344,7 +4345,7 @@ def format(q) end def ===(other) - other.is_a?(Block) && opening === other.opening && + other.is_a?(BlockNode) && opening === other.opening && block_var === other.block_var && bodystmt === other.bodystmt end @@ -4376,7 +4377,7 @@ def unchangeable_bounds?(q) # use the do..end bounds. def forced_do_end_bounds?(q) case q.parent.call - when Break, Next, Return, Super + when Break, Next, ReturnNode, Super true else false @@ -4392,7 +4393,7 @@ def forced_brace_bounds?(q) when Paren, Statements # If we hit certain breakpoints then we know we're safe. return false - when If, IfOp, Unless, While, Until + when IfNode, IfOp, UnlessNode, WhileNode, UntilNode return true if parent.predicate == previous end @@ -4443,7 +4444,7 @@ def format_flat(q, flat_opening, flat_closing) end end - # RangeLiteral represents using the .. or the ... operator between two + # RangeNode represents using the .. or the ... operator between two # expressions. Usually this is to create a range object. # # 1..2 @@ -4454,7 +4455,7 @@ def format_flat(q, flat_opening, flat_closing) # end # # One of the sides of the expression may be nil, but not both. - class RangeLiteral < Node + class RangeNode < Node # [nil | untyped] the left side of the expression attr_reader :left @@ -4476,7 +4477,7 @@ def initialize(left:, operator:, right:, location:) end def accept(visitor) - visitor.visit_range_literal(self) + visitor.visit_range(self) end def child_nodes @@ -4485,7 +4486,7 @@ def child_nodes def copy(left: nil, operator: nil, right: nil, location: nil) node = - RangeLiteral.new( + RangeNode.new( left: left || self.left, operator: operator || self.operator, right: right || self.right, @@ -4512,7 +4513,7 @@ def format(q) q.format(left) if left case q.parent - when If, Unless + when IfNode, UnlessNode q.text(" #{operator.value} ") else q.text(operator.value) @@ -4522,7 +4523,7 @@ def format(q) end def ===(other) - other.is_a?(RangeLiteral) && left === other.left && + other.is_a?(RangeNode) && left === other.left && operator === other.operator && right === other.right end end @@ -6182,9 +6183,10 @@ def call(q, node) # and default instead to breaking them into multiple lines. def ternaryable?(statement) case statement - when Alias, Assign, Break, Command, CommandCall, Heredoc, If, IfOp, - Lambda, MAssign, Next, OpAssign, RescueMod, Return, Super, Undef, - Unless, Until, VoidStmt, While, Yield, ZSuper + when AliasNode, Assign, Break, Command, CommandCall, Heredoc, IfNode, + IfOp, Lambda, MAssign, Next, OpAssign, RescueMod, ReturnNode, + Super, Undef, UnlessNode, UntilNode, VoidStmt, WhileNode, + YieldNode, ZSuper # This is a list of nodes that should not be allowed to be a part of a # ternary clause. false @@ -6343,7 +6345,7 @@ def contains_conditional? return false if statements.length != 1 case statements.first - when If, IfOp, Unless + when IfNode, IfOp, UnlessNode true else false @@ -6356,7 +6358,7 @@ def contains_conditional? # if predicate # end # - class If < Node + class IfNode < Node # [untyped] the expression to be checked attr_reader :predicate @@ -6387,7 +6389,7 @@ def child_nodes def copy(predicate: nil, statements: nil, consequent: nil, location: nil) node = - If.new( + IfNode.new( predicate: predicate || self.predicate, statements: statements || self.statements, consequent: consequent || self.consequent, @@ -6415,7 +6417,7 @@ def format(q) end def ===(other) - other.is_a?(If) && predicate === other.predicate && + other.is_a?(IfNode) && predicate === other.predicate && statements === other.statements && consequent === other.consequent end @@ -6485,9 +6487,9 @@ def deconstruct_keys(_keys) def format(q) force_flat = [ - Alias, Assign, Break, Command, CommandCall, Heredoc, If, IfOp, Lambda, - MAssign, Next, OpAssign, RescueMod, Return, Super, Undef, Unless, - VoidStmt, Yield, ZSuper + AliasNode, Assign, Break, Command, CommandCall, Heredoc, IfNode, IfOp, + Lambda, MAssign, Next, OpAssign, RescueMod, ReturnNode, Super, Undef, + UnlessNode, VoidStmt, YieldNode, ZSuper ] if q.parent.is_a?(Paren) || force_flat.include?(truthy.class) || @@ -8039,7 +8041,7 @@ module Parentheses Assign, Assoc, Binary, - Call, + CallNode, Defined, MAssign, OpAssign @@ -8283,7 +8285,7 @@ def format(q) return end - if q.parent.is_a?(Def) + if q.parent.is_a?(DefNode) q.nest(0) do q.text("(") q.group do @@ -9550,7 +9552,7 @@ def ===(other) # # return value # - class Return < Node + class ReturnNode < Node # [nil | Args] the arguments being passed to the keyword attr_reader :arguments @@ -9573,7 +9575,7 @@ def child_nodes def copy(arguments: nil, location: nil) node = - Return.new( + ReturnNode.new( arguments: arguments || self.arguments, location: location || self.location ) @@ -9593,7 +9595,7 @@ def format(q) end def ===(other) - other.is_a?(Return) && arguments === other.arguments + other.is_a?(ReturnNode) && arguments === other.arguments end end @@ -10955,7 +10957,7 @@ def format(q) else grandparent = q.grandparent ternary = - (grandparent.is_a?(If) || grandparent.is_a?(Unless)) && + (grandparent.is_a?(IfNode) || grandparent.is_a?(UnlessNode)) && Ternaryable.call(q, grandparent) if ternary @@ -11127,7 +11129,7 @@ def ===(other) # unless predicate # end # - class Unless < Node + class UnlessNode < Node # [untyped] the expression to be checked attr_reader :predicate @@ -11158,7 +11160,7 @@ def child_nodes def copy(predicate: nil, statements: nil, consequent: nil, location: nil) node = - Unless.new( + UnlessNode.new( predicate: predicate || self.predicate, statements: statements || self.statements, consequent: consequent || self.consequent, @@ -11186,7 +11188,7 @@ def format(q) end def ===(other) - other.is_a?(Unless) && predicate === other.predicate && + other.is_a?(UnlessNode) && predicate === other.predicate && statements === other.statements && consequent === other.consequent end @@ -11273,7 +11275,7 @@ def format_break(q) # until predicate # end # - class Until < Node + class UntilNode < Node # [untyped] the expression to be checked attr_reader :predicate @@ -11300,7 +11302,7 @@ def child_nodes def copy(predicate: nil, statements: nil, location: nil) node = - Until.new( + UntilNode.new( predicate: predicate || self.predicate, statements: statements || self.statements, location: location || self.location @@ -11326,7 +11328,7 @@ def format(q) end def ===(other) - other.is_a?(Until) && predicate === other.predicate && + other.is_a?(UntilNode) && predicate === other.predicate && statements === other.statements end @@ -11723,7 +11725,7 @@ def format(q) # last argument to the predicate is and endless range, then you are # forced to use the "then" keyword to make it parse properly. last = arguments.parts.last - q.text(" then") if last.is_a?(RangeLiteral) && !last.right + q.text(" then") if last.is_a?(RangeNode) && !last.right end end @@ -11752,7 +11754,7 @@ def ===(other) # while predicate # end # - class While < Node + class WhileNode < Node # [untyped] the expression to be checked attr_reader :predicate @@ -11779,7 +11781,7 @@ def child_nodes def copy(predicate: nil, statements: nil, location: nil) node = - While.new( + WhileNode.new( predicate: predicate || self.predicate, statements: statements || self.statements, location: location || self.location @@ -11805,7 +11807,7 @@ def format(q) end def ===(other) - other.is_a?(While) && predicate === other.predicate && + other.is_a?(WhileNode) && predicate === other.predicate && statements === other.statements end @@ -12090,7 +12092,7 @@ def ===(other) # # yield value # - class Yield < Node + class YieldNode < Node # [nil | Args | Paren] the arguments passed to the yield attr_reader :arguments @@ -12113,7 +12115,7 @@ def child_nodes def copy(arguments: nil, location: nil) node = - Yield.new( + YieldNode.new( arguments: arguments || self.arguments, location: location || self.location ) @@ -12152,7 +12154,7 @@ def format(q) end def ===(other) - other.is_a?(Yield) && arguments === other.arguments + other.is_a?(YieldNode) && arguments === other.arguments end end diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index cd14672e..23a3196c 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -421,11 +421,11 @@ def on___end__(value) # on_alias: ( # (DynaSymbol | SymbolLiteral) left, # (DynaSymbol | SymbolLiteral) right - # ) -> Alias + # ) -> AliasNode def on_alias(left, right) keyword = consume_keyword(:alias) - Alias.new( + AliasNode.new( left: left, right: right, location: keyword.location.to(right.location) @@ -920,7 +920,7 @@ def on_bodystmt(statements, rescue_clause, else_clause, ensure_clause) # on_brace_block: ( # (nil | BlockVar) block_var, # Statements statements - # ) -> Block + # ) -> BlockNode def on_brace_block(block_var, statements) lbrace = consume_token(LBrace) rbrace = consume_token(RBrace) @@ -947,7 +947,7 @@ def on_brace_block(block_var, statements) end_column: rbrace.location.end_column ) - Block.new( + BlockNode.new( opening: lbrace, block_var: block_var, bodystmt: statements, @@ -971,7 +971,7 @@ def on_break(arguments) # untyped receiver, # (:"::" | Op | Period) operator, # (:call | Backtick | Const | Ident | Op) message - # ) -> Call + # ) -> CallNode def on_call(receiver, operator, message) ending = if message != :call @@ -982,7 +982,7 @@ def on_call(receiver, operator, message) receiver end - Call.new( + CallNode.new( receiver: receiver, operator: operator, message: message, @@ -1183,7 +1183,7 @@ def on_cvar(value) # (Backtick | Const | Ident | Kw | Op) name, # (nil | Params | Paren) params, # untyped bodystmt - # ) -> Def + # ) -> DefNode def on_def(name, params, bodystmt) # Make sure to delete this token in case you're defining something like # def class which would lead to this being a kw and causing all kinds of @@ -1225,7 +1225,7 @@ def on_def(name, params, bodystmt) ending.location.start_column ) - Def.new( + DefNode.new( target: nil, operator: nil, name: name, @@ -1238,7 +1238,7 @@ def on_def(name, params, bodystmt) # the statements list. Before, it was just the individual statement. statement = bodystmt.is_a?(BodyStmt) ? bodystmt.statements : bodystmt - Def.new( + DefNode.new( target: nil, operator: nil, name: name, @@ -1274,7 +1274,7 @@ def on_defined(value) # (Backtick | Const | Ident | Kw | Op) name, # (Params | Paren) params, # BodyStmt bodystmt - # ) -> Def + # ) -> DefNode def on_defs(target, operator, name, params, bodystmt) # Make sure to delete this token in case you're defining something # like def class which would lead to this being a kw and causing all kinds @@ -1313,7 +1313,7 @@ def on_defs(target, operator, name, params, bodystmt) ending.location.start_column ) - Def.new( + DefNode.new( target: target, operator: operator, name: name, @@ -1326,7 +1326,7 @@ def on_defs(target, operator, name, params, bodystmt) # the statements list. Before, it was just the individual statement. statement = bodystmt.is_a?(BodyStmt) ? bodystmt.statements : bodystmt - Def.new( + DefNode.new( target: target, operator: operator, name: name, @@ -1338,7 +1338,7 @@ def on_defs(target, operator, name, params, bodystmt) end # :call-seq: - # on_do_block: (BlockVar block_var, BodyStmt bodystmt) -> Block + # on_do_block: (BlockVar block_var, BodyStmt bodystmt) -> BlockNode def on_do_block(block_var, bodystmt) beginning = consume_keyword(:do) ending = consume_keyword(:end) @@ -1352,7 +1352,7 @@ def on_do_block(block_var, bodystmt) ending.location.start_column ) - Block.new( + BlockNode.new( opening: beginning, block_var: block_var, bodystmt: bodystmt, @@ -1361,14 +1361,14 @@ def on_do_block(block_var, bodystmt) end # :call-seq: - # on_dot2: ((nil | untyped) left, (nil | untyped) right) -> RangeLiteral + # on_dot2: ((nil | untyped) left, (nil | untyped) right) -> RangeNode def on_dot2(left, right) operator = consume_operator(:"..") beginning = left || operator ending = right || operator - RangeLiteral.new( + RangeNode.new( left: left, operator: operator, right: right, @@ -1377,14 +1377,14 @@ def on_dot2(left, right) end # :call-seq: - # on_dot3: ((nil | untyped) left, (nil | untyped) right) -> RangeLiteral + # on_dot3: ((nil | untyped) left, (nil | untyped) right) -> RangeNode def on_dot3(left, right) operator = consume_operator(:"...") beginning = left || operator ending = right || operator - RangeLiteral.new( + RangeNode.new( left: left, operator: operator, right: right, @@ -1614,9 +1614,9 @@ def on_excessed_comma(*) end # :call-seq: - # on_fcall: ((Const | Ident) value) -> Call + # on_fcall: ((Const | Ident) value) -> CallNode def on_fcall(value) - Call.new( + CallNode.new( receiver: nil, operator: nil, message: value, @@ -1890,7 +1890,7 @@ def on_ident(value) # untyped predicate, # Statements statements, # (nil | Elsif | Else) consequent - # ) -> If + # ) -> IfNode def on_if(predicate, statements, consequent) beginning = consume_keyword(:if) ending = consequent || consume_keyword(:end) @@ -1903,7 +1903,7 @@ def on_if(predicate, statements, consequent) ending.location.start_column ) - If.new( + IfNode.new( predicate: predicate, statements: statements, consequent: consequent, @@ -1923,11 +1923,11 @@ def on_ifop(predicate, truthy, falsy) end # :call-seq: - # on_if_mod: (untyped predicate, untyped statement) -> If + # on_if_mod: (untyped predicate, untyped statement) -> IfNode def on_if_mod(predicate, statement) consume_keyword(:if) - If.new( + IfNode.new( predicate: predicate, statements: Statements.new(self, body: [statement], location: statement.location), @@ -2319,14 +2319,14 @@ def on_massign(target, value) # :call-seq: # on_method_add_arg: ( - # Call call, + # CallNode call, # (ArgParen | Args) arguments - # ) -> Call + # ) -> CallNode def on_method_add_arg(call, arguments) location = call.location location = location.to(arguments.location) if arguments.is_a?(ArgParen) - Call.new( + CallNode.new( receiver: call.receiver, operator: call.operator, message: call.message, @@ -2341,29 +2341,11 @@ def on_method_add_arg(call, arguments) # Block block # ) -> MethodAddBlock def on_method_add_block(call, block) - case call - when Command - node = - Command.new( - message: call.message, - arguments: call.arguments, - block: block, - location: call.location.to(block.location) - ) - - node.comments.concat(call.comments) - node - when CommandCall - node = - CommandCall.new( - receiver: call.receiver, - operator: call.operator, - message: call.message, - arguments: call.arguments, - block: block, - location: call.location.to(block.location) - ) + location = call.location.to(block.location) + case call + when Command, CommandCall + node = call.copy(block: block, location: location) node.comments.concat(call.comments) node else @@ -3110,22 +3092,22 @@ def on_retry end # :call-seq: - # on_return: (Args arguments) -> Return + # on_return: (Args arguments) -> ReturnNode def on_return(arguments) keyword = consume_keyword(:return) - Return.new( + ReturnNode.new( arguments: arguments, location: keyword.location.to(arguments.location) ) end # :call-seq: - # on_return0: () -> Return + # on_return0: () -> ReturnNode def on_return0 keyword = consume_keyword(:return) - Return.new(arguments: nil, location: keyword.location) + ReturnNode.new(arguments: nil, location: keyword.location) end # :call-seq: @@ -3602,7 +3584,7 @@ def on_undef(symbols) # untyped predicate, # Statements statements, # ((nil | Elsif | Else) consequent) - # ) -> Unless + # ) -> UnlessNode def on_unless(predicate, statements, consequent) beginning = consume_keyword(:unless) ending = consequent || consume_keyword(:end) @@ -3615,7 +3597,7 @@ def on_unless(predicate, statements, consequent) ending.location.start_column ) - Unless.new( + UnlessNode.new( predicate: predicate, statements: statements, consequent: consequent, @@ -3624,11 +3606,11 @@ def on_unless(predicate, statements, consequent) end # :call-seq: - # on_unless_mod: (untyped predicate, untyped statement) -> Unless + # on_unless_mod: (untyped predicate, untyped statement) -> UnlessNode def on_unless_mod(predicate, statement) consume_keyword(:unless) - Unless.new( + UnlessNode.new( predicate: predicate, statements: Statements.new(self, body: [statement], location: statement.location), @@ -3638,7 +3620,7 @@ def on_unless_mod(predicate, statement) end # :call-seq: - # on_until: (untyped predicate, Statements statements) -> Until + # on_until: (untyped predicate, Statements statements) -> UntilNode def on_until(predicate, statements) beginning = consume_keyword(:until) ending = consume_keyword(:end) @@ -3660,7 +3642,7 @@ def on_until(predicate, statements) ending.location.start_column ) - Until.new( + UntilNode.new( predicate: predicate, statements: statements, location: beginning.location.to(ending.location) @@ -3668,11 +3650,11 @@ def on_until(predicate, statements) end # :call-seq: - # on_until_mod: (untyped predicate, untyped statement) -> Until + # on_until_mod: (untyped predicate, untyped statement) -> UntilNode def on_until_mod(predicate, statement) consume_keyword(:until) - Until.new( + UntilNode.new( predicate: predicate, statements: Statements.new(self, body: [statement], location: statement.location), @@ -3681,11 +3663,11 @@ def on_until_mod(predicate, statement) end # :call-seq: - # on_var_alias: (GVar left, (Backref | GVar) right) -> Alias + # on_var_alias: (GVar left, (Backref | GVar) right) -> AliasNode def on_var_alias(left, right) keyword = consume_keyword(:alias) - Alias.new( + AliasNode.new( left: left, right: right, location: keyword.location.to(right.location) @@ -3765,7 +3747,7 @@ def on_when(arguments, statements, consequent) end # :call-seq: - # on_while: (untyped predicate, Statements statements) -> While + # on_while: (untyped predicate, Statements statements) -> WhileNode def on_while(predicate, statements) beginning = consume_keyword(:while) ending = consume_keyword(:end) @@ -3787,7 +3769,7 @@ def on_while(predicate, statements) ending.location.start_column ) - While.new( + WhileNode.new( predicate: predicate, statements: statements, location: beginning.location.to(ending.location) @@ -3795,11 +3777,11 @@ def on_while(predicate, statements) end # :call-seq: - # on_while_mod: (untyped predicate, untyped statement) -> While + # on_while_mod: (untyped predicate, untyped statement) -> WhileNode def on_while_mod(predicate, statement) consume_keyword(:while) - While.new( + WhileNode.new( predicate: predicate, statements: Statements.new(self, body: [statement], location: statement.location), @@ -3925,22 +3907,22 @@ def on_xstring_literal(xstring) end # :call-seq: - # on_yield: ((Args | Paren) arguments) -> Yield + # on_yield: ((Args | Paren) arguments) -> YieldNode def on_yield(arguments) keyword = consume_keyword(:yield) - Yield.new( + YieldNode.new( arguments: arguments, location: keyword.location.to(arguments.location) ) end # :call-seq: - # on_yield0: () -> Yield + # on_yield0: () -> YieldNode def on_yield0 keyword = consume_keyword(:yield) - Yield.new(arguments: nil, location: keyword.location) + YieldNode.new(arguments: nil, location: keyword.location) end # :call-seq: diff --git a/lib/syntax_tree/visitor.rb b/lib/syntax_tree/visitor.rb index 57aca619..eb57acd2 100644 --- a/lib/syntax_tree/visitor.rb +++ b/lib/syntax_tree/visitor.rb @@ -11,7 +11,7 @@ class Visitor < BasicVisitor # Visit an ARefField node. alias visit_aref_field visit_child_nodes - # Visit an Alias node. + # Visit an AliasNode node. alias visit_alias visit_child_nodes # Visit an ArgBlock node. @@ -185,7 +185,7 @@ class Visitor < BasicVisitor # Visit an Ident node. alias visit_ident visit_child_nodes - # Visit an If node. + # Visit an IfNode node. alias visit_if visit_child_nodes # Visit an IfOp node. @@ -290,8 +290,8 @@ class Visitor < BasicVisitor # Visit a QWordsBeg node. alias visit_qwords_beg visit_child_nodes - # Visit a RangeLiteral node - alias visit_range_literal visit_child_nodes + # Visit a RangeNode node + alias visit_range visit_child_nodes # Visit a RAssign node. alias visit_rassign visit_child_nodes @@ -407,10 +407,10 @@ class Visitor < BasicVisitor # Visit an Undef node. alias visit_undef visit_child_nodes - # Visit an Unless node. + # Visit an UnlessNode node. alias visit_unless visit_child_nodes - # Visit an Until node. + # Visit an UntilNode node. alias visit_until visit_child_nodes # Visit a VarField node. @@ -428,7 +428,7 @@ class Visitor < BasicVisitor # Visit a When node. alias visit_when visit_child_nodes - # Visit a While node. + # Visit a WhileNode node. alias visit_while visit_child_nodes # Visit a Word node. @@ -446,7 +446,7 @@ class Visitor < BasicVisitor # Visit a XStringLiteral node. alias visit_xstring_literal visit_child_nodes - # Visit a Yield node. + # Visit a YieldNode node. alias visit_yield visit_child_nodes # Visit a ZSuper node. diff --git a/lib/syntax_tree/visitor/field_visitor.rb b/lib/syntax_tree/visitor/field_visitor.rb index b56d771c..6e643e09 100644 --- a/lib/syntax_tree/visitor/field_visitor.rb +++ b/lib/syntax_tree/visitor/field_visitor.rb @@ -684,8 +684,8 @@ def visit_qwords_beg(node) node(node, "qwords_beg") { field("value", node.value) } end - def visit_range_literal(node) - node(node, "range_literal") do + def visit_range(node) + node(node, "range") do field("left", node.left) if node.left field("operator", node.operator) field("right", node.right) if node.right diff --git a/lib/syntax_tree/visitor/mutation_visitor.rb b/lib/syntax_tree/visitor/mutation_visitor.rb index 6e7d4ff2..65f8c5ba 100644 --- a/lib/syntax_tree/visitor/mutation_visitor.rb +++ b/lib/syntax_tree/visitor/mutation_visitor.rb @@ -60,7 +60,7 @@ def visit___end__(node) node.copy end - # Visit a Alias node. + # Visit a AliasNode node. def visit_alias(node) node.copy(left: visit(node.left), right: visit(node.right)) end @@ -300,8 +300,8 @@ def visit_block(node) ) end - # Visit a RangeLiteral node. - def visit_range_literal(node) + # Visit a RangeNode node. + def visit_range(node) node.copy( left: visit(node.left), operator: visit(node.operator), @@ -435,7 +435,7 @@ def visit_ident(node) node.copy end - # Visit a If node. + # Visit a IfNode node. def visit_if(node) node.copy( predicate: visit(node.predicate), @@ -823,7 +823,7 @@ def visit_undef(node) node.copy(symbols: visit_all(node.symbols)) end - # Visit a Unless node. + # Visit a UnlessNode node. def visit_unless(node) node.copy( predicate: visit(node.predicate), @@ -832,7 +832,7 @@ def visit_unless(node) ) end - # Visit a Until node. + # Visit a UntilNode node. def visit_until(node) node.copy( predicate: visit(node.predicate), @@ -874,7 +874,7 @@ def visit_when(node) ) end - # Visit a While node. + # Visit a WhileNode node. def visit_while(node) node.copy( predicate: visit(node.predicate), @@ -910,7 +910,7 @@ def visit_xstring_literal(node) node.copy(parts: visit_all(node.parts)) end - # Visit a Yield node. + # Visit a YieldNode node. def visit_yield(node) node.copy(arguments: visit(node.arguments)) end diff --git a/test/mutation_test.rb b/test/mutation_test.rb index ab607beb..ab9dd019 100644 --- a/test/mutation_test.rb +++ b/test/mutation_test.rb @@ -25,7 +25,7 @@ def test_mutates_based_on_patterns def build_mutation SyntaxTree.mutation do |mutation| - mutation.mutate("If[predicate: Assign | OpAssign]") do |node| + mutation.mutate("IfNode[predicate: Assign | OpAssign]") do |node| # Get the existing If's predicate node predicate = node.predicate diff --git a/test/node_test.rb b/test/node_test.rb index cbfc6173..15826be0 100644 --- a/test/node_test.rb +++ b/test/node_test.rb @@ -32,7 +32,7 @@ def test___end__ end def test_alias - assert_node(Alias, "alias left right") + assert_node(AliasNode, "alias left right") end def test_aref @@ -276,7 +276,7 @@ def test_brace_block source = "method { |variable| variable + 1 }" at = location(chars: 7..34) - assert_node(Block, source, at: at, &:block) + assert_node(BlockNode, source, at: at, &:block) end def test_break @@ -284,7 +284,7 @@ def test_break end def test_call - assert_node(Call, "receiver.message") + assert_node(CallNode, "receiver.message") end def test_case @@ -365,7 +365,7 @@ def test_cvar end def test_def - assert_node(Def, "def method(param) result end") + assert_node(DefNode, "def method(param) result end") end def test_def_paramless @@ -374,18 +374,18 @@ def method end SOURCE - assert_node(Def, source) + assert_node(DefNode, source) end guard_version("3.0.0") do def test_def_endless - assert_node(Def, "def method = result") + assert_node(DefNode, "def method = result") end end guard_version("3.1.0") do def test_def_endless_command - assert_node(Def, "def method = result argument") + assert_node(DefNode, "def method = result argument") end end @@ -394,7 +394,7 @@ def test_defined end def test_defs - assert_node(Def, "def object.method(param) result end") + assert_node(DefNode, "def object.method(param) result end") end def test_defs_paramless @@ -403,22 +403,22 @@ def object.method end SOURCE - assert_node(Def, source) + assert_node(DefNode, source) end def test_do_block source = "method do |variable| variable + 1 end" at = location(chars: 7..37) - assert_node(Block, source, at: at, &:block) + assert_node(BlockNode, source, at: at, &:block) end def test_dot2 - assert_node(RangeLiteral, "1..3") + assert_node(RangeNode, "1..3") end def test_dot3 - assert_node(RangeLiteral, "1...3") + assert_node(RangeNode, "1...3") end def test_dyna_symbol @@ -487,7 +487,7 @@ def test_excessed_comma end def test_fcall - assert_node(Call, "method(argument)") + assert_node(CallNode, "method(argument)") end def test_field @@ -575,7 +575,7 @@ def test_ident end def test_if - assert_node(If, "if value then else end") + assert_node(IfNode, "if value then else end") end def test_if_op @@ -583,7 +583,7 @@ def test_if_op end def test_if_mod - assert_node(If, "expression if predicate") + assert_node(IfNode, "expression if predicate") end def test_imaginary @@ -837,11 +837,11 @@ def test_retry end def test_return - assert_node(Return, "return value") + assert_node(ReturnNode, "return value") end def test_return0 - assert_node(Return, "return") + assert_node(ReturnNode, "return") end def test_sclass @@ -923,19 +923,23 @@ def test_undef end def test_unless - assert_node(Unless, "unless value then else end") + assert_node(UnlessNode, "unless value then else end") + end + + def test_unless_mod + assert_node(UnlessNode, "expression unless predicate") end def test_until - assert_node(Until, "until value do end") + assert_node(UntilNode, "until value do end") end def test_until_mod - assert_node(Until, "expression until predicate") + assert_node(UntilNode, "expression until predicate") end def test_var_alias - assert_node(Alias, "alias $new $old") + assert_node(AliasNode, "alias $new $old") end def test_var_field @@ -977,11 +981,11 @@ def test_when end def test_while - assert_node(While, "while value do end") + assert_node(WhileNode, "while value do end") end def test_while_mod - assert_node(While, "expression while predicate") + assert_node(WhileNode, "expression while predicate") end def test_word @@ -1009,11 +1013,11 @@ def test_xstring_heredoc end def test_yield - assert_node(Yield, "yield value") + assert_node(YieldNode, "yield value") end def test_yield0 - assert_node(Yield, "yield") + assert_node(YieldNode, "yield") end def test_zsuper From 8faf11edfd407237c1eb07420bef0bb9a395c121 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 9 Nov 2022 11:58:05 -0500 Subject: [PATCH 202/536] Make the test suite silent --- test/ractor_test.rb | 21 ++++++++++++++++++--- test/rake_test.rb | 7 +++---- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/test/ractor_test.rb b/test/ractor_test.rb index e697b48e..bcdb2a51 100644 --- a/test/ractor_test.rb +++ b/test/ractor_test.rb @@ -20,12 +20,14 @@ def test_formatting source = SyntaxTree.read(filepath) program = SyntaxTree.parse(source) - Ractor.new(source, program, name: filepath) do |source, program| - SyntaxTree::Formatter.format(source, program) + with_silenced_warnings do + Ractor.new(source, program, name: filepath) do |source, program| + SyntaxTree::Formatter.format(source, program) + end end end - ractors.each(&:take) + ractors.each { |ractor| assert_kind_of String, ractor.take } end private @@ -33,5 +35,18 @@ def test_formatting def filepaths Dir.glob(File.expand_path("../lib/syntax_tree/{node,parser}.rb", __dir__)) end + + # Ractors still warn about usage, so I'm disabling that warning here just to + # have clean test output. + def with_silenced_warnings + previous = $VERBOSE + + begin + $VERBOSE = nil + yield + ensure + $VERBOSE = previous + end + end end end diff --git a/test/rake_test.rb b/test/rake_test.rb index d07fc49c..90662519 100644 --- a/test/rake_test.rb +++ b/test/rake_test.rb @@ -46,12 +46,11 @@ def invoke(task_name) invocation = nil stub = ->(args) { invocation = Invocation.new(args) } - begin + assert_raises SystemExit do SyntaxTree::CLI.stub(:run, stub) { ::Rake::Task[task_name].invoke } - flunk - rescue SystemExit - invocation end + + invocation end end end From a1981d70011efa9e7c8a5b97f121635df5cbbeee Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 9 Nov 2022 12:07:43 -0500 Subject: [PATCH 203/536] Document ignoring code --- README.md | 45 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 050877ee..a6b01362 100644 --- a/README.md +++ b/README.md @@ -46,8 +46,8 @@ It is built with only standard library dependencies. It additionally ships with - [textDocument/formatting](#textdocumentformatting) - [textDocument/inlayHint](#textdocumentinlayhint) - [syntaxTree/visualizing](#syntaxtreevisualizing) -- [Plugins](#plugins) - - [Customization](#customization) +- [Customization](#customization) + - [Plugins](#plugins) - [Languages](#languages) - [Integration](#integration) - [Rake](#rake) @@ -320,6 +320,10 @@ Baked into this syntax is the ability to provide exceptions to file name pattern stree write "**/{[!schema]*,*}.rb" ``` +## Formatting + + + ## Library Syntax Tree can be used as a library to access the syntax tree underlying Ruby source code. @@ -619,18 +623,45 @@ Implicity, the `2 * 3` is going to be executed first because the `*` operator ha The language server additionally includes this custom request to return a textual representation of the syntax tree underlying the source code of a file. Language server clients can use this to (for example) open an additional tab with this information displayed. -## Plugins +## Customization + +There are multiple ways to customize Syntax Tree's behavior when parsing and formatting code. You can ignore certain sections of the source code, you can register plugins to provide custom formatting behavior, and you can register additional languages to be parsed and formatted. + +### Ignoring code + +To ignore a section of source code, you can a special `# stree-ignore` comment. This comment should be placed immediately above the code that you want to ignore. For example: + +```ruby +numbers = [ + 10000, + 20000, + 30000 +] +``` + +Normally the snippet above would be formatted as `numbers = [10_000, 20_000, 30_000]`. However, sometimes you want to keep the original formatting to improve readability or maintainability. In that case, you can put the ignore comment before it, as in: + +```ruby +# stree-ignore +numbers = [ + 10000, + 20000, + 30000 +] +``` + +Now when Syntax Tree goes to format that code, it will copy the source code exactly as it is, including the newlines and indentation. -You can register additional customization and additional languages that can flow through the same CLI with Syntax Tree's plugin system. When invoking the CLI, you pass through the list of plugins with the `--plugins` options to the commands that accept them. They should be a comma-delimited list. When the CLI first starts, it will require the files corresponding to those names. +### Plugins -### Customization +You can register additional customization that can flow through the same CLI with Syntax Tree's plugin system. When invoking the CLI, you pass through the list of plugins with the `--plugins` options to the commands that accept them. They should be a comma-delimited list. When the CLI first starts, it will require the files corresponding to those names. -To register additional customization, define a file somewhere in your load path named `syntax_tree/my_plugin`. Then when invoking the CLI, you will pass `--plugins=my_plugin`. To require multiple, separate them by a comma. In this way, you can modify Syntax Tree however you would like. Some plugins ship with Syntax Tree itself. They are: +To register plugins, define a file somewhere in your load path named `syntax_tree/my_plugin`. Then when invoking the CLI, you will pass `--plugins=my_plugin`. To require multiple, separate them by a comma. In this way, you can modify Syntax Tree however you would like. Some plugins ship with Syntax Tree itself. They are: * `plugin/single_quotes` - This will change all of your string literals to use single quotes instead of the default double quotes. * `plugin/trailing_comma` - This will put trailing commas into multiline array literals, hash literals, and method calls that can support trailing commas. -If you're using Syntax Tree as a library, you should require those files directly. +If you're using Syntax Tree as a library, you can require those files directly or manually pass those options to the formatter initializer through the `SyntaxTree::Formatter::Options` class. ### Languages From cdede9bed6fd0951d92c4587a2fe691c79e87b4c Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 9 Nov 2022 12:13:43 -0500 Subject: [PATCH 204/536] Update documentation in the README --- README.md | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index a6b01362..db047bb1 100644 --- a/README.md +++ b/README.md @@ -42,11 +42,13 @@ It is built with only standard library dependencies. It additionally ships with - [visit_method](#visit_method) - [BasicVisitor](#basicvisitor) - [MutationVisitor](#mutationvisitor) + - [WithEnvironment](#withenvironment) - [Language server](#language-server) - [textDocument/formatting](#textdocumentformatting) - [textDocument/inlayHint](#textdocumentinlayhint) - [syntaxTree/visualizing](#syntaxtreevisualizing) - [Customization](#customization) + - [Ignoring code](#ignoring-code) - [Plugins](#plugins) - [Languages](#languages) - [Integration](#integration) @@ -320,10 +322,6 @@ Baked into this syntax is the ability to provide exceptions to file name pattern stree write "**/{[!schema]*,*}.rb" ``` -## Formatting - - - ## Library Syntax Tree can be used as a library to access the syntax tree underlying Ruby source code. @@ -580,13 +578,13 @@ class MyVisitor < Visitor include WithEnvironment def visit_ident(node) - # find_local will return a Local for any local variables or arguments present in the current environment or nil if - # the identifier is not a local + # find_local will return a Local for any local variables or arguments + # present in the current environment or nil if the identifier is not a local local = current_environment.find_local(node) - puts local.type # print the type of the local (:variable or :argument) - puts local.definitions # print the array of locations where this local is defined - puts local.usages # print the array of locations where this local occurs + puts local.type # the type of the local (:variable or :argument) + puts local.definitions # the array of locations where this local is defined + puts local.usages # the array of locations where this local occurs end end ``` From a651a4e8acbf4b338e97c950ec26034f917a19c9 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 9 Nov 2022 12:14:57 -0500 Subject: [PATCH 205/536] Fix typos in READMe --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index db047bb1..0f1b626a 100644 --- a/README.md +++ b/README.md @@ -627,7 +627,7 @@ There are multiple ways to customize Syntax Tree's behavior when parsing and for ### Ignoring code -To ignore a section of source code, you can a special `# stree-ignore` comment. This comment should be placed immediately above the code that you want to ignore. For example: +To ignore a section of source code, you can use a special `# stree-ignore` comment. This comment should be placed immediately above the code that you want to ignore. For example: ```ruby numbers = [ From 64f045bafcb66f6c7459b1c1e8069ad1f1787347 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 9 Nov 2022 12:18:59 -0500 Subject: [PATCH 206/536] Bump to v5.0.0 --- CHANGELOG.md | 5 ++++- Gemfile.lock | 2 +- lib/syntax_tree/version.rb | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e4783e4..034fe2b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +## [5.0.0] - 2022-11-09 + ### Added - Every node now implements the `#copy(**)` method, which provides a copy of the node with the given attributes replaced. @@ -448,7 +450,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - 🎉 Initial release! 🎉 -[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v4.3.0...HEAD +[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v5.0.0...HEAD +[5.0.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v4.3.0...v5.0.0 [4.3.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v4.2.0...v4.3.0 [4.2.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v4.1.0...v4.2.0 [4.1.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v4.0.2...v4.1.0 diff --git a/Gemfile.lock b/Gemfile.lock index 351c842a..d9067ba9 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - syntax_tree (4.3.0) + syntax_tree (5.0.0) prettier_print (>= 1.1.0) GEM diff --git a/lib/syntax_tree/version.rb b/lib/syntax_tree/version.rb index a12c472d..29a413d9 100644 --- a/lib/syntax_tree/version.rb +++ b/lib/syntax_tree/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SyntaxTree - VERSION = "4.3.0" + VERSION = "5.0.0" end From 0d0b851c5f383a3d389b04d8d8ced97d1afe0288 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 10 Nov 2022 09:39:52 -0500 Subject: [PATCH 207/536] Fix plugins parsing for 5.0.0 --- lib/syntax_tree.rb | 4 ++++ lib/syntax_tree/cli.rb | 14 ++++++++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index c2cb3484..418468a9 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -40,6 +40,10 @@ module SyntaxTree # optional second argument to ::format. DEFAULT_PRINT_WIDTH = 80 + # This is the default ruby version that we're going to target for formatting. + # It shouldn't really be changed except in very niche circumstances. + DEFAULT_RUBY_VERSION = Formatter::SemanticVersion.new(RUBY_VERSION).freeze + # This is a hook provided so that plugins can register themselves as the # handler for a particular file type. def self.register_handler(extension, handler) diff --git a/lib/syntax_tree/cli.rb b/lib/syntax_tree/cli.rb index 3975df18..392dd627 100644 --- a/lib/syntax_tree/cli.rb +++ b/lib/syntax_tree/cli.rb @@ -377,14 +377,19 @@ class Options :plugins, :print_width, :scripts, - :formatter_options + :target_ruby_version def initialize @ignore_files = [] @plugins = [] @print_width = DEFAULT_PRINT_WIDTH @scripts = [] - @formatter_options = Formatter::Options.new + @target_ruby_version = DEFAULT_RUBY_VERSION + end + + def formatter_options + @formatter_options ||= + Formatter::Options.new(target_ruby_version: target_ruby_version) end def parse(arguments) @@ -430,10 +435,7 @@ def parser # If there is a target ruby version specified on the command line, # parse that out and use it when formatting. opts.on("--target-ruby-version=VERSION") do |version| - @formatter_options = - Formatter::Options.new( - target_ruby_version: Formatter::SemanticVersion.new(version) - ) + @target_ruby_version = Formatter::SemanticVersion.new(version) end end end From cb5ca2220d59639c92eddb8a12a18f7bbe62d0c4 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 10 Nov 2022 09:42:28 -0500 Subject: [PATCH 208/536] Bump to v5.0.1 --- CHANGELOG.md | 9 ++++++++- Gemfile.lock | 2 +- lib/syntax_tree/version.rb | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 034fe2b7..e320cd82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +## [5.0.1] - 2022-11-10 + +### Changed + +- Fix the plugin parsing on the CLI so that they are respected. + ## [5.0.0] - 2022-11-09 ### Added @@ -450,7 +456,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - 🎉 Initial release! 🎉 -[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v5.0.0...HEAD +[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v5.0.1...HEAD +[5.0.1]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v5.0.0...v5.0.1 [5.0.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v4.3.0...v5.0.0 [4.3.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v4.2.0...v4.3.0 [4.2.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v4.1.0...v4.2.0 diff --git a/Gemfile.lock b/Gemfile.lock index d9067ba9..ffbdc5d1 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - syntax_tree (5.0.0) + syntax_tree (5.0.1) prettier_print (>= 1.1.0) GEM diff --git a/lib/syntax_tree/version.rb b/lib/syntax_tree/version.rb index 29a413d9..340bbbdf 100644 --- a/lib/syntax_tree/version.rb +++ b/lib/syntax_tree/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SyntaxTree - VERSION = "5.0.0" + VERSION = "5.0.1" end From 3b662a92bc0f37952a283a24fa05a40916600a8f Mon Sep 17 00:00:00 2001 From: Vinicius Stock Date: Thu, 10 Nov 2022 16:30:23 -0500 Subject: [PATCH 209/536] Allow formatting code with a different base level of indentation Being able to override the base level of indentation allows us to format parts of a document that may be nested. --- lib/syntax_tree.rb | 7 ++++++- lib/syntax_tree/formatter.rb | 4 ++-- test/formatting_test.rb | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index 418468a9..bdb4a931 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -44,6 +44,10 @@ module SyntaxTree # It shouldn't really be changed except in very niche circumstances. DEFAULT_RUBY_VERSION = Formatter::SemanticVersion.new(RUBY_VERSION).freeze + # The default indentation level for formatting. We allow changing this so + # that Syntax Tree can format arbitrary parts of a document. + DEFAULT_INDENTATION = 0 + # This is a hook provided so that plugins can register themselves as the # handler for a particular file type. def self.register_handler(extension, handler) @@ -61,12 +65,13 @@ def self.parse(source) def self.format( source, maxwidth = DEFAULT_PRINT_WIDTH, + base_indentation = DEFAULT_INDENTATION, options: Formatter::Options.new ) formatter = Formatter.new(source, [], maxwidth, options: options) parse(source).format(formatter) - formatter.flush + formatter.flush(base_indentation) formatter.output.join end diff --git a/lib/syntax_tree/formatter.rb b/lib/syntax_tree/formatter.rb index d5d251c6..fddc06fe 100644 --- a/lib/syntax_tree/formatter.rb +++ b/lib/syntax_tree/formatter.rb @@ -84,10 +84,10 @@ def initialize(source, *args, options: Options.new) @target_ruby_version = options.target_ruby_version end - def self.format(source, node) + def self.format(source, node, base_indentation = 0) q = new(source, []) q.format(node) - q.flush + q.flush(base_indentation) q.output.join end diff --git a/test/formatting_test.rb b/test/formatting_test.rb index eff7ef71..37ca29e1 100644 --- a/test/formatting_test.rb +++ b/test/formatting_test.rb @@ -27,5 +27,37 @@ def test_stree_ignore assert_equal(source, SyntaxTree.format(source)) end + + def test_formatting_with_different_indentation_level + source = <<~SOURCE + def foo + puts "a" + end + SOURCE + + # Default indentation + assert_equal(source, SyntaxTree.format(source)) + + # Level 2 + assert_equal(<<-EXPECTED.chomp, SyntaxTree.format(source, 80, 2).rstrip) + def foo + puts "a" + end + EXPECTED + + # Level 4 + assert_equal(<<-EXPECTED.chomp, SyntaxTree.format(source, 80, 4).rstrip) + def foo + puts "a" + end + EXPECTED + + # Level 6 + assert_equal(<<-EXPECTED.chomp, SyntaxTree.format(source, 80, 6).rstrip) + def foo + puts "a" + end + EXPECTED + end end end From d76043bcb8952a05f1167dc822efd2a74810a099 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 11 Nov 2022 15:39:29 -0500 Subject: [PATCH 210/536] Anonymous kwargs --- CHANGELOG.md | 4 ++++ lib/syntax_tree/node.rb | 4 ++-- lib/syntax_tree/parser.rb | 2 +- test/fixtures/assoc_splat.rb | 4 ++++ 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e320cd82..20808e3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +### Changed + +- Support forwarding anonymous keyword arguments with `**`. + ## [5.0.1] - 2022-11-10 ### Changed diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index f32789a3..53fb3905 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -1604,7 +1604,7 @@ def format_contents(q) # { **pairs } # class AssocSplat < Node - # [untyped] the expression that is being splatted + # [nil | untyped] the expression that is being splatted attr_reader :value # [Array[ Comment | EmbDoc ]] the comments attached to this node @@ -1643,7 +1643,7 @@ def deconstruct_keys(_keys) def format(q) q.text("**") - q.format(value) + q.format(value) if value end def ===(other) diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index 23a3196c..5b093a87 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -744,7 +744,7 @@ def on_assoc_splat(value) AssocSplat.new( value: value, - location: operator.location.to(value.location) + location: operator.location.to((value || operator).location) ) end diff --git a/test/fixtures/assoc_splat.rb b/test/fixtures/assoc_splat.rb index 2182c2ed..8b595ce9 100644 --- a/test/fixtures/assoc_splat.rb +++ b/test/fixtures/assoc_splat.rb @@ -12,3 +12,7 @@ } - { **foo } +% # >= 3.2.0 +def foo(**) + bar(**) +end From b0b47198baf43d56201cb15341446a9f0674f0cb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Nov 2022 17:08:50 +0000 Subject: [PATCH 211/536] Bump rubocop from 1.38.0 to 1.39.0 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.38.0 to 1.39.0. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.38.0...v1.39.0) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index ffbdc5d1..0e81e5ff 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -19,7 +19,7 @@ GEM rake (13.0.6) regexp_parser (2.6.0) rexml (3.2.5) - rubocop (1.38.0) + rubocop (1.39.0) json (~> 2.3) parallel (~> 1.10) parser (>= 3.1.2.1) From fbc64f5d61ad2e7032312fc4a9f75596565ddfdb Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 10 Nov 2022 09:23:00 -0500 Subject: [PATCH 212/536] Compile to YARV --- .rubocop.yml | 12 + lib/syntax_tree.rb | 1 + lib/syntax_tree/visitor/compiler.rb | 1830 +++++++++++++++++++++++++++ test/compiler_test.rb | 342 +++++ 4 files changed, 2185 insertions(+) create mode 100644 lib/syntax_tree/visitor/compiler.rb create mode 100644 test/compiler_test.rb diff --git a/.rubocop.yml b/.rubocop.yml index 6c9be677..22f1bbef 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -16,6 +16,9 @@ Layout/LineLength: Lint/AmbiguousBlockAssociation: Enabled: false +Lint/BooleanSymbol: + Enabled: false + Lint/DuplicateBranch: Enabled: false @@ -46,9 +49,15 @@ Naming/MethodParameterName: Naming/RescuedExceptionsVariableName: PreferredName: error +Naming/VariableNumber: + Enabled: false + Style/CaseEquality: Enabled: false +Style/CaseLikeIf: + Enabled: false + Style/ExplicitBlockArgument: Enabled: false @@ -88,6 +97,9 @@ Style/ParallelAssignment: Style/PerlBackrefs: Enabled: false +Style/SafeNavigation: + Enabled: false + Style/SpecialGlobalVars: Enabled: false diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index 418468a9..aea21d8e 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -13,6 +13,7 @@ require_relative "syntax_tree/basic_visitor" require_relative "syntax_tree/visitor" +require_relative "syntax_tree/visitor/compiler" require_relative "syntax_tree/visitor/field_visitor" require_relative "syntax_tree/visitor/json_visitor" require_relative "syntax_tree/visitor/match_visitor" diff --git a/lib/syntax_tree/visitor/compiler.rb b/lib/syntax_tree/visitor/compiler.rb new file mode 100644 index 00000000..fac19831 --- /dev/null +++ b/lib/syntax_tree/visitor/compiler.rb @@ -0,0 +1,1830 @@ +# frozen_string_literal: true + +module SyntaxTree + class Visitor + # This class is an experiment in transforming Syntax Tree nodes into their + # corresponding YARV instruction sequences. It attempts to mirror the + # behavior of RubyVM::InstructionSequence.compile. + # + # You use this as with any other visitor. First you parse code into a tree, + # then you visit it with this compiler. Visiting the root node of the tree + # will return a SyntaxTree::Visitor::Compiler::InstructionSequence object. + # With that object you can call #to_a on it, which will return a serialized + # form of the instruction sequence as an array. This array _should_ mirror + # the array given by RubyVM::InstructionSequence#to_a. + # + # As an example, here is how you would compile a single expression: + # + # program = SyntaxTree.parse("1 + 2") + # program.accept(SyntaxTree::Visitor::Compiler.new).to_a + # + # [ + # "YARVInstructionSequence/SimpleDataFormat", + # 3, + # 1, + # 1, + # {:arg_size=>0, :local_size=>0, :stack_max=>2}, + # "", + # "", + # "", + # 1, + # :top, + # [], + # {}, + # [], + # [ + # [:putobject_INT2FIX_1_], + # [:putobject, 2], + # [:opt_plus, {:mid=>:+, :flag=>16, :orig_argc=>1}], + # [:leave] + # ] + # ] + # + # Note that this is the same output as calling: + # + # RubyVM::InstructionSequence.compile("1 + 2").to_a + # + class Compiler < BasicVisitor + # This visitor is responsible for converting Syntax Tree nodes into their + # corresponding Ruby structures. This is used to convert the operands of + # some instructions like putobject that push a Ruby object directly onto + # the stack. It is only used when the entire structure can be represented + # at compile-time, as opposed to constructed at run-time. + class RubyVisitor < BasicVisitor + # This error is raised whenever a node cannot be converted into a Ruby + # object at compile-time. + class CompilationError < StandardError + end + + def visit_array(node) + visit_all(node.contents.parts) + end + + def visit_bare_assoc_hash(node) + node.assocs.to_h do |assoc| + # We can only convert regular key-value pairs. A double splat ** + # operator means it has to be converted at run-time. + raise CompilationError unless assoc.is_a?(Assoc) + [visit(assoc.key), visit(assoc.value)] + end + end + + def visit_float(node) + node.value.to_f + end + + alias visit_hash visit_bare_assoc_hash + + def visit_imaginary(node) + node.value.to_c + end + + def visit_int(node) + node.value.to_i + end + + def visit_label(node) + node.value.chomp(":").to_sym + end + + def visit_qsymbols(node) + node.elements.map { |element| visit(element).to_sym } + end + + def visit_range(node) + left, right = [visit(node.left), visit(node.right)] + node.operator.value === ".." ? left..right : left...right + end + + def visit_rational(node) + node.value.to_r + end + + def visit_regexp_literal(node) + if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) + Regexp.new(node.parts.first.value, visit_regexp_literal_flags(node)) + else + # Any interpolation of expressions or variables will result in the + # regular expression being constructed at run-time. + raise CompilationError + end + end + + # This isn't actually a visit method, though maybe it should be. It is + # responsible for converting the set of string options on a regular + # expression into its equivalent integer. + def visit_regexp_literal_flags(node) + node + .options + .chars + .inject(0) do |accum, option| + accum | + case option + when "i" + Regexp::IGNORECASE + when "x" + Regexp::EXTENDED + when "m" + Regexp::MULTILINE + else + raise "Unknown regexp option: #{option}" + end + end + end + + def visit_symbol_literal(node) + node.value.value.to_sym + end + + def visit_symbols(node) + node.elements.map { |element| visit(element).to_sym } + end + + def visit_tstring_content(node) + node.value + end + + def visit_word(node) + if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) + node.parts.first.value + else + # Any interpolation of expressions or variables will result in the + # string being constructed at run-time. + raise CompilationError + end + end + + def visit_unsupported(_node) + raise CompilationError + end + + # Please forgive the metaprogramming here. This is used to create visit + # methods for every node that we did not explicitly handle. By default + # each of these methods will raise a CompilationError. + handled = instance_methods(false) + (Visitor.instance_methods(false) - handled).each do |method| + alias_method method, :visit_unsupported + end + end + + # This object is used to track the size of the stack at any given time. It + # is effectively a mini symbolic interpreter. It's necessary because when + # instruction sequences get serialized they include a :stack_max field on + # them. This field is used to determine how much stack space to allocate + # for the instruction sequence. + class Stack + attr_reader :current_size, :maximum_size + + def initialize + @current_size = 0 + @maximum_size = 0 + end + + def change_by(value) + @current_size += value + @maximum_size = @current_size if @current_size > @maximum_size + end + end + + # This class is meant to mirror RubyVM::InstructionSequence. It contains a + # list of instructions along with the metadata pertaining to them. It also + # functions as a builder for the instruction sequence. + class InstructionSequence + # This is a small data class that captures the level of a local variable + # table (the number of scopes to traverse) and the index of the local + # variable in that table. + class LocalVariable + attr_reader :level, :index + + def initialize(level, index) + @level = level + @index = index + end + end + + # The name of the instruction sequence. + attr_reader :name + + # The parent instruction sequence, if there is one. + attr_reader :parent_iseq + + # The location of the root node of this instruction sequence. + attr_reader :location + + # The list of instructions for this instruction sequence. + attr_reader :insns + + # The array of symbols corresponding to the local variables of this + # instruction sequence. + attr_reader :local_variables + + # The hash of names of instance and class variables pointing to the + # index of their associated inline storage. + attr_reader :inline_storages + + # The index of the next inline storage that will be created. + attr_reader :storage_index + + # An object that will track the current size of the stack and the + # maximum size of the stack for this instruction sequence. + attr_reader :stack + + def initialize(name, parent_iseq, location) + @name = name + @parent_iseq = parent_iseq + @location = location + + @local_variables = [] + @inline_storages = {} + + @insns = [] + @storage_index = 0 + @stack = Stack.new + end + + def local_variable(name, level = 0) + if (index = local_variables.index(name)) + LocalVariable.new(level, index) + elsif parent_iseq + parent_iseq.local_variable(name, level + 1) + else + raise "Unknown local variable: #{name}" + end + end + + def push(insn) + insns << insn + insn + end + + def inline_storage + storage = storage_index + @storage_index += 1 + storage + end + + def inline_storage_for(name) + unless inline_storages.key?(name) + inline_storages[name] = inline_storage + end + + inline_storages[name] + end + + def length + insns.sum(&:length) + end + + def each_child + insns.each do |insn| + insn[1..].each do |operand| + yield operand if operand.is_a?(InstructionSequence) + end + end + end + + def to_a + versions = RUBY_VERSION.split(".").map(&:to_i) + + [ + "YARVInstructionSequence/SimpleDataFormat", + versions[0], + versions[1], + 1, + { + arg_size: 0, + local_size: local_variables.length, + stack_max: stack.maximum_size + }, + name, + "", + "", + 1, + :top, + local_variables, + {}, + [], + insns.map do |insn| + case insn[0] + when :getlocal_WC_0, :setlocal_WC_0 + # Here we need to map the local variable index to the offset + # from the top of the stack where it will be stored. + [insn[0], local_variables.length - (insn[1] - 3) - 1] + when :getlocal_WC_1, :setlocal_WC_1 + # Here we're going to do the same thing as with _WC_0 except + # we're looking at the parent scope. + [ + insn[0], + parent_iseq.local_variables.length - (insn[1] - 3) - 1 + ] + when :getlocal, :setlocal + # Here we're going to do the same thing as the other local + # instructions except that we'll traverse up the instruction + # sequences first. + iseq = self + insn[2].times { iseq = iseq.parent_iseq } + [insn[0], iseq.local_variables.length - (insn[1] - 3) - 1] + when :send + # For any instructions that push instruction sequences onto the + # stack, we need to call #to_a on them as well. + [insn[0], insn[1], (insn[2].to_a if insn[2])] + else + insn + end + end + ] + end + end + + # This class serves as a layer of indirection between the instruction + # sequence and the compiler. It allows us to provide different behavior + # for certain instructions depending on the Ruby version. For example, + # class variable reads and writes gained an inline cache in Ruby 3.0. So + # we place the logic for checking the Ruby version in this class. + class Builder + attr_reader :iseq, :stack + + def initialize(iseq) + @iseq = iseq + @stack = iseq.stack + end + + # This creates a new label at the current length of the instruction + # sequence. It is used as the operand for jump instructions. + def label + :"label_#{iseq.length}" + end + + def adjuststack(number) + stack.change_by(-number) + iseq.push([:adjuststack, number]) + end + + def anytostring + stack.change_by(-2 + 1) + iseq.push([:anytostring]) + end + + def branchif(index) + stack.change_by(-1) + iseq.push([:branchif, index]) + end + + def branchunless(index) + stack.change_by(-1) + iseq.push([:branchunless, index]) + end + + def concatstrings(number) + stack.change_by(-number + 1) + iseq.push([:concatstrings, number]) + end + + def defined(type, name, message) + stack.change_by(-1 + 1) + iseq.push([:defined, type, name, message]) + end + + def dup + stack.change_by(-1 + 2) + iseq.push([:dup]) + end + + def duparray(object) + stack.change_by(+1) + iseq.push([:duparray, object]) + end + + def duphash(object) + stack.change_by(+1) + iseq.push([:duphash, object]) + end + + def dupn(number) + stack.change_by(+number) + iseq.push([:dupn, number]) + end + + def getclassvariable(name) + stack.change_by(+1) + + if RUBY_VERSION >= "3.0" + iseq.push([:getclassvariable, name, iseq.inline_storage_for(name)]) + else + iseq.push([:getclassvariable, name]) + end + end + + def getconstant(name) + stack.change_by(-2 + 1) + iseq.push([:getconstant, name]) + end + + def getglobal(name) + stack.change_by(+1) + iseq.push([:getglobal, name]) + end + + def getinstancevariable(name) + stack.change_by(+1) + + if RUBY_VERSION >= "3.2" + iseq.push([:getinstancevariable, name, iseq.inline_storage]) + else + iseq.push( + [:getinstancevariable, name, iseq.inline_storage_for(name)] + ) + end + end + + def getlocal(index, level) + stack.change_by(+1) + + # Specialize the getlocal instruction based on the level of the + # local variable. If it's 0 or 1, then there's a specialized + # instruction that will look at the current scope or the parent + # scope, respectively, and requires fewer operands. + case level + when 0 + iseq.push([:getlocal_WC_0, index]) + when 1 + iseq.push([:getlocal_WC_1, index]) + else + iseq.push([:getlocal, index, level]) + end + end + + def getspecial(key, type) + stack.change_by(-0 + 1) + iseq.push([:getspecial, key, type]) + end + + def intern + stack.change_by(-1 + 1) + iseq.push([:intern]) + end + + def invokesuper(method_id, argc, flag, block_iseq) + stack.change_by(-(argc + 1) + 1) + iseq.push( + [:invokesuper, call_data(method_id, argc, flag), block_iseq] + ) + end + + def jump(index) + stack.change_by(0) + iseq.push([:jump, index]) + end + + def leave + stack.change_by(-1) + iseq.push([:leave]) + end + + def newarray(length) + stack.change_by(-length + 1) + iseq.push([:newarray, length]) + end + + def newhash(length) + stack.change_by(-length + 1) + iseq.push([:newhash, length]) + end + + def newrange(flag) + stack.change_by(-2 + 1) + iseq.push([:newrange, flag]) + end + + def objtostring(method_id, argc, flag) + stack.change_by(-1 + 1) + iseq.push([:objtostring, call_data(method_id, argc, flag)]) + end + + def opt_getconstant_path(names) + if RUBY_VERSION >= "3.2" + stack.change_by(+1) + iseq.push([:opt_getconstant_path, names]) + else + inline_storage = iseq.inline_storage + getinlinecache = opt_getinlinecache(-1, inline_storage) + + if names[0] == :"" + names.shift + pop + putobject(Object) + end + + names.each_with_index do |name, index| + putobject(index == 0) + getconstant(name) + end + + opt_setinlinecache(inline_storage) + getinlinecache[1] = label + end + end + + def opt_getinlinecache(offset, inline_storage) + stack.change_by(+1) + iseq.push([:opt_getinlinecache, offset, inline_storage]) + end + + def opt_setinlinecache(inline_storage) + stack.change_by(-1 + 1) + iseq.push([:opt_setinlinecache, inline_storage]) + end + + def pop + stack.change_by(-1) + iseq.push([:pop]) + end + + def putnil + stack.change_by(+1) + iseq.push([:putnil]) + end + + def putobject(object) + stack.change_by(+1) + + # Specialize the putobject instruction based on the value of the + # object. If it's 0 or 1, then there's a specialized instruction + # that will push the object onto the stack and requires fewer + # operands. + if object.eql?(0) + iseq.push([:putobject_INT2FIX_0_]) + elsif object.eql?(1) + iseq.push([:putobject_INT2FIX_1_]) + else + iseq.push([:putobject, object]) + end + end + + def putself + stack.change_by(+1) + iseq.push([:putself]) + end + + def putspecialobject(object) + stack.change_by(+1) + iseq.push([:putspecialobject, object]) + end + + def putstring(object) + stack.change_by(+1) + iseq.push([:putstring, object]) + end + + def send(method_id, argc, flag, block_iseq = nil) + stack.change_by(-(argc + 1) + 1) + cdata = call_data(method_id, argc, flag) + + # Specialize the send instruction. If it doesn't have a block + # attached, then we will replace it with an opt_send_without_block + # and do further specializations based on the called method and the + # number of arguments. + + # stree-ignore + if !block_iseq && (flag & VM_CALL_ARGS_BLOCKARG) == 0 + case [method_id, argc] + when [:length, 0] then iseq.push([:opt_length, cdata]) + when [:size, 0] then iseq.push([:opt_size, cdata]) + when [:empty?, 0] then iseq.push([:opt_empty_p, cdata]) + when [:nil?, 0] then iseq.push([:opt_nil_p, cdata]) + when [:succ, 0] then iseq.push([:opt_succ, cdata]) + when [:!, 0] then iseq.push([:opt_not, cdata]) + when [:+, 1] then iseq.push([:opt_plus, cdata]) + when [:-, 1] then iseq.push([:opt_minus, cdata]) + when [:*, 1] then iseq.push([:opt_mult, cdata]) + when [:/, 1] then iseq.push([:opt_div, cdata]) + when [:%, 1] then iseq.push([:opt_mod, cdata]) + when [:==, 1] then iseq.push([:opt_eq, cdata]) + when [:=~, 1] then iseq.push([:opt_regexpmatch2, cdata]) + when [:<, 1] then iseq.push([:opt_lt, cdata]) + when [:<=, 1] then iseq.push([:opt_le, cdata]) + when [:>, 1] then iseq.push([:opt_gt, cdata]) + when [:>=, 1] then iseq.push([:opt_ge, cdata]) + when [:<<, 1] then iseq.push([:opt_ltlt, cdata]) + when [:[], 1] then iseq.push([:opt_aref, cdata]) + when [:&, 1] then iseq.push([:opt_and, cdata]) + when [:|, 1] then iseq.push([:opt_or, cdata]) + when [:[]=, 2] then iseq.push([:opt_aset, cdata]) + when [:!=, 1] + eql_data = call_data(:==, 1, VM_CALL_ARGS_SIMPLE) + iseq.push([:opt_neq, eql_data, cdata]) + else + iseq.push([:opt_send_without_block, cdata]) + end + else + iseq.push([:send, cdata, block_iseq]) + end + end + + def setclassvariable(name) + stack.change_by(-1) + + if RUBY_VERSION >= "3.0" + iseq.push([:setclassvariable, name, iseq.inline_storage_for(name)]) + else + iseq.push([:setclassvariable, name]) + end + end + + def setconstant(name) + stack.change_by(-2) + iseq.push([:setconstant, name]) + end + + def setglobal(name) + stack.change_by(-1) + iseq.push([:setglobal, name]) + end + + def setinstancevariable(name) + stack.change_by(-1) + + if RUBY_VERSION >= "3.2" + iseq.push([:setinstancevariable, name, iseq.inline_storage]) + else + iseq.push( + [:setinstancevariable, name, iseq.inline_storage_for(name)] + ) + end + end + + def setlocal(index, level) + stack.change_by(-1) + + # Specialize the setlocal instruction based on the level of the + # local variable. If it's 0 or 1, then there's a specialized + # instruction that will write to the current scope or the parent + # scope, respectively, and requires fewer operands. + case level + when 0 + iseq.push([:setlocal_WC_0, index]) + when 1 + iseq.push([:setlocal_WC_1, index]) + else + iseq.push([:setlocal, index, level]) + end + end + + def setn(number) + stack.change_by(-1 + 1) + iseq.push([:setn, number]) + end + + def splatarray(flag) + stack.change_by(-1 + 1) + iseq.push([:splatarray, flag]) + end + + def swap + stack.change_by(-2 + 2) + iseq.push([:swap]) + end + + def topn(number) + stack.change_by(+1) + iseq.push([:topn, number]) + end + + private + + # This creates a call data object that is used as the operand for the + # send, invokesuper, and objtostring instructions. + def call_data(method_id, argc, flag) + { mid: method_id, flag: flag, orig_argc: argc } + end + end + + # These constants correspond to the putspecialobject instruction. They are + # used to represent special objects that are pushed onto the stack. + VM_SPECIAL_OBJECT_VMCORE = 1 + VM_SPECIAL_OBJECT_CBASE = 2 + VM_SPECIAL_OBJECT_CONST_BASE = 3 + + # These constants correspond to the flag passed as part of the call data + # structure on the send instruction. They are used to represent various + # metadata about the callsite (e.g., were keyword arguments used?, was a + # block given?, etc.). + VM_CALL_ARGS_SPLAT = 1 << 0 + VM_CALL_ARGS_BLOCKARG = 1 << 1 + VM_CALL_FCALL = 1 << 2 + VM_CALL_VCALL = 1 << 3 + VM_CALL_ARGS_SIMPLE = 1 << 4 + VM_CALL_BLOCKISEQ = 1 << 5 + VM_CALL_KWARG = 1 << 6 + VM_CALL_KW_SPLAT = 1 << 7 + VM_CALL_TAILCALL = 1 << 8 + VM_CALL_SUPER = 1 << 9 + VM_CALL_ZSUPER = 1 << 10 + VM_CALL_OPT_SEND = 1 << 11 + VM_CALL_KW_SPLAT_MUT = 1 << 12 + + # These constants correspond to the value passed as part of the defined + # instruction. It's an enum defined in the CRuby codebase that tells that + # instruction what kind of defined check to perform. + DEFINED_NIL = 1 + DEFINED_IVAR = 2 + DEFINED_LVAR = 3 + DEFINED_GVAR = 4 + DEFINED_CVAR = 5 + DEFINED_CONST = 6 + DEFINED_METHOD = 7 + DEFINED_YIELD = 8 + DEFINED_ZSUPER = 9 + DEFINED_SELF = 10 + DEFINED_TRUE = 11 + DEFINED_FALSE = 12 + DEFINED_ASGN = 13 + DEFINED_EXPR = 14 + DEFINED_REF = 15 + DEFINED_FUNC = 16 + DEFINED_CONST_FROM = 17 + + # The current instruction sequence that is being compiled. + attr_reader :current_iseq + + # This is the current builder that is being used to construct the current + # instruction sequence. + attr_reader :builder + + # A boolean that tracks whether or not we're currently compiling and + # inline storage for a constant lookup. + attr_reader :writing_storage + + # A boolean to track if we're currently compiling the last statement + # within a set of statements. This information is necessary to determine + # if we need to return the value of the last statement. + attr_reader :last_statement + + # Whether or not the frozen_string_literal pragma has been set. + attr_reader :frozen_string_literal + + def initialize + @current_iseq = nil + @builder = nil + @writing_storage = false + @last_statement = false + @frozen_string_literal = false + end + + def visit_CHAR(node) + if frozen_string_literal + builder.putobject(node.value[1..]) + else + builder.putstring(node.value[1..]) + end + end + + def visit_alias(node) + builder.putspecialobject(VM_SPECIAL_OBJECT_VMCORE) + builder.putspecialobject(VM_SPECIAL_OBJECT_CBASE) + visit(node.left) + visit(node.right) + builder.send(:"core#set_method_alias", 3, VM_CALL_ARGS_SIMPLE) + end + + def visit_aref(node) + visit(node.collection) + visit(node.index) + builder.send(:[], 1, VM_CALL_ARGS_SIMPLE) + end + + def visit_arg_block(node) + visit(node.value) + end + + def visit_arg_paren(node) + visit(node.arguments) + end + + def visit_arg_star(node) + visit(node.value) + builder.splatarray(false) + end + + def visit_args(node) + visit_all(node.parts) + end + + def visit_array(node) + builder.duparray(node.accept(RubyVisitor.new)) + rescue RubyVisitor::CompilationError + visit_all(node.contents.parts) + builder.newarray(node.contents.parts.length) + end + + def visit_assign(node) + case node.target + when ARefField + builder.putnil + visit(node.target.collection) + visit(node.target.index) + visit(node.value) + builder.setn(3) + builder.send(:[]=, 2, VM_CALL_ARGS_SIMPLE) + builder.pop + when ConstPathField + names = constant_names(node.target) + name = names.pop + + if RUBY_VERSION >= "3.2" + builder.opt_getconstant_path(names) + visit(node.value) + builder.swap + builder.topn(1) + builder.swap + builder.setconstant(name) + else + visit(node.value) + builder.dup if last_statement? + builder.opt_getconstant_path(names) + builder.setconstant(name) + end + when Field + builder.putnil + visit(node.target) + visit(node.value) + builder.setn(2) + builder.send(:"#{node.target.name.value}=", 1, VM_CALL_ARGS_SIMPLE) + builder.pop + when TopConstField + name = node.target.constant.value.to_sym + + if RUBY_VERSION >= "3.2" + builder.putobject(Object) + visit(node.value) + builder.swap + builder.topn(1) + builder.swap + builder.setconstant(name) + else + visit(node.value) + builder.dup if last_statement? + builder.putobject(Object) + builder.setconstant(name) + end + when VarField + visit(node.value) + builder.dup if last_statement? + + case node.target.value + when Const + builder.putspecialobject(VM_SPECIAL_OBJECT_CONST_BASE) + builder.setconstant(node.target.value.value.to_sym) + when CVar + builder.setclassvariable(node.target.value.value.to_sym) + when GVar + builder.setglobal(node.target.value.value.to_sym) + when Ident + local_variable = visit(node.target) + builder.setlocal(local_variable.index, local_variable.level) + when IVar + builder.setinstancevariable(node.target.value.value.to_sym) + end + end + end + + def visit_assoc(node) + visit(node.key) + visit(node.value) + end + + def visit_assoc_splat(node) + visit(node.value) + end + + def visit_backref(node) + builder.getspecial(1, 2 * node.value[1..].to_i) + end + + def visit_bare_assoc_hash(node) + builder.duphash(node.accept(RubyVisitor.new)) + rescue RubyVisitor::CompilationError + visit_all(node.assocs) + end + + def visit_binary(node) + case node.operator + when :"&&" + visit(node.left) + builder.dup + + branchunless = builder.branchunless(-1) + builder.pop + + visit(node.right) + branchunless[1] = builder.label + when :"||" + visit(node.left) + builder.dup + + branchif = builder.branchif(-1) + builder.pop + + visit(node.right) + branchif[1] = builder.label + else + visit(node.left) + visit(node.right) + builder.send(node.operator, 1, VM_CALL_ARGS_SIMPLE) + end + end + + def visit_call(node) + node.receiver ? visit(node.receiver) : builder.putself + + visit(node.arguments) + arg_parts = argument_parts(node.arguments) + + if arg_parts.last.is_a?(ArgBlock) + flag = node.receiver.nil? ? VM_CALL_FCALL : 0 + flag |= VM_CALL_ARGS_BLOCKARG + + if arg_parts.any? { |part| part.is_a?(ArgStar) } + flag |= VM_CALL_ARGS_SPLAT + end + + if arg_parts.any? { |part| part.is_a?(BareAssocHash) } + flag |= VM_CALL_KW_SPLAT + end + + builder.send(node.message.value.to_sym, arg_parts.length - 1, flag) + else + flag = 0 + arg_parts.each do |arg_part| + case arg_part + when ArgStar + flag |= VM_CALL_ARGS_SPLAT + when BareAssocHash + flag |= VM_CALL_KW_SPLAT + end + end + + flag |= VM_CALL_ARGS_SIMPLE if flag == 0 + flag |= VM_CALL_FCALL if node.receiver.nil? + builder.send(node.message.value.to_sym, arg_parts.length, flag) + end + end + + def visit_command(node) + call_node = + CallNode.new( + receiver: nil, + operator: nil, + message: node.message, + arguments: node.arguments, + location: node.location + ) + + call_node.comments.concat(node.comments) + visit_call(call_node) + end + + def visit_command_call(node) + call_node = + CallNode.new( + receiver: node.receiver, + operator: node.operator, + message: node.message, + arguments: node.arguments, + location: node.location + ) + + call_node.comments.concat(node.comments) + visit_call(call_node) + end + + def visit_const_path_field(node) + visit(node.parent) + end + + def visit_const_path_ref(node) + names = constant_names(node) + builder.opt_getconstant_path(names) + end + + def visit_defined(node) + case node.value + when Assign + # If we're assigning to a local variable, then we need to make sure + # that we put it into the local table. + if node.value.target.is_a?(VarField) && + node.value.target.value.is_a?(Ident) + name = node.value.target.value.value.to_sym + unless current_iseq.local_variables.include?(name) + current_iseq.local_variables << name + end + end + + builder.putobject("assignment") + when VarRef + value = node.value.value + name = value.value.to_sym + + case value + when Const + builder.putnil + builder.defined(DEFINED_CONST, name, "constant") + when CVar + builder.putnil + builder.defined(DEFINED_CVAR, name, "class variable") + when GVar + builder.putnil + builder.defined(DEFINED_GVAR, name, "global-variable") + when Ident + builder.putobject("local-variable") + when IVar + builder.putnil + builder.defined(DEFINED_IVAR, name, "instance-variable") + when Kw + case name + when :false + builder.putobject("false") + when :nil + builder.putobject("nil") + when :self + builder.putobject("self") + when :true + builder.putobject("true") + end + end + when VCall + builder.putself + + name = node.value.value.value.to_sym + builder.defined(DEFINED_FUNC, name, "method") + when YieldNode + builder.putnil + builder.defined(DEFINED_YIELD, false, "yield") + when ZSuper + builder.putnil + builder.defined(DEFINED_ZSUPER, false, "super") + else + builder.putobject("expression") + end + end + + def visit_dyna_symbol(node) + if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) + builder.putobject(node.parts.first.value.to_sym) + end + end + + def visit_else(node) + visit(node.statements) + builder.pop unless last_statement? + end + + def visit_field(node) + visit(node.parent) + end + + def visit_float(node) + builder.putobject(node.accept(RubyVisitor.new)) + end + + def visit_for(node) + visit(node.collection) + + # Be sure we set up the local table before we start compiling the body + # of the for loop. + if node.index.is_a?(VarField) && node.index.value.is_a?(Ident) + name = node.index.value.value.to_sym + unless current_iseq.local_variables.include?(name) + current_iseq.local_variables << name + end + end + + block_iseq = + with_instruction_sequence( + "block in #{current_iseq.name}", + current_iseq, + node.statements + ) do + visit(node.statements) + builder.leave + end + + builder.send(:each, 0, 0, block_iseq) + end + + def visit_hash(node) + builder.duphash(node.accept(RubyVisitor.new)) + rescue RubyVisitor::CompilationError + visit_all(node.assocs) + builder.newhash(node.assocs.length * 2) + end + + def visit_heredoc(node) + if node.beginning.value.end_with?("`") + visit_xstring_literal(node) + elsif node.parts.length == 1 && node.parts.first.is_a?(TStringContent) + visit(node.parts.first) + else + visit_string_parts(node) + builder.concatstrings(node.parts.length) + end + end + + def visit_if(node) + visit(node.predicate) + branchunless = builder.branchunless(-1) + visit(node.statements) + + if last_statement? + builder.leave + branchunless[1] = builder.label + + node.consequent ? visit(node.consequent) : builder.putnil + else + builder.pop + + if node.consequent + jump = builder.jump(-1) + branchunless[1] = builder.label + visit(node.consequent) + jump[1] = builder.label + else + branchunless[1] = builder.label + end + end + end + + def visit_imaginary(node) + builder.putobject(node.accept(RubyVisitor.new)) + end + + def visit_int(node) + builder.putobject(node.accept(RubyVisitor.new)) + end + + def visit_label(node) + builder.putobject(node.accept(RubyVisitor.new)) + end + + def visit_not(node) + visit(node.statement) + builder.send(:!, 0, VM_CALL_ARGS_SIMPLE) + end + + def visit_opassign(node) + flag = VM_CALL_ARGS_SIMPLE + if node.target.is_a?(ConstPathField) || node.target.is_a?(TopConstField) + flag |= VM_CALL_FCALL + end + + case (operator = node.operator.value.chomp("=").to_sym) + when :"&&" + branchunless = nil + + with_opassign(node) do + builder.dup + branchunless = builder.branchunless(-1) + builder.pop + visit(node.value) + end + + case node.target + when ARefField + builder.leave + branchunless[1] = builder.label + builder.setn(3) + builder.adjuststack(3) + when ConstPathField, TopConstField + branchunless[1] = builder.label + builder.swap + builder.pop + else + branchunless[1] = builder.label + end + when :"||" + if node.target.is_a?(ConstPathField) || + node.target.is_a?(TopConstField) + opassign_defined(node) + builder.swap + builder.pop + elsif node.target.is_a?(VarField) && + [Const, CVar, GVar].include?(node.target.value.class) + opassign_defined(node) + else + branchif = nil + + with_opassign(node) do + builder.dup + branchif = builder.branchif(-1) + builder.pop + visit(node.value) + end + + if node.target.is_a?(ARefField) + builder.leave + branchif[1] = builder.label + builder.setn(3) + builder.adjuststack(3) + else + branchif[1] = builder.label + end + end + else + with_opassign(node) do + visit(node.value) + builder.send(operator, 1, flag) + end + end + end + + def visit_paren(node) + visit(node.contents) + end + + def visit_program(node) + node.statements.body.each do |statement| + break unless statement.is_a?(Comment) + + if statement.value == "# frozen_string_literal: true" + @frozen_string_literal = true + end + end + + statements = + node.statements.body.select do |statement| + case statement + when Comment, EmbDoc, EndContent, VoidStmt + false + else + true + end + end + + with_instruction_sequence("", nil, node) do + if statements.empty? + builder.putnil + else + *statements, last_statement = statements + visit_all(statements) + with_last_statement { visit(last_statement) } + end + + builder.leave + end + end + + def visit_qsymbols(node) + builder.duparray(node.accept(RubyVisitor.new)) + end + + def visit_qwords(node) + visit_all(node.elements) + builder.newarray(node.elements.length) + end + + def visit_range(node) + builder.putobject(node.accept(RubyVisitor.new)) + rescue RubyVisitor::CompilationError + visit(node.left) + visit(node.right) + builder.newrange(node.operator.value == ".." ? 0 : 1) + end + + def visit_rational(node) + builder.putobject(node.accept(RubyVisitor.new)) + end + + def visit_regexp_literal(node) + builder.putobject(node.accept(RubyVisitor.new)) + rescue RubyVisitor::CompilationError + visit_string_parts(node) + + flags = RubyVisitor.new.visit_regexp_literal_flags(node) + builder.toregexp(flags, node.parts.length) + end + + def visit_statements(node) + statements = + node.body.select do |statement| + case statement + when Comment, EmbDoc, EndContent, VoidStmt + false + else + true + end + end + + statements.empty? ? builder.putnil : visit_all(statements) + end + + def visit_string_concat(node) + value = node.left.parts.first.value + node.right.parts.first.value + content = TStringContent.new(value: value, location: node.location) + + literal = + StringLiteral.new( + parts: [content], + quote: node.left.quote, + location: node.location + ) + visit_string_literal(literal) + end + + def visit_string_embexpr(node) + visit(node.statements) + end + + def visit_string_literal(node) + if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) + visit(node.parts.first) + else + visit_string_parts(node) + builder.concatstrings(node.parts.length) + end + end + + def visit_symbol_literal(node) + builder.putobject(node.accept(RubyVisitor.new)) + end + + def visit_symbols(node) + builder.duparray(node.accept(RubyVisitor.new)) + rescue RubyVisitor::CompilationError + node.elements.each do |element| + if element.parts.length == 1 && + element.parts.first.is_a?(TStringContent) + builder.putobject(element.parts.first.value.to_sym) + else + length = element.parts.length + unless element.parts.first.is_a?(TStringContent) + builder.putobject("") + length += 1 + end + + visit_string_parts(element) + builder.concatstrings(length) + builder.intern + end + end + + builder.newarray(node.elements.length) + end + + def visit_top_const_ref(node) + builder.opt_getconstant_path(constant_names(node)) + end + + def visit_tstring_content(node) + if frozen_string_literal + builder.putobject(node.accept(RubyVisitor.new)) + else + builder.putstring(node.accept(RubyVisitor.new)) + end + end + + def visit_unary(node) + visit(node.statement) + + method_id = + case node.operator + when "+", "-" + :"#{node.operator}@" + else + node.operator.to_sym + end + + builder.send(method_id, 0, VM_CALL_ARGS_SIMPLE) + end + + def visit_undef(node) + node.symbols.each_with_index do |symbol, index| + builder.pop if index != 0 + builder.putspecialobject(VM_SPECIAL_OBJECT_VMCORE) + builder.putspecialobject(VM_SPECIAL_OBJECT_CBASE) + visit(symbol) + builder.send(:"core#undef_method", 2, VM_CALL_ARGS_SIMPLE) + end + end + + def visit_var_field(node) + case node.value + when CVar, IVar + name = node.value.value.to_sym + current_iseq.inline_storage_for(name) + when Ident + name = node.value.value.to_sym + unless current_iseq.local_variables.include?(name) + current_iseq.local_variables << name + end + current_iseq.local_variable(name) + end + end + + def visit_var_ref(node) + case node.value + when Const + builder.opt_getconstant_path(constant_names(node)) + when CVar + name = node.value.value.to_sym + builder.getclassvariable(name) + when GVar + builder.getglobal(node.value.value.to_sym) + when Ident + local_variable = current_iseq.local_variable(node.value.value.to_sym) + builder.getlocal(local_variable.index, local_variable.level) + when IVar + name = node.value.value.to_sym + builder.getinstancevariable(name) + when Kw + case node.value.value + when "false" + builder.putobject(false) + when "nil" + builder.putnil + when "self" + builder.putself + when "true" + builder.putobject(true) + end + end + end + + def visit_vcall(node) + builder.putself + + flag = VM_CALL_FCALL | VM_CALL_VCALL | VM_CALL_ARGS_SIMPLE + builder.send(node.value.value.to_sym, 0, flag) + end + + def visit_while(node) + jumps = [] + + jumps << builder.jump(-1) + builder.putnil + builder.pop + jumps << builder.jump(-1) + + label = builder.label + visit(node.statements) + builder.pop + jumps.each { |jump| jump[1] = builder.label } + + visit(node.predicate) + builder.branchif(label) + builder.putnil if last_statement? + end + + def visit_word(node) + if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) + visit(node.parts.first) + else + length = node.parts.length + unless node.parts.first.is_a?(TStringContent) + builder.putobject("") + length += 1 + end + + visit_string_parts(node) + builder.concatstrings(length) + end + end + + def visit_words(node) + visit_all(node.elements) + builder.newarray(node.elements.length) + end + + def visit_xstring_literal(node) + builder.putself + visit_string_parts(node) + builder.concatstrings(node.parts.length) if node.parts.length > 1 + builder.send(:`, 1, VM_CALL_FCALL | VM_CALL_ARGS_SIMPLE) + end + + def visit_zsuper(_node) + builder.putself + builder.invokesuper( + nil, + 0, + VM_CALL_FCALL | VM_CALL_ARGS_SIMPLE | VM_CALL_SUPER | VM_CALL_ZSUPER, + nil + ) + end + + private + + # This is a helper that is used in places where arguments may be present + # or they may be wrapped in parentheses. It's meant to descend down the + # tree and return an array of argument nodes. + def argument_parts(node) + case node + when nil + [] + when Args + node.parts + when ArgParen + node.arguments.parts + end + end + + # Constant names when they are being assigned or referenced come in as a + # tree, but it's more convenient to work with them as an array. This + # method converts them into that array. This is nice because it's the + # operand that goes to opt_getconstant_path in Ruby 3.2. + def constant_names(node) + current = node + names = [] + + while current.is_a?(ConstPathField) || current.is_a?(ConstPathRef) + names.unshift(current.constant.value.to_sym) + current = current.parent + end + + case current + when VarField, VarRef + names.unshift(current.value.value.to_sym) + when TopConstRef + names.unshift(current.constant.value.to_sym) + names.unshift(:"") + end + + names + end + + # For the most part when an OpAssign (operator assignment) node with a ||= + # operator is being compiled it's a matter of reading the target, checking + # if the value should be evaluated, evaluating it if so, and then writing + # the result back to the target. + # + # However, in certain kinds of assignments (X, ::X, X::Y, @@x, and $x) we + # first check if the value is defined using the defined instruction. I + # don't know why it is necessary, and suspect that it isn't. + def opassign_defined(node) + case node.target + when ConstPathField + visit(node.target.parent) + name = node.target.constant.value.to_sym + + builder.dup + builder.defined(DEFINED_CONST_FROM, name, true) + when TopConstField + name = node.target.constant.value.to_sym + + builder.putobject(Object) + builder.dup + builder.defined(DEFINED_CONST_FROM, name, true) + when VarField + name = node.target.value.value.to_sym + builder.putnil + + case node.target.value + when Const + builder.defined(DEFINED_CONST, name, true) + when CVar + builder.defined(DEFINED_CVAR, name, true) + when GVar + builder.defined(DEFINED_GVAR, name, true) + end + end + + branchunless = builder.branchunless(-1) + + case node.target + when ConstPathField, TopConstField + builder.dup + builder.putobject(true) + builder.getconstant(name) + when VarField + case node.target.value + when Const + builder.opt_getconstant_path(constant_names(node.target)) + when CVar + builder.getclassvariable(name) + when GVar + builder.getglobal(name) + end + end + + builder.dup + branchif = builder.branchif(-1) + builder.pop + + branchunless[1] = builder.label + visit(node.value) + + case node.target + when ConstPathField, TopConstField + builder.dupn(2) + builder.swap + builder.setconstant(name) + when VarField + builder.dup + + case node.target.value + when Const + builder.putspecialobject(VM_SPECIAL_OBJECT_CONST_BASE) + builder.setconstant(name) + when CVar + builder.setclassvariable(name) + when GVar + builder.setglobal(name) + end + end + + branchif[1] = builder.label + end + + # Whenever a value is interpolated into a string-like structure, these + # three instructions are pushed. + def push_interpolate + builder.dup + builder.objtostring(:to_s, 0, VM_CALL_FCALL | VM_CALL_ARGS_SIMPLE) + builder.anytostring + end + + # There are a lot of nodes in the AST that act as contains of parts of + # strings. This includes things like string literals, regular expressions, + # heredocs, etc. This method will visit all the parts of a string within + # those containers. + def visit_string_parts(node) + node.parts.each do |part| + case part + when StringDVar + visit(part.variable) + push_interpolate + when StringEmbExpr + visit(part) + push_interpolate + when TStringContent + builder.putobject(part.accept(RubyVisitor.new)) + end + end + end + + # This is a helper method for compiling a constant lookup. In order to + # avoid having to look up the tree to determine if the constant is part of + # a larger path or not, we store a boolean flag that indicates that we're + # already in the middle of a constant lookup. That way we only get one set + # of opt_getinlinecache..opt_setinlinecache instructions. + def with_inline_storage + return yield if writing_storage + + @writing_storage = true + inline_storage = current_iseq.inline_storage + + getinlinecache = builder.opt_getinlinecache(-1, inline_storage) + yield + builder.opt_setinlinecache(inline_storage) + + getinlinecache[1] = builder.label + @writing_storage = false + end + + # The current instruction sequence that we're compiling is always stored + # on the compiler. When we descend into a node that has its own + # instruction sequence, this method can be called to temporarily set the + # new value of the instruction sequence, yield, and then set it back. + def with_instruction_sequence(name, parent_iseq, node) + previous_iseq = current_iseq + previous_builder = builder + + begin + iseq = InstructionSequence.new(name, parent_iseq, node.location) + @current_iseq = iseq + @builder = Builder.new(iseq) + yield + iseq + ensure + @current_iseq = previous_iseq + @builder = previous_builder + end + end + + # When we're compiling the last statement of a set of statements within a + # scope, the instructions sometimes change from pops to leaves. These + # kinds of peephole optimizations can reduce the overall number of + # instructions. Therefore, we keep track of whether we're compiling the + # last statement of a scope and allow visit methods to query that + # information. + def with_last_statement + @last_statement = true + + begin + yield + ensure + @last_statement = false + end + end + + def last_statement? + @last_statement + end + + # OpAssign nodes can have a number of different kinds of nodes as their + # "target" (i.e., the left-hand side of the assignment). When compiling + # these nodes we typically need to first fetch the current value of the + # variable, then perform some kind of action, then store the result back + # into the variable. This method handles that by first fetching the value, + # then yielding to the block, then storing the result. + def with_opassign(node) + case node.target + when ARefField + builder.putnil + visit(node.target.collection) + visit(node.target.index) + + builder.dupn(2) + builder.send(:[], 1, VM_CALL_ARGS_SIMPLE) + + yield + + builder.setn(3) + builder.send(:[]=, 2, VM_CALL_ARGS_SIMPLE) + builder.pop + when ConstPathField + name = node.target.constant.value.to_sym + + visit(node.target.parent) + builder.dup + builder.putobject(true) + builder.getconstant(name) + + yield + + if node.operator.value == "&&=" + builder.dupn(2) + else + builder.swap + builder.topn(1) + end + + builder.swap + builder.setconstant(name) + when TopConstField + name = node.target.constant.value.to_sym + + builder.putobject(Object) + builder.dup + builder.putobject(true) + builder.getconstant(name) + + yield + + if node.operator.value == "&&=" + builder.dupn(2) + else + builder.swap + builder.topn(1) + end + + builder.swap + builder.setconstant(name) + when VarField + case node.target.value + when Const + names = constant_names(node.target) + builder.opt_getconstant_path(names) + + yield + + builder.dup + builder.putspecialobject(VM_SPECIAL_OBJECT_CONST_BASE) + builder.setconstant(names.last) + when CVar + name = node.target.value.value.to_sym + builder.getclassvariable(name) + + yield + + builder.dup + builder.setclassvariable(name) + when GVar + name = node.target.value.value.to_sym + builder.getglobal(name) + + yield + + builder.dup + builder.setglobal(name) + when Ident + local_variable = visit(node.target) + builder.getlocal(local_variable.index, local_variable.level) + + yield + + builder.dup + builder.setlocal(local_variable.index, local_variable.level) + when IVar + name = node.target.value.value.to_sym + builder.getinstancevariable(name) + + yield + + builder.dup + builder.setinstancevariable(name) + end + end + end + end + end +end diff --git a/test/compiler_test.rb b/test/compiler_test.rb new file mode 100644 index 00000000..4ed5bd0b --- /dev/null +++ b/test/compiler_test.rb @@ -0,0 +1,342 @@ +# frozen_string_literal: true + +return if !defined?(RubyVM::InstructionSequence) || RUBY_VERSION < "3.1" +require_relative "test_helper" + +module SyntaxTree + class CompilerTest < Minitest::Test + CASES = [ + # Various literals placed on the stack + "true", + "false", + "nil", + "self", + "0", + "1", + "2", + "1.0", + "1i", + "1r", + "1..2", + "1...2", + "(1)", + "%w[foo bar baz]", + "%W[foo bar baz]", + "%i[foo bar baz]", + "%I[foo bar baz]", + "{ foo: 1, bar: 1.0, baz: 1i }", + "'foo'", + "\"foo\"", + "\"foo\#{bar}\"", + "\"foo\#@bar\"", + "%q[foo]", + "%Q[foo]", + <<~RUBY, + "foo" \\ + "bar" + RUBY + <<~RUBY, + < 2", + "1 >= 2", + "1 == 2", + "1 != 2", + "1 & 2", + "1 | 2", + "1 << 2", + "1 ^ 2", + "foo.empty?", + "foo.length", + "foo.nil?", + "foo.size", + "foo.succ", + "/foo/ =~ \"foo\" && $1", + # Various method calls + "foo?", + "foo.bar", + "foo.bar(baz)", + "foo bar", + "foo.bar baz", + "foo(*bar)", + "foo(**bar)", + "foo(&bar)", + "foo.bar = baz", + "not foo", + "!foo", + "~foo", + "+foo", + "-foo", + "`foo`", + "`foo \#{bar} baz`", + # Local variables + "foo", + "foo = 1", + "foo = 1; bar = 2; baz = 3", + "foo = 1; foo", + "foo += 1", + "foo -= 1", + "foo *= 1", + "foo /= 1", + "foo %= 1", + "foo &= 1", + "foo |= 1", + "foo &&= 1", + "foo ||= 1", + "foo <<= 1", + "foo ^= 1", + # Instance variables + "@foo", + "@foo = 1", + "@foo = 1; @bar = 2; @baz = 3", + "@foo = 1; @foo", + "@foo += 1", + "@foo -= 1", + "@foo *= 1", + "@foo /= 1", + "@foo %= 1", + "@foo &= 1", + "@foo |= 1", + "@foo &&= 1", + "@foo ||= 1", + "@foo <<= 1", + "@foo ^= 1", + # Class variables + "@@foo", + "@@foo = 1", + "@@foo = 1; @@bar = 2; @@baz = 3", + "@@foo = 1; @@foo", + "@@foo += 1", + "@@foo -= 1", + "@@foo *= 1", + "@@foo /= 1", + "@@foo %= 1", + "@@foo &= 1", + "@@foo |= 1", + "@@foo &&= 1", + "@@foo ||= 1", + "@@foo <<= 1", + "@@foo ^= 1", + # Global variables + "$foo", + "$foo = 1", + "$foo = 1; $bar = 2; $baz = 3", + "$foo = 1; $foo", + "$foo += 1", + "$foo -= 1", + "$foo *= 1", + "$foo /= 1", + "$foo %= 1", + "$foo &= 1", + "$foo |= 1", + "$foo &&= 1", + "$foo ||= 1", + "$foo <<= 1", + "$foo ^= 1", + # Index access + "foo[bar]", + "foo[bar] = 1", + "foo[bar] += 1", + "foo[bar] -= 1", + "foo[bar] *= 1", + "foo[bar] /= 1", + "foo[bar] %= 1", + "foo[bar] &= 1", + "foo[bar] |= 1", + "foo[bar] &&= 1", + "foo[bar] ||= 1", + "foo[bar] <<= 1", + "foo[bar] ^= 1", + # Constants (single) + "Foo", + "Foo = 1", + "Foo += 1", + "Foo -= 1", + "Foo *= 1", + "Foo /= 1", + "Foo %= 1", + "Foo &= 1", + "Foo |= 1", + "Foo &&= 1", + "Foo ||= 1", + "Foo <<= 1", + "Foo ^= 1", + # Constants (top) + "::Foo", + "::Foo = 1", + "::Foo += 1", + "::Foo -= 1", + "::Foo *= 1", + "::Foo /= 1", + "::Foo %= 1", + "::Foo &= 1", + "::Foo |= 1", + "::Foo &&= 1", + "::Foo ||= 1", + "::Foo <<= 1", + "::Foo ^= 1", + # Constants (nested) + "Foo::Bar::Baz", + "Foo::Bar::Baz += 1", + "Foo::Bar::Baz -= 1", + "Foo::Bar::Baz *= 1", + "Foo::Bar::Baz /= 1", + "Foo::Bar::Baz %= 1", + "Foo::Bar::Baz &= 1", + "Foo::Bar::Baz |= 1", + "Foo::Bar::Baz &&= 1", + "Foo::Bar::Baz ||= 1", + "Foo::Bar::Baz <<= 1", + "Foo::Bar::Baz ^= 1", + # Constants (top nested) + "::Foo::Bar::Baz", + "::Foo::Bar::Baz = 1", + "::Foo::Bar::Baz += 1", + "::Foo::Bar::Baz -= 1", + "::Foo::Bar::Baz *= 1", + "::Foo::Bar::Baz /= 1", + "::Foo::Bar::Baz %= 1", + "::Foo::Bar::Baz &= 1", + "::Foo::Bar::Baz |= 1", + "::Foo::Bar::Baz &&= 1", + "::Foo::Bar::Baz ||= 1", + "::Foo::Bar::Baz <<= 1", + "::Foo::Bar::Baz ^= 1", + # Constants (calls) + "Foo::Bar.baz", + "::Foo::Bar.baz", + "Foo::Bar.baz = 1", + "::Foo::Bar.baz = 1", + # Control flow + "1 && 2", + "1 || 2", + "if foo then bar end", + "if foo then bar else baz end", + "foo if bar", + "foo while bar", + # Constructed values + "foo..bar", + "foo...bar", + "[1, 1.0, 1i, 1r]", + "[foo, bar, baz]", + "[@foo, @bar, @baz]", + "[@@foo, @@bar, @@baz]", + "[$foo, $bar, $baz]", + "%W[foo \#{bar} baz]", + "%I[foo \#{bar} baz]", + "[foo, bar] + [baz, qux]", + "{ foo: bar, baz: qux }", + "{ :foo => bar, :baz => qux }", + "{ foo => bar, baz => qux }", + "%s[foo]", + "[$1, $2, $3, $4, $5, $6, $7, $8, $9]", + # Core method calls + "alias foo bar", + "alias :foo :bar", + "undef foo", + "undef :foo", + "undef foo, bar, baz", + "undef :foo, :bar, :baz", + "super", + # defined? usage + "defined?(foo)", + "defined?(\"foo\")", + "defined?(:foo)", + "defined?(@foo)", + "defined?(@@foo)", + "defined?($foo)", + "defined?(Foo)", + "defined?(yield)", + "defined?(super)", + "foo = 1; defined?(foo)", + "defined?(self)", + "defined?(true)", + "defined?(false)", + "defined?(nil)", + "defined?(foo = 1)", + # Ignored content + ";;;", + "# comment", + "=begin\nfoo\n=end", + <<~RUBY + __END__ + RUBY + ] + + CASES.each do |source| + define_method(:"test_#{source}") { assert_compiles source } + end + + private + + def serialize_iseq(iseq) + serialized = iseq.to_a + + serialized[4].delete(:node_id) + serialized[4].delete(:code_location) + serialized[4].delete(:node_ids) + + serialized[13] = serialized[13].filter_map do |insn| + next unless insn.is_a?(Array) + + insn.map do |operand| + if operand.is_a?(Array) && + operand[0] == "YARVInstructionSequence/SimpleDataFormat" + serialize_iseq(operand) + else + operand + end + end + end + + serialized + end + + def assert_compiles(source) + assert_equal( + serialize_iseq(RubyVM::InstructionSequence.compile(source)), + serialize_iseq(SyntaxTree.parse(source).accept(Visitor::Compiler.new)) + ) + end + end +end From f51e211a76df9b8398709d7760132cb736561e1f Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 14 Nov 2022 18:34:04 -0500 Subject: [PATCH 213/536] Remove unused writing_storage --- lib/syntax_tree/visitor/compiler.rb | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/lib/syntax_tree/visitor/compiler.rb b/lib/syntax_tree/visitor/compiler.rb index fac19831..002614b5 100644 --- a/lib/syntax_tree/visitor/compiler.rb +++ b/lib/syntax_tree/visitor/compiler.rb @@ -751,10 +751,6 @@ def call_data(method_id, argc, flag) # instruction sequence. attr_reader :builder - # A boolean that tracks whether or not we're currently compiling and - # inline storage for a constant lookup. - attr_reader :writing_storage - # A boolean to track if we're currently compiling the last statement # within a set of statements. This information is necessary to determine # if we need to return the value of the last statement. @@ -766,7 +762,6 @@ def call_data(method_id, argc, flag) def initialize @current_iseq = nil @builder = nil - @writing_storage = false @last_statement = false @frozen_string_literal = false end @@ -1661,25 +1656,6 @@ def visit_string_parts(node) end end - # This is a helper method for compiling a constant lookup. In order to - # avoid having to look up the tree to determine if the constant is part of - # a larger path or not, we store a boolean flag that indicates that we're - # already in the middle of a constant lookup. That way we only get one set - # of opt_getinlinecache..opt_setinlinecache instructions. - def with_inline_storage - return yield if writing_storage - - @writing_storage = true - inline_storage = current_iseq.inline_storage - - getinlinecache = builder.opt_getinlinecache(-1, inline_storage) - yield - builder.opt_setinlinecache(inline_storage) - - getinlinecache[1] = builder.label - @writing_storage = false - end - # The current instruction sequence that we're compiling is always stored # on the compiler. When we descend into a node that has its own # instruction sequence, this method can be called to temporarily set the From 8464fc7a775969d26bbcedd819041ceecee0595e Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 14 Nov 2022 22:11:26 -0500 Subject: [PATCH 214/536] Handle various compilation options --- .rubocop.yml | 6 + lib/syntax_tree/visitor/compiler.rb | 373 +++++++++++++++++++--------- test/compiler_test.rb | 39 ++- 3 files changed, 293 insertions(+), 125 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index 22f1bbef..d0bf0830 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -34,6 +34,9 @@ Lint/MissingSuper: Lint/RedundantRequireStatement: Enabled: false +Lint/SuppressedException: + Enabled: false + Lint/UnusedMethodArgument: AllowUnusedKeywordArguments: true @@ -52,6 +55,9 @@ Naming/RescuedExceptionsVariableName: Naming/VariableNumber: Enabled: false +Style/AccessorGrouping: + Enabled: false + Style/CaseEquality: Enabled: false diff --git a/lib/syntax_tree/visitor/compiler.rb b/lib/syntax_tree/visitor/compiler.rb index 002614b5..14e277ae 100644 --- a/lib/syntax_tree/visitor/compiler.rb +++ b/lib/syntax_tree/visitor/compiler.rb @@ -91,6 +91,10 @@ def visit_qsymbols(node) node.elements.map { |element| visit(element).to_sym } end + def visit_qwords(node) + visit_all(node.elements) + end + def visit_range(node) left, right = [visit(node.left), visit(node.right)] node.operator.value === ".." ? left..right : left...right @@ -154,6 +158,10 @@ def visit_word(node) end end + def visit_words(node) + visit_all(node.elements) + end + def visit_unsupported(_node) raise CompilationError end @@ -202,6 +210,9 @@ def initialize(level, index) end end + # The type of the instruction sequence. + attr_reader :type + # The name of the instruction sequence. attr_reader :name @@ -211,6 +222,11 @@ def initialize(level, index) # The location of the root node of this instruction sequence. attr_reader :location + # This is the list of information about the arguments to this + # instruction sequence. + attr_accessor :argument_size + attr_reader :argument_options + # The list of instructions for this instruction sequence. attr_reader :insns @@ -229,14 +245,17 @@ def initialize(level, index) # maximum size of the stack for this instruction sequence. attr_reader :stack - def initialize(name, parent_iseq, location) + def initialize(type, name, parent_iseq, location) + @type = type @name = name @parent_iseq = parent_iseq @location = location + @argument_size = 0 + @argument_options = {} + @local_variables = [] @inline_storages = {} - @insns = [] @storage_index = 0 @stack = Stack.new @@ -292,7 +311,7 @@ def to_a versions[1], 1, { - arg_size: 0, + arg_size: argument_size, local_size: local_variables.length, stack_max: stack.maximum_size }, @@ -300,40 +319,43 @@ def to_a "", "", 1, - :top, + type, local_variables, - {}, + argument_options, [], - insns.map do |insn| - case insn[0] - when :getlocal_WC_0, :setlocal_WC_0 - # Here we need to map the local variable index to the offset - # from the top of the stack where it will be stored. - [insn[0], local_variables.length - (insn[1] - 3) - 1] - when :getlocal_WC_1, :setlocal_WC_1 - # Here we're going to do the same thing as with _WC_0 except - # we're looking at the parent scope. - [ - insn[0], - parent_iseq.local_variables.length - (insn[1] - 3) - 1 - ] - when :getlocal, :setlocal - # Here we're going to do the same thing as the other local - # instructions except that we'll traverse up the instruction - # sequences first. - iseq = self - insn[2].times { iseq = iseq.parent_iseq } - [insn[0], iseq.local_variables.length - (insn[1] - 3) - 1] - when :send - # For any instructions that push instruction sequences onto the - # stack, we need to call #to_a on them as well. - [insn[0], insn[1], (insn[2].to_a if insn[2])] - else - insn - end - end + insns.map { |insn| serialize(insn) } ] end + + private + + def serialize(insn) + case insn[0] + when :getlocal_WC_0, :getlocal_WC_1, :getlocal, :setlocal_WC_0, + :setlocal_WC_1, :setlocal + iseq = self + + case insn[0] + when :getlocal_WC_1, :setlocal_WC_1 + iseq = iseq.parent_iseq + when :getlocal, :setlocal + insn[2].times { iseq = iseq.parent_iseq } + end + + # Here we need to map the local variable index to the offset + # from the top of the stack where it will be stored. + index = iseq.local_variables.length - (insn[1] - 3) - 1 + [insn[0], index, *insn[2..]] + when :definemethod + [insn[0], insn[1], insn[2].to_a] + when :send + # For any instructions that push instruction sequences onto the + # stack, we need to call #to_a on them as well. + [insn[0], insn[1], (insn[2].to_a if insn[2])] + else + insn + end + end end # This class serves as a layer of indirection between the instruction @@ -343,10 +365,22 @@ def to_a # we place the logic for checking the Ruby version in this class. class Builder attr_reader :iseq, :stack - - def initialize(iseq) + attr_reader :frozen_string_literal, + :operands_unification, + :specialized_instruction + + def initialize( + iseq, + frozen_string_literal: false, + operands_unification: true, + specialized_instruction: true + ) @iseq = iseq @stack = iseq.stack + + @frozen_string_literal = frozen_string_literal + @operands_unification = operands_unification + @specialized_instruction = specialized_instruction end # This creates a new label at the current length of the instruction @@ -385,6 +419,11 @@ def defined(type, name, message) iseq.push([:defined, type, name, message]) end + def definemethod(name, method_iseq) + stack.change_by(0) + iseq.push([:definemethod, name, method_iseq]) + end + def dup stack.change_by(-1 + 2) iseq.push([:dup]) @@ -431,24 +470,27 @@ def getinstancevariable(name) if RUBY_VERSION >= "3.2" iseq.push([:getinstancevariable, name, iseq.inline_storage]) else - iseq.push( - [:getinstancevariable, name, iseq.inline_storage_for(name)] - ) + inline_storage = iseq.inline_storage_for(name) + iseq.push([:getinstancevariable, name, inline_storage]) end end def getlocal(index, level) stack.change_by(+1) - # Specialize the getlocal instruction based on the level of the - # local variable. If it's 0 or 1, then there's a specialized - # instruction that will look at the current scope or the parent - # scope, respectively, and requires fewer operands. - case level - when 0 - iseq.push([:getlocal_WC_0, index]) - when 1 - iseq.push([:getlocal_WC_1, index]) + if operands_unification + # Specialize the getlocal instruction based on the level of the + # local variable. If it's 0 or 1, then there's a specialized + # instruction that will look at the current scope or the parent + # scope, respectively, and requires fewer operands. + case level + when 0 + iseq.push([:getlocal_WC_0, index]) + when 1 + iseq.push([:getlocal_WC_1, index]) + else + iseq.push([:getlocal, index, level]) + end else iseq.push([:getlocal, index, level]) end @@ -466,9 +508,9 @@ def intern def invokesuper(method_id, argc, flag, block_iseq) stack.change_by(-(argc + 1) + 1) - iseq.push( - [:invokesuper, call_data(method_id, argc, flag), block_iseq] - ) + + cdata = call_data(method_id, argc, flag) + iseq.push([:invokesuper, cdata, block_iseq]) end def jump(index) @@ -548,14 +590,18 @@ def putnil def putobject(object) stack.change_by(+1) - # Specialize the putobject instruction based on the value of the - # object. If it's 0 or 1, then there's a specialized instruction - # that will push the object onto the stack and requires fewer - # operands. - if object.eql?(0) - iseq.push([:putobject_INT2FIX_0_]) - elsif object.eql?(1) - iseq.push([:putobject_INT2FIX_1_]) + if operands_unification + # Specialize the putobject instruction based on the value of the + # object. If it's 0 or 1, then there's a specialized instruction + # that will push the object onto the stack and requires fewer + # operands. + if object.eql?(0) + iseq.push([:putobject_INT2FIX_0_]) + elsif object.eql?(1) + iseq.push([:putobject_INT2FIX_1_]) + else + iseq.push([:putobject, object]) + end else iseq.push([:putobject, object]) end @@ -580,41 +626,45 @@ def send(method_id, argc, flag, block_iseq = nil) stack.change_by(-(argc + 1) + 1) cdata = call_data(method_id, argc, flag) - # Specialize the send instruction. If it doesn't have a block - # attached, then we will replace it with an opt_send_without_block - # and do further specializations based on the called method and the - # number of arguments. - - # stree-ignore - if !block_iseq && (flag & VM_CALL_ARGS_BLOCKARG) == 0 - case [method_id, argc] - when [:length, 0] then iseq.push([:opt_length, cdata]) - when [:size, 0] then iseq.push([:opt_size, cdata]) - when [:empty?, 0] then iseq.push([:opt_empty_p, cdata]) - when [:nil?, 0] then iseq.push([:opt_nil_p, cdata]) - when [:succ, 0] then iseq.push([:opt_succ, cdata]) - when [:!, 0] then iseq.push([:opt_not, cdata]) - when [:+, 1] then iseq.push([:opt_plus, cdata]) - when [:-, 1] then iseq.push([:opt_minus, cdata]) - when [:*, 1] then iseq.push([:opt_mult, cdata]) - when [:/, 1] then iseq.push([:opt_div, cdata]) - when [:%, 1] then iseq.push([:opt_mod, cdata]) - when [:==, 1] then iseq.push([:opt_eq, cdata]) - when [:=~, 1] then iseq.push([:opt_regexpmatch2, cdata]) - when [:<, 1] then iseq.push([:opt_lt, cdata]) - when [:<=, 1] then iseq.push([:opt_le, cdata]) - when [:>, 1] then iseq.push([:opt_gt, cdata]) - when [:>=, 1] then iseq.push([:opt_ge, cdata]) - when [:<<, 1] then iseq.push([:opt_ltlt, cdata]) - when [:[], 1] then iseq.push([:opt_aref, cdata]) - when [:&, 1] then iseq.push([:opt_and, cdata]) - when [:|, 1] then iseq.push([:opt_or, cdata]) - when [:[]=, 2] then iseq.push([:opt_aset, cdata]) - when [:!=, 1] - eql_data = call_data(:==, 1, VM_CALL_ARGS_SIMPLE) - iseq.push([:opt_neq, eql_data, cdata]) + if specialized_instruction + # Specialize the send instruction. If it doesn't have a block + # attached, then we will replace it with an opt_send_without_block + # and do further specializations based on the called method and the + # number of arguments. + + # stree-ignore + if !block_iseq && (flag & VM_CALL_ARGS_BLOCKARG) == 0 + case [method_id, argc] + when [:length, 0] then iseq.push([:opt_length, cdata]) + when [:size, 0] then iseq.push([:opt_size, cdata]) + when [:empty?, 0] then iseq.push([:opt_empty_p, cdata]) + when [:nil?, 0] then iseq.push([:opt_nil_p, cdata]) + when [:succ, 0] then iseq.push([:opt_succ, cdata]) + when [:!, 0] then iseq.push([:opt_not, cdata]) + when [:+, 1] then iseq.push([:opt_plus, cdata]) + when [:-, 1] then iseq.push([:opt_minus, cdata]) + when [:*, 1] then iseq.push([:opt_mult, cdata]) + when [:/, 1] then iseq.push([:opt_div, cdata]) + when [:%, 1] then iseq.push([:opt_mod, cdata]) + when [:==, 1] then iseq.push([:opt_eq, cdata]) + when [:=~, 1] then iseq.push([:opt_regexpmatch2, cdata]) + when [:<, 1] then iseq.push([:opt_lt, cdata]) + when [:<=, 1] then iseq.push([:opt_le, cdata]) + when [:>, 1] then iseq.push([:opt_gt, cdata]) + when [:>=, 1] then iseq.push([:opt_ge, cdata]) + when [:<<, 1] then iseq.push([:opt_ltlt, cdata]) + when [:[], 1] then iseq.push([:opt_aref, cdata]) + when [:&, 1] then iseq.push([:opt_and, cdata]) + when [:|, 1] then iseq.push([:opt_or, cdata]) + when [:[]=, 2] then iseq.push([:opt_aset, cdata]) + when [:!=, 1] + eql_data = call_data(:==, 1, VM_CALL_ARGS_SIMPLE) + iseq.push([:opt_neq, eql_data, cdata]) + else + iseq.push([:opt_send_without_block, cdata]) + end else - iseq.push([:opt_send_without_block, cdata]) + iseq.push([:send, cdata, block_iseq]) end else iseq.push([:send, cdata, block_iseq]) @@ -647,24 +697,27 @@ def setinstancevariable(name) if RUBY_VERSION >= "3.2" iseq.push([:setinstancevariable, name, iseq.inline_storage]) else - iseq.push( - [:setinstancevariable, name, iseq.inline_storage_for(name)] - ) + inline_storage = iseq.inline_storage_for(name) + iseq.push([:setinstancevariable, name, inline_storage]) end end def setlocal(index, level) stack.change_by(-1) - # Specialize the setlocal instruction based on the level of the - # local variable. If it's 0 or 1, then there's a specialized - # instruction that will write to the current scope or the parent - # scope, respectively, and requires fewer operands. - case level - when 0 - iseq.push([:setlocal_WC_0, index]) - when 1 - iseq.push([:setlocal_WC_1, index]) + if operands_unification + # Specialize the setlocal instruction based on the level of the + # local variable. If it's 0 or 1, then there's a specialized + # instruction that will write to the current scope or the parent + # scope, respectively, and requires fewer operands. + case level + when 0 + iseq.push([:setlocal_WC_0, index]) + when 1 + iseq.push([:setlocal_WC_1, index]) + else + iseq.push([:setlocal, index, level]) + end else iseq.push([:setlocal, index, level]) end @@ -744,6 +797,12 @@ def call_data(method_id, argc, flag) DEFINED_FUNC = 16 DEFINED_CONST_FROM = 17 + # These options mirror the compilation options that we currently support + # that can be also passed to RubyVM::InstructionSequence.compile. + attr_reader :frozen_string_literal, + :operands_unification, + :specialized_instruction + # The current instruction sequence that is being compiled. attr_reader :current_iseq @@ -756,14 +815,18 @@ def call_data(method_id, argc, flag) # if we need to return the value of the last statement. attr_reader :last_statement - # Whether or not the frozen_string_literal pragma has been set. - attr_reader :frozen_string_literal + def initialize( + frozen_string_literal: false, + operands_unification: true, + specialized_instruction: true + ) + @frozen_string_literal = frozen_string_literal + @operands_unification = operands_unification + @specialized_instruction = specialized_instruction - def initialize @current_iseq = nil @builder = nil @last_statement = false - @frozen_string_literal = false end def visit_CHAR(node) @@ -929,6 +992,10 @@ def visit_binary(node) end end + def visit_bodystmt(node) + visit(node.statements) + end + def visit_call(node) node.receiver ? visit(node.receiver) : builder.putself @@ -1002,6 +1069,52 @@ def visit_const_path_ref(node) builder.opt_getconstant_path(names) end + def visit_def(node) + params = node.params + params = params.contents if params.is_a?(Paren) + + method_iseq = + with_instruction_sequence( + :method, + node.name.value, + current_iseq, + node + ) do |iseq| + if params + params.requireds.each do |required| + iseq.local_variables << required.value.to_sym + iseq.argument_size += 1 + + iseq.argument_options[:lead_num] ||= 0 + iseq.argument_options[:lead_num] += 1 + end + + params.optionals.each do |(optional, value)| + index = iseq.local_variables.length + name = optional.value.to_sym + + iseq.local_variables << name + iseq.argument_size += 1 + + unless iseq.argument_options.key?(:opt) + iseq.argument_options[:opt] = [builder.label] + end + + visit(value) + builder.setlocal(index, 0) + iseq.argument_options[:opt] << builder.label + end + end + + visit(node.bodystmt) + builder.leave + end + + name = node.name.value.to_sym + builder.definemethod(name, method_iseq) + builder.putobject(name) + end + def visit_defined(node) case node.value when Assign @@ -1096,6 +1209,7 @@ def visit_for(node) block_iseq = with_instruction_sequence( + :block, "block in #{current_iseq.name}", current_iseq, node.statements @@ -1255,7 +1369,7 @@ def visit_program(node) end end - with_instruction_sequence("", nil, node) do + with_instruction_sequence(:top, "", nil, node) do if statements.empty? builder.putnil else @@ -1273,8 +1387,12 @@ def visit_qsymbols(node) end def visit_qwords(node) - visit_all(node.elements) - builder.newarray(node.elements.length) + if frozen_string_literal + builder.duparray(node.accept(RubyVisitor.new)) + else + visit_all(node.elements) + builder.newarray(node.elements.length) + end end def visit_range(node) @@ -1485,8 +1603,21 @@ def visit_word(node) end def visit_words(node) - visit_all(node.elements) - builder.newarray(node.elements.length) + converted = nil + + if frozen_string_literal + begin + converted = node.accept(RubyVisitor.new) + rescue RubyVisitor::CompilationError + end + end + + if converted + builder.duparray(converted) + else + visit_all(node.elements) + builder.newarray(node.elements.length) + end end def visit_xstring_literal(node) @@ -1660,15 +1791,23 @@ def visit_string_parts(node) # on the compiler. When we descend into a node that has its own # instruction sequence, this method can be called to temporarily set the # new value of the instruction sequence, yield, and then set it back. - def with_instruction_sequence(name, parent_iseq, node) + def with_instruction_sequence(type, name, parent_iseq, node) previous_iseq = current_iseq previous_builder = builder begin - iseq = InstructionSequence.new(name, parent_iseq, node.location) + iseq = InstructionSequence.new(type, name, parent_iseq, node.location) + @current_iseq = iseq - @builder = Builder.new(iseq) - yield + @builder = + Builder.new( + iseq, + frozen_string_literal: frozen_string_literal, + operands_unification: operands_unification, + specialized_instruction: specialized_instruction + ) + + yield iseq iseq ensure @current_iseq = previous_iseq diff --git a/test/compiler_test.rb b/test/compiler_test.rb index 4ed5bd0b..e44b35aa 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -248,8 +248,8 @@ class CompilerTest < Minitest::Test "Foo::Bar.baz = 1", "::Foo::Bar.baz = 1", # Control flow - "1 && 2", - "1 || 2", + "foo && bar", + "foo || bar", "if foo then bar end", "if foo then bar else baz end", "foo if bar", @@ -298,13 +298,34 @@ class CompilerTest < Minitest::Test ";;;", "# comment", "=begin\nfoo\n=end", - <<~RUBY + <<~RUBY, __END__ RUBY + # Method definitions + "def foo; end", + "def foo(bar); end", + "def foo(bar, baz); end", + "def foo(bar = 1); end", + "def foo(bar = 1, baz = 2); end" + ] + + # These are the combinations of instructions that we're going to test. + OPTIONS = [ + {}, + { frozen_string_literal: true }, + { operands_unification: false }, + { specialized_instruction: false }, + { operands_unification: false, specialized_instruction: false } ] - CASES.each do |source| - define_method(:"test_#{source}") { assert_compiles source } + OPTIONS.each do |options| + suffix = options.inspect + + CASES.each do |source| + define_method(:"test_#{source}_#{suffix}") do + assert_compiles(source, **options) + end + end end private @@ -332,10 +353,12 @@ def serialize_iseq(iseq) serialized end - def assert_compiles(source) + def assert_compiles(source, **options) + program = SyntaxTree.parse(source) + assert_equal( - serialize_iseq(RubyVM::InstructionSequence.compile(source)), - serialize_iseq(SyntaxTree.parse(source).accept(Visitor::Compiler.new)) + serialize_iseq(RubyVM::InstructionSequence.compile(source, **options)), + serialize_iseq(program.accept(Visitor::Compiler.new(**options))) ) end end From 6c0bbe60a1bec109e52b440fd36a468e3db15653 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 14 Nov 2022 23:20:26 -0500 Subject: [PATCH 215/536] Define modules --- lib/syntax_tree/visitor/compiler.rb | 45 +++++++++++++++++++++++++++++ test/compiler_test.rb | 8 ++++- 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/lib/syntax_tree/visitor/compiler.rb b/lib/syntax_tree/visitor/compiler.rb index 14e277ae..ebc98a14 100644 --- a/lib/syntax_tree/visitor/compiler.rb +++ b/lib/syntax_tree/visitor/compiler.rb @@ -346,6 +346,8 @@ def serialize(insn) # from the top of the stack where it will be stored. index = iseq.local_variables.length - (insn[1] - 3) - 1 [insn[0], index, *insn[2..]] + when :defineclass + [insn[0], insn[1], insn[2].to_a, insn[3]] when :definemethod [insn[0], insn[1], insn[2].to_a] when :send @@ -419,6 +421,11 @@ def defined(type, name, message) iseq.push([:defined, type, name, message]) end + def defineclass(name, class_iseq, flags) + stack.change_by(-2 + 1) + iseq.push([:defineclass, name, class_iseq, flags]) + end + def definemethod(name, method_iseq) stack.change_by(0) iseq.push([:definemethod, name, method_iseq]) @@ -797,6 +804,14 @@ def call_data(method_id, argc, flag) DEFINED_FUNC = 16 DEFINED_CONST_FROM = 17 + # These constants correspond to the value passed in the flags as part of + # the defineclass instruction. + VM_DEFINECLASS_TYPE_CLASS = 0 + VM_DEFINECLASS_TYPE_SINGLETON_CLASS = 1 + VM_DEFINECLASS_TYPE_MODULE = 2 + VM_DEFINECLASS_FLAG_SCOPED = 8 + VM_DEFINECLASS_FLAG_HAS_SUPERCLASS = 16 + # These options mirror the compilation options that we currently support # that can be also passed to RubyVM::InstructionSequence.compile. attr_reader :frozen_string_literal, @@ -1275,6 +1290,36 @@ def visit_label(node) builder.putobject(node.accept(RubyVisitor.new)) end + def visit_module(node) + name = node.constant.constant.value.to_sym + module_iseq = + with_instruction_sequence( + :class, + "", + current_iseq, + node + ) do + visit(node.bodystmt) + builder.leave + end + + flags = VM_DEFINECLASS_TYPE_MODULE + + case node.constant + when ConstPathRef + flags |= VM_DEFINECLASS_FLAG_SCOPED + visit(node.constant.parent) + when ConstRef + builder.putspecialobject(VM_SPECIAL_OBJECT_CONST_BASE) + when TopConstRef + flags |= VM_DEFINECLASS_FLAG_SCOPED + builder.putobject(Object) + end + + builder.putnil + builder.defineclass(name, module_iseq, flags) + end + def visit_not(node) visit(node.statement) builder.send(:!, 0, VM_CALL_ARGS_SIMPLE) diff --git a/test/compiler_test.rb b/test/compiler_test.rb index e44b35aa..08316ee3 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -306,7 +306,13 @@ class CompilerTest < Minitest::Test "def foo(bar); end", "def foo(bar, baz); end", "def foo(bar = 1); end", - "def foo(bar = 1, baz = 2); end" + "def foo(bar = 1, baz = 2); end", + # Class/module definitions + "module Foo; end", + "module ::Foo; end", + "module Foo::Bar; end", + "module ::Foo::Bar; end", + "module Foo; module Bar; end; end" ] # These are the combinations of instructions that we're going to test. From ebfddc3a1ea713708085da96210897773cda2cf7 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 14 Nov 2022 23:20:26 -0500 Subject: [PATCH 216/536] Visit params --- lib/syntax_tree/visitor/compiler.rb | 83 ++++++++++++++++++----------- test/compiler_test.rb | 5 ++ 2 files changed, 57 insertions(+), 31 deletions(-) diff --git a/lib/syntax_tree/visitor/compiler.rb b/lib/syntax_tree/visitor/compiler.rb index ebc98a14..ac4bc7c0 100644 --- a/lib/syntax_tree/visitor/compiler.rb +++ b/lib/syntax_tree/visitor/compiler.rb @@ -1085,42 +1085,14 @@ def visit_const_path_ref(node) end def visit_def(node) - params = node.params - params = params.contents if params.is_a?(Paren) - method_iseq = with_instruction_sequence( :method, node.name.value, current_iseq, node - ) do |iseq| - if params - params.requireds.each do |required| - iseq.local_variables << required.value.to_sym - iseq.argument_size += 1 - - iseq.argument_options[:lead_num] ||= 0 - iseq.argument_options[:lead_num] += 1 - end - - params.optionals.each do |(optional, value)| - index = iseq.local_variables.length - name = optional.value.to_sym - - iseq.local_variables << name - iseq.argument_size += 1 - - unless iseq.argument_options.key?(:opt) - iseq.argument_options[:opt] = [builder.label] - end - - visit(value) - builder.setlocal(index, 0) - iseq.argument_options[:opt] << builder.label - end - end - + ) do + visit(node.params) if node.params visit(node.bodystmt) builder.leave end @@ -1391,6 +1363,49 @@ def visit_opassign(node) end end + def visit_params(node) + argument_options = current_iseq.argument_options + + if node.requireds.any? + argument_options[:lead_num] = 0 + + node.requireds.each do |required| + current_iseq.local_variables << required.value.to_sym + current_iseq.argument_size += 1 + argument_options[:lead_num] += 1 + end + end + + node.optionals.each do |(optional, value)| + index = current_iseq.local_variables.length + name = optional.value.to_sym + + current_iseq.local_variables << name + current_iseq.argument_size += 1 + + unless argument_options.key?(:opt) + argument_options[:opt] = [builder.label] + end + + visit(value) + builder.setlocal(index, 0) + current_iseq.argument_options[:opt] << builder.label + end + + visit(node.rest) if node.rest + + if node.posts.any? + argument_options[:post_start] = current_iseq.argument_size + argument_options[:post_num] = 0 + + node.posts.each do |post| + current_iseq.local_variables << post.value.to_sym + current_iseq.argument_size += 1 + argument_options[:post_num] += 1 + end + end + end + def visit_paren(node) visit(node.contents) end @@ -1461,6 +1476,12 @@ def visit_regexp_literal(node) builder.toregexp(flags, node.parts.length) end + def visit_rest_param(node) + current_iseq.local_variables << node.name.value.to_sym + current_iseq.argument_options[:rest_start] = current_iseq.argument_size + current_iseq.argument_size += 1 + end + def visit_statements(node) statements = node.body.select do |statement| @@ -1852,7 +1873,7 @@ def with_instruction_sequence(type, name, parent_iseq, node) specialized_instruction: specialized_instruction ) - yield iseq + yield iseq ensure @current_iseq = previous_iseq diff --git a/test/compiler_test.rb b/test/compiler_test.rb index 08316ee3..81a01777 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -307,6 +307,11 @@ class CompilerTest < Minitest::Test "def foo(bar, baz); end", "def foo(bar = 1); end", "def foo(bar = 1, baz = 2); end", + "def foo(*bar); end", + "def foo(bar, *baz); end", + "def foo(*bar, baz, qux); end", + "def foo(bar, *baz, qux); end", + "def foo(bar, baz, *qux, quaz); end", # Class/module definitions "module Foo; end", "module ::Foo; end", From 72e527391e889f62f4e628495da42005ba7244e2 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 15 Nov 2022 00:02:17 -0500 Subject: [PATCH 217/536] Fix implementing toregexp --- lib/syntax_tree/visitor/compiler.rb | 5 +++++ test/compiler_test.rb | 2 ++ 2 files changed, 7 insertions(+) diff --git a/lib/syntax_tree/visitor/compiler.rb b/lib/syntax_tree/visitor/compiler.rb index ac4bc7c0..4aa50fc1 100644 --- a/lib/syntax_tree/visitor/compiler.rb +++ b/lib/syntax_tree/visitor/compiler.rb @@ -750,6 +750,11 @@ def topn(number) iseq.push([:topn, number]) end + def toregexp(options, length) + stack.change_by(-length + 1) + iseq.push([:toregexp, options, length]) + end + private # This creates a call data object that is used as the operand for the diff --git a/test/compiler_test.rb b/test/compiler_test.rb index 81a01777..ca4d4898 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -270,6 +270,8 @@ class CompilerTest < Minitest::Test "{ foo => bar, baz => qux }", "%s[foo]", "[$1, $2, $3, $4, $5, $6, $7, $8, $9]", + "/foo \#{bar} baz/", + "%r{foo \#{bar} baz}", # Core method calls "alias foo bar", "alias :foo :bar", From e9b767c2bc4b6beea7d88234619a0ab16813006b Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 15 Nov 2022 00:11:31 -0500 Subject: [PATCH 218/536] Compile for loops --- lib/syntax_tree/visitor/compiler.rb | 28 +++++++++++++++++++++------- test/compiler_test.rb | 1 + 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/lib/syntax_tree/visitor/compiler.rb b/lib/syntax_tree/visitor/compiler.rb index 4aa50fc1..780495d2 100644 --- a/lib/syntax_tree/visitor/compiler.rb +++ b/lib/syntax_tree/visitor/compiler.rb @@ -545,6 +545,11 @@ def newrange(flag) iseq.push([:newrange, flag]) end + def nop + stack.change_by(0) + iseq.push([:nop]) + end + def objtostring(method_id, argc, flag) stack.change_by(-1 + 1) iseq.push([:objtostring, call_data(method_id, argc, flag)]) @@ -1190,13 +1195,9 @@ def visit_float(node) def visit_for(node) visit(node.collection) - # Be sure we set up the local table before we start compiling the body - # of the for loop. - if node.index.is_a?(VarField) && node.index.value.is_a?(Ident) - name = node.index.value.value.to_sym - unless current_iseq.local_variables.include?(name) - current_iseq.local_variables << name - end + name = node.index.value.value.to_sym + unless current_iseq.local_variables.include?(name) + current_iseq.local_variables << name end block_iseq = @@ -1206,6 +1207,19 @@ def visit_for(node) current_iseq, node.statements ) do + current_iseq.argument_options[:lead_num] ||= 0 + current_iseq.argument_options[:lead_num] += 1 + current_iseq.argument_options[:ambiguous_param0] = true + + current_iseq.argument_size += 1 + current_iseq.local_variables << 2 + + builder.getlocal(0, 0) + + local_variable = current_iseq.local_variable(name) + builder.setlocal(local_variable.index, local_variable.level) + builder.nop + visit(node.statements) builder.leave end diff --git a/test/compiler_test.rb b/test/compiler_test.rb index ca4d4898..85d62b5b 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -254,6 +254,7 @@ class CompilerTest < Minitest::Test "if foo then bar else baz end", "foo if bar", "foo while bar", + "for i in [1, 2, 3] do i end", # Constructed values "foo..bar", "foo...bar", From 07afc36ddbbcefddcec5d8062e918d28d479c0b8 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 15 Nov 2022 00:24:20 -0500 Subject: [PATCH 219/536] Handle super with arguments --- lib/syntax_tree/visitor/compiler.rb | 11 +++++++++++ test/compiler_test.rb | 2 ++ 2 files changed, 13 insertions(+) diff --git a/lib/syntax_tree/visitor/compiler.rb b/lib/syntax_tree/visitor/compiler.rb index 780495d2..c56e553d 100644 --- a/lib/syntax_tree/visitor/compiler.rb +++ b/lib/syntax_tree/visitor/compiler.rb @@ -1541,6 +1541,17 @@ def visit_string_literal(node) end end + def visit_super(node) + builder.putself + visit(node.arguments) + builder.invokesuper( + nil, + argument_parts(node.arguments).length, + VM_CALL_FCALL | VM_CALL_ARGS_SIMPLE | VM_CALL_SUPER, + nil + ) + end + def visit_symbol_literal(node) builder.putobject(node.accept(RubyVisitor.new)) end diff --git a/test/compiler_test.rb b/test/compiler_test.rb index 85d62b5b..1c6cde38 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -281,6 +281,8 @@ class CompilerTest < Minitest::Test "undef foo, bar, baz", "undef :foo, :bar, :baz", "super", + "super(1)", + "super(1, 2, 3)", # defined? usage "defined?(foo)", "defined?(\"foo\")", From a8555eee14eafe504c9c60272f1a7e0dbfb3146c Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 16 Nov 2022 09:11:41 -0500 Subject: [PATCH 220/536] Handle block params --- lib/syntax_tree/visitor/compiler.rb | 8 ++++++++ test/compiler_test.rb | 2 ++ 2 files changed, 10 insertions(+) diff --git a/lib/syntax_tree/visitor/compiler.rb b/lib/syntax_tree/visitor/compiler.rb index c56e553d..a3fc7bbe 100644 --- a/lib/syntax_tree/visitor/compiler.rb +++ b/lib/syntax_tree/visitor/compiler.rb @@ -1017,6 +1017,12 @@ def visit_binary(node) end end + def visit_blockarg(node) + current_iseq.argument_options[:block_start] = current_iseq.argument_size + current_iseq.local_variables << node.name.value.to_sym + current_iseq.argument_size += 1 + end + def visit_bodystmt(node) visit(node.statements) end @@ -1423,6 +1429,8 @@ def visit_params(node) argument_options[:post_num] += 1 end end + + visit(node.block) if node.block end def visit_paren(node) diff --git a/test/compiler_test.rb b/test/compiler_test.rb index 1c6cde38..8534a3d0 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -317,6 +317,8 @@ class CompilerTest < Minitest::Test "def foo(*bar, baz, qux); end", "def foo(bar, *baz, qux); end", "def foo(bar, baz, *qux, quaz); end", + "def foo(bar, baz, &qux); end", + "def foo(bar, *baz, &qux); end", # Class/module definitions "module Foo; end", "module ::Foo; end", From 72534c52bbf48002f07c0a4eff63a17b465b27c8 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 16 Nov 2022 10:07:51 -0500 Subject: [PATCH 221/536] Block call --- lib/syntax_tree/visitor/compiler.rb | 72 ++++++++++++++++++++--------- test/compiler_test.rb | 1 + 2 files changed, 51 insertions(+), 22 deletions(-) diff --git a/lib/syntax_tree/visitor/compiler.rb b/lib/syntax_tree/visitor/compiler.rb index a3fc7bbe..870eb4c9 100644 --- a/lib/syntax_tree/visitor/compiler.rb +++ b/lib/syntax_tree/visitor/compiler.rb @@ -201,15 +201,32 @@ class InstructionSequence # This is a small data class that captures the level of a local variable # table (the number of scopes to traverse) and the index of the local # variable in that table. - class LocalVariable - attr_reader :level, :index + class LocalVariableLookup + attr_reader :local_variable, :level, :index - def initialize(level, index) + def initialize(local_variable, level, index) + @local_variable = local_variable @level = level @index = index end end + class PlainLocalVariable + attr_reader :name + + def initialize(name) + @name = name + end + end + + class BlockProxyLocalVariable + attr_reader :name + + def initialize(name) + @name = name + end + end + # The type of the instruction sequence. attr_reader :type @@ -262,8 +279,8 @@ def initialize(type, name, parent_iseq, location) end def local_variable(name, level = 0) - if (index = local_variables.index(name)) - LocalVariable.new(level, index) + if (index = local_variables.index { |local_variable| local_variable.name == name }) + LocalVariableLookup.new(local_variables[index], level, index) elsif parent_iseq parent_iseq.local_variable(name, level + 1) else @@ -320,7 +337,7 @@ def to_a "", 1, type, - local_variables, + local_variables.map(&:name), argument_options, [], insns.map { |insn| serialize(insn) } @@ -331,14 +348,14 @@ def to_a def serialize(insn) case insn[0] - when :getlocal_WC_0, :getlocal_WC_1, :getlocal, :setlocal_WC_0, - :setlocal_WC_1, :setlocal + when :getblockparamproxy, :getlocal_WC_0, :getlocal_WC_1, :getlocal, + :setlocal_WC_0, :setlocal_WC_1, :setlocal iseq = self case insn[0] when :getlocal_WC_1, :setlocal_WC_1 iseq = iseq.parent_iseq - when :getlocal, :setlocal + when :getblockparamproxy, :getlocal, :setlocal insn[2].times { iseq = iseq.parent_iseq } end @@ -451,6 +468,11 @@ def dupn(number) iseq.push([:dupn, number]) end + def getblockparamproxy(index, level) + stack.change_by(+1) + iseq.push([:getblockparamproxy, index, level]) + end + def getclassvariable(name) stack.change_by(+1) @@ -1019,7 +1041,7 @@ def visit_binary(node) def visit_blockarg(node) current_iseq.argument_options[:block_start] = current_iseq.argument_size - current_iseq.local_variables << node.name.value.to_sym + current_iseq.local_variables << InstructionSequence::BlockProxyLocalVariable.new(node.name.value.to_sym) current_iseq.argument_size += 1 end @@ -1126,8 +1148,8 @@ def visit_defined(node) if node.value.target.is_a?(VarField) && node.value.target.value.is_a?(Ident) name = node.value.target.value.value.to_sym - unless current_iseq.local_variables.include?(name) - current_iseq.local_variables << name + unless current_iseq.local_variables.any? { |local_variable| local_variable.name == name } + current_iseq.local_variables << InstructionSequence::PlainLocalVariable.new(name) end end @@ -1202,8 +1224,8 @@ def visit_for(node) visit(node.collection) name = node.index.value.value.to_sym - unless current_iseq.local_variables.include?(name) - current_iseq.local_variables << name + unless current_iseq.local_variables.any? { |local_variable| local_variable.name == name } + current_iseq.local_variables << InstructionSequence::PlainLocalVariable.new(name) end block_iseq = @@ -1218,7 +1240,7 @@ def visit_for(node) current_iseq.argument_options[:ambiguous_param0] = true current_iseq.argument_size += 1 - current_iseq.local_variables << 2 + current_iseq.local_variables << InstructionSequence::PlainLocalVariable.new(2) builder.getlocal(0, 0) @@ -1395,7 +1417,7 @@ def visit_params(node) argument_options[:lead_num] = 0 node.requireds.each do |required| - current_iseq.local_variables << required.value.to_sym + current_iseq.local_variables << InstructionSequence::PlainLocalVariable.new(required.value.to_sym) current_iseq.argument_size += 1 argument_options[:lead_num] += 1 end @@ -1405,7 +1427,7 @@ def visit_params(node) index = current_iseq.local_variables.length name = optional.value.to_sym - current_iseq.local_variables << name + current_iseq.local_variables << InstructionSequence::PlainLocalVariable.new(name) current_iseq.argument_size += 1 unless argument_options.key?(:opt) @@ -1424,7 +1446,7 @@ def visit_params(node) argument_options[:post_num] = 0 node.posts.each do |post| - current_iseq.local_variables << post.value.to_sym + current_iseq.local_variables << InstructionSequence::PlainLocalVariable.new(post.value.to_sym) current_iseq.argument_size += 1 argument_options[:post_num] += 1 end @@ -1504,7 +1526,7 @@ def visit_regexp_literal(node) end def visit_rest_param(node) - current_iseq.local_variables << node.name.value.to_sym + current_iseq.local_variables << InstructionSequence::PlainLocalVariable.new(node.name.value.to_sym) current_iseq.argument_options[:rest_start] = current_iseq.argument_size current_iseq.argument_size += 1 end @@ -1630,8 +1652,8 @@ def visit_var_field(node) current_iseq.inline_storage_for(name) when Ident name = node.value.value.to_sym - unless current_iseq.local_variables.include?(name) - current_iseq.local_variables << name + unless current_iseq.local_variables.any? { |local_variable| local_variable.name == name } + current_iseq.local_variables << InstructionSequence::PlainLocalVariable.new(name) end current_iseq.local_variable(name) end @@ -1648,7 +1670,13 @@ def visit_var_ref(node) builder.getglobal(node.value.value.to_sym) when Ident local_variable = current_iseq.local_variable(node.value.value.to_sym) - builder.getlocal(local_variable.index, local_variable.level) + + case local_variable.local_variable + when InstructionSequence::BlockProxyLocalVariable + builder.getblockparamproxy(local_variable.index, local_variable.level) + when InstructionSequence::PlainLocalVariable + builder.getlocal(local_variable.index, local_variable.level) + end when IVar name = node.value.value.to_sym builder.getinstancevariable(name) diff --git a/test/compiler_test.rb b/test/compiler_test.rb index 8534a3d0..c473e7be 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -319,6 +319,7 @@ class CompilerTest < Minitest::Test "def foo(bar, baz, *qux, quaz); end", "def foo(bar, baz, &qux); end", "def foo(bar, *baz, &qux); end", + "def foo(&qux); qux.call; end", # Class/module definitions "module Foo; end", "module ::Foo; end", From b96ffb51b3524ce2deea7ad81a95c35d5c6a558f Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 16 Nov 2022 14:09:06 -0500 Subject: [PATCH 222/536] Refactor local variables --- lib/syntax_tree/visitor/compiler.rb | 142 ++++++++++++++++++---------- 1 file changed, 90 insertions(+), 52 deletions(-) diff --git a/lib/syntax_tree/visitor/compiler.rb b/lib/syntax_tree/visitor/compiler.rb index 870eb4c9..f569ddcd 100644 --- a/lib/syntax_tree/visitor/compiler.rb +++ b/lib/syntax_tree/visitor/compiler.rb @@ -194,24 +194,15 @@ def change_by(value) end end - # This class is meant to mirror RubyVM::InstructionSequence. It contains a - # list of instructions along with the metadata pertaining to them. It also - # functions as a builder for the instruction sequence. - class InstructionSequence - # This is a small data class that captures the level of a local variable - # table (the number of scopes to traverse) and the index of the local - # variable in that table. - class LocalVariableLookup - attr_reader :local_variable, :level, :index - - def initialize(local_variable, level, index) - @local_variable = local_variable - @level = level - @index = index - end - end - - class PlainLocalVariable + # This represents every local variable associated with an instruction + # sequence. There are two kinds of locals: plain locals that are what you + # expect, and block proxy locals, which represent local variables + # associated with blocks that were passed into the current instruction + # sequence. + class LocalTable + # A local representing a block passed into the current instruction + # sequence. + class BlockProxyLocal attr_reader :name def initialize(name) @@ -219,7 +210,8 @@ def initialize(name) end end - class BlockProxyLocalVariable + # A regular local variable. + class PlainLocal attr_reader :name def initialize(name) @@ -227,6 +219,61 @@ def initialize(name) end end + # The result of looking up a local variable in the current local table. + class Lookup + attr_reader :local, :index, :level + + def initialize(local, index, level) + @local = local + @index = index + @level = level + end + end + + attr_reader :locals + + def initialize + @locals = [] + end + + def find(name, level) + index = locals.index { |local| local.name == name } + Lookup.new(locals[index], index, level) if index + end + + def has?(name) + locals.any? { |local| local.name == name } + end + + def names + locals.map(&:name) + end + + def size + locals.length + end + + # Add a BlockProxyLocal to the local table. + def block_proxy(name) + locals << BlockProxyLocal.new(name) unless has?(name) + end + + # Add a PlainLocal to the local table. + def plain(name) + locals << PlainLocal.new(name) unless has?(name) + end + + # This is the offset from the top of the stack where this local variable + # lives. + def offset(index) + size - (index - 3) - 1 + end + end + + # This class is meant to mirror RubyVM::InstructionSequence. It contains a + # list of instructions along with the metadata pertaining to them. It also + # functions as a builder for the instruction sequence. + class InstructionSequence # The type of the instruction sequence. attr_reader :type @@ -247,9 +294,8 @@ def initialize(name) # The list of instructions for this instruction sequence. attr_reader :insns - # The array of symbols corresponding to the local variables of this - # instruction sequence. - attr_reader :local_variables + # The table of local variables. + attr_reader :local_table # The hash of names of instance and class variables pointing to the # index of their associated inline storage. @@ -271,7 +317,7 @@ def initialize(type, name, parent_iseq, location) @argument_size = 0 @argument_options = {} - @local_variables = [] + @local_table = LocalTable.new @inline_storages = {} @insns = [] @storage_index = 0 @@ -279,8 +325,8 @@ def initialize(type, name, parent_iseq, location) end def local_variable(name, level = 0) - if (index = local_variables.index { |local_variable| local_variable.name == name }) - LocalVariableLookup.new(local_variables[index], level, index) + if (lookup = local_table.find(name, level)) + lookup elsif parent_iseq parent_iseq.local_variable(name, level + 1) else @@ -329,7 +375,7 @@ def to_a 1, { arg_size: argument_size, - local_size: local_variables.length, + local_size: local_table.size, stack_max: stack.maximum_size }, name, @@ -337,7 +383,7 @@ def to_a "", 1, type, - local_variables.map(&:name), + local_table.names, argument_options, [], insns.map { |insn| serialize(insn) } @@ -361,8 +407,7 @@ def serialize(insn) # Here we need to map the local variable index to the offset # from the top of the stack where it will be stored. - index = iseq.local_variables.length - (insn[1] - 3) - 1 - [insn[0], index, *insn[2..]] + [insn[0], iseq.local_table.offset(insn[1]), *insn[2..]] when :defineclass [insn[0], insn[1], insn[2].to_a, insn[3]] when :definemethod @@ -1041,7 +1086,7 @@ def visit_binary(node) def visit_blockarg(node) current_iseq.argument_options[:block_start] = current_iseq.argument_size - current_iseq.local_variables << InstructionSequence::BlockProxyLocalVariable.new(node.name.value.to_sym) + current_iseq.local_table.block_proxy(node.name.value.to_sym) current_iseq.argument_size += 1 end @@ -1147,10 +1192,7 @@ def visit_defined(node) # that we put it into the local table. if node.value.target.is_a?(VarField) && node.value.target.value.is_a?(Ident) - name = node.value.target.value.value.to_sym - unless current_iseq.local_variables.any? { |local_variable| local_variable.name == name } - current_iseq.local_variables << InstructionSequence::PlainLocalVariable.new(name) - end + current_iseq.local_table.plain(node.value.target.value.value.to_sym) end builder.putobject("assignment") @@ -1224,9 +1266,7 @@ def visit_for(node) visit(node.collection) name = node.index.value.value.to_sym - unless current_iseq.local_variables.any? { |local_variable| local_variable.name == name } - current_iseq.local_variables << InstructionSequence::PlainLocalVariable.new(name) - end + current_iseq.local_table.plain(name) block_iseq = with_instruction_sequence( @@ -1240,7 +1280,7 @@ def visit_for(node) current_iseq.argument_options[:ambiguous_param0] = true current_iseq.argument_size += 1 - current_iseq.local_variables << InstructionSequence::PlainLocalVariable.new(2) + current_iseq.local_table.plain(2) builder.getlocal(0, 0) @@ -1417,17 +1457,17 @@ def visit_params(node) argument_options[:lead_num] = 0 node.requireds.each do |required| - current_iseq.local_variables << InstructionSequence::PlainLocalVariable.new(required.value.to_sym) + current_iseq.local_table.plain(required.value.to_sym) current_iseq.argument_size += 1 argument_options[:lead_num] += 1 end end node.optionals.each do |(optional, value)| - index = current_iseq.local_variables.length + index = current_iseq.local_table.size name = optional.value.to_sym - current_iseq.local_variables << InstructionSequence::PlainLocalVariable.new(name) + current_iseq.local_table.plain(name) current_iseq.argument_size += 1 unless argument_options.key?(:opt) @@ -1446,7 +1486,7 @@ def visit_params(node) argument_options[:post_num] = 0 node.posts.each do |post| - current_iseq.local_variables << InstructionSequence::PlainLocalVariable.new(post.value.to_sym) + current_iseq.local_table.plain(post.value.to_sym) current_iseq.argument_size += 1 argument_options[:post_num] += 1 end @@ -1526,7 +1566,7 @@ def visit_regexp_literal(node) end def visit_rest_param(node) - current_iseq.local_variables << InstructionSequence::PlainLocalVariable.new(node.name.value.to_sym) + current_iseq.local_table.plain(node.name.value.to_sym) current_iseq.argument_options[:rest_start] = current_iseq.argument_size current_iseq.argument_size += 1 end @@ -1652,9 +1692,7 @@ def visit_var_field(node) current_iseq.inline_storage_for(name) when Ident name = node.value.value.to_sym - unless current_iseq.local_variables.any? { |local_variable| local_variable.name == name } - current_iseq.local_variables << InstructionSequence::PlainLocalVariable.new(name) - end + current_iseq.local_table.plain(name) current_iseq.local_variable(name) end end @@ -1669,13 +1707,13 @@ def visit_var_ref(node) when GVar builder.getglobal(node.value.value.to_sym) when Ident - local_variable = current_iseq.local_variable(node.value.value.to_sym) + lookup = current_iseq.local_variable(node.value.value.to_sym) - case local_variable.local_variable - when InstructionSequence::BlockProxyLocalVariable - builder.getblockparamproxy(local_variable.index, local_variable.level) - when InstructionSequence::PlainLocalVariable - builder.getlocal(local_variable.index, local_variable.level) + case lookup.local + when LocalTable::BlockProxyLocal + builder.getblockparamproxy(lookup.index, lookup.level) + when LocalTable::PlainLocal + builder.getlocal(lookup.index, lookup.level) end when IVar name = node.value.value.to_sym From ed83ff84973a354577e20595ea8cac43b35431ec Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 16 Nov 2022 14:44:35 -0500 Subject: [PATCH 223/536] Handle required keyword parameters --- lib/syntax_tree/visitor/compiler.rb | 18 ++++++++++++++++++ test/compiler_test.rb | 2 ++ 2 files changed, 20 insertions(+) diff --git a/lib/syntax_tree/visitor/compiler.rb b/lib/syntax_tree/visitor/compiler.rb index f569ddcd..c624dd56 100644 --- a/lib/syntax_tree/visitor/compiler.rb +++ b/lib/syntax_tree/visitor/compiler.rb @@ -1492,6 +1492,24 @@ def visit_params(node) end end + if node.keywords.any? + argument_options[:kwbits] = 0 + argument_options[:keyword] = [] + + node.keywords.each do |(keyword, value)| + name = keyword.value.chomp(":").to_sym + + current_iseq.local_table.plain(name) + current_iseq.argument_size += 1 + + argument_options[:kwbits] += 1 + argument_options[:keyword] << name + end + + current_iseq.argument_size += 1 + current_iseq.local_table.plain(2) + end + visit(node.block) if node.block end diff --git a/test/compiler_test.rb b/test/compiler_test.rb index c473e7be..53975926 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -320,6 +320,8 @@ class CompilerTest < Minitest::Test "def foo(bar, baz, &qux); end", "def foo(bar, *baz, &qux); end", "def foo(&qux); qux.call; end", + "def foo(bar:); end", + "def foo(bar:, baz:); end", # Class/module definitions "module Foo; end", "module ::Foo; end", From 1f7758e5fc1cbc8d3f78f263296326a8ed241802 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 16 Nov 2022 14:58:48 -0500 Subject: [PATCH 224/536] Handle optional keyword parameters --- lib/syntax_tree/visitor/compiler.rb | 35 ++++++++++++++++++++++++----- test/compiler_test.rb | 6 +++++ 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/lib/syntax_tree/visitor/compiler.rb b/lib/syntax_tree/visitor/compiler.rb index c624dd56..6cbcb272 100644 --- a/lib/syntax_tree/visitor/compiler.rb +++ b/lib/syntax_tree/visitor/compiler.rb @@ -394,8 +394,9 @@ def to_a def serialize(insn) case insn[0] - when :getblockparamproxy, :getlocal_WC_0, :getlocal_WC_1, :getlocal, - :setlocal_WC_0, :setlocal_WC_1, :setlocal + when :checkkeyword, :getblockparamproxy, :getlocal_WC_0, + :getlocal_WC_1, :getlocal, :setlocal_WC_0, :setlocal_WC_1, + :setlocal iseq = self case insn[0] @@ -473,6 +474,11 @@ def branchunless(index) iseq.push([:branchunless, index]) end + def checkkeyword(index, keyword_index) + stack.change_by(+1) + iseq.push([:checkkeyword, index, keyword_index]) + end + def concatstrings(number) stack.change_by(-number + 1) iseq.push([:concatstrings, number]) @@ -1495,19 +1501,38 @@ def visit_params(node) if node.keywords.any? argument_options[:kwbits] = 0 argument_options[:keyword] = [] + checkkeywords = [] - node.keywords.each do |(keyword, value)| + node.keywords.each_with_index do |(keyword, value), keyword_index| name = keyword.value.chomp(":").to_sym + index = current_iseq.local_table.size current_iseq.local_table.plain(name) current_iseq.argument_size += 1 - argument_options[:kwbits] += 1 - argument_options[:keyword] << name + + if value.nil? + argument_options[:keyword] << name + else + begin + compiled = value.accept(RubyVisitor.new) + argument_options[:keyword] << [name, compiled] + rescue RubyVisitor::CompilationError + argument_options[:keyword] << [name] + checkkeywords << builder.checkkeyword(-1, keyword_index) + branchif = builder.branchif(-1) + visit(value) + builder.setlocal(index, 0) + branchif[1] = builder.label + end + end end current_iseq.argument_size += 1 current_iseq.local_table.plain(2) + + lookup = current_iseq.local_table.find(2, 0) + checkkeywords.each { |checkkeyword| checkkeyword[1] = lookup.index } end visit(node.block) if node.block diff --git a/test/compiler_test.rb b/test/compiler_test.rb index 53975926..2a053d7f 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -322,6 +322,12 @@ class CompilerTest < Minitest::Test "def foo(&qux); qux.call; end", "def foo(bar:); end", "def foo(bar:, baz:); end", + "def foo(bar: 1); end", + "def foo(bar: 1, baz: 2); end", + "def foo(bar: baz); end", + "def foo(bar: 1, baz: qux); end", + "def foo(bar: qux, baz: 1); end", + "def foo(bar: baz, qux: qaz); end", # Class/module definitions "module Foo; end", "module ::Foo; end", From a453b7e1092a97bdff26ea795906993a92cab469 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 16 Nov 2022 15:03:27 -0500 Subject: [PATCH 225/536] Handle splat keyword parameters --- lib/syntax_tree/visitor/compiler.rb | 12 ++++++++++-- test/compiler_test.rb | 9 +++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/lib/syntax_tree/visitor/compiler.rb b/lib/syntax_tree/visitor/compiler.rb index 6cbcb272..a0b757e4 100644 --- a/lib/syntax_tree/visitor/compiler.rb +++ b/lib/syntax_tree/visitor/compiler.rb @@ -1351,6 +1351,12 @@ def visit_int(node) builder.putobject(node.accept(RubyVisitor.new)) end + def visit_kwrest_param(node) + current_iseq.argument_options[:kwrest] = current_iseq.argument_size + current_iseq.argument_size += 1 + current_iseq.local_table.plain(node.name.value.to_sym) + end + def visit_label(node) builder.putobject(node.accept(RubyVisitor.new)) end @@ -1528,13 +1534,15 @@ def visit_params(node) end end + name = node.keyword_rest ? 3 : 2 current_iseq.argument_size += 1 - current_iseq.local_table.plain(2) + current_iseq.local_table.plain(name) - lookup = current_iseq.local_table.find(2, 0) + lookup = current_iseq.local_table.find(name, 0) checkkeywords.each { |checkkeyword| checkkeyword[1] = lookup.index } end + visit(node.keyword_rest) if node.keyword_rest visit(node.block) if node.block end diff --git a/test/compiler_test.rb b/test/compiler_test.rb index 2a053d7f..8c933ce6 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -328,6 +328,15 @@ class CompilerTest < Minitest::Test "def foo(bar: 1, baz: qux); end", "def foo(bar: qux, baz: 1); end", "def foo(bar: baz, qux: qaz); end", + "def foo(**rest); end", + "def foo(bar:, **rest); end", + "def foo(bar:, baz:, **rest); end", + "def foo(bar: 1, **rest); end", + "def foo(bar: 1, baz: 2, **rest); end", + "def foo(bar: baz, **rest); end", + "def foo(bar: 1, baz: qux, **rest); end", + "def foo(bar: qux, baz: 1, **rest); end", + "def foo(bar: baz, qux: qaz, **rest); end", # Class/module definitions "module Foo; end", "module ::Foo; end", From 22b3bd2efe803973f0f0786080e7d622fcb908cd Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 16 Nov 2022 15:10:05 -0500 Subject: [PATCH 226/536] Handle yield without arguments --- lib/syntax_tree/visitor/compiler.rb | 9 +++++++++ test/compiler_test.rb | 7 ++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/lib/syntax_tree/visitor/compiler.rb b/lib/syntax_tree/visitor/compiler.rb index a0b757e4..bc7f6e1e 100644 --- a/lib/syntax_tree/visitor/compiler.rb +++ b/lib/syntax_tree/visitor/compiler.rb @@ -586,6 +586,11 @@ def intern iseq.push([:intern]) end + def invokeblock(method_id, argc, flag) + stack.change_by(-argc + 1) + iseq.push([:invokeblock, call_data(method_id, argc, flag)]) + end + def invokesuper(method_id, argc, flag, block_iseq) stack.change_by(-(argc + 1) + 1) @@ -1848,6 +1853,10 @@ def visit_xstring_literal(node) builder.send(:`, 1, VM_CALL_FCALL | VM_CALL_ARGS_SIMPLE) end + def visit_yield(node) + builder.invokeblock(nil, 0, VM_CALL_ARGS_SIMPLE) + end + def visit_zsuper(_node) builder.putself builder.invokesuper( diff --git a/test/compiler_test.rb b/test/compiler_test.rb index 8c933ce6..9912e436 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -276,13 +276,14 @@ class CompilerTest < Minitest::Test # Core method calls "alias foo bar", "alias :foo :bar", + "super", + "super(1)", + "super(1, 2, 3)", "undef foo", "undef :foo", "undef foo, bar, baz", "undef :foo, :bar, :baz", - "super", - "super(1)", - "super(1, 2, 3)", + "def foo; yield; end", # defined? usage "defined?(foo)", "defined?(\"foo\")", From 63793deacdc331de69db52ab736e884798164dc0 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 16 Nov 2022 15:12:35 -0500 Subject: [PATCH 227/536] Handle yield with arguments --- lib/syntax_tree/visitor/compiler.rb | 6 +++++- test/compiler_test.rb | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/syntax_tree/visitor/compiler.rb b/lib/syntax_tree/visitor/compiler.rb index bc7f6e1e..9d932f94 100644 --- a/lib/syntax_tree/visitor/compiler.rb +++ b/lib/syntax_tree/visitor/compiler.rb @@ -1854,7 +1854,9 @@ def visit_xstring_literal(node) end def visit_yield(node) - builder.invokeblock(nil, 0, VM_CALL_ARGS_SIMPLE) + parts = argument_parts(node.arguments) + visit_all(parts) + builder.invokeblock(nil, parts.length, VM_CALL_ARGS_SIMPLE) end def visit_zsuper(_node) @@ -1880,6 +1882,8 @@ def argument_parts(node) node.parts when ArgParen node.arguments.parts + when Paren + node.contents.parts end end diff --git a/test/compiler_test.rb b/test/compiler_test.rb index 9912e436..98928304 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -284,6 +284,8 @@ class CompilerTest < Minitest::Test "undef foo, bar, baz", "undef :foo, :bar, :baz", "def foo; yield; end", + "def foo; yield(1); end", + "def foo; yield(1, 2, 3); end", # defined? usage "defined?(foo)", "defined?(\"foo\")", From ae4f7ddf51c8cdc4403ae5a67b9d36cc0d2253fc Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 16 Nov 2022 15:19:40 -0500 Subject: [PATCH 228/536] Handle class definitions --- lib/syntax_tree/visitor/compiler.rb | 36 +++++++++++++++++++++++++++++ test/compiler_test.rb | 13 ++++++++++- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/lib/syntax_tree/visitor/compiler.rb b/lib/syntax_tree/visitor/compiler.rb index 9d932f94..1582f9ff 100644 --- a/lib/syntax_tree/visitor/compiler.rb +++ b/lib/syntax_tree/visitor/compiler.rb @@ -1141,6 +1141,42 @@ def visit_call(node) end end + def visit_class(node) + name = node.constant.constant.value.to_sym + class_iseq = + with_instruction_sequence( + :class, + "", + current_iseq, + node + ) do + visit(node.bodystmt) + builder.leave + end + + flags = VM_DEFINECLASS_TYPE_CLASS + + case node.constant + when ConstPathRef + flags |= VM_DEFINECLASS_FLAG_SCOPED + visit(node.constant.parent) + when ConstRef + builder.putspecialobject(VM_SPECIAL_OBJECT_CONST_BASE) + when TopConstRef + flags |= VM_DEFINECLASS_FLAG_SCOPED + builder.putobject(Object) + end + + if node.superclass + flags |= VM_DEFINECLASS_FLAG_HAS_SUPERCLASS + visit(node.superclass) + else + builder.putnil + end + + builder.defineclass(name, class_iseq, flags) + end + def visit_command(node) call_node = CallNode.new( diff --git a/test/compiler_test.rb b/test/compiler_test.rb index 98928304..795d7d13 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -345,7 +345,18 @@ class CompilerTest < Minitest::Test "module ::Foo; end", "module Foo::Bar; end", "module ::Foo::Bar; end", - "module Foo; module Bar; end; end" + "module Foo; module Bar; end; end", + "class Foo; end", + "class ::Foo; end", + "class Foo::Bar; end", + "class ::Foo::Bar; end", + "class Foo; class Bar; end; end", + "class Foo < Baz; end", + "class ::Foo < Baz; end", + "class Foo::Bar < Baz; end", + "class ::Foo::Bar < Baz; end", + "class Foo; class Bar < Baz; end; end", + "class Foo < baz; end" ] # These are the combinations of instructions that we're going to test. From 30d434f053a383cddd1333d3e3649168991372d3 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 16 Nov 2022 15:25:27 -0500 Subject: [PATCH 229/536] Handle splats within array --- lib/syntax_tree/visitor/compiler.rb | 26 ++++++++++++++++++++++++-- test/compiler_test.rb | 1 + 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/lib/syntax_tree/visitor/compiler.rb b/lib/syntax_tree/visitor/compiler.rb index 1582f9ff..9eda29b5 100644 --- a/lib/syntax_tree/visitor/compiler.rb +++ b/lib/syntax_tree/visitor/compiler.rb @@ -479,6 +479,11 @@ def checkkeyword(index, keyword_index) iseq.push([:checkkeyword, index, keyword_index]) end + def concatarray + stack.change_by(-2 + 1) + iseq.push([:concatarray]) + end + def concatstrings(number) stack.change_by(-number + 1) iseq.push([:concatstrings, number]) @@ -974,8 +979,25 @@ def visit_args(node) def visit_array(node) builder.duparray(node.accept(RubyVisitor.new)) rescue RubyVisitor::CompilationError - visit_all(node.contents.parts) - builder.newarray(node.contents.parts.length) + length = 0 + + node.contents.parts.each do |part| + if part.is_a?(ArgStar) + if length > 0 + builder.newarray(length) + length = 0 + end + + visit(part.value) + builder.concatarray + else + visit(part) + length += 1 + end + end + + builder.newarray(length) + builder.concatarray if length != node.contents.parts.length end def visit_assign(node) diff --git a/test/compiler_test.rb b/test/compiler_test.rb index 795d7d13..1fe690d9 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -266,6 +266,7 @@ class CompilerTest < Minitest::Test "%W[foo \#{bar} baz]", "%I[foo \#{bar} baz]", "[foo, bar] + [baz, qux]", + "[foo, bar, *baz, qux]", "{ foo: bar, baz: qux }", "{ :foo => bar, :baz => qux }", "{ foo => bar, baz => qux }", From b80c9ffae6253a9e4136084f237c13d973c5f6a5 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 16 Nov 2022 15:44:14 -0500 Subject: [PATCH 230/536] opt_newarray_min and opt_newarray_max --- lib/syntax_tree/visitor/compiler.rb | 59 +++++++++++++++++++++++++++-- test/compiler_test.rb | 6 +++ 2 files changed, 61 insertions(+), 4 deletions(-) diff --git a/lib/syntax_tree/visitor/compiler.rb b/lib/syntax_tree/visitor/compiler.rb index 9eda29b5..b99b56f8 100644 --- a/lib/syntax_tree/visitor/compiler.rb +++ b/lib/syntax_tree/visitor/compiler.rb @@ -667,6 +667,26 @@ def opt_getinlinecache(offset, inline_storage) iseq.push([:opt_getinlinecache, offset, inline_storage]) end + def opt_newarray_max(length) + if specialized_instruction + stack.change_by(-length + 1) + iseq.push([:opt_newarray_max, length]) + else + newarray(length) + send(:max, 0, VM_CALL_ARGS_SIMPLE) + end + end + + def opt_newarray_min(length) + if specialized_instruction + stack.change_by(-length + 1) + iseq.push([:opt_newarray_min, length]) + else + newarray(length) + send(:min, 0, VM_CALL_ARGS_SIMPLE) + end + end + def opt_setinlinecache(inline_storage) stack.change_by(-1 + 1) iseq.push([:opt_setinlinecache, inline_storage]) @@ -996,8 +1016,8 @@ def visit_array(node) end end - builder.newarray(length) - builder.concatarray if length != node.contents.parts.length + builder.newarray(length) if length > 0 + builder.concatarray if length > 0 && length != node.contents.parts.length end def visit_assign(node) @@ -1128,10 +1148,41 @@ def visit_bodystmt(node) end def visit_call(node) - node.receiver ? visit(node.receiver) : builder.putself + arg_parts = argument_parts(node.arguments) + + # First we're going to check if we're calling a method on an array + # literal without any arguments. In that case there are some + # specializations we might be able to perform. + if node.receiver.is_a?(ArrayLiteral) && arg_parts.length == 0 && node.message.is_a?(Ident) + parts = node.receiver.contents&.parts || [] + + unless parts.any? { |part| part.is_a?(ArgStar) } + begin + # If we can compile the receiver, then we won't be attempting to + # specialize the instruction. Otherwise we will. + node.receiver.accept(RubyVisitor.new) + rescue RubyVisitor::CompilationError + case node.message.value + when "max" + visit(node.receiver.contents) + builder.opt_newarray_max(parts.length) + return + when "min" + visit(node.receiver.contents) + builder.opt_newarray_min(parts.length) + return + end + end + end + end + + if node.receiver + visit(node.receiver) + else + builder.putself + end visit(node.arguments) - arg_parts = argument_parts(node.arguments) if arg_parts.last.is_a?(ArgBlock) flag = node.receiver.nil? ? VM_CALL_FCALL : 0 diff --git a/test/compiler_test.rb b/test/compiler_test.rb index 1fe690d9..2af625f6 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -274,6 +274,12 @@ class CompilerTest < Minitest::Test "[$1, $2, $3, $4, $5, $6, $7, $8, $9]", "/foo \#{bar} baz/", "%r{foo \#{bar} baz}", + "[1, 2, 3].max", + "[foo, bar, baz].max", + "[foo, bar, baz].max(1)", + "[1, 2, 3].min", + "[foo, bar, baz].min", + "[foo, bar, baz].min(1)", # Core method calls "alias foo bar", "alias :foo :bar", From b12953867d1608bd4d943c5172da0392f27e7071 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 16 Nov 2022 15:49:31 -0500 Subject: [PATCH 231/536] New RubyVisitor.compile --- lib/syntax_tree/visitor/compiler.rb | 51 ++++++++++++++++++----------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/lib/syntax_tree/visitor/compiler.rb b/lib/syntax_tree/visitor/compiler.rb index b99b56f8..de970461 100644 --- a/lib/syntax_tree/visitor/compiler.rb +++ b/lib/syntax_tree/visitor/compiler.rb @@ -56,6 +56,13 @@ class RubyVisitor < BasicVisitor class CompilationError < StandardError end + # This will attempt to compile the given node. If it's possible, then + # it will return the compiled object. Otherwise it will return nil. + def self.compile(node) + node.accept(new) + rescue CompilationError + end + def visit_array(node) visit_all(node.contents.parts) end @@ -997,27 +1004,29 @@ def visit_args(node) end def visit_array(node) - builder.duparray(node.accept(RubyVisitor.new)) - rescue RubyVisitor::CompilationError - length = 0 + if compiled = RubyVisitor.compile(node) + builder.duparray(compiled) + else + length = 0 - node.contents.parts.each do |part| - if part.is_a?(ArgStar) - if length > 0 - builder.newarray(length) - length = 0 - end + node.contents.parts.each do |part| + if part.is_a?(ArgStar) + if length > 0 + builder.newarray(length) + length = 0 + end - visit(part.value) - builder.concatarray - else - visit(part) - length += 1 + visit(part.value) + builder.concatarray + else + visit(part) + length += 1 + end end - end - builder.newarray(length) if length > 0 - builder.concatarray if length > 0 && length != node.contents.parts.length + builder.newarray(length) if length > 0 + builder.concatarray if length > 0 && length != node.contents.parts.length + end end def visit_assign(node) @@ -1105,9 +1114,11 @@ def visit_backref(node) end def visit_bare_assoc_hash(node) - builder.duphash(node.accept(RubyVisitor.new)) - rescue RubyVisitor::CompilationError - visit_all(node.assocs) + if compiled = RubyVisitor.compile(node) + builder.duphash(compiled) + else + visit_all(node.assocs) + end end def visit_binary(node) From f9a86c537e59b86dabdfb80d7e45d03f863544b1 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 16 Nov 2022 15:56:30 -0500 Subject: [PATCH 232/536] opt_str_freeze --- lib/syntax_tree/visitor/compiler.rb | 33 +++++++++++++++++++++-------- test/compiler_test.rb | 2 ++ 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/lib/syntax_tree/visitor/compiler.rb b/lib/syntax_tree/visitor/compiler.rb index de970461..00201618 100644 --- a/lib/syntax_tree/visitor/compiler.rb +++ b/lib/syntax_tree/visitor/compiler.rb @@ -699,6 +699,16 @@ def opt_setinlinecache(inline_storage) iseq.push([:opt_setinlinecache, inline_storage]) end + def opt_str_freeze(value) + if specialized_instruction + stack.change_by(+1) + iseq.push([:opt_str_freeze, value, call_data(:freeze, 0, VM_CALL_ARGS_SIMPLE)]) + else + putstring(value) + send(:freeze, 0, VM_CALL_ARGS_SIMPLE) + end + end + def pop stack.change_by(-1) iseq.push([:pop]) @@ -1164,15 +1174,12 @@ def visit_call(node) # First we're going to check if we're calling a method on an array # literal without any arguments. In that case there are some # specializations we might be able to perform. - if node.receiver.is_a?(ArrayLiteral) && arg_parts.length == 0 && node.message.is_a?(Ident) - parts = node.receiver.contents&.parts || [] - - unless parts.any? { |part| part.is_a?(ArgStar) } - begin - # If we can compile the receiver, then we won't be attempting to - # specialize the instruction. Otherwise we will. - node.receiver.accept(RubyVisitor.new) - rescue RubyVisitor::CompilationError + if arg_parts.length == 0 && node.message.is_a?(Ident) + case node.receiver + when ArrayLiteral + parts = node.receiver.contents&.parts || [] + + if parts.none? { |part| part.is_a?(ArgStar) } && RubyVisitor.compile(node.receiver).nil? case node.message.value when "max" visit(node.receiver.contents) @@ -1184,6 +1191,14 @@ def visit_call(node) return end end + when StringLiteral + if RubyVisitor.compile(node.receiver).nil? + case node.message.value + when "freeze" + builder.opt_str_freeze(node.receiver.parts.first.value) + return + end + end end end diff --git a/test/compiler_test.rb b/test/compiler_test.rb index 2af625f6..62a7cc7c 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -92,6 +92,8 @@ class CompilerTest < Minitest::Test "foo.size", "foo.succ", "/foo/ =~ \"foo\" && $1", + "\"foo\".freeze", + "\"foo\".freeze(1)", # Various method calls "foo?", "foo.bar", From 7fa75c946c14b2d546c1b2c781455ccab91ba437 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 16 Nov 2022 17:42:57 -0500 Subject: [PATCH 233/536] opt_str_uminus --- lib/syntax_tree/visitor/compiler.rb | 43 +++++++++++++++++++---------- test/compiler_test.rb | 3 ++ 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/lib/syntax_tree/visitor/compiler.rb b/lib/syntax_tree/visitor/compiler.rb index 00201618..4d77314e 100644 --- a/lib/syntax_tree/visitor/compiler.rb +++ b/lib/syntax_tree/visitor/compiler.rb @@ -709,6 +709,16 @@ def opt_str_freeze(value) end end + def opt_str_uminus(value) + if specialized_instruction + stack.change_by(+1) + iseq.push([:opt_str_uminus, value, call_data(:-@, 0, VM_CALL_ARGS_SIMPLE)]) + else + putstring(value) + send(:-@, 0, VM_CALL_ARGS_SIMPLE) + end + end + def pop stack.change_by(-1) iseq.push([:pop]) @@ -1174,7 +1184,7 @@ def visit_call(node) # First we're going to check if we're calling a method on an array # literal without any arguments. In that case there are some # specializations we might be able to perform. - if arg_parts.length == 0 && node.message.is_a?(Ident) + if arg_parts.length == 0 && (node.message.is_a?(Ident) || node.message.is_a?(Op)) case node.receiver when ArrayLiteral parts = node.receiver.contents&.parts || [] @@ -1194,6 +1204,9 @@ def visit_call(node) when StringLiteral if RubyVisitor.compile(node.receiver).nil? case node.message.value + when "-@" + builder.opt_str_uminus(node.receiver.parts.first.value) + return when "freeze" builder.opt_str_freeze(node.receiver.parts.first.value) return @@ -1277,7 +1290,7 @@ def visit_class(node) end def visit_command(node) - call_node = + visit_call( CallNode.new( receiver: nil, operator: nil, @@ -1285,13 +1298,11 @@ def visit_command(node) arguments: node.arguments, location: node.location ) - - call_node.comments.concat(node.comments) - visit_call(call_node) + ) end def visit_command_call(node) - call_node = + visit_call( CallNode.new( receiver: node.receiver, operator: node.operator, @@ -1299,9 +1310,7 @@ def visit_command_call(node) arguments: node.arguments, location: node.location ) - - call_node.comments.concat(node.comments) - visit_call(call_node) + ) end def visit_const_path_field(node) @@ -1853,17 +1862,23 @@ def visit_tstring_content(node) end def visit_unary(node) - visit(node.statement) - method_id = case node.operator when "+", "-" - :"#{node.operator}@" + "#{node.operator}@" else - node.operator.to_sym + node.operator end - builder.send(method_id, 0, VM_CALL_ARGS_SIMPLE) + visit_call( + CallNode.new( + receiver: node.statement, + operator: nil, + message: Ident.new(value: method_id, location: node.location), + arguments: nil, + location: node.location + ) + ) end def visit_undef(node) diff --git a/test/compiler_test.rb b/test/compiler_test.rb index 62a7cc7c..cdb9e72a 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -94,6 +94,9 @@ class CompilerTest < Minitest::Test "/foo/ =~ \"foo\" && $1", "\"foo\".freeze", "\"foo\".freeze(1)", + "-\"foo\"", + "\"foo\".-@", + "\"foo\".-@(1)", # Various method calls "foo?", "foo.bar", From 1bec2b4ee634c60aa68f45dd81a4a026c68076d1 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 16 Nov 2022 17:51:28 -0500 Subject: [PATCH 234/536] Handle ternaries --- lib/syntax_tree/visitor/compiler.rb | 20 ++++++++++++++++++-- test/compiler_test.rb | 1 + 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/lib/syntax_tree/visitor/compiler.rb b/lib/syntax_tree/visitor/compiler.rb index 4d77314e..be476816 100644 --- a/lib/syntax_tree/visitor/compiler.rb +++ b/lib/syntax_tree/visitor/compiler.rb @@ -1492,6 +1492,22 @@ def visit_if(node) end end + def visit_if_op(node) + visit_if( + IfNode.new( + predicate: node.predicate, + statements: node.truthy, + consequent: + Else.new( + keyword: Kw.new(value: "else", location: Location.default), + statements: node.falsy, + location: Location.default + ), + location: Location.default + ) + ) + end + def visit_imaginary(node) builder.putobject(node.accept(RubyVisitor.new)) end @@ -1874,9 +1890,9 @@ def visit_unary(node) CallNode.new( receiver: node.statement, operator: nil, - message: Ident.new(value: method_id, location: node.location), + message: Ident.new(value: method_id, location: Location.default), arguments: nil, - location: node.location + location: Location.default ) ) end diff --git a/test/compiler_test.rb b/test/compiler_test.rb index cdb9e72a..720e18be 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -260,6 +260,7 @@ class CompilerTest < Minitest::Test "foo if bar", "foo while bar", "for i in [1, 2, 3] do i end", + "foo ? bar : baz", # Constructed values "foo..bar", "foo...bar", From 61924f8f2c0a458e990c5ab1a1c9f688f416cabb Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 16 Nov 2022 17:54:04 -0500 Subject: [PATCH 235/536] Handle elsif --- lib/syntax_tree/visitor/compiler.rb | 11 +++++++++++ test/compiler_test.rb | 1 + 2 files changed, 12 insertions(+) diff --git a/lib/syntax_tree/visitor/compiler.rb b/lib/syntax_tree/visitor/compiler.rb index be476816..32bead26 100644 --- a/lib/syntax_tree/visitor/compiler.rb +++ b/lib/syntax_tree/visitor/compiler.rb @@ -1409,6 +1409,17 @@ def visit_else(node) builder.pop unless last_statement? end + def visit_elsif(node) + visit_if( + IfNode.new( + predicate: node.predicate, + statements: node.statements, + consequent: node.consequent, + location: node.location + ) + ) + end + def visit_field(node) visit(node.parent) end diff --git a/test/compiler_test.rb b/test/compiler_test.rb index 720e18be..fdca6985 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -257,6 +257,7 @@ class CompilerTest < Minitest::Test "foo || bar", "if foo then bar end", "if foo then bar else baz end", + "if foo then bar elsif baz then qux end", "foo if bar", "foo while bar", "for i in [1, 2, 3] do i end", From 76428d06c625ad0ab8bd59e47366656ce80efff8 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 17 Nov 2022 11:26:47 -0500 Subject: [PATCH 236/536] Compile blocks --- lib/syntax_tree/visitor/compiler.rb | 100 +++++++++++++++++++++++----- test/compiler_test.rb | 8 ++- 2 files changed, 89 insertions(+), 19 deletions(-) diff --git a/lib/syntax_tree/visitor/compiler.rb b/lib/syntax_tree/visitor/compiler.rb index 32bead26..10c59a77 100644 --- a/lib/syntax_tree/visitor/compiler.rb +++ b/lib/syntax_tree/visitor/compiler.rb @@ -702,7 +702,13 @@ def opt_setinlinecache(inline_storage) def opt_str_freeze(value) if specialized_instruction stack.change_by(+1) - iseq.push([:opt_str_freeze, value, call_data(:freeze, 0, VM_CALL_ARGS_SIMPLE)]) + iseq.push( + [ + :opt_str_freeze, + value, + call_data(:freeze, 0, VM_CALL_ARGS_SIMPLE) + ] + ) else putstring(value) send(:freeze, 0, VM_CALL_ARGS_SIMPLE) @@ -712,7 +718,9 @@ def opt_str_freeze(value) def opt_str_uminus(value) if specialized_instruction stack.change_by(+1) - iseq.push([:opt_str_uminus, value, call_data(:-@, 0, VM_CALL_ARGS_SIMPLE)]) + iseq.push( + [:opt_str_uminus, value, call_data(:-@, 0, VM_CALL_ARGS_SIMPLE)] + ) else putstring(value) send(:-@, 0, VM_CALL_ARGS_SIMPLE) @@ -1024,7 +1032,7 @@ def visit_args(node) end def visit_array(node) - if compiled = RubyVisitor.compile(node) + if (compiled = RubyVisitor.compile(node)) builder.duparray(compiled) else length = 0 @@ -1045,7 +1053,9 @@ def visit_array(node) end builder.newarray(length) if length > 0 - builder.concatarray if length > 0 && length != node.contents.parts.length + if length > 0 && length != node.contents.parts.length + builder.concatarray + end end end @@ -1134,7 +1144,7 @@ def visit_backref(node) end def visit_bare_assoc_hash(node) - if compiled = RubyVisitor.compile(node) + if (compiled = RubyVisitor.compile(node)) builder.duphash(compiled) else visit_all(node.assocs) @@ -1168,6 +1178,35 @@ def visit_binary(node) end end + def visit_block(node) + with_instruction_sequence( + :block, + "block in #{current_iseq.name}", + current_iseq, + node + ) do + visit(node.block_var) + visit(node.bodystmt) + builder.leave + end + end + + def visit_block_var(node) + params = node.params + + if params.requireds.length == 1 && params.optionals.empty? && + !params.rest && params.posts.empty? && params.keywords.empty? && + !params.keyword_rest && !params.block + current_iseq.argument_options[:ambiguous_param0] = true + end + + visit(node.params) + + node.locals.each do |local| + current_iseq.local_table.plain(local.value.to_sym) + end + end + def visit_blockarg(node) current_iseq.argument_options[:block_start] = current_iseq.argument_size current_iseq.local_table.block_proxy(node.name.value.to_sym) @@ -1184,12 +1223,14 @@ def visit_call(node) # First we're going to check if we're calling a method on an array # literal without any arguments. In that case there are some # specializations we might be able to perform. - if arg_parts.length == 0 && (node.message.is_a?(Ident) || node.message.is_a?(Op)) + if arg_parts.empty? && + (node.message.is_a?(Ident) || node.message.is_a?(Op)) case node.receiver when ArrayLiteral parts = node.receiver.contents&.parts || [] - if parts.none? { |part| part.is_a?(ArgStar) } && RubyVisitor.compile(node.receiver).nil? + if parts.none? { |part| part.is_a?(ArgStar) } && + RubyVisitor.compile(node.receiver).nil? case node.message.value when "max" visit(node.receiver.contents) @@ -1215,13 +1256,10 @@ def visit_call(node) end end - if node.receiver - visit(node.receiver) - else - builder.putself - end + node.receiver ? visit(node.receiver) : builder.putself visit(node.arguments) + block_iseq = visit(node.block) if node.respond_to?(:block) && node.block if arg_parts.last.is_a?(ArgBlock) flag = node.receiver.nil? ? VM_CALL_FCALL : 0 @@ -1235,7 +1273,12 @@ def visit_call(node) flag |= VM_CALL_KW_SPLAT end - builder.send(node.message.value.to_sym, arg_parts.length - 1, flag) + builder.send( + node.message.value.to_sym, + arg_parts.length - 1, + flag, + block_iseq + ) else flag = 0 arg_parts.each do |arg_part| @@ -1247,9 +1290,14 @@ def visit_call(node) end end - flag |= VM_CALL_ARGS_SIMPLE if flag == 0 + flag |= VM_CALL_ARGS_SIMPLE if block_iseq.nil? && flag == 0 flag |= VM_CALL_FCALL if node.receiver.nil? - builder.send(node.message.value.to_sym, arg_parts.length, flag) + builder.send( + node.message.value.to_sym, + arg_parts.length, + flag, + block_iseq + ) end end @@ -1291,11 +1339,12 @@ def visit_class(node) def visit_command(node) visit_call( - CallNode.new( + CommandCall.new( receiver: nil, operator: nil, message: node.message, arguments: node.arguments, + block: node.block, location: node.location ) ) @@ -1303,11 +1352,12 @@ def visit_command(node) def visit_command_call(node) visit_call( - CallNode.new( + CommandCall.new( receiver: node.receiver, operator: node.operator, message: node.message, arguments: node.arguments, + block: node.block, location: node.location ) ) @@ -1537,6 +1587,19 @@ def visit_label(node) builder.putobject(node.accept(RubyVisitor.new)) end + def visit_method_add_block(node) + visit_call( + CommandCall.new( + receiver: node.call.receiver, + operator: node.call.operator, + message: node.call.message, + arguments: node.call.arguments, + block: node.block, + location: node.location + ) + ) + end + def visit_module(node) name = node.constant.constant.value.to_sym module_iseq = @@ -1898,11 +1961,12 @@ def visit_unary(node) end visit_call( - CallNode.new( + CommandCall.new( receiver: node.statement, operator: nil, message: Ident.new(value: method_id, location: Location.default), arguments: nil, + block: nil, location: Location.default ) ) diff --git a/test/compiler_test.rb b/test/compiler_test.rb index fdca6985..fe0bd1f6 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -370,7 +370,13 @@ class CompilerTest < Minitest::Test "class Foo::Bar < Baz; end", "class ::Foo::Bar < Baz; end", "class Foo; class Bar < Baz; end; end", - "class Foo < baz; end" + "class Foo < baz; end", + # Block + "foo do end", + "foo {}", + "foo do |bar| end", + "foo { |bar| }", + "foo { |bar; baz| }" ] # These are the combinations of instructions that we're going to test. From 0b2d012beddc2cb0ed6f4dbdca8486a8ce396b23 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 18 Nov 2022 10:32:57 -0500 Subject: [PATCH 237/536] Handle args forwarding --- lib/syntax_tree/visitor/compiler.rb | 99 +++++++++++++++++------------ test/compiler_test.rb | 4 ++ 2 files changed, 63 insertions(+), 40 deletions(-) diff --git a/lib/syntax_tree/visitor/compiler.rb b/lib/syntax_tree/visitor/compiler.rb index 10c59a77..029d858a 100644 --- a/lib/syntax_tree/visitor/compiler.rb +++ b/lib/syntax_tree/visitor/compiler.rb @@ -1218,13 +1218,26 @@ def visit_bodystmt(node) end def visit_call(node) + if node.is_a?(CallNode) + return visit_call( + CommandCall.new( + receiver: node.receiver, + operator: node.operator, + message: node.message, + arguments: node.arguments, + block: nil, + location: node.location + ) + ) + end + arg_parts = argument_parts(node.arguments) + argc = arg_parts.length # First we're going to check if we're calling a method on an array # literal without any arguments. In that case there are some # specializations we might be able to perform. - if arg_parts.empty? && - (node.message.is_a?(Ident) || node.message.is_a?(Op)) + if argc == 0 && (node.message.is_a?(Ident) || node.message.is_a?(Op)) case node.receiver when ArrayLiteral parts = node.receiver.contents&.parts || [] @@ -1257,48 +1270,39 @@ def visit_call(node) end node.receiver ? visit(node.receiver) : builder.putself - - visit(node.arguments) - block_iseq = visit(node.block) if node.respond_to?(:block) && node.block - - if arg_parts.last.is_a?(ArgBlock) - flag = node.receiver.nil? ? VM_CALL_FCALL : 0 - flag |= VM_CALL_ARGS_BLOCKARG - - if arg_parts.any? { |part| part.is_a?(ArgStar) } + flag = 0 + + arg_parts.each do |arg_part| + case arg_part + when ArgBlock + argc -= 1 + flag |= VM_CALL_ARGS_BLOCKARG + visit(arg_part) + when ArgStar flag |= VM_CALL_ARGS_SPLAT - end + visit(arg_part) + when ArgsForward + flag |= VM_CALL_ARGS_SPLAT | VM_CALL_ARGS_BLOCKARG - if arg_parts.any? { |part| part.is_a?(BareAssocHash) } + lookup = current_iseq.local_table.find(:*, 0) + builder.getlocal(lookup.index, lookup.level) + builder.splatarray(arg_parts.length != 1) + + lookup = current_iseq.local_table.find(:&, 0) + builder.getblockparamproxy(lookup.index, lookup.level) + when BareAssocHash flag |= VM_CALL_KW_SPLAT + visit(arg_part) + else + visit(arg_part) end + end - builder.send( - node.message.value.to_sym, - arg_parts.length - 1, - flag, - block_iseq - ) - else - flag = 0 - arg_parts.each do |arg_part| - case arg_part - when ArgStar - flag |= VM_CALL_ARGS_SPLAT - when BareAssocHash - flag |= VM_CALL_KW_SPLAT - end - end + block_iseq = visit(node.block) if node.block + flag |= VM_CALL_ARGS_SIMPLE if block_iseq.nil? && flag == 0 + flag |= VM_CALL_FCALL if node.receiver.nil? - flag |= VM_CALL_ARGS_SIMPLE if block_iseq.nil? && flag == 0 - flag |= VM_CALL_FCALL if node.receiver.nil? - builder.send( - node.message.value.to_sym, - arg_parts.length, - flag, - block_iseq - ) - end + builder.send(node.message.value.to_sym, argc, flag, block_iseq) end def visit_class(node) @@ -1781,7 +1785,18 @@ def visit_params(node) checkkeywords.each { |checkkeyword| checkkeyword[1] = lookup.index } end - visit(node.keyword_rest) if node.keyword_rest + if node.keyword_rest.is_a?(ArgsForward) + current_iseq.local_table.plain(:*) + current_iseq.local_table.plain(:&) + + current_iseq.argument_options[:rest_start] = current_iseq.argument_size + current_iseq.argument_options[:block_start] = current_iseq.argument_size + 1 + + current_iseq.argument_size += 2 + elsif node.keyword_rest + visit(node.keyword_rest) + end + visit(node.block) if node.block end @@ -2122,7 +2137,11 @@ def argument_parts(node) when Args node.parts when ArgParen - node.arguments.parts + if node.arguments.is_a?(ArgsForward) + [node.arguments] + else + node.arguments.parts + end when Paren node.contents.parts end diff --git a/test/compiler_test.rb b/test/compiler_test.rb index fe0bd1f6..ec3766e2 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -354,6 +354,10 @@ class CompilerTest < Minitest::Test "def foo(bar: 1, baz: qux, **rest); end", "def foo(bar: qux, baz: 1, **rest); end", "def foo(bar: baz, qux: qaz, **rest); end", + "def foo(...); end", + "def foo(bar, ...); end", + "def foo(...); bar(...); end", + "def foo(bar, ...); baz(1, 2, 3, ...); end", # Class/module definitions "module Foo; end", "module ::Foo; end", From c59c58550b19186a7d8e61fd3b6bee3796760263 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 18 Nov 2022 10:35:12 -0500 Subject: [PATCH 238/536] Handle until --- lib/syntax_tree/visitor/compiler.rb | 18 ++++++++++++++++++ test/compiler_test.rb | 3 +++ 2 files changed, 21 insertions(+) diff --git a/lib/syntax_tree/visitor/compiler.rb b/lib/syntax_tree/visitor/compiler.rb index 029d858a..1fe4365f 100644 --- a/lib/syntax_tree/visitor/compiler.rb +++ b/lib/syntax_tree/visitor/compiler.rb @@ -1997,6 +1997,24 @@ def visit_undef(node) end end + def visit_until(node) + jumps = [] + + jumps << builder.jump(-1) + builder.putnil + builder.pop + jumps << builder.jump(-1) + + label = builder.label + visit(node.statements) + builder.pop + jumps.each { |jump| jump[1] = builder.label } + + visit(node.predicate) + builder.branchunless(label) + builder.putnil if last_statement? + end + def visit_var_field(node) case node.value when CVar, IVar diff --git a/test/compiler_test.rb b/test/compiler_test.rb index ec3766e2..9fd3cfe9 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -260,6 +260,9 @@ class CompilerTest < Minitest::Test "if foo then bar elsif baz then qux end", "foo if bar", "foo while bar", + "while foo do bar end", + "foo until bar", + "until foo do bar end", "for i in [1, 2, 3] do i end", "foo ? bar : baz", # Constructed values From 2f78d142b15a2aefae11d4dbf94961ad9c4fee28 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 18 Nov 2022 10:38:49 -0500 Subject: [PATCH 239/536] Handle sclass --- lib/syntax_tree/visitor/compiler.rb | 13 +++++++++++++ test/compiler_test.rb | 2 ++ 2 files changed, 15 insertions(+) diff --git a/lib/syntax_tree/visitor/compiler.rb b/lib/syntax_tree/visitor/compiler.rb index 1fe4365f..d93021ef 100644 --- a/lib/syntax_tree/visitor/compiler.rb +++ b/lib/syntax_tree/visitor/compiler.rb @@ -1876,6 +1876,19 @@ def visit_rest_param(node) current_iseq.argument_size += 1 end + def visit_sclass(node) + visit(node.target) + builder.putnil + + singleton_iseq = + with_instruction_sequence(:class, "singleton class", current_iseq, node) do + visit(node.bodystmt) + builder.leave + end + + builder.defineclass(:singletonclass, singleton_iseq, VM_DEFINECLASS_TYPE_SINGLETON_CLASS) + end + def visit_statements(node) statements = node.body.select do |statement| diff --git a/test/compiler_test.rb b/test/compiler_test.rb index 9fd3cfe9..aba9cff5 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -378,6 +378,8 @@ class CompilerTest < Minitest::Test "class ::Foo::Bar < Baz; end", "class Foo; class Bar < Baz; end; end", "class Foo < baz; end", + "class << Object; end", + "class << ::String; end", # Block "foo do end", "foo {}", From 69bf4bcf447180e5bfc5daf20a282815cbbad307 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 18 Nov 2022 10:44:00 -0500 Subject: [PATCH 240/536] Handle lambda --- lib/syntax_tree/visitor/compiler.rb | 16 ++++++++++++++++ test/compiler_test.rb | 7 ++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/lib/syntax_tree/visitor/compiler.rb b/lib/syntax_tree/visitor/compiler.rb index d93021ef..e5807c0f 100644 --- a/lib/syntax_tree/visitor/compiler.rb +++ b/lib/syntax_tree/visitor/compiler.rb @@ -1591,6 +1591,22 @@ def visit_label(node) builder.putobject(node.accept(RubyVisitor.new)) end + def visit_lambda(node) + lambda_iseq = + with_instruction_sequence(:block, "block in #{current_iseq.name}", current_iseq, node) do + visit(node.params) + visit(node.statements) + builder.leave + end + + builder.putspecialobject(VM_SPECIAL_OBJECT_VMCORE) + builder.send(:lambda, 0, VM_CALL_FCALL, lambda_iseq) + end + + def visit_lambda_var(node) + visit_block_var(node) + end + def visit_method_add_block(node) visit_call( CommandCall.new( diff --git a/test/compiler_test.rb b/test/compiler_test.rb index aba9cff5..d0fed6fd 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -385,7 +385,12 @@ class CompilerTest < Minitest::Test "foo {}", "foo do |bar| end", "foo { |bar| }", - "foo { |bar; baz| }" + "foo { |bar; baz| }", + "-> do end", + "-> {}", + "-> (bar) do end", + "-> (bar) {}", + "-> (bar; baz) { }" ] # These are the combinations of instructions that we're going to test. From 5fa8a6f44800a638b14c886a0a11b7a2e9bd21bb Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 18 Nov 2022 10:47:49 -0500 Subject: [PATCH 241/536] Handle unless --- lib/syntax_tree/visitor/compiler.rb | 24 ++++++++++++++++++++++++ test/compiler_test.rb | 3 +++ 2 files changed, 27 insertions(+) diff --git a/lib/syntax_tree/visitor/compiler.rb b/lib/syntax_tree/visitor/compiler.rb index e5807c0f..fab538ae 100644 --- a/lib/syntax_tree/visitor/compiler.rb +++ b/lib/syntax_tree/visitor/compiler.rb @@ -2026,6 +2026,30 @@ def visit_undef(node) end end + def visit_unless(node) + visit(node.predicate) + branchunless = builder.branchunless(-1) + node.consequent ? visit(node.consequent) : builder.putnil + + if last_statement? + builder.leave + branchunless[1] = builder.label + + visit(node.statements) + else + builder.pop + + if node.consequent + jump = builder.jump(-1) + branchunless[1] = builder.label + visit(node.consequent) + jump[1] = builder.label + else + branchunless[1] = builder.label + end + end + end + def visit_until(node) jumps = [] diff --git a/test/compiler_test.rb b/test/compiler_test.rb index d0fed6fd..c9476b81 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -259,6 +259,9 @@ class CompilerTest < Minitest::Test "if foo then bar else baz end", "if foo then bar elsif baz then qux end", "foo if bar", + "unless foo then bar end", + "unless foo then bar else baz end", + "foo unless bar", "foo while bar", "while foo do bar end", "foo until bar", From 8e88b0d09fe32f759339fa6fc61d1efdf2e5d1cb Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 18 Nov 2022 10:51:06 -0500 Subject: [PATCH 242/536] definesmethod --- lib/syntax_tree/visitor/compiler.rb | 16 ++++++++++++++-- test/compiler_test.rb | 2 ++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/lib/syntax_tree/visitor/compiler.rb b/lib/syntax_tree/visitor/compiler.rb index fab538ae..c8e5b3e8 100644 --- a/lib/syntax_tree/visitor/compiler.rb +++ b/lib/syntax_tree/visitor/compiler.rb @@ -418,7 +418,7 @@ def serialize(insn) [insn[0], iseq.local_table.offset(insn[1]), *insn[2..]] when :defineclass [insn[0], insn[1], insn[2].to_a, insn[3]] - when :definemethod + when :definemethod, :definesmethod [insn[0], insn[1], insn[2].to_a] when :send # For any instructions that push instruction sequences onto the @@ -511,6 +511,11 @@ def definemethod(name, method_iseq) iseq.push([:definemethod, name, method_iseq]) end + def definesmethod(name, method_iseq) + stack.change_by(-1) + iseq.push([:definesmethod, name, method_iseq]) + end + def dup stack.change_by(-1 + 2) iseq.push([:dup]) @@ -1390,7 +1395,14 @@ def visit_def(node) end name = node.name.value.to_sym - builder.definemethod(name, method_iseq) + + if node.target + visit(node.target) + builder.definesmethod(name, method_iseq) + else + builder.definemethod(name, method_iseq) + end + builder.putobject(name) end diff --git a/test/compiler_test.rb b/test/compiler_test.rb index c9476b81..af42fe0a 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -364,6 +364,8 @@ class CompilerTest < Minitest::Test "def foo(bar, ...); end", "def foo(...); bar(...); end", "def foo(bar, ...); baz(1, 2, 3, ...); end", + "def self.foo; end", + "def foo.bar(baz); end", # Class/module definitions "module Foo; end", "module ::Foo; end", From f8ac1227dad0fc0c54db01c38e916d4e0f47b109 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 18 Nov 2022 10:57:28 -0500 Subject: [PATCH 243/536] getblockparam --- lib/syntax_tree/visitor/compiler.rb | 38 ++++++++++++++++++++--------- test/compiler_test.rb | 1 + 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/lib/syntax_tree/visitor/compiler.rb b/lib/syntax_tree/visitor/compiler.rb index c8e5b3e8..df5e4838 100644 --- a/lib/syntax_tree/visitor/compiler.rb +++ b/lib/syntax_tree/visitor/compiler.rb @@ -209,7 +209,7 @@ def change_by(value) class LocalTable # A local representing a block passed into the current instruction # sequence. - class BlockProxyLocal + class BlockLocal attr_reader :name def initialize(name) @@ -260,9 +260,9 @@ def size locals.length end - # Add a BlockProxyLocal to the local table. - def block_proxy(name) - locals << BlockProxyLocal.new(name) unless has?(name) + # Add a BlockLocal to the local table. + def block(name) + locals << BlockLocal.new(name) unless has?(name) end # Add a PlainLocal to the local table. @@ -401,15 +401,15 @@ def to_a def serialize(insn) case insn[0] - when :checkkeyword, :getblockparamproxy, :getlocal_WC_0, - :getlocal_WC_1, :getlocal, :setlocal_WC_0, :setlocal_WC_1, - :setlocal + when :checkkeyword, :getblockparam, :getblockparamproxy, + :getlocal_WC_0, :getlocal_WC_1, :getlocal, + :setlocal_WC_0, :setlocal_WC_1, :setlocal iseq = self case insn[0] when :getlocal_WC_1, :setlocal_WC_1 iseq = iseq.parent_iseq - when :getblockparamproxy, :getlocal, :setlocal + when :getblockparam, :getblockparamproxy, :getlocal, :setlocal insn[2].times { iseq = iseq.parent_iseq } end @@ -536,6 +536,11 @@ def dupn(number) iseq.push([:dupn, number]) end + def getblockparam(index, level) + stack.change_by(+1) + iseq.push([:getblockparam, index, level]) + end + def getblockparamproxy(index, level) stack.change_by(+1) iseq.push([:getblockparamproxy, index, level]) @@ -1214,7 +1219,7 @@ def visit_block_var(node) def visit_blockarg(node) current_iseq.argument_options[:block_start] = current_iseq.argument_size - current_iseq.local_table.block_proxy(node.name.value.to_sym) + current_iseq.local_table.block(node.name.value.to_sym) current_iseq.argument_size += 1 end @@ -1274,7 +1279,16 @@ def visit_call(node) end end - node.receiver ? visit(node.receiver) : builder.putself + if node.receiver + if node.receiver.is_a?(VarRef) && (lookup = current_iseq.local_variable(node.receiver.value.value.to_sym)) && lookup.local.is_a?(LocalTable::BlockLocal) + builder.getblockparamproxy(lookup.index, lookup.level) + else + visit(node.receiver) + end + else + builder.putself + end + flag = 0 arg_parts.each do |arg_part| @@ -2105,8 +2119,8 @@ def visit_var_ref(node) lookup = current_iseq.local_variable(node.value.value.to_sym) case lookup.local - when LocalTable::BlockProxyLocal - builder.getblockparamproxy(lookup.index, lookup.level) + when LocalTable::BlockLocal + builder.getblockparam(lookup.index, lookup.level) when LocalTable::PlainLocal builder.getlocal(lookup.index, lookup.level) end diff --git a/test/compiler_test.rb b/test/compiler_test.rb index af42fe0a..5f320ef2 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -342,6 +342,7 @@ class CompilerTest < Minitest::Test "def foo(bar, baz, *qux, quaz); end", "def foo(bar, baz, &qux); end", "def foo(bar, *baz, &qux); end", + "def foo(&qux); qux; end", "def foo(&qux); qux.call; end", "def foo(bar:); end", "def foo(bar:, baz:); end", From db88d3309390f061c62fa2896bd8d07d9ab81f61 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 18 Nov 2022 11:17:51 -0500 Subject: [PATCH 244/536] Handle case/when --- lib/syntax_tree/visitor/compiler.rb | 47 +++++++++++++++++++++++++++++ test/compiler_test.rb | 2 ++ 2 files changed, 49 insertions(+) diff --git a/lib/syntax_tree/visitor/compiler.rb b/lib/syntax_tree/visitor/compiler.rb index df5e4838..331a937e 100644 --- a/lib/syntax_tree/visitor/compiler.rb +++ b/lib/syntax_tree/visitor/compiler.rb @@ -1324,6 +1324,49 @@ def visit_call(node) builder.send(node.message.value.to_sym, argc, flag, block_iseq) end + def visit_case(node) + visit(node.value) if node.value + + clauses = [] + else_clause = nil + + current = node.consequent + + while current + clauses << current + + if (current = current.consequent).is_a?(Else) + else_clause = current + break + end + end + + branches = + clauses.map do |clause| + visit(clause.arguments) + builder.topn(1) + builder.send(:===, 1, VM_CALL_FCALL | VM_CALL_ARGS_SIMPLE) + [clause, builder.branchif(:label_00)] + end + + builder.pop + + if else_clause + visit(else_clause) + else + builder.putnil + end + + builder.leave + + branches.each_with_index do |(clause, branchif), index| + builder.leave if index != 0 + branchif[1] = builder.label + builder.pop + visit(clause) + end + end + def visit_class(node) name = node.constant.constant.value.to_sym class_iseq = @@ -2148,6 +2191,10 @@ def visit_vcall(node) builder.send(node.value.value.to_sym, 0, flag) end + def visit_when(node) + visit(node.statements) + end + def visit_while(node) jumps = [] diff --git a/test/compiler_test.rb b/test/compiler_test.rb index 5f320ef2..8b95c07a 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -268,6 +268,8 @@ class CompilerTest < Minitest::Test "until foo do bar end", "for i in [1, 2, 3] do i end", "foo ? bar : baz", + "case foo when bar then 1 end", + "case foo when bar then 1 else 2 end", # Constructed values "foo..bar", "foo...bar", From a7e78259dccd31606e1e2078ae72c3a22ccf5634 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 18 Nov 2022 11:41:39 -0500 Subject: [PATCH 245/536] Handle BEGIN{} and END{} --- lib/syntax_tree/visitor/compiler.rb | 70 +++++++++++++++++++++++------ 1 file changed, 56 insertions(+), 14 deletions(-) diff --git a/lib/syntax_tree/visitor/compiler.rb b/lib/syntax_tree/visitor/compiler.rb index 331a937e..8114a9c5 100644 --- a/lib/syntax_tree/visitor/compiler.rb +++ b/lib/syntax_tree/visitor/compiler.rb @@ -336,8 +336,6 @@ def local_variable(name, level = 0) lookup elsif parent_iseq parent_iseq.local_variable(name, level + 1) - else - raise "Unknown local variable: #{name}" end end @@ -388,7 +386,7 @@ def to_a name, "", "", - 1, + location.start_line, type, local_table.names, argument_options, @@ -424,6 +422,8 @@ def serialize(insn) # For any instructions that push instruction sequences onto the # stack, we need to call #to_a on them as well. [insn[0], insn[1], (insn[2].to_a if insn[2])] + when :once + [insn[0], insn[1].to_a, insn[2]] else insn end @@ -655,6 +655,11 @@ def objtostring(method_id, argc, flag) iseq.push([:objtostring, call_data(method_id, argc, flag)]) end + def once(postexe_iseq, inline_storage) + stack.change_by(+1) + iseq.push([:once, postexe_iseq, inline_storage]) + end + def opt_getconstant_path(names) if RUBY_VERSION >= "3.2" stack.change_by(+1) @@ -1002,6 +1007,10 @@ def initialize( @last_statement = false end + def visit_BEGIN(node) + visit(node.statements) + end + def visit_CHAR(node) if frozen_string_literal builder.putobject(node.value[1..]) @@ -1010,6 +1019,27 @@ def visit_CHAR(node) end end + def visit_END(node) + name = "block in #{current_iseq.name}" + once_iseq = + with_instruction_sequence(:block, name, current_iseq, node) do + postexe_iseq = + with_instruction_sequence(:block, name, current_iseq, node) do + *statements, last_statement = node.statements.body + visit_all(statements) + with_last_statement { visit(last_statement) } + builder.leave + end + + builder.putspecialobject(VM_SPECIAL_OBJECT_VMCORE) + builder.send(:"core#set_postexe", 0, VM_CALL_FCALL, postexe_iseq) + builder.leave + end + + builder.once(once_iseq, current_iseq.inline_storage) + builder.pop + end + def visit_alias(node) builder.putspecialobject(VM_SPECIAL_OBJECT_VMCORE) builder.putspecialobject(VM_SPECIAL_OBJECT_CBASE) @@ -1898,17 +1928,23 @@ def visit_program(node) end end - statements = - node.statements.body.select do |statement| - case statement - when Comment, EmbDoc, EndContent, VoidStmt - false - else - true - end + preexes = [] + statements = [] + + node.statements.body.each do |statement| + case statement + when Comment, EmbDoc, EndContent, VoidStmt + # ignore + when BEGINBlock + preexes << statement + else + statements << statement end + end with_instruction_sequence(:top, "", nil, node) do + visit_all(preexes) + if statements.empty? builder.putnil else @@ -2144,8 +2180,13 @@ def visit_var_field(node) current_iseq.inline_storage_for(name) when Ident name = node.value.value.to_sym - current_iseq.local_table.plain(name) - current_iseq.local_variable(name) + + if (local_variable = current_iseq.local_variable(name)) + local_variable + else + current_iseq.local_table.plain(name) + current_iseq.local_variable(name) + end end end @@ -2460,12 +2501,13 @@ def with_instruction_sequence(type, name, parent_iseq, node) # last statement of a scope and allow visit methods to query that # information. def with_last_statement + previous = @last_statement @last_statement = true begin yield ensure - @last_statement = false + @last_statement = previous end end From b022d5617560f57cc23a367946a4722b5dd6a9fd Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 18 Nov 2022 11:50:43 -0500 Subject: [PATCH 246/536] Handle mrhs, mlhs, massign --- lib/syntax_tree/visitor/compiler.rb | 41 +++++++++++++++++++++++++++++ test/compiler_test.rb | 6 +++++ 2 files changed, 47 insertions(+) diff --git a/lib/syntax_tree/visitor/compiler.rb b/lib/syntax_tree/visitor/compiler.rb index 8114a9c5..c09efe84 100644 --- a/lib/syntax_tree/visitor/compiler.rb +++ b/lib/syntax_tree/visitor/compiler.rb @@ -94,6 +94,10 @@ def visit_label(node) node.value.chomp(":").to_sym end + def visit_mrhs(node) + visit_all(node.parts) + end + def visit_qsymbols(node) node.elements.map { |element| visit(element).to_sym } end @@ -536,6 +540,11 @@ def dupn(number) iseq.push([:dupn, number]) end + def expandarray(length, flag) + stack.change_by(-1 + length) + iseq.push([:expandarray, length, flag]) + end + def getblockparam(index, level) stack.change_by(+1) iseq.push([:getblockparam, index, level]) @@ -1706,6 +1715,12 @@ def visit_lambda_var(node) visit_block_var(node) end + def visit_massign(node) + visit(node.value) + builder.dup + visit(node.target) + end + def visit_method_add_block(node) visit_call( CommandCall.new( @@ -1719,6 +1734,23 @@ def visit_method_add_block(node) ) end + def visit_mlhs(node) + lookups = [] + + node.parts.each do |part| + case part + when VarField + lookups << visit(part) + end + end + + builder.expandarray(lookups.length, 0) + + lookups.each do |lookup| + builder.setlocal(lookup.index, lookup.level) + end + end + def visit_module(node) name = node.constant.constant.value.to_sym module_iseq = @@ -1749,6 +1781,15 @@ def visit_module(node) builder.defineclass(name, module_iseq, flags) end + def visit_mrhs(node) + if (compiled = RubyVisitor.compile(node)) + builder.duparray(compiled) + else + visit_all(node.parts) + builder.newarray(node.parts.length) + end + end + def visit_not(node) visit(node.statement) builder.send(:!, 0, VM_CALL_ARGS_SIMPLE) diff --git a/test/compiler_test.rb b/test/compiler_test.rb index 8b95c07a..8868b801 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -130,6 +130,12 @@ class CompilerTest < Minitest::Test "foo ||= 1", "foo <<= 1", "foo ^= 1", + "foo, bar = 1, 2", + "foo, bar, = 1, 2", + "foo, bar, baz = 1, 2", + "foo, bar = 1, 2, 3", + "foo = 1, 2, 3", + "foo, * = 1, 2, 3", # Instance variables "@foo", "@foo = 1", From 358d029abd6bfa9bafee78154f89a03c95999d04 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 18 Nov 2022 11:55:54 -0500 Subject: [PATCH 247/536] Better handle visit_string_parts --- lib/syntax_tree/visitor/compiler.rb | 44 ++++++++++++++--------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/lib/syntax_tree/visitor/compiler.rb b/lib/syntax_tree/visitor/compiler.rb index c09efe84..8b42613c 100644 --- a/lib/syntax_tree/visitor/compiler.rb +++ b/lib/syntax_tree/visitor/compiler.rb @@ -1636,8 +1636,8 @@ def visit_heredoc(node) elsif node.parts.length == 1 && node.parts.first.is_a?(TStringContent) visit(node.parts.first) else - visit_string_parts(node) - builder.concatstrings(node.parts.length) + length = visit_string_parts(node) + builder.concatstrings(length) end end @@ -2026,10 +2026,9 @@ def visit_rational(node) def visit_regexp_literal(node) builder.putobject(node.accept(RubyVisitor.new)) rescue RubyVisitor::CompilationError - visit_string_parts(node) - flags = RubyVisitor.new.visit_regexp_literal_flags(node) - builder.toregexp(flags, node.parts.length) + length = visit_string_parts(node) + builder.toregexp(flags, length) end def visit_rest_param(node) @@ -2086,8 +2085,8 @@ def visit_string_literal(node) if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) visit(node.parts.first) else - visit_string_parts(node) - builder.concatstrings(node.parts.length) + length = visit_string_parts(node) + builder.concatstrings(length) end end @@ -2114,13 +2113,7 @@ def visit_symbols(node) element.parts.first.is_a?(TStringContent) builder.putobject(element.parts.first.value.to_sym) else - length = element.parts.length - unless element.parts.first.is_a?(TStringContent) - builder.putobject("") - length += 1 - end - - visit_string_parts(element) + length = visit_string_parts(element) builder.concatstrings(length) builder.intern end @@ -2299,13 +2292,7 @@ def visit_word(node) if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) visit(node.parts.first) else - length = node.parts.length - unless node.parts.first.is_a?(TStringContent) - builder.putobject("") - length += 1 - end - - visit_string_parts(node) + length = visit_string_parts(node) builder.concatstrings(length) end end @@ -2330,8 +2317,8 @@ def visit_words(node) def visit_xstring_literal(node) builder.putself - visit_string_parts(node) - builder.concatstrings(node.parts.length) if node.parts.length > 1 + length = visit_string_parts(node) + builder.concatstrings(node.parts.length) if length > 1 builder.send(:`, 1, VM_CALL_FCALL | VM_CALL_ARGS_SIMPLE) end @@ -2493,6 +2480,13 @@ def push_interpolate # heredocs, etc. This method will visit all the parts of a string within # those containers. def visit_string_parts(node) + length = 0 + + unless node.parts.first.is_a?(TStringContent) + builder.putobject("") + length += 1 + end + node.parts.each do |part| case part when StringDVar @@ -2504,7 +2498,11 @@ def visit_string_parts(node) when TStringContent builder.putobject(part.accept(RubyVisitor.new)) end + + length += 1 end + + length end # The current instruction sequence that we're compiling is always stored From 593486dff299c01b39bf16a0ad4cd40a9147e45f Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 18 Nov 2022 11:59:56 -0500 Subject: [PATCH 248/536] Handle the &. operator --- lib/syntax_tree/visitor/compiler.rb | 82 ++++++++++++++++++++--------- test/compiler_test.rb | 4 ++ 2 files changed, 60 insertions(+), 26 deletions(-) diff --git a/lib/syntax_tree/visitor/compiler.rb b/lib/syntax_tree/visitor/compiler.rb index 8b42613c..bac8b914 100644 --- a/lib/syntax_tree/visitor/compiler.rb +++ b/lib/syntax_tree/visitor/compiler.rb @@ -404,8 +404,8 @@ def to_a def serialize(insn) case insn[0] when :checkkeyword, :getblockparam, :getblockparamproxy, - :getlocal_WC_0, :getlocal_WC_1, :getlocal, - :setlocal_WC_0, :setlocal_WC_1, :setlocal + :getlocal_WC_0, :getlocal_WC_1, :getlocal, :setlocal_WC_0, + :setlocal_WC_1, :setlocal iseq = self case insn[0] @@ -480,6 +480,11 @@ def branchif(index) iseq.push([:branchif, index]) end + def branchnil(index) + stack.change_by(-1) + iseq.push([:branchnil, index]) + end + def branchunless(index) stack.change_by(-1) iseq.push([:branchunless, index]) @@ -1268,14 +1273,16 @@ def visit_bodystmt(node) def visit_call(node) if node.is_a?(CallNode) - return visit_call( - CommandCall.new( - receiver: node.receiver, - operator: node.operator, - message: node.message, - arguments: node.arguments, - block: nil, - location: node.location + return( + visit_call( + CommandCall.new( + receiver: node.receiver, + operator: node.operator, + message: node.message, + arguments: node.arguments, + block: nil, + location: node.location + ) ) ) end @@ -1319,7 +1326,11 @@ def visit_call(node) end if node.receiver - if node.receiver.is_a?(VarRef) && (lookup = current_iseq.local_variable(node.receiver.value.value.to_sym)) && lookup.local.is_a?(LocalTable::BlockLocal) + if node.receiver.is_a?(VarRef) && + ( + lookup = + current_iseq.local_variable(node.receiver.value.value.to_sym) + ) && lookup.local.is_a?(LocalTable::BlockLocal) builder.getblockparamproxy(lookup.index, lookup.level) else visit(node.receiver) @@ -1328,6 +1339,12 @@ def visit_call(node) builder.putself end + branchnil = + if node.operator&.value == "&." + builder.dup + builder.branchnil(-1) + end + flag = 0 arg_parts.each do |arg_part| @@ -1361,6 +1378,7 @@ def visit_call(node) flag |= VM_CALL_FCALL if node.receiver.nil? builder.send(node.message.value.to_sym, argc, flag, block_iseq) + branchnil[1] = builder.label if branchnil end def visit_case(node) @@ -1390,11 +1408,7 @@ def visit_case(node) builder.pop - if else_clause - visit(else_clause) - else - builder.putnil - end + else_clause ? visit(else_clause) : builder.putnil builder.leave @@ -1701,7 +1715,12 @@ def visit_label(node) def visit_lambda(node) lambda_iseq = - with_instruction_sequence(:block, "block in #{current_iseq.name}", current_iseq, node) do + with_instruction_sequence( + :block, + "block in #{current_iseq.name}", + current_iseq, + node + ) do visit(node.params) visit(node.statements) builder.leave @@ -1746,9 +1765,7 @@ def visit_mlhs(node) builder.expandarray(lookups.length, 0) - lookups.each do |lookup| - builder.setlocal(lookup.index, lookup.level) - end + lookups.each { |lookup| builder.setlocal(lookup.index, lookup.level) } end def visit_module(node) @@ -1944,10 +1961,14 @@ def visit_params(node) if node.keyword_rest.is_a?(ArgsForward) current_iseq.local_table.plain(:*) current_iseq.local_table.plain(:&) - - current_iseq.argument_options[:rest_start] = current_iseq.argument_size - current_iseq.argument_options[:block_start] = current_iseq.argument_size + 1 - + + current_iseq.argument_options[ + :rest_start + ] = current_iseq.argument_size + current_iseq.argument_options[ + :block_start + ] = current_iseq.argument_size + 1 + current_iseq.argument_size += 2 elsif node.keyword_rest visit(node.keyword_rest) @@ -2042,12 +2063,21 @@ def visit_sclass(node) builder.putnil singleton_iseq = - with_instruction_sequence(:class, "singleton class", current_iseq, node) do + with_instruction_sequence( + :class, + "singleton class", + current_iseq, + node + ) do visit(node.bodystmt) builder.leave end - builder.defineclass(:singletonclass, singleton_iseq, VM_DEFINECLASS_TYPE_SINGLETON_CLASS) + builder.defineclass( + :singletonclass, + singleton_iseq, + VM_DEFINECLASS_TYPE_SINGLETON_CLASS + ) end def visit_statements(node) diff --git a/test/compiler_test.rb b/test/compiler_test.rb index 8868b801..7afd920e 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -259,6 +259,10 @@ class CompilerTest < Minitest::Test "Foo::Bar.baz = 1", "::Foo::Bar.baz = 1", # Control flow + "foo&.bar", + "foo&.bar(1)", + "foo&.bar 1, 2, 3", + "foo&.bar {}", "foo && bar", "foo || bar", "if foo then bar end", From 7c58e9204e12c84f17825175fa65bc67f3489bd0 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 18 Nov 2022 13:48:11 -0500 Subject: [PATCH 249/536] Test evaluation --- test/compiler_test.rb | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/test/compiler_test.rb b/test/compiler_test.rb index 7afd920e..632b3e55 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -2,9 +2,17 @@ return if !defined?(RubyVM::InstructionSequence) || RUBY_VERSION < "3.1" require_relative "test_helper" +require "fiddle" module SyntaxTree class CompilerTest < Minitest::Test + ISEQ_LOAD = + Fiddle::Function.new( + Fiddle::Handle::DEFAULT["rb_iseq_load"], + [Fiddle::TYPE_VOIDP] * 3, + Fiddle::TYPE_VOIDP + ) + CASES = [ # Various literals placed on the stack "true", @@ -430,6 +438,11 @@ class CompilerTest < Minitest::Test end end + def test_evaluation + assert_evaluates 5, "2 + 3" + assert_evaluates 5, "a = 2; b = 3; a + b" + end + private def serialize_iseq(iseq) @@ -463,5 +476,17 @@ def assert_compiles(source, **options) serialize_iseq(program.accept(Visitor::Compiler.new(**options))) ) end + + def assert_evaluates(expected, source, **options) + program = SyntaxTree.parse(source) + compiled = program.accept(Visitor::Compiler.new(**options)).to_a + + # Temporary hack until we get these working. + compiled[4][:node_id] = 11 + compiled[4][:node_ids] = [1, 0, 3, 2, 6, 7, 9, -1] + + iseq = Fiddle.dlunwrap(ISEQ_LOAD.call(Fiddle.dlwrap(compiled), 0, nil)) + assert_equal expected, iseq.eval + end end end From af8c5203f92e5b8f45ba07c60690aa43ad17e7f4 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 18 Nov 2022 14:04:42 -0500 Subject: [PATCH 250/536] Handle tracepoint events except line --- lib/syntax_tree/visitor/compiler.rb | 31 ++++++++++++++++++++++++++--- test/compiler_test.rb | 21 +++++++++++-------- 2 files changed, 41 insertions(+), 11 deletions(-) diff --git a/lib/syntax_tree/visitor/compiler.rb b/lib/syntax_tree/visitor/compiler.rb index bac8b914..82155d37 100644 --- a/lib/syntax_tree/visitor/compiler.rb +++ b/lib/syntax_tree/visitor/compiler.rb @@ -285,6 +285,8 @@ def offset(index) # list of instructions along with the metadata pertaining to them. It also # functions as a builder for the instruction sequence. class InstructionSequence + MAGIC = "YARVInstructionSequence/SimpleDataFormat" + # The type of the instruction sequence. attr_reader :type @@ -363,7 +365,9 @@ def inline_storage_for(name) end def length - insns.sum(&:length) + insns.inject(0) do |sum, insn| + insn.is_a?(Array) ? sum + insn.length : sum + end end def each_child @@ -378,7 +382,7 @@ def to_a versions = RUBY_VERSION.split(".").map(&:to_i) [ - "YARVInstructionSequence/SimpleDataFormat", + MAGIC, versions[0], versions[1], 1, @@ -462,7 +466,13 @@ def initialize( # This creates a new label at the current length of the instruction # sequence. It is used as the operand for jump instructions. def label - :"label_#{iseq.length}" + name = :"label_#{iseq.length}" + iseq.insns.last == name ? name : event(name) + end + + def event(name) + iseq.push(name) + name end def adjuststack(number) @@ -1239,8 +1249,10 @@ def visit_block(node) current_iseq, node ) do + builder.event(:RUBY_EVENT_B_CALL) visit(node.block_var) visit(node.bodystmt) + builder.event(:RUBY_EVENT_B_RETURN) builder.leave end end @@ -1429,7 +1441,9 @@ def visit_class(node) current_iseq, node ) do + builder.event(:RUBY_EVENT_CLASS) visit(node.bodystmt) + builder.event(:RUBY_EVENT_END) builder.leave end @@ -1500,7 +1514,9 @@ def visit_def(node) node ) do visit(node.params) if node.params + builder.event(:RUBY_EVENT_CALL) visit(node.bodystmt) + builder.event(:RUBY_EVENT_RETURN) builder.leave end @@ -1628,9 +1644,12 @@ def visit_for(node) local_variable = current_iseq.local_variable(name) builder.setlocal(local_variable.index, local_variable.level) + + builder.event(:RUBY_EVENT_B_CALL) builder.nop visit(node.statements) + builder.event(:RUBY_EVENT_B_RETURN) builder.leave end @@ -1721,8 +1740,10 @@ def visit_lambda(node) current_iseq, node ) do + builder.event(:RUBY_EVENT_B_CALL) visit(node.params) visit(node.statements) + builder.event(:RUBY_EVENT_B_RETURN) builder.leave end @@ -1777,7 +1798,9 @@ def visit_module(node) current_iseq, node ) do + builder.event(:RUBY_EVENT_CLASS) visit(node.bodystmt) + builder.event(:RUBY_EVENT_END) builder.leave end @@ -2069,7 +2092,9 @@ def visit_sclass(node) current_iseq, node ) do + builder.event(:RUBY_EVENT_CLASS) visit(node.bodystmt) + builder.event(:RUBY_EVENT_END) builder.leave end diff --git a/test/compiler_test.rb b/test/compiler_test.rb index 632b3e55..cf0667bb 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -453,15 +453,20 @@ def serialize_iseq(iseq) serialized[4].delete(:node_ids) serialized[13] = serialized[13].filter_map do |insn| - next unless insn.is_a?(Array) - - insn.map do |operand| - if operand.is_a?(Array) && - operand[0] == "YARVInstructionSequence/SimpleDataFormat" - serialize_iseq(operand) - else - operand + case insn + when Array + insn.map do |operand| + if operand.is_a?(Array) && + operand[0] == Visitor::Compiler::InstructionSequence::MAGIC + serialize_iseq(operand) + else + operand + end end + when Integer, :RUBY_EVENT_LINE + # ignore these for now + else + insn end end From f40ae12519f52b32a78dd60e87fe69e4f3fa12ce Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 18 Nov 2022 20:00:55 -0500 Subject: [PATCH 251/536] Move compiler to its own file --- lib/syntax_tree.rb | 4 +- lib/syntax_tree/compiler.rb | 2737 +++++++++++++++++++++++++++ lib/syntax_tree/visitor/compiler.rb | 2719 -------------------------- test/compiler_test.rb | 21 +- 4 files changed, 2743 insertions(+), 2738 deletions(-) create mode 100644 lib/syntax_tree/compiler.rb delete mode 100644 lib/syntax_tree/visitor/compiler.rb diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index aea21d8e..c62132e6 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require "etc" +require "fiddle" require "json" require "pp" require "prettier_print" @@ -13,7 +14,6 @@ require_relative "syntax_tree/basic_visitor" require_relative "syntax_tree/visitor" -require_relative "syntax_tree/visitor/compiler" require_relative "syntax_tree/visitor/field_visitor" require_relative "syntax_tree/visitor/json_visitor" require_relative "syntax_tree/visitor/match_visitor" @@ -26,6 +26,8 @@ require_relative "syntax_tree/pattern" require_relative "syntax_tree/search" +require_relative "syntax_tree/compiler" + # Syntax Tree is a suite of tools built on top of the internal CRuby parser. It # provides the ability to generate a syntax tree from source, as well as the # tools necessary to inspect and manipulate that syntax tree. It can be used to diff --git a/lib/syntax_tree/compiler.rb b/lib/syntax_tree/compiler.rb new file mode 100644 index 00000000..d9b7e787 --- /dev/null +++ b/lib/syntax_tree/compiler.rb @@ -0,0 +1,2737 @@ +# frozen_string_literal: true + +module SyntaxTree + # This class is an experiment in transforming Syntax Tree nodes into their + # corresponding YARV instruction sequences. It attempts to mirror the + # behavior of RubyVM::InstructionSequence.compile. + # + # You use this as with any other visitor. First you parse code into a tree, + # then you visit it with this compiler. Visiting the root node of the tree + # will return a SyntaxTree::Visitor::Compiler::InstructionSequence object. + # With that object you can call #to_a on it, which will return a serialized + # form of the instruction sequence as an array. This array _should_ mirror + # the array given by RubyVM::InstructionSequence#to_a. + # + # As an example, here is how you would compile a single expression: + # + # program = SyntaxTree.parse("1 + 2") + # program.accept(SyntaxTree::Visitor::Compiler.new).to_a + # + # [ + # "YARVInstructionSequence/SimpleDataFormat", + # 3, + # 1, + # 1, + # {:arg_size=>0, :local_size=>0, :stack_max=>2}, + # "", + # "", + # "", + # 1, + # :top, + # [], + # {}, + # [], + # [ + # [:putobject_INT2FIX_1_], + # [:putobject, 2], + # [:opt_plus, {:mid=>:+, :flag=>16, :orig_argc=>1}], + # [:leave] + # ] + # ] + # + # Note that this is the same output as calling: + # + # RubyVM::InstructionSequence.compile("1 + 2").to_a + # + class Compiler < BasicVisitor + # This visitor is responsible for converting Syntax Tree nodes into their + # corresponding Ruby structures. This is used to convert the operands of + # some instructions like putobject that push a Ruby object directly onto + # the stack. It is only used when the entire structure can be represented + # at compile-time, as opposed to constructed at run-time. + class RubyVisitor < BasicVisitor + # This error is raised whenever a node cannot be converted into a Ruby + # object at compile-time. + class CompilationError < StandardError + end + + # This will attempt to compile the given node. If it's possible, then + # it will return the compiled object. Otherwise it will return nil. + def self.compile(node) + node.accept(new) + rescue CompilationError + end + + def visit_array(node) + visit_all(node.contents.parts) + end + + def visit_bare_assoc_hash(node) + node.assocs.to_h do |assoc| + # We can only convert regular key-value pairs. A double splat ** + # operator means it has to be converted at run-time. + raise CompilationError unless assoc.is_a?(Assoc) + [visit(assoc.key), visit(assoc.value)] + end + end + + def visit_float(node) + node.value.to_f + end + + alias visit_hash visit_bare_assoc_hash + + def visit_imaginary(node) + node.value.to_c + end + + def visit_int(node) + node.value.to_i + end + + def visit_label(node) + node.value.chomp(":").to_sym + end + + def visit_mrhs(node) + visit_all(node.parts) + end + + def visit_qsymbols(node) + node.elements.map { |element| visit(element).to_sym } + end + + def visit_qwords(node) + visit_all(node.elements) + end + + def visit_range(node) + left, right = [visit(node.left), visit(node.right)] + node.operator.value === ".." ? left..right : left...right + end + + def visit_rational(node) + node.value.to_r + end + + def visit_regexp_literal(node) + if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) + Regexp.new(node.parts.first.value, visit_regexp_literal_flags(node)) + else + # Any interpolation of expressions or variables will result in the + # regular expression being constructed at run-time. + raise CompilationError + end + end + + # This isn't actually a visit method, though maybe it should be. It is + # responsible for converting the set of string options on a regular + # expression into its equivalent integer. + def visit_regexp_literal_flags(node) + node + .options + .chars + .inject(0) do |accum, option| + accum | + case option + when "i" + Regexp::IGNORECASE + when "x" + Regexp::EXTENDED + when "m" + Regexp::MULTILINE + else + raise "Unknown regexp option: #{option}" + end + end + end + + def visit_symbol_literal(node) + node.value.value.to_sym + end + + def visit_symbols(node) + node.elements.map { |element| visit(element).to_sym } + end + + def visit_tstring_content(node) + node.value + end + + def visit_word(node) + if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) + node.parts.first.value + else + # Any interpolation of expressions or variables will result in the + # string being constructed at run-time. + raise CompilationError + end + end + + def visit_words(node) + visit_all(node.elements) + end + + def visit_unsupported(_node) + raise CompilationError + end + + # Please forgive the metaprogramming here. This is used to create visit + # methods for every node that we did not explicitly handle. By default + # each of these methods will raise a CompilationError. + handled = instance_methods(false) + (Visitor.instance_methods(false) - handled).each do |method| + alias_method method, :visit_unsupported + end + end + + # This object is used to track the size of the stack at any given time. It + # is effectively a mini symbolic interpreter. It's necessary because when + # instruction sequences get serialized they include a :stack_max field on + # them. This field is used to determine how much stack space to allocate + # for the instruction sequence. + class Stack + attr_reader :current_size, :maximum_size + + def initialize + @current_size = 0 + @maximum_size = 0 + end + + def change_by(value) + @current_size += value + @maximum_size = @current_size if @current_size > @maximum_size + end + end + + # This represents every local variable associated with an instruction + # sequence. There are two kinds of locals: plain locals that are what you + # expect, and block proxy locals, which represent local variables + # associated with blocks that were passed into the current instruction + # sequence. + class LocalTable + # A local representing a block passed into the current instruction + # sequence. + class BlockLocal + attr_reader :name + + def initialize(name) + @name = name + end + end + + # A regular local variable. + class PlainLocal + attr_reader :name + + def initialize(name) + @name = name + end + end + + # The result of looking up a local variable in the current local table. + class Lookup + attr_reader :local, :index, :level + + def initialize(local, index, level) + @local = local + @index = index + @level = level + end + end + + attr_reader :locals + + def initialize + @locals = [] + end + + def find(name, level) + index = locals.index { |local| local.name == name } + Lookup.new(locals[index], index, level) if index + end + + def has?(name) + locals.any? { |local| local.name == name } + end + + def names + locals.map(&:name) + end + + def size + locals.length + end + + # Add a BlockLocal to the local table. + def block(name) + locals << BlockLocal.new(name) unless has?(name) + end + + # Add a PlainLocal to the local table. + def plain(name) + locals << PlainLocal.new(name) unless has?(name) + end + + # This is the offset from the top of the stack where this local variable + # lives. + def offset(index) + size - (index - 3) - 1 + end + end + + # This class is meant to mirror RubyVM::InstructionSequence. It contains a + # list of instructions along with the metadata pertaining to them. It also + # functions as a builder for the instruction sequence. + class InstructionSequence + MAGIC = "YARVInstructionSequence/SimpleDataFormat" + + # This provides a handle to the rb_iseq_load function, which allows you to + # pass a serialized iseq to Ruby and have it return a + # RubyVM::InstructionSequence object. + ISEQ_LOAD = + Fiddle::Function.new( + Fiddle::Handle::DEFAULT["rb_iseq_load"], + [Fiddle::TYPE_VOIDP] * 3, + Fiddle::TYPE_VOIDP + ) + + # The type of the instruction sequence. + attr_reader :type + + # The name of the instruction sequence. + attr_reader :name + + # The parent instruction sequence, if there is one. + attr_reader :parent_iseq + + # The location of the root node of this instruction sequence. + attr_reader :location + + # This is the list of information about the arguments to this + # instruction sequence. + attr_accessor :argument_size + attr_reader :argument_options + + # The list of instructions for this instruction sequence. + attr_reader :insns + + # The table of local variables. + attr_reader :local_table + + # The hash of names of instance and class variables pointing to the + # index of their associated inline storage. + attr_reader :inline_storages + + # The index of the next inline storage that will be created. + attr_reader :storage_index + + # An object that will track the current size of the stack and the + # maximum size of the stack for this instruction sequence. + attr_reader :stack + + def initialize(type, name, parent_iseq, location) + @type = type + @name = name + @parent_iseq = parent_iseq + @location = location + + @argument_size = 0 + @argument_options = {} + + @local_table = LocalTable.new + @inline_storages = {} + @insns = [] + @storage_index = 0 + @stack = Stack.new + end + + def local_variable(name, level = 0) + if (lookup = local_table.find(name, level)) + lookup + elsif parent_iseq + parent_iseq.local_variable(name, level + 1) + end + end + + def push(insn) + insns << insn + insn + end + + def inline_storage + storage = storage_index + @storage_index += 1 + storage + end + + def inline_storage_for(name) + unless inline_storages.key?(name) + inline_storages[name] = inline_storage + end + + inline_storages[name] + end + + def length + insns.inject(0) do |sum, insn| + insn.is_a?(Array) ? sum + insn.length : sum + end + end + + def each_child + insns.each do |insn| + insn[1..].each do |operand| + yield operand if operand.is_a?(InstructionSequence) + end + end + end + + def eval + compiled = to_a + + # Temporary hack until we get these working. + compiled[4][:node_id] = 11 + compiled[4][:node_ids] = [1, 0, 3, 2, 6, 7, 9, -1] + + Fiddle.dlunwrap(ISEQ_LOAD.call(Fiddle.dlwrap(compiled), 0, nil)).eval + end + + def to_a + versions = RUBY_VERSION.split(".").map(&:to_i) + + [ + MAGIC, + versions[0], + versions[1], + 1, + { + arg_size: argument_size, + local_size: local_table.size, + stack_max: stack.maximum_size + }, + name, + "", + "", + location.start_line, + type, + local_table.names, + argument_options, + [], + insns.map { |insn| serialize(insn) } + ] + end + + private + + def serialize(insn) + case insn[0] + when :checkkeyword, :getblockparam, :getblockparamproxy, + :getlocal_WC_0, :getlocal_WC_1, :getlocal, :setlocal_WC_0, + :setlocal_WC_1, :setlocal + iseq = self + + case insn[0] + when :getlocal_WC_1, :setlocal_WC_1 + iseq = iseq.parent_iseq + when :getblockparam, :getblockparamproxy, :getlocal, :setlocal + insn[2].times { iseq = iseq.parent_iseq } + end + + # Here we need to map the local variable index to the offset + # from the top of the stack where it will be stored. + [insn[0], iseq.local_table.offset(insn[1]), *insn[2..]] + when :defineclass + [insn[0], insn[1], insn[2].to_a, insn[3]] + when :definemethod, :definesmethod + [insn[0], insn[1], insn[2].to_a] + when :send + # For any instructions that push instruction sequences onto the + # stack, we need to call #to_a on them as well. + [insn[0], insn[1], (insn[2].to_a if insn[2])] + when :once + [insn[0], insn[1].to_a, insn[2]] + else + insn + end + end + end + + # This class serves as a layer of indirection between the instruction + # sequence and the compiler. It allows us to provide different behavior + # for certain instructions depending on the Ruby version. For example, + # class variable reads and writes gained an inline cache in Ruby 3.0. So + # we place the logic for checking the Ruby version in this class. + class Builder + attr_reader :iseq, :stack + attr_reader :frozen_string_literal, + :operands_unification, + :specialized_instruction + + def initialize( + iseq, + frozen_string_literal: false, + operands_unification: true, + specialized_instruction: true + ) + @iseq = iseq + @stack = iseq.stack + + @frozen_string_literal = frozen_string_literal + @operands_unification = operands_unification + @specialized_instruction = specialized_instruction + end + + # This creates a new label at the current length of the instruction + # sequence. It is used as the operand for jump instructions. + def label + name = :"label_#{iseq.length}" + iseq.insns.last == name ? name : event(name) + end + + def event(name) + iseq.push(name) + name + end + + def adjuststack(number) + stack.change_by(-number) + iseq.push([:adjuststack, number]) + end + + def anytostring + stack.change_by(-2 + 1) + iseq.push([:anytostring]) + end + + def branchif(index) + stack.change_by(-1) + iseq.push([:branchif, index]) + end + + def branchnil(index) + stack.change_by(-1) + iseq.push([:branchnil, index]) + end + + def branchunless(index) + stack.change_by(-1) + iseq.push([:branchunless, index]) + end + + def checkkeyword(index, keyword_index) + stack.change_by(+1) + iseq.push([:checkkeyword, index, keyword_index]) + end + + def concatarray + stack.change_by(-2 + 1) + iseq.push([:concatarray]) + end + + def concatstrings(number) + stack.change_by(-number + 1) + iseq.push([:concatstrings, number]) + end + + def defined(type, name, message) + stack.change_by(-1 + 1) + iseq.push([:defined, type, name, message]) + end + + def defineclass(name, class_iseq, flags) + stack.change_by(-2 + 1) + iseq.push([:defineclass, name, class_iseq, flags]) + end + + def definemethod(name, method_iseq) + stack.change_by(0) + iseq.push([:definemethod, name, method_iseq]) + end + + def definesmethod(name, method_iseq) + stack.change_by(-1) + iseq.push([:definesmethod, name, method_iseq]) + end + + def dup + stack.change_by(-1 + 2) + iseq.push([:dup]) + end + + def duparray(object) + stack.change_by(+1) + iseq.push([:duparray, object]) + end + + def duphash(object) + stack.change_by(+1) + iseq.push([:duphash, object]) + end + + def dupn(number) + stack.change_by(+number) + iseq.push([:dupn, number]) + end + + def expandarray(length, flag) + stack.change_by(-1 + length) + iseq.push([:expandarray, length, flag]) + end + + def getblockparam(index, level) + stack.change_by(+1) + iseq.push([:getblockparam, index, level]) + end + + def getblockparamproxy(index, level) + stack.change_by(+1) + iseq.push([:getblockparamproxy, index, level]) + end + + def getclassvariable(name) + stack.change_by(+1) + + if RUBY_VERSION >= "3.0" + iseq.push([:getclassvariable, name, iseq.inline_storage_for(name)]) + else + iseq.push([:getclassvariable, name]) + end + end + + def getconstant(name) + stack.change_by(-2 + 1) + iseq.push([:getconstant, name]) + end + + def getglobal(name) + stack.change_by(+1) + iseq.push([:getglobal, name]) + end + + def getinstancevariable(name) + stack.change_by(+1) + + if RUBY_VERSION >= "3.2" + iseq.push([:getinstancevariable, name, iseq.inline_storage]) + else + inline_storage = iseq.inline_storage_for(name) + iseq.push([:getinstancevariable, name, inline_storage]) + end + end + + def getlocal(index, level) + stack.change_by(+1) + + if operands_unification + # Specialize the getlocal instruction based on the level of the + # local variable. If it's 0 or 1, then there's a specialized + # instruction that will look at the current scope or the parent + # scope, respectively, and requires fewer operands. + case level + when 0 + iseq.push([:getlocal_WC_0, index]) + when 1 + iseq.push([:getlocal_WC_1, index]) + else + iseq.push([:getlocal, index, level]) + end + else + iseq.push([:getlocal, index, level]) + end + end + + def getspecial(key, type) + stack.change_by(-0 + 1) + iseq.push([:getspecial, key, type]) + end + + def intern + stack.change_by(-1 + 1) + iseq.push([:intern]) + end + + def invokeblock(method_id, argc, flag) + stack.change_by(-argc + 1) + iseq.push([:invokeblock, call_data(method_id, argc, flag)]) + end + + def invokesuper(method_id, argc, flag, block_iseq) + stack.change_by(-(argc + 1) + 1) + + cdata = call_data(method_id, argc, flag) + iseq.push([:invokesuper, cdata, block_iseq]) + end + + def jump(index) + stack.change_by(0) + iseq.push([:jump, index]) + end + + def leave + stack.change_by(-1) + iseq.push([:leave]) + end + + def newarray(length) + stack.change_by(-length + 1) + iseq.push([:newarray, length]) + end + + def newhash(length) + stack.change_by(-length + 1) + iseq.push([:newhash, length]) + end + + def newrange(flag) + stack.change_by(-2 + 1) + iseq.push([:newrange, flag]) + end + + def nop + stack.change_by(0) + iseq.push([:nop]) + end + + def objtostring(method_id, argc, flag) + stack.change_by(-1 + 1) + iseq.push([:objtostring, call_data(method_id, argc, flag)]) + end + + def once(postexe_iseq, inline_storage) + stack.change_by(+1) + iseq.push([:once, postexe_iseq, inline_storage]) + end + + def opt_getconstant_path(names) + if RUBY_VERSION >= "3.2" + stack.change_by(+1) + iseq.push([:opt_getconstant_path, names]) + else + inline_storage = iseq.inline_storage + getinlinecache = opt_getinlinecache(-1, inline_storage) + + if names[0] == :"" + names.shift + pop + putobject(Object) + end + + names.each_with_index do |name, index| + putobject(index == 0) + getconstant(name) + end + + opt_setinlinecache(inline_storage) + getinlinecache[1] = label + end + end + + def opt_getinlinecache(offset, inline_storage) + stack.change_by(+1) + iseq.push([:opt_getinlinecache, offset, inline_storage]) + end + + def opt_newarray_max(length) + if specialized_instruction + stack.change_by(-length + 1) + iseq.push([:opt_newarray_max, length]) + else + newarray(length) + send(:max, 0, VM_CALL_ARGS_SIMPLE) + end + end + + def opt_newarray_min(length) + if specialized_instruction + stack.change_by(-length + 1) + iseq.push([:opt_newarray_min, length]) + else + newarray(length) + send(:min, 0, VM_CALL_ARGS_SIMPLE) + end + end + + def opt_setinlinecache(inline_storage) + stack.change_by(-1 + 1) + iseq.push([:opt_setinlinecache, inline_storage]) + end + + def opt_str_freeze(value) + if specialized_instruction + stack.change_by(+1) + iseq.push( + [ + :opt_str_freeze, + value, + call_data(:freeze, 0, VM_CALL_ARGS_SIMPLE) + ] + ) + else + putstring(value) + send(:freeze, 0, VM_CALL_ARGS_SIMPLE) + end + end + + def opt_str_uminus(value) + if specialized_instruction + stack.change_by(+1) + iseq.push( + [:opt_str_uminus, value, call_data(:-@, 0, VM_CALL_ARGS_SIMPLE)] + ) + else + putstring(value) + send(:-@, 0, VM_CALL_ARGS_SIMPLE) + end + end + + def pop + stack.change_by(-1) + iseq.push([:pop]) + end + + def putnil + stack.change_by(+1) + iseq.push([:putnil]) + end + + def putobject(object) + stack.change_by(+1) + + if operands_unification + # Specialize the putobject instruction based on the value of the + # object. If it's 0 or 1, then there's a specialized instruction + # that will push the object onto the stack and requires fewer + # operands. + if object.eql?(0) + iseq.push([:putobject_INT2FIX_0_]) + elsif object.eql?(1) + iseq.push([:putobject_INT2FIX_1_]) + else + iseq.push([:putobject, object]) + end + else + iseq.push([:putobject, object]) + end + end + + def putself + stack.change_by(+1) + iseq.push([:putself]) + end + + def putspecialobject(object) + stack.change_by(+1) + iseq.push([:putspecialobject, object]) + end + + def putstring(object) + stack.change_by(+1) + iseq.push([:putstring, object]) + end + + def send(method_id, argc, flag, block_iseq = nil) + stack.change_by(-(argc + 1) + 1) + cdata = call_data(method_id, argc, flag) + + if specialized_instruction + # Specialize the send instruction. If it doesn't have a block + # attached, then we will replace it with an opt_send_without_block + # and do further specializations based on the called method and the + # number of arguments. + + # stree-ignore + if !block_iseq && (flag & VM_CALL_ARGS_BLOCKARG) == 0 + case [method_id, argc] + when [:length, 0] then iseq.push([:opt_length, cdata]) + when [:size, 0] then iseq.push([:opt_size, cdata]) + when [:empty?, 0] then iseq.push([:opt_empty_p, cdata]) + when [:nil?, 0] then iseq.push([:opt_nil_p, cdata]) + when [:succ, 0] then iseq.push([:opt_succ, cdata]) + when [:!, 0] then iseq.push([:opt_not, cdata]) + when [:+, 1] then iseq.push([:opt_plus, cdata]) + when [:-, 1] then iseq.push([:opt_minus, cdata]) + when [:*, 1] then iseq.push([:opt_mult, cdata]) + when [:/, 1] then iseq.push([:opt_div, cdata]) + when [:%, 1] then iseq.push([:opt_mod, cdata]) + when [:==, 1] then iseq.push([:opt_eq, cdata]) + when [:=~, 1] then iseq.push([:opt_regexpmatch2, cdata]) + when [:<, 1] then iseq.push([:opt_lt, cdata]) + when [:<=, 1] then iseq.push([:opt_le, cdata]) + when [:>, 1] then iseq.push([:opt_gt, cdata]) + when [:>=, 1] then iseq.push([:opt_ge, cdata]) + when [:<<, 1] then iseq.push([:opt_ltlt, cdata]) + when [:[], 1] then iseq.push([:opt_aref, cdata]) + when [:&, 1] then iseq.push([:opt_and, cdata]) + when [:|, 1] then iseq.push([:opt_or, cdata]) + when [:[]=, 2] then iseq.push([:opt_aset, cdata]) + when [:!=, 1] + eql_data = call_data(:==, 1, VM_CALL_ARGS_SIMPLE) + iseq.push([:opt_neq, eql_data, cdata]) + else + iseq.push([:opt_send_without_block, cdata]) + end + else + iseq.push([:send, cdata, block_iseq]) + end + else + iseq.push([:send, cdata, block_iseq]) + end + end + + def setclassvariable(name) + stack.change_by(-1) + + if RUBY_VERSION >= "3.0" + iseq.push([:setclassvariable, name, iseq.inline_storage_for(name)]) + else + iseq.push([:setclassvariable, name]) + end + end + + def setconstant(name) + stack.change_by(-2) + iseq.push([:setconstant, name]) + end + + def setglobal(name) + stack.change_by(-1) + iseq.push([:setglobal, name]) + end + + def setinstancevariable(name) + stack.change_by(-1) + + if RUBY_VERSION >= "3.2" + iseq.push([:setinstancevariable, name, iseq.inline_storage]) + else + inline_storage = iseq.inline_storage_for(name) + iseq.push([:setinstancevariable, name, inline_storage]) + end + end + + def setlocal(index, level) + stack.change_by(-1) + + if operands_unification + # Specialize the setlocal instruction based on the level of the + # local variable. If it's 0 or 1, then there's a specialized + # instruction that will write to the current scope or the parent + # scope, respectively, and requires fewer operands. + case level + when 0 + iseq.push([:setlocal_WC_0, index]) + when 1 + iseq.push([:setlocal_WC_1, index]) + else + iseq.push([:setlocal, index, level]) + end + else + iseq.push([:setlocal, index, level]) + end + end + + def setn(number) + stack.change_by(-1 + 1) + iseq.push([:setn, number]) + end + + def splatarray(flag) + stack.change_by(-1 + 1) + iseq.push([:splatarray, flag]) + end + + def swap + stack.change_by(-2 + 2) + iseq.push([:swap]) + end + + def topn(number) + stack.change_by(+1) + iseq.push([:topn, number]) + end + + def toregexp(options, length) + stack.change_by(-length + 1) + iseq.push([:toregexp, options, length]) + end + + private + + # This creates a call data object that is used as the operand for the + # send, invokesuper, and objtostring instructions. + def call_data(method_id, argc, flag) + { mid: method_id, flag: flag, orig_argc: argc } + end + end + + # These constants correspond to the putspecialobject instruction. They are + # used to represent special objects that are pushed onto the stack. + VM_SPECIAL_OBJECT_VMCORE = 1 + VM_SPECIAL_OBJECT_CBASE = 2 + VM_SPECIAL_OBJECT_CONST_BASE = 3 + + # These constants correspond to the flag passed as part of the call data + # structure on the send instruction. They are used to represent various + # metadata about the callsite (e.g., were keyword arguments used?, was a + # block given?, etc.). + VM_CALL_ARGS_SPLAT = 1 << 0 + VM_CALL_ARGS_BLOCKARG = 1 << 1 + VM_CALL_FCALL = 1 << 2 + VM_CALL_VCALL = 1 << 3 + VM_CALL_ARGS_SIMPLE = 1 << 4 + VM_CALL_BLOCKISEQ = 1 << 5 + VM_CALL_KWARG = 1 << 6 + VM_CALL_KW_SPLAT = 1 << 7 + VM_CALL_TAILCALL = 1 << 8 + VM_CALL_SUPER = 1 << 9 + VM_CALL_ZSUPER = 1 << 10 + VM_CALL_OPT_SEND = 1 << 11 + VM_CALL_KW_SPLAT_MUT = 1 << 12 + + # These constants correspond to the value passed as part of the defined + # instruction. It's an enum defined in the CRuby codebase that tells that + # instruction what kind of defined check to perform. + DEFINED_NIL = 1 + DEFINED_IVAR = 2 + DEFINED_LVAR = 3 + DEFINED_GVAR = 4 + DEFINED_CVAR = 5 + DEFINED_CONST = 6 + DEFINED_METHOD = 7 + DEFINED_YIELD = 8 + DEFINED_ZSUPER = 9 + DEFINED_SELF = 10 + DEFINED_TRUE = 11 + DEFINED_FALSE = 12 + DEFINED_ASGN = 13 + DEFINED_EXPR = 14 + DEFINED_REF = 15 + DEFINED_FUNC = 16 + DEFINED_CONST_FROM = 17 + + # These constants correspond to the value passed in the flags as part of + # the defineclass instruction. + VM_DEFINECLASS_TYPE_CLASS = 0 + VM_DEFINECLASS_TYPE_SINGLETON_CLASS = 1 + VM_DEFINECLASS_TYPE_MODULE = 2 + VM_DEFINECLASS_FLAG_SCOPED = 8 + VM_DEFINECLASS_FLAG_HAS_SUPERCLASS = 16 + + # These options mirror the compilation options that we currently support + # that can be also passed to RubyVM::InstructionSequence.compile. + attr_reader :frozen_string_literal, + :operands_unification, + :specialized_instruction + + # The current instruction sequence that is being compiled. + attr_reader :current_iseq + + # This is the current builder that is being used to construct the current + # instruction sequence. + attr_reader :builder + + # A boolean to track if we're currently compiling the last statement + # within a set of statements. This information is necessary to determine + # if we need to return the value of the last statement. + attr_reader :last_statement + + def initialize( + frozen_string_literal: false, + operands_unification: true, + specialized_instruction: true + ) + @frozen_string_literal = frozen_string_literal + @operands_unification = operands_unification + @specialized_instruction = specialized_instruction + + @current_iseq = nil + @builder = nil + @last_statement = false + end + + def visit_BEGIN(node) + visit(node.statements) + end + + def visit_CHAR(node) + if frozen_string_literal + builder.putobject(node.value[1..]) + else + builder.putstring(node.value[1..]) + end + end + + def visit_END(node) + name = "block in #{current_iseq.name}" + once_iseq = + with_instruction_sequence(:block, name, current_iseq, node) do + postexe_iseq = + with_instruction_sequence(:block, name, current_iseq, node) do + *statements, last_statement = node.statements.body + visit_all(statements) + with_last_statement { visit(last_statement) } + builder.leave + end + + builder.putspecialobject(VM_SPECIAL_OBJECT_VMCORE) + builder.send(:"core#set_postexe", 0, VM_CALL_FCALL, postexe_iseq) + builder.leave + end + + builder.once(once_iseq, current_iseq.inline_storage) + builder.pop + end + + def visit_alias(node) + builder.putspecialobject(VM_SPECIAL_OBJECT_VMCORE) + builder.putspecialobject(VM_SPECIAL_OBJECT_CBASE) + visit(node.left) + visit(node.right) + builder.send(:"core#set_method_alias", 3, VM_CALL_ARGS_SIMPLE) + end + + def visit_aref(node) + visit(node.collection) + visit(node.index) + builder.send(:[], 1, VM_CALL_ARGS_SIMPLE) + end + + def visit_arg_block(node) + visit(node.value) + end + + def visit_arg_paren(node) + visit(node.arguments) + end + + def visit_arg_star(node) + visit(node.value) + builder.splatarray(false) + end + + def visit_args(node) + visit_all(node.parts) + end + + def visit_array(node) + if (compiled = RubyVisitor.compile(node)) + builder.duparray(compiled) + else + length = 0 + + node.contents.parts.each do |part| + if part.is_a?(ArgStar) + if length > 0 + builder.newarray(length) + length = 0 + end + + visit(part.value) + builder.concatarray + else + visit(part) + length += 1 + end + end + + builder.newarray(length) if length > 0 + if length > 0 && length != node.contents.parts.length + builder.concatarray + end + end + end + + def visit_assign(node) + case node.target + when ARefField + builder.putnil + visit(node.target.collection) + visit(node.target.index) + visit(node.value) + builder.setn(3) + builder.send(:[]=, 2, VM_CALL_ARGS_SIMPLE) + builder.pop + when ConstPathField + names = constant_names(node.target) + name = names.pop + + if RUBY_VERSION >= "3.2" + builder.opt_getconstant_path(names) + visit(node.value) + builder.swap + builder.topn(1) + builder.swap + builder.setconstant(name) + else + visit(node.value) + builder.dup if last_statement? + builder.opt_getconstant_path(names) + builder.setconstant(name) + end + when Field + builder.putnil + visit(node.target) + visit(node.value) + builder.setn(2) + builder.send(:"#{node.target.name.value}=", 1, VM_CALL_ARGS_SIMPLE) + builder.pop + when TopConstField + name = node.target.constant.value.to_sym + + if RUBY_VERSION >= "3.2" + builder.putobject(Object) + visit(node.value) + builder.swap + builder.topn(1) + builder.swap + builder.setconstant(name) + else + visit(node.value) + builder.dup if last_statement? + builder.putobject(Object) + builder.setconstant(name) + end + when VarField + visit(node.value) + builder.dup if last_statement? + + case node.target.value + when Const + builder.putspecialobject(VM_SPECIAL_OBJECT_CONST_BASE) + builder.setconstant(node.target.value.value.to_sym) + when CVar + builder.setclassvariable(node.target.value.value.to_sym) + when GVar + builder.setglobal(node.target.value.value.to_sym) + when Ident + local_variable = visit(node.target) + builder.setlocal(local_variable.index, local_variable.level) + when IVar + builder.setinstancevariable(node.target.value.value.to_sym) + end + end + end + + def visit_assoc(node) + visit(node.key) + visit(node.value) + end + + def visit_assoc_splat(node) + visit(node.value) + end + + def visit_backref(node) + builder.getspecial(1, 2 * node.value[1..].to_i) + end + + def visit_bare_assoc_hash(node) + if (compiled = RubyVisitor.compile(node)) + builder.duphash(compiled) + else + visit_all(node.assocs) + end + end + + def visit_binary(node) + case node.operator + when :"&&" + visit(node.left) + builder.dup + + branchunless = builder.branchunless(-1) + builder.pop + + visit(node.right) + branchunless[1] = builder.label + when :"||" + visit(node.left) + builder.dup + + branchif = builder.branchif(-1) + builder.pop + + visit(node.right) + branchif[1] = builder.label + else + visit(node.left) + visit(node.right) + builder.send(node.operator, 1, VM_CALL_ARGS_SIMPLE) + end + end + + def visit_block(node) + with_instruction_sequence( + :block, + "block in #{current_iseq.name}", + current_iseq, + node + ) do + builder.event(:RUBY_EVENT_B_CALL) + visit(node.block_var) + visit(node.bodystmt) + builder.event(:RUBY_EVENT_B_RETURN) + builder.leave + end + end + + def visit_block_var(node) + params = node.params + + if params.requireds.length == 1 && params.optionals.empty? && + !params.rest && params.posts.empty? && params.keywords.empty? && + !params.keyword_rest && !params.block + current_iseq.argument_options[:ambiguous_param0] = true + end + + visit(node.params) + + node.locals.each do |local| + current_iseq.local_table.plain(local.value.to_sym) + end + end + + def visit_blockarg(node) + current_iseq.argument_options[:block_start] = current_iseq.argument_size + current_iseq.local_table.block(node.name.value.to_sym) + current_iseq.argument_size += 1 + end + + def visit_bodystmt(node) + visit(node.statements) + end + + def visit_call(node) + if node.is_a?(CallNode) + return( + visit_call( + CommandCall.new( + receiver: node.receiver, + operator: node.operator, + message: node.message, + arguments: node.arguments, + block: nil, + location: node.location + ) + ) + ) + end + + arg_parts = argument_parts(node.arguments) + argc = arg_parts.length + + # First we're going to check if we're calling a method on an array + # literal without any arguments. In that case there are some + # specializations we might be able to perform. + if argc == 0 && (node.message.is_a?(Ident) || node.message.is_a?(Op)) + case node.receiver + when ArrayLiteral + parts = node.receiver.contents&.parts || [] + + if parts.none? { |part| part.is_a?(ArgStar) } && + RubyVisitor.compile(node.receiver).nil? + case node.message.value + when "max" + visit(node.receiver.contents) + builder.opt_newarray_max(parts.length) + return + when "min" + visit(node.receiver.contents) + builder.opt_newarray_min(parts.length) + return + end + end + when StringLiteral + if RubyVisitor.compile(node.receiver).nil? + case node.message.value + when "-@" + builder.opt_str_uminus(node.receiver.parts.first.value) + return + when "freeze" + builder.opt_str_freeze(node.receiver.parts.first.value) + return + end + end + end + end + + if node.receiver + if node.receiver.is_a?(VarRef) && + ( + lookup = + current_iseq.local_variable(node.receiver.value.value.to_sym) + ) && lookup.local.is_a?(LocalTable::BlockLocal) + builder.getblockparamproxy(lookup.index, lookup.level) + else + visit(node.receiver) + end + else + builder.putself + end + + branchnil = + if node.operator&.value == "&." + builder.dup + builder.branchnil(-1) + end + + flag = 0 + + arg_parts.each do |arg_part| + case arg_part + when ArgBlock + argc -= 1 + flag |= VM_CALL_ARGS_BLOCKARG + visit(arg_part) + when ArgStar + flag |= VM_CALL_ARGS_SPLAT + visit(arg_part) + when ArgsForward + flag |= VM_CALL_ARGS_SPLAT | VM_CALL_ARGS_BLOCKARG + + lookup = current_iseq.local_table.find(:*, 0) + builder.getlocal(lookup.index, lookup.level) + builder.splatarray(arg_parts.length != 1) + + lookup = current_iseq.local_table.find(:&, 0) + builder.getblockparamproxy(lookup.index, lookup.level) + when BareAssocHash + flag |= VM_CALL_KW_SPLAT + visit(arg_part) + else + visit(arg_part) + end + end + + block_iseq = visit(node.block) if node.block + flag |= VM_CALL_ARGS_SIMPLE if block_iseq.nil? && flag == 0 + flag |= VM_CALL_FCALL if node.receiver.nil? + + builder.send(node.message.value.to_sym, argc, flag, block_iseq) + branchnil[1] = builder.label if branchnil + end + + def visit_case(node) + visit(node.value) if node.value + + clauses = [] + else_clause = nil + + current = node.consequent + + while current + clauses << current + + if (current = current.consequent).is_a?(Else) + else_clause = current + break + end + end + + branches = + clauses.map do |clause| + visit(clause.arguments) + builder.topn(1) + builder.send(:===, 1, VM_CALL_FCALL | VM_CALL_ARGS_SIMPLE) + [clause, builder.branchif(:label_00)] + end + + builder.pop + + else_clause ? visit(else_clause) : builder.putnil + + builder.leave + + branches.each_with_index do |(clause, branchif), index| + builder.leave if index != 0 + branchif[1] = builder.label + builder.pop + visit(clause) + end + end + + def visit_class(node) + name = node.constant.constant.value.to_sym + class_iseq = + with_instruction_sequence( + :class, + "", + current_iseq, + node + ) do + builder.event(:RUBY_EVENT_CLASS) + visit(node.bodystmt) + builder.event(:RUBY_EVENT_END) + builder.leave + end + + flags = VM_DEFINECLASS_TYPE_CLASS + + case node.constant + when ConstPathRef + flags |= VM_DEFINECLASS_FLAG_SCOPED + visit(node.constant.parent) + when ConstRef + builder.putspecialobject(VM_SPECIAL_OBJECT_CONST_BASE) + when TopConstRef + flags |= VM_DEFINECLASS_FLAG_SCOPED + builder.putobject(Object) + end + + if node.superclass + flags |= VM_DEFINECLASS_FLAG_HAS_SUPERCLASS + visit(node.superclass) + else + builder.putnil + end + + builder.defineclass(name, class_iseq, flags) + end + + def visit_command(node) + visit_call( + CommandCall.new( + receiver: nil, + operator: nil, + message: node.message, + arguments: node.arguments, + block: node.block, + location: node.location + ) + ) + end + + def visit_command_call(node) + visit_call( + CommandCall.new( + receiver: node.receiver, + operator: node.operator, + message: node.message, + arguments: node.arguments, + block: node.block, + location: node.location + ) + ) + end + + def visit_const_path_field(node) + visit(node.parent) + end + + def visit_const_path_ref(node) + names = constant_names(node) + builder.opt_getconstant_path(names) + end + + def visit_def(node) + method_iseq = + with_instruction_sequence( + :method, + node.name.value, + current_iseq, + node + ) do + visit(node.params) if node.params + builder.event(:RUBY_EVENT_CALL) + visit(node.bodystmt) + builder.event(:RUBY_EVENT_RETURN) + builder.leave + end + + name = node.name.value.to_sym + + if node.target + visit(node.target) + builder.definesmethod(name, method_iseq) + else + builder.definemethod(name, method_iseq) + end + + builder.putobject(name) + end + + def visit_defined(node) + case node.value + when Assign + # If we're assigning to a local variable, then we need to make sure + # that we put it into the local table. + if node.value.target.is_a?(VarField) && + node.value.target.value.is_a?(Ident) + current_iseq.local_table.plain(node.value.target.value.value.to_sym) + end + + builder.putobject("assignment") + when VarRef + value = node.value.value + name = value.value.to_sym + + case value + when Const + builder.putnil + builder.defined(DEFINED_CONST, name, "constant") + when CVar + builder.putnil + builder.defined(DEFINED_CVAR, name, "class variable") + when GVar + builder.putnil + builder.defined(DEFINED_GVAR, name, "global-variable") + when Ident + builder.putobject("local-variable") + when IVar + builder.putnil + builder.defined(DEFINED_IVAR, name, "instance-variable") + when Kw + case name + when :false + builder.putobject("false") + when :nil + builder.putobject("nil") + when :self + builder.putobject("self") + when :true + builder.putobject("true") + end + end + when VCall + builder.putself + + name = node.value.value.value.to_sym + builder.defined(DEFINED_FUNC, name, "method") + when YieldNode + builder.putnil + builder.defined(DEFINED_YIELD, false, "yield") + when ZSuper + builder.putnil + builder.defined(DEFINED_ZSUPER, false, "super") + else + builder.putobject("expression") + end + end + + def visit_dyna_symbol(node) + if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) + builder.putobject(node.parts.first.value.to_sym) + end + end + + def visit_else(node) + visit(node.statements) + builder.pop unless last_statement? + end + + def visit_elsif(node) + visit_if( + IfNode.new( + predicate: node.predicate, + statements: node.statements, + consequent: node.consequent, + location: node.location + ) + ) + end + + def visit_field(node) + visit(node.parent) + end + + def visit_float(node) + builder.putobject(node.accept(RubyVisitor.new)) + end + + def visit_for(node) + visit(node.collection) + + name = node.index.value.value.to_sym + current_iseq.local_table.plain(name) + + block_iseq = + with_instruction_sequence( + :block, + "block in #{current_iseq.name}", + current_iseq, + node.statements + ) do + current_iseq.argument_options[:lead_num] ||= 0 + current_iseq.argument_options[:lead_num] += 1 + current_iseq.argument_options[:ambiguous_param0] = true + + current_iseq.argument_size += 1 + current_iseq.local_table.plain(2) + + builder.getlocal(0, 0) + + local_variable = current_iseq.local_variable(name) + builder.setlocal(local_variable.index, local_variable.level) + + builder.event(:RUBY_EVENT_B_CALL) + builder.nop + + visit(node.statements) + builder.event(:RUBY_EVENT_B_RETURN) + builder.leave + end + + builder.send(:each, 0, 0, block_iseq) + end + + def visit_hash(node) + builder.duphash(node.accept(RubyVisitor.new)) + rescue RubyVisitor::CompilationError + visit_all(node.assocs) + builder.newhash(node.assocs.length * 2) + end + + def visit_heredoc(node) + if node.beginning.value.end_with?("`") + visit_xstring_literal(node) + elsif node.parts.length == 1 && node.parts.first.is_a?(TStringContent) + visit(node.parts.first) + else + length = visit_string_parts(node) + builder.concatstrings(length) + end + end + + def visit_if(node) + visit(node.predicate) + branchunless = builder.branchunless(-1) + visit(node.statements) + + if last_statement? + builder.leave + branchunless[1] = builder.label + + node.consequent ? visit(node.consequent) : builder.putnil + else + builder.pop + + if node.consequent + jump = builder.jump(-1) + branchunless[1] = builder.label + visit(node.consequent) + jump[1] = builder.label + else + branchunless[1] = builder.label + end + end + end + + def visit_if_op(node) + visit_if( + IfNode.new( + predicate: node.predicate, + statements: node.truthy, + consequent: + Else.new( + keyword: Kw.new(value: "else", location: Location.default), + statements: node.falsy, + location: Location.default + ), + location: Location.default + ) + ) + end + + def visit_imaginary(node) + builder.putobject(node.accept(RubyVisitor.new)) + end + + def visit_int(node) + builder.putobject(node.accept(RubyVisitor.new)) + end + + def visit_kwrest_param(node) + current_iseq.argument_options[:kwrest] = current_iseq.argument_size + current_iseq.argument_size += 1 + current_iseq.local_table.plain(node.name.value.to_sym) + end + + def visit_label(node) + builder.putobject(node.accept(RubyVisitor.new)) + end + + def visit_lambda(node) + lambda_iseq = + with_instruction_sequence( + :block, + "block in #{current_iseq.name}", + current_iseq, + node + ) do + builder.event(:RUBY_EVENT_B_CALL) + visit(node.params) + visit(node.statements) + builder.event(:RUBY_EVENT_B_RETURN) + builder.leave + end + + builder.putspecialobject(VM_SPECIAL_OBJECT_VMCORE) + builder.send(:lambda, 0, VM_CALL_FCALL, lambda_iseq) + end + + def visit_lambda_var(node) + visit_block_var(node) + end + + def visit_massign(node) + visit(node.value) + builder.dup + visit(node.target) + end + + def visit_method_add_block(node) + visit_call( + CommandCall.new( + receiver: node.call.receiver, + operator: node.call.operator, + message: node.call.message, + arguments: node.call.arguments, + block: node.block, + location: node.location + ) + ) + end + + def visit_mlhs(node) + lookups = [] + + node.parts.each do |part| + case part + when VarField + lookups << visit(part) + end + end + + builder.expandarray(lookups.length, 0) + + lookups.each { |lookup| builder.setlocal(lookup.index, lookup.level) } + end + + def visit_module(node) + name = node.constant.constant.value.to_sym + module_iseq = + with_instruction_sequence( + :class, + "", + current_iseq, + node + ) do + builder.event(:RUBY_EVENT_CLASS) + visit(node.bodystmt) + builder.event(:RUBY_EVENT_END) + builder.leave + end + + flags = VM_DEFINECLASS_TYPE_MODULE + + case node.constant + when ConstPathRef + flags |= VM_DEFINECLASS_FLAG_SCOPED + visit(node.constant.parent) + when ConstRef + builder.putspecialobject(VM_SPECIAL_OBJECT_CONST_BASE) + when TopConstRef + flags |= VM_DEFINECLASS_FLAG_SCOPED + builder.putobject(Object) + end + + builder.putnil + builder.defineclass(name, module_iseq, flags) + end + + def visit_mrhs(node) + if (compiled = RubyVisitor.compile(node)) + builder.duparray(compiled) + else + visit_all(node.parts) + builder.newarray(node.parts.length) + end + end + + def visit_not(node) + visit(node.statement) + builder.send(:!, 0, VM_CALL_ARGS_SIMPLE) + end + + def visit_opassign(node) + flag = VM_CALL_ARGS_SIMPLE + if node.target.is_a?(ConstPathField) || node.target.is_a?(TopConstField) + flag |= VM_CALL_FCALL + end + + case (operator = node.operator.value.chomp("=").to_sym) + when :"&&" + branchunless = nil + + with_opassign(node) do + builder.dup + branchunless = builder.branchunless(-1) + builder.pop + visit(node.value) + end + + case node.target + when ARefField + builder.leave + branchunless[1] = builder.label + builder.setn(3) + builder.adjuststack(3) + when ConstPathField, TopConstField + branchunless[1] = builder.label + builder.swap + builder.pop + else + branchunless[1] = builder.label + end + when :"||" + if node.target.is_a?(ConstPathField) || + node.target.is_a?(TopConstField) + opassign_defined(node) + builder.swap + builder.pop + elsif node.target.is_a?(VarField) && + [Const, CVar, GVar].include?(node.target.value.class) + opassign_defined(node) + else + branchif = nil + + with_opassign(node) do + builder.dup + branchif = builder.branchif(-1) + builder.pop + visit(node.value) + end + + if node.target.is_a?(ARefField) + builder.leave + branchif[1] = builder.label + builder.setn(3) + builder.adjuststack(3) + else + branchif[1] = builder.label + end + end + else + with_opassign(node) do + visit(node.value) + builder.send(operator, 1, flag) + end + end + end + + def visit_params(node) + argument_options = current_iseq.argument_options + + if node.requireds.any? + argument_options[:lead_num] = 0 + + node.requireds.each do |required| + current_iseq.local_table.plain(required.value.to_sym) + current_iseq.argument_size += 1 + argument_options[:lead_num] += 1 + end + end + + node.optionals.each do |(optional, value)| + index = current_iseq.local_table.size + name = optional.value.to_sym + + current_iseq.local_table.plain(name) + current_iseq.argument_size += 1 + + unless argument_options.key?(:opt) + argument_options[:opt] = [builder.label] + end + + visit(value) + builder.setlocal(index, 0) + current_iseq.argument_options[:opt] << builder.label + end + + visit(node.rest) if node.rest + + if node.posts.any? + argument_options[:post_start] = current_iseq.argument_size + argument_options[:post_num] = 0 + + node.posts.each do |post| + current_iseq.local_table.plain(post.value.to_sym) + current_iseq.argument_size += 1 + argument_options[:post_num] += 1 + end + end + + if node.keywords.any? + argument_options[:kwbits] = 0 + argument_options[:keyword] = [] + checkkeywords = [] + + node.keywords.each_with_index do |(keyword, value), keyword_index| + name = keyword.value.chomp(":").to_sym + index = current_iseq.local_table.size + + current_iseq.local_table.plain(name) + current_iseq.argument_size += 1 + argument_options[:kwbits] += 1 + + if value.nil? + argument_options[:keyword] << name + else + begin + compiled = value.accept(RubyVisitor.new) + argument_options[:keyword] << [name, compiled] + rescue RubyVisitor::CompilationError + argument_options[:keyword] << [name] + checkkeywords << builder.checkkeyword(-1, keyword_index) + branchif = builder.branchif(-1) + visit(value) + builder.setlocal(index, 0) + branchif[1] = builder.label + end + end + end + + name = node.keyword_rest ? 3 : 2 + current_iseq.argument_size += 1 + current_iseq.local_table.plain(name) + + lookup = current_iseq.local_table.find(name, 0) + checkkeywords.each { |checkkeyword| checkkeyword[1] = lookup.index } + end + + if node.keyword_rest.is_a?(ArgsForward) + current_iseq.local_table.plain(:*) + current_iseq.local_table.plain(:&) + + current_iseq.argument_options[ + :rest_start + ] = current_iseq.argument_size + current_iseq.argument_options[ + :block_start + ] = current_iseq.argument_size + 1 + + current_iseq.argument_size += 2 + elsif node.keyword_rest + visit(node.keyword_rest) + end + + visit(node.block) if node.block + end + + def visit_paren(node) + visit(node.contents) + end + + def visit_program(node) + node.statements.body.each do |statement| + break unless statement.is_a?(Comment) + + if statement.value == "# frozen_string_literal: true" + @frozen_string_literal = true + end + end + + preexes = [] + statements = [] + + node.statements.body.each do |statement| + case statement + when Comment, EmbDoc, EndContent, VoidStmt + # ignore + when BEGINBlock + preexes << statement + else + statements << statement + end + end + + with_instruction_sequence(:top, "", nil, node) do + visit_all(preexes) + + if statements.empty? + builder.putnil + else + *statements, last_statement = statements + visit_all(statements) + with_last_statement { visit(last_statement) } + end + + builder.leave + end + end + + def visit_qsymbols(node) + builder.duparray(node.accept(RubyVisitor.new)) + end + + def visit_qwords(node) + if frozen_string_literal + builder.duparray(node.accept(RubyVisitor.new)) + else + visit_all(node.elements) + builder.newarray(node.elements.length) + end + end + + def visit_range(node) + builder.putobject(node.accept(RubyVisitor.new)) + rescue RubyVisitor::CompilationError + visit(node.left) + visit(node.right) + builder.newrange(node.operator.value == ".." ? 0 : 1) + end + + def visit_rational(node) + builder.putobject(node.accept(RubyVisitor.new)) + end + + def visit_regexp_literal(node) + builder.putobject(node.accept(RubyVisitor.new)) + rescue RubyVisitor::CompilationError + flags = RubyVisitor.new.visit_regexp_literal_flags(node) + length = visit_string_parts(node) + builder.toregexp(flags, length) + end + + def visit_rest_param(node) + current_iseq.local_table.plain(node.name.value.to_sym) + current_iseq.argument_options[:rest_start] = current_iseq.argument_size + current_iseq.argument_size += 1 + end + + def visit_sclass(node) + visit(node.target) + builder.putnil + + singleton_iseq = + with_instruction_sequence( + :class, + "singleton class", + current_iseq, + node + ) do + builder.event(:RUBY_EVENT_CLASS) + visit(node.bodystmt) + builder.event(:RUBY_EVENT_END) + builder.leave + end + + builder.defineclass( + :singletonclass, + singleton_iseq, + VM_DEFINECLASS_TYPE_SINGLETON_CLASS + ) + end + + def visit_statements(node) + statements = + node.body.select do |statement| + case statement + when Comment, EmbDoc, EndContent, VoidStmt + false + else + true + end + end + + statements.empty? ? builder.putnil : visit_all(statements) + end + + def visit_string_concat(node) + value = node.left.parts.first.value + node.right.parts.first.value + content = TStringContent.new(value: value, location: node.location) + + literal = + StringLiteral.new( + parts: [content], + quote: node.left.quote, + location: node.location + ) + visit_string_literal(literal) + end + + def visit_string_embexpr(node) + visit(node.statements) + end + + def visit_string_literal(node) + if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) + visit(node.parts.first) + else + length = visit_string_parts(node) + builder.concatstrings(length) + end + end + + def visit_super(node) + builder.putself + visit(node.arguments) + builder.invokesuper( + nil, + argument_parts(node.arguments).length, + VM_CALL_FCALL | VM_CALL_ARGS_SIMPLE | VM_CALL_SUPER, + nil + ) + end + + def visit_symbol_literal(node) + builder.putobject(node.accept(RubyVisitor.new)) + end + + def visit_symbols(node) + builder.duparray(node.accept(RubyVisitor.new)) + rescue RubyVisitor::CompilationError + node.elements.each do |element| + if element.parts.length == 1 && + element.parts.first.is_a?(TStringContent) + builder.putobject(element.parts.first.value.to_sym) + else + length = visit_string_parts(element) + builder.concatstrings(length) + builder.intern + end + end + + builder.newarray(node.elements.length) + end + + def visit_top_const_ref(node) + builder.opt_getconstant_path(constant_names(node)) + end + + def visit_tstring_content(node) + if frozen_string_literal + builder.putobject(node.accept(RubyVisitor.new)) + else + builder.putstring(node.accept(RubyVisitor.new)) + end + end + + def visit_unary(node) + method_id = + case node.operator + when "+", "-" + "#{node.operator}@" + else + node.operator + end + + visit_call( + CommandCall.new( + receiver: node.statement, + operator: nil, + message: Ident.new(value: method_id, location: Location.default), + arguments: nil, + block: nil, + location: Location.default + ) + ) + end + + def visit_undef(node) + node.symbols.each_with_index do |symbol, index| + builder.pop if index != 0 + builder.putspecialobject(VM_SPECIAL_OBJECT_VMCORE) + builder.putspecialobject(VM_SPECIAL_OBJECT_CBASE) + visit(symbol) + builder.send(:"core#undef_method", 2, VM_CALL_ARGS_SIMPLE) + end + end + + def visit_unless(node) + visit(node.predicate) + branchunless = builder.branchunless(-1) + node.consequent ? visit(node.consequent) : builder.putnil + + if last_statement? + builder.leave + branchunless[1] = builder.label + + visit(node.statements) + else + builder.pop + + if node.consequent + jump = builder.jump(-1) + branchunless[1] = builder.label + visit(node.consequent) + jump[1] = builder.label + else + branchunless[1] = builder.label + end + end + end + + def visit_until(node) + jumps = [] + + jumps << builder.jump(-1) + builder.putnil + builder.pop + jumps << builder.jump(-1) + + label = builder.label + visit(node.statements) + builder.pop + jumps.each { |jump| jump[1] = builder.label } + + visit(node.predicate) + builder.branchunless(label) + builder.putnil if last_statement? + end + + def visit_var_field(node) + case node.value + when CVar, IVar + name = node.value.value.to_sym + current_iseq.inline_storage_for(name) + when Ident + name = node.value.value.to_sym + + if (local_variable = current_iseq.local_variable(name)) + local_variable + else + current_iseq.local_table.plain(name) + current_iseq.local_variable(name) + end + end + end + + def visit_var_ref(node) + case node.value + when Const + builder.opt_getconstant_path(constant_names(node)) + when CVar + name = node.value.value.to_sym + builder.getclassvariable(name) + when GVar + builder.getglobal(node.value.value.to_sym) + when Ident + lookup = current_iseq.local_variable(node.value.value.to_sym) + + case lookup.local + when LocalTable::BlockLocal + builder.getblockparam(lookup.index, lookup.level) + when LocalTable::PlainLocal + builder.getlocal(lookup.index, lookup.level) + end + when IVar + name = node.value.value.to_sym + builder.getinstancevariable(name) + when Kw + case node.value.value + when "false" + builder.putobject(false) + when "nil" + builder.putnil + when "self" + builder.putself + when "true" + builder.putobject(true) + end + end + end + + def visit_vcall(node) + builder.putself + + flag = VM_CALL_FCALL | VM_CALL_VCALL | VM_CALL_ARGS_SIMPLE + builder.send(node.value.value.to_sym, 0, flag) + end + + def visit_when(node) + visit(node.statements) + end + + def visit_while(node) + jumps = [] + + jumps << builder.jump(-1) + builder.putnil + builder.pop + jumps << builder.jump(-1) + + label = builder.label + visit(node.statements) + builder.pop + jumps.each { |jump| jump[1] = builder.label } + + visit(node.predicate) + builder.branchif(label) + builder.putnil if last_statement? + end + + def visit_word(node) + if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) + visit(node.parts.first) + else + length = visit_string_parts(node) + builder.concatstrings(length) + end + end + + def visit_words(node) + converted = nil + + if frozen_string_literal + begin + converted = node.accept(RubyVisitor.new) + rescue RubyVisitor::CompilationError + end + end + + if converted + builder.duparray(converted) + else + visit_all(node.elements) + builder.newarray(node.elements.length) + end + end + + def visit_xstring_literal(node) + builder.putself + length = visit_string_parts(node) + builder.concatstrings(node.parts.length) if length > 1 + builder.send(:`, 1, VM_CALL_FCALL | VM_CALL_ARGS_SIMPLE) + end + + def visit_yield(node) + parts = argument_parts(node.arguments) + visit_all(parts) + builder.invokeblock(nil, parts.length, VM_CALL_ARGS_SIMPLE) + end + + def visit_zsuper(_node) + builder.putself + builder.invokesuper( + nil, + 0, + VM_CALL_FCALL | VM_CALL_ARGS_SIMPLE | VM_CALL_SUPER | VM_CALL_ZSUPER, + nil + ) + end + + private + + # This is a helper that is used in places where arguments may be present + # or they may be wrapped in parentheses. It's meant to descend down the + # tree and return an array of argument nodes. + def argument_parts(node) + case node + when nil + [] + when Args + node.parts + when ArgParen + if node.arguments.is_a?(ArgsForward) + [node.arguments] + else + node.arguments.parts + end + when Paren + node.contents.parts + end + end + + # Constant names when they are being assigned or referenced come in as a + # tree, but it's more convenient to work with them as an array. This + # method converts them into that array. This is nice because it's the + # operand that goes to opt_getconstant_path in Ruby 3.2. + def constant_names(node) + current = node + names = [] + + while current.is_a?(ConstPathField) || current.is_a?(ConstPathRef) + names.unshift(current.constant.value.to_sym) + current = current.parent + end + + case current + when VarField, VarRef + names.unshift(current.value.value.to_sym) + when TopConstRef + names.unshift(current.constant.value.to_sym) + names.unshift(:"") + end + + names + end + + # For the most part when an OpAssign (operator assignment) node with a ||= + # operator is being compiled it's a matter of reading the target, checking + # if the value should be evaluated, evaluating it if so, and then writing + # the result back to the target. + # + # However, in certain kinds of assignments (X, ::X, X::Y, @@x, and $x) we + # first check if the value is defined using the defined instruction. I + # don't know why it is necessary, and suspect that it isn't. + def opassign_defined(node) + case node.target + when ConstPathField + visit(node.target.parent) + name = node.target.constant.value.to_sym + + builder.dup + builder.defined(DEFINED_CONST_FROM, name, true) + when TopConstField + name = node.target.constant.value.to_sym + + builder.putobject(Object) + builder.dup + builder.defined(DEFINED_CONST_FROM, name, true) + when VarField + name = node.target.value.value.to_sym + builder.putnil + + case node.target.value + when Const + builder.defined(DEFINED_CONST, name, true) + when CVar + builder.defined(DEFINED_CVAR, name, true) + when GVar + builder.defined(DEFINED_GVAR, name, true) + end + end + + branchunless = builder.branchunless(-1) + + case node.target + when ConstPathField, TopConstField + builder.dup + builder.putobject(true) + builder.getconstant(name) + when VarField + case node.target.value + when Const + builder.opt_getconstant_path(constant_names(node.target)) + when CVar + builder.getclassvariable(name) + when GVar + builder.getglobal(name) + end + end + + builder.dup + branchif = builder.branchif(-1) + builder.pop + + branchunless[1] = builder.label + visit(node.value) + + case node.target + when ConstPathField, TopConstField + builder.dupn(2) + builder.swap + builder.setconstant(name) + when VarField + builder.dup + + case node.target.value + when Const + builder.putspecialobject(VM_SPECIAL_OBJECT_CONST_BASE) + builder.setconstant(name) + when CVar + builder.setclassvariable(name) + when GVar + builder.setglobal(name) + end + end + + branchif[1] = builder.label + end + + # Whenever a value is interpolated into a string-like structure, these + # three instructions are pushed. + def push_interpolate + builder.dup + builder.objtostring(:to_s, 0, VM_CALL_FCALL | VM_CALL_ARGS_SIMPLE) + builder.anytostring + end + + # There are a lot of nodes in the AST that act as contains of parts of + # strings. This includes things like string literals, regular expressions, + # heredocs, etc. This method will visit all the parts of a string within + # those containers. + def visit_string_parts(node) + length = 0 + + unless node.parts.first.is_a?(TStringContent) + builder.putobject("") + length += 1 + end + + node.parts.each do |part| + case part + when StringDVar + visit(part.variable) + push_interpolate + when StringEmbExpr + visit(part) + push_interpolate + when TStringContent + builder.putobject(part.accept(RubyVisitor.new)) + end + + length += 1 + end + + length + end + + # The current instruction sequence that we're compiling is always stored + # on the compiler. When we descend into a node that has its own + # instruction sequence, this method can be called to temporarily set the + # new value of the instruction sequence, yield, and then set it back. + def with_instruction_sequence(type, name, parent_iseq, node) + previous_iseq = current_iseq + previous_builder = builder + + begin + iseq = InstructionSequence.new(type, name, parent_iseq, node.location) + + @current_iseq = iseq + @builder = + Builder.new( + iseq, + frozen_string_literal: frozen_string_literal, + operands_unification: operands_unification, + specialized_instruction: specialized_instruction + ) + + yield + iseq + ensure + @current_iseq = previous_iseq + @builder = previous_builder + end + end + + # When we're compiling the last statement of a set of statements within a + # scope, the instructions sometimes change from pops to leaves. These + # kinds of peephole optimizations can reduce the overall number of + # instructions. Therefore, we keep track of whether we're compiling the + # last statement of a scope and allow visit methods to query that + # information. + def with_last_statement + previous = @last_statement + @last_statement = true + + begin + yield + ensure + @last_statement = previous + end + end + + def last_statement? + @last_statement + end + + # OpAssign nodes can have a number of different kinds of nodes as their + # "target" (i.e., the left-hand side of the assignment). When compiling + # these nodes we typically need to first fetch the current value of the + # variable, then perform some kind of action, then store the result back + # into the variable. This method handles that by first fetching the value, + # then yielding to the block, then storing the result. + def with_opassign(node) + case node.target + when ARefField + builder.putnil + visit(node.target.collection) + visit(node.target.index) + + builder.dupn(2) + builder.send(:[], 1, VM_CALL_ARGS_SIMPLE) + + yield + + builder.setn(3) + builder.send(:[]=, 2, VM_CALL_ARGS_SIMPLE) + builder.pop + when ConstPathField + name = node.target.constant.value.to_sym + + visit(node.target.parent) + builder.dup + builder.putobject(true) + builder.getconstant(name) + + yield + + if node.operator.value == "&&=" + builder.dupn(2) + else + builder.swap + builder.topn(1) + end + + builder.swap + builder.setconstant(name) + when TopConstField + name = node.target.constant.value.to_sym + + builder.putobject(Object) + builder.dup + builder.putobject(true) + builder.getconstant(name) + + yield + + if node.operator.value == "&&=" + builder.dupn(2) + else + builder.swap + builder.topn(1) + end + + builder.swap + builder.setconstant(name) + when VarField + case node.target.value + when Const + names = constant_names(node.target) + builder.opt_getconstant_path(names) + + yield + + builder.dup + builder.putspecialobject(VM_SPECIAL_OBJECT_CONST_BASE) + builder.setconstant(names.last) + when CVar + name = node.target.value.value.to_sym + builder.getclassvariable(name) + + yield + + builder.dup + builder.setclassvariable(name) + when GVar + name = node.target.value.value.to_sym + builder.getglobal(name) + + yield + + builder.dup + builder.setglobal(name) + when Ident + local_variable = visit(node.target) + builder.getlocal(local_variable.index, local_variable.level) + + yield + + builder.dup + builder.setlocal(local_variable.index, local_variable.level) + when IVar + name = node.target.value.value.to_sym + builder.getinstancevariable(name) + + yield + + builder.dup + builder.setinstancevariable(name) + end + end + end + end +end diff --git a/lib/syntax_tree/visitor/compiler.rb b/lib/syntax_tree/visitor/compiler.rb deleted file mode 100644 index 82155d37..00000000 --- a/lib/syntax_tree/visitor/compiler.rb +++ /dev/null @@ -1,2719 +0,0 @@ -# frozen_string_literal: true - -module SyntaxTree - class Visitor - # This class is an experiment in transforming Syntax Tree nodes into their - # corresponding YARV instruction sequences. It attempts to mirror the - # behavior of RubyVM::InstructionSequence.compile. - # - # You use this as with any other visitor. First you parse code into a tree, - # then you visit it with this compiler. Visiting the root node of the tree - # will return a SyntaxTree::Visitor::Compiler::InstructionSequence object. - # With that object you can call #to_a on it, which will return a serialized - # form of the instruction sequence as an array. This array _should_ mirror - # the array given by RubyVM::InstructionSequence#to_a. - # - # As an example, here is how you would compile a single expression: - # - # program = SyntaxTree.parse("1 + 2") - # program.accept(SyntaxTree::Visitor::Compiler.new).to_a - # - # [ - # "YARVInstructionSequence/SimpleDataFormat", - # 3, - # 1, - # 1, - # {:arg_size=>0, :local_size=>0, :stack_max=>2}, - # "", - # "", - # "", - # 1, - # :top, - # [], - # {}, - # [], - # [ - # [:putobject_INT2FIX_1_], - # [:putobject, 2], - # [:opt_plus, {:mid=>:+, :flag=>16, :orig_argc=>1}], - # [:leave] - # ] - # ] - # - # Note that this is the same output as calling: - # - # RubyVM::InstructionSequence.compile("1 + 2").to_a - # - class Compiler < BasicVisitor - # This visitor is responsible for converting Syntax Tree nodes into their - # corresponding Ruby structures. This is used to convert the operands of - # some instructions like putobject that push a Ruby object directly onto - # the stack. It is only used when the entire structure can be represented - # at compile-time, as opposed to constructed at run-time. - class RubyVisitor < BasicVisitor - # This error is raised whenever a node cannot be converted into a Ruby - # object at compile-time. - class CompilationError < StandardError - end - - # This will attempt to compile the given node. If it's possible, then - # it will return the compiled object. Otherwise it will return nil. - def self.compile(node) - node.accept(new) - rescue CompilationError - end - - def visit_array(node) - visit_all(node.contents.parts) - end - - def visit_bare_assoc_hash(node) - node.assocs.to_h do |assoc| - # We can only convert regular key-value pairs. A double splat ** - # operator means it has to be converted at run-time. - raise CompilationError unless assoc.is_a?(Assoc) - [visit(assoc.key), visit(assoc.value)] - end - end - - def visit_float(node) - node.value.to_f - end - - alias visit_hash visit_bare_assoc_hash - - def visit_imaginary(node) - node.value.to_c - end - - def visit_int(node) - node.value.to_i - end - - def visit_label(node) - node.value.chomp(":").to_sym - end - - def visit_mrhs(node) - visit_all(node.parts) - end - - def visit_qsymbols(node) - node.elements.map { |element| visit(element).to_sym } - end - - def visit_qwords(node) - visit_all(node.elements) - end - - def visit_range(node) - left, right = [visit(node.left), visit(node.right)] - node.operator.value === ".." ? left..right : left...right - end - - def visit_rational(node) - node.value.to_r - end - - def visit_regexp_literal(node) - if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) - Regexp.new(node.parts.first.value, visit_regexp_literal_flags(node)) - else - # Any interpolation of expressions or variables will result in the - # regular expression being constructed at run-time. - raise CompilationError - end - end - - # This isn't actually a visit method, though maybe it should be. It is - # responsible for converting the set of string options on a regular - # expression into its equivalent integer. - def visit_regexp_literal_flags(node) - node - .options - .chars - .inject(0) do |accum, option| - accum | - case option - when "i" - Regexp::IGNORECASE - when "x" - Regexp::EXTENDED - when "m" - Regexp::MULTILINE - else - raise "Unknown regexp option: #{option}" - end - end - end - - def visit_symbol_literal(node) - node.value.value.to_sym - end - - def visit_symbols(node) - node.elements.map { |element| visit(element).to_sym } - end - - def visit_tstring_content(node) - node.value - end - - def visit_word(node) - if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) - node.parts.first.value - else - # Any interpolation of expressions or variables will result in the - # string being constructed at run-time. - raise CompilationError - end - end - - def visit_words(node) - visit_all(node.elements) - end - - def visit_unsupported(_node) - raise CompilationError - end - - # Please forgive the metaprogramming here. This is used to create visit - # methods for every node that we did not explicitly handle. By default - # each of these methods will raise a CompilationError. - handled = instance_methods(false) - (Visitor.instance_methods(false) - handled).each do |method| - alias_method method, :visit_unsupported - end - end - - # This object is used to track the size of the stack at any given time. It - # is effectively a mini symbolic interpreter. It's necessary because when - # instruction sequences get serialized they include a :stack_max field on - # them. This field is used to determine how much stack space to allocate - # for the instruction sequence. - class Stack - attr_reader :current_size, :maximum_size - - def initialize - @current_size = 0 - @maximum_size = 0 - end - - def change_by(value) - @current_size += value - @maximum_size = @current_size if @current_size > @maximum_size - end - end - - # This represents every local variable associated with an instruction - # sequence. There are two kinds of locals: plain locals that are what you - # expect, and block proxy locals, which represent local variables - # associated with blocks that were passed into the current instruction - # sequence. - class LocalTable - # A local representing a block passed into the current instruction - # sequence. - class BlockLocal - attr_reader :name - - def initialize(name) - @name = name - end - end - - # A regular local variable. - class PlainLocal - attr_reader :name - - def initialize(name) - @name = name - end - end - - # The result of looking up a local variable in the current local table. - class Lookup - attr_reader :local, :index, :level - - def initialize(local, index, level) - @local = local - @index = index - @level = level - end - end - - attr_reader :locals - - def initialize - @locals = [] - end - - def find(name, level) - index = locals.index { |local| local.name == name } - Lookup.new(locals[index], index, level) if index - end - - def has?(name) - locals.any? { |local| local.name == name } - end - - def names - locals.map(&:name) - end - - def size - locals.length - end - - # Add a BlockLocal to the local table. - def block(name) - locals << BlockLocal.new(name) unless has?(name) - end - - # Add a PlainLocal to the local table. - def plain(name) - locals << PlainLocal.new(name) unless has?(name) - end - - # This is the offset from the top of the stack where this local variable - # lives. - def offset(index) - size - (index - 3) - 1 - end - end - - # This class is meant to mirror RubyVM::InstructionSequence. It contains a - # list of instructions along with the metadata pertaining to them. It also - # functions as a builder for the instruction sequence. - class InstructionSequence - MAGIC = "YARVInstructionSequence/SimpleDataFormat" - - # The type of the instruction sequence. - attr_reader :type - - # The name of the instruction sequence. - attr_reader :name - - # The parent instruction sequence, if there is one. - attr_reader :parent_iseq - - # The location of the root node of this instruction sequence. - attr_reader :location - - # This is the list of information about the arguments to this - # instruction sequence. - attr_accessor :argument_size - attr_reader :argument_options - - # The list of instructions for this instruction sequence. - attr_reader :insns - - # The table of local variables. - attr_reader :local_table - - # The hash of names of instance and class variables pointing to the - # index of their associated inline storage. - attr_reader :inline_storages - - # The index of the next inline storage that will be created. - attr_reader :storage_index - - # An object that will track the current size of the stack and the - # maximum size of the stack for this instruction sequence. - attr_reader :stack - - def initialize(type, name, parent_iseq, location) - @type = type - @name = name - @parent_iseq = parent_iseq - @location = location - - @argument_size = 0 - @argument_options = {} - - @local_table = LocalTable.new - @inline_storages = {} - @insns = [] - @storage_index = 0 - @stack = Stack.new - end - - def local_variable(name, level = 0) - if (lookup = local_table.find(name, level)) - lookup - elsif parent_iseq - parent_iseq.local_variable(name, level + 1) - end - end - - def push(insn) - insns << insn - insn - end - - def inline_storage - storage = storage_index - @storage_index += 1 - storage - end - - def inline_storage_for(name) - unless inline_storages.key?(name) - inline_storages[name] = inline_storage - end - - inline_storages[name] - end - - def length - insns.inject(0) do |sum, insn| - insn.is_a?(Array) ? sum + insn.length : sum - end - end - - def each_child - insns.each do |insn| - insn[1..].each do |operand| - yield operand if operand.is_a?(InstructionSequence) - end - end - end - - def to_a - versions = RUBY_VERSION.split(".").map(&:to_i) - - [ - MAGIC, - versions[0], - versions[1], - 1, - { - arg_size: argument_size, - local_size: local_table.size, - stack_max: stack.maximum_size - }, - name, - "", - "", - location.start_line, - type, - local_table.names, - argument_options, - [], - insns.map { |insn| serialize(insn) } - ] - end - - private - - def serialize(insn) - case insn[0] - when :checkkeyword, :getblockparam, :getblockparamproxy, - :getlocal_WC_0, :getlocal_WC_1, :getlocal, :setlocal_WC_0, - :setlocal_WC_1, :setlocal - iseq = self - - case insn[0] - when :getlocal_WC_1, :setlocal_WC_1 - iseq = iseq.parent_iseq - when :getblockparam, :getblockparamproxy, :getlocal, :setlocal - insn[2].times { iseq = iseq.parent_iseq } - end - - # Here we need to map the local variable index to the offset - # from the top of the stack where it will be stored. - [insn[0], iseq.local_table.offset(insn[1]), *insn[2..]] - when :defineclass - [insn[0], insn[1], insn[2].to_a, insn[3]] - when :definemethod, :definesmethod - [insn[0], insn[1], insn[2].to_a] - when :send - # For any instructions that push instruction sequences onto the - # stack, we need to call #to_a on them as well. - [insn[0], insn[1], (insn[2].to_a if insn[2])] - when :once - [insn[0], insn[1].to_a, insn[2]] - else - insn - end - end - end - - # This class serves as a layer of indirection between the instruction - # sequence and the compiler. It allows us to provide different behavior - # for certain instructions depending on the Ruby version. For example, - # class variable reads and writes gained an inline cache in Ruby 3.0. So - # we place the logic for checking the Ruby version in this class. - class Builder - attr_reader :iseq, :stack - attr_reader :frozen_string_literal, - :operands_unification, - :specialized_instruction - - def initialize( - iseq, - frozen_string_literal: false, - operands_unification: true, - specialized_instruction: true - ) - @iseq = iseq - @stack = iseq.stack - - @frozen_string_literal = frozen_string_literal - @operands_unification = operands_unification - @specialized_instruction = specialized_instruction - end - - # This creates a new label at the current length of the instruction - # sequence. It is used as the operand for jump instructions. - def label - name = :"label_#{iseq.length}" - iseq.insns.last == name ? name : event(name) - end - - def event(name) - iseq.push(name) - name - end - - def adjuststack(number) - stack.change_by(-number) - iseq.push([:adjuststack, number]) - end - - def anytostring - stack.change_by(-2 + 1) - iseq.push([:anytostring]) - end - - def branchif(index) - stack.change_by(-1) - iseq.push([:branchif, index]) - end - - def branchnil(index) - stack.change_by(-1) - iseq.push([:branchnil, index]) - end - - def branchunless(index) - stack.change_by(-1) - iseq.push([:branchunless, index]) - end - - def checkkeyword(index, keyword_index) - stack.change_by(+1) - iseq.push([:checkkeyword, index, keyword_index]) - end - - def concatarray - stack.change_by(-2 + 1) - iseq.push([:concatarray]) - end - - def concatstrings(number) - stack.change_by(-number + 1) - iseq.push([:concatstrings, number]) - end - - def defined(type, name, message) - stack.change_by(-1 + 1) - iseq.push([:defined, type, name, message]) - end - - def defineclass(name, class_iseq, flags) - stack.change_by(-2 + 1) - iseq.push([:defineclass, name, class_iseq, flags]) - end - - def definemethod(name, method_iseq) - stack.change_by(0) - iseq.push([:definemethod, name, method_iseq]) - end - - def definesmethod(name, method_iseq) - stack.change_by(-1) - iseq.push([:definesmethod, name, method_iseq]) - end - - def dup - stack.change_by(-1 + 2) - iseq.push([:dup]) - end - - def duparray(object) - stack.change_by(+1) - iseq.push([:duparray, object]) - end - - def duphash(object) - stack.change_by(+1) - iseq.push([:duphash, object]) - end - - def dupn(number) - stack.change_by(+number) - iseq.push([:dupn, number]) - end - - def expandarray(length, flag) - stack.change_by(-1 + length) - iseq.push([:expandarray, length, flag]) - end - - def getblockparam(index, level) - stack.change_by(+1) - iseq.push([:getblockparam, index, level]) - end - - def getblockparamproxy(index, level) - stack.change_by(+1) - iseq.push([:getblockparamproxy, index, level]) - end - - def getclassvariable(name) - stack.change_by(+1) - - if RUBY_VERSION >= "3.0" - iseq.push([:getclassvariable, name, iseq.inline_storage_for(name)]) - else - iseq.push([:getclassvariable, name]) - end - end - - def getconstant(name) - stack.change_by(-2 + 1) - iseq.push([:getconstant, name]) - end - - def getglobal(name) - stack.change_by(+1) - iseq.push([:getglobal, name]) - end - - def getinstancevariable(name) - stack.change_by(+1) - - if RUBY_VERSION >= "3.2" - iseq.push([:getinstancevariable, name, iseq.inline_storage]) - else - inline_storage = iseq.inline_storage_for(name) - iseq.push([:getinstancevariable, name, inline_storage]) - end - end - - def getlocal(index, level) - stack.change_by(+1) - - if operands_unification - # Specialize the getlocal instruction based on the level of the - # local variable. If it's 0 or 1, then there's a specialized - # instruction that will look at the current scope or the parent - # scope, respectively, and requires fewer operands. - case level - when 0 - iseq.push([:getlocal_WC_0, index]) - when 1 - iseq.push([:getlocal_WC_1, index]) - else - iseq.push([:getlocal, index, level]) - end - else - iseq.push([:getlocal, index, level]) - end - end - - def getspecial(key, type) - stack.change_by(-0 + 1) - iseq.push([:getspecial, key, type]) - end - - def intern - stack.change_by(-1 + 1) - iseq.push([:intern]) - end - - def invokeblock(method_id, argc, flag) - stack.change_by(-argc + 1) - iseq.push([:invokeblock, call_data(method_id, argc, flag)]) - end - - def invokesuper(method_id, argc, flag, block_iseq) - stack.change_by(-(argc + 1) + 1) - - cdata = call_data(method_id, argc, flag) - iseq.push([:invokesuper, cdata, block_iseq]) - end - - def jump(index) - stack.change_by(0) - iseq.push([:jump, index]) - end - - def leave - stack.change_by(-1) - iseq.push([:leave]) - end - - def newarray(length) - stack.change_by(-length + 1) - iseq.push([:newarray, length]) - end - - def newhash(length) - stack.change_by(-length + 1) - iseq.push([:newhash, length]) - end - - def newrange(flag) - stack.change_by(-2 + 1) - iseq.push([:newrange, flag]) - end - - def nop - stack.change_by(0) - iseq.push([:nop]) - end - - def objtostring(method_id, argc, flag) - stack.change_by(-1 + 1) - iseq.push([:objtostring, call_data(method_id, argc, flag)]) - end - - def once(postexe_iseq, inline_storage) - stack.change_by(+1) - iseq.push([:once, postexe_iseq, inline_storage]) - end - - def opt_getconstant_path(names) - if RUBY_VERSION >= "3.2" - stack.change_by(+1) - iseq.push([:opt_getconstant_path, names]) - else - inline_storage = iseq.inline_storage - getinlinecache = opt_getinlinecache(-1, inline_storage) - - if names[0] == :"" - names.shift - pop - putobject(Object) - end - - names.each_with_index do |name, index| - putobject(index == 0) - getconstant(name) - end - - opt_setinlinecache(inline_storage) - getinlinecache[1] = label - end - end - - def opt_getinlinecache(offset, inline_storage) - stack.change_by(+1) - iseq.push([:opt_getinlinecache, offset, inline_storage]) - end - - def opt_newarray_max(length) - if specialized_instruction - stack.change_by(-length + 1) - iseq.push([:opt_newarray_max, length]) - else - newarray(length) - send(:max, 0, VM_CALL_ARGS_SIMPLE) - end - end - - def opt_newarray_min(length) - if specialized_instruction - stack.change_by(-length + 1) - iseq.push([:opt_newarray_min, length]) - else - newarray(length) - send(:min, 0, VM_CALL_ARGS_SIMPLE) - end - end - - def opt_setinlinecache(inline_storage) - stack.change_by(-1 + 1) - iseq.push([:opt_setinlinecache, inline_storage]) - end - - def opt_str_freeze(value) - if specialized_instruction - stack.change_by(+1) - iseq.push( - [ - :opt_str_freeze, - value, - call_data(:freeze, 0, VM_CALL_ARGS_SIMPLE) - ] - ) - else - putstring(value) - send(:freeze, 0, VM_CALL_ARGS_SIMPLE) - end - end - - def opt_str_uminus(value) - if specialized_instruction - stack.change_by(+1) - iseq.push( - [:opt_str_uminus, value, call_data(:-@, 0, VM_CALL_ARGS_SIMPLE)] - ) - else - putstring(value) - send(:-@, 0, VM_CALL_ARGS_SIMPLE) - end - end - - def pop - stack.change_by(-1) - iseq.push([:pop]) - end - - def putnil - stack.change_by(+1) - iseq.push([:putnil]) - end - - def putobject(object) - stack.change_by(+1) - - if operands_unification - # Specialize the putobject instruction based on the value of the - # object. If it's 0 or 1, then there's a specialized instruction - # that will push the object onto the stack and requires fewer - # operands. - if object.eql?(0) - iseq.push([:putobject_INT2FIX_0_]) - elsif object.eql?(1) - iseq.push([:putobject_INT2FIX_1_]) - else - iseq.push([:putobject, object]) - end - else - iseq.push([:putobject, object]) - end - end - - def putself - stack.change_by(+1) - iseq.push([:putself]) - end - - def putspecialobject(object) - stack.change_by(+1) - iseq.push([:putspecialobject, object]) - end - - def putstring(object) - stack.change_by(+1) - iseq.push([:putstring, object]) - end - - def send(method_id, argc, flag, block_iseq = nil) - stack.change_by(-(argc + 1) + 1) - cdata = call_data(method_id, argc, flag) - - if specialized_instruction - # Specialize the send instruction. If it doesn't have a block - # attached, then we will replace it with an opt_send_without_block - # and do further specializations based on the called method and the - # number of arguments. - - # stree-ignore - if !block_iseq && (flag & VM_CALL_ARGS_BLOCKARG) == 0 - case [method_id, argc] - when [:length, 0] then iseq.push([:opt_length, cdata]) - when [:size, 0] then iseq.push([:opt_size, cdata]) - when [:empty?, 0] then iseq.push([:opt_empty_p, cdata]) - when [:nil?, 0] then iseq.push([:opt_nil_p, cdata]) - when [:succ, 0] then iseq.push([:opt_succ, cdata]) - when [:!, 0] then iseq.push([:opt_not, cdata]) - when [:+, 1] then iseq.push([:opt_plus, cdata]) - when [:-, 1] then iseq.push([:opt_minus, cdata]) - when [:*, 1] then iseq.push([:opt_mult, cdata]) - when [:/, 1] then iseq.push([:opt_div, cdata]) - when [:%, 1] then iseq.push([:opt_mod, cdata]) - when [:==, 1] then iseq.push([:opt_eq, cdata]) - when [:=~, 1] then iseq.push([:opt_regexpmatch2, cdata]) - when [:<, 1] then iseq.push([:opt_lt, cdata]) - when [:<=, 1] then iseq.push([:opt_le, cdata]) - when [:>, 1] then iseq.push([:opt_gt, cdata]) - when [:>=, 1] then iseq.push([:opt_ge, cdata]) - when [:<<, 1] then iseq.push([:opt_ltlt, cdata]) - when [:[], 1] then iseq.push([:opt_aref, cdata]) - when [:&, 1] then iseq.push([:opt_and, cdata]) - when [:|, 1] then iseq.push([:opt_or, cdata]) - when [:[]=, 2] then iseq.push([:opt_aset, cdata]) - when [:!=, 1] - eql_data = call_data(:==, 1, VM_CALL_ARGS_SIMPLE) - iseq.push([:opt_neq, eql_data, cdata]) - else - iseq.push([:opt_send_without_block, cdata]) - end - else - iseq.push([:send, cdata, block_iseq]) - end - else - iseq.push([:send, cdata, block_iseq]) - end - end - - def setclassvariable(name) - stack.change_by(-1) - - if RUBY_VERSION >= "3.0" - iseq.push([:setclassvariable, name, iseq.inline_storage_for(name)]) - else - iseq.push([:setclassvariable, name]) - end - end - - def setconstant(name) - stack.change_by(-2) - iseq.push([:setconstant, name]) - end - - def setglobal(name) - stack.change_by(-1) - iseq.push([:setglobal, name]) - end - - def setinstancevariable(name) - stack.change_by(-1) - - if RUBY_VERSION >= "3.2" - iseq.push([:setinstancevariable, name, iseq.inline_storage]) - else - inline_storage = iseq.inline_storage_for(name) - iseq.push([:setinstancevariable, name, inline_storage]) - end - end - - def setlocal(index, level) - stack.change_by(-1) - - if operands_unification - # Specialize the setlocal instruction based on the level of the - # local variable. If it's 0 or 1, then there's a specialized - # instruction that will write to the current scope or the parent - # scope, respectively, and requires fewer operands. - case level - when 0 - iseq.push([:setlocal_WC_0, index]) - when 1 - iseq.push([:setlocal_WC_1, index]) - else - iseq.push([:setlocal, index, level]) - end - else - iseq.push([:setlocal, index, level]) - end - end - - def setn(number) - stack.change_by(-1 + 1) - iseq.push([:setn, number]) - end - - def splatarray(flag) - stack.change_by(-1 + 1) - iseq.push([:splatarray, flag]) - end - - def swap - stack.change_by(-2 + 2) - iseq.push([:swap]) - end - - def topn(number) - stack.change_by(+1) - iseq.push([:topn, number]) - end - - def toregexp(options, length) - stack.change_by(-length + 1) - iseq.push([:toregexp, options, length]) - end - - private - - # This creates a call data object that is used as the operand for the - # send, invokesuper, and objtostring instructions. - def call_data(method_id, argc, flag) - { mid: method_id, flag: flag, orig_argc: argc } - end - end - - # These constants correspond to the putspecialobject instruction. They are - # used to represent special objects that are pushed onto the stack. - VM_SPECIAL_OBJECT_VMCORE = 1 - VM_SPECIAL_OBJECT_CBASE = 2 - VM_SPECIAL_OBJECT_CONST_BASE = 3 - - # These constants correspond to the flag passed as part of the call data - # structure on the send instruction. They are used to represent various - # metadata about the callsite (e.g., were keyword arguments used?, was a - # block given?, etc.). - VM_CALL_ARGS_SPLAT = 1 << 0 - VM_CALL_ARGS_BLOCKARG = 1 << 1 - VM_CALL_FCALL = 1 << 2 - VM_CALL_VCALL = 1 << 3 - VM_CALL_ARGS_SIMPLE = 1 << 4 - VM_CALL_BLOCKISEQ = 1 << 5 - VM_CALL_KWARG = 1 << 6 - VM_CALL_KW_SPLAT = 1 << 7 - VM_CALL_TAILCALL = 1 << 8 - VM_CALL_SUPER = 1 << 9 - VM_CALL_ZSUPER = 1 << 10 - VM_CALL_OPT_SEND = 1 << 11 - VM_CALL_KW_SPLAT_MUT = 1 << 12 - - # These constants correspond to the value passed as part of the defined - # instruction. It's an enum defined in the CRuby codebase that tells that - # instruction what kind of defined check to perform. - DEFINED_NIL = 1 - DEFINED_IVAR = 2 - DEFINED_LVAR = 3 - DEFINED_GVAR = 4 - DEFINED_CVAR = 5 - DEFINED_CONST = 6 - DEFINED_METHOD = 7 - DEFINED_YIELD = 8 - DEFINED_ZSUPER = 9 - DEFINED_SELF = 10 - DEFINED_TRUE = 11 - DEFINED_FALSE = 12 - DEFINED_ASGN = 13 - DEFINED_EXPR = 14 - DEFINED_REF = 15 - DEFINED_FUNC = 16 - DEFINED_CONST_FROM = 17 - - # These constants correspond to the value passed in the flags as part of - # the defineclass instruction. - VM_DEFINECLASS_TYPE_CLASS = 0 - VM_DEFINECLASS_TYPE_SINGLETON_CLASS = 1 - VM_DEFINECLASS_TYPE_MODULE = 2 - VM_DEFINECLASS_FLAG_SCOPED = 8 - VM_DEFINECLASS_FLAG_HAS_SUPERCLASS = 16 - - # These options mirror the compilation options that we currently support - # that can be also passed to RubyVM::InstructionSequence.compile. - attr_reader :frozen_string_literal, - :operands_unification, - :specialized_instruction - - # The current instruction sequence that is being compiled. - attr_reader :current_iseq - - # This is the current builder that is being used to construct the current - # instruction sequence. - attr_reader :builder - - # A boolean to track if we're currently compiling the last statement - # within a set of statements. This information is necessary to determine - # if we need to return the value of the last statement. - attr_reader :last_statement - - def initialize( - frozen_string_literal: false, - operands_unification: true, - specialized_instruction: true - ) - @frozen_string_literal = frozen_string_literal - @operands_unification = operands_unification - @specialized_instruction = specialized_instruction - - @current_iseq = nil - @builder = nil - @last_statement = false - end - - def visit_BEGIN(node) - visit(node.statements) - end - - def visit_CHAR(node) - if frozen_string_literal - builder.putobject(node.value[1..]) - else - builder.putstring(node.value[1..]) - end - end - - def visit_END(node) - name = "block in #{current_iseq.name}" - once_iseq = - with_instruction_sequence(:block, name, current_iseq, node) do - postexe_iseq = - with_instruction_sequence(:block, name, current_iseq, node) do - *statements, last_statement = node.statements.body - visit_all(statements) - with_last_statement { visit(last_statement) } - builder.leave - end - - builder.putspecialobject(VM_SPECIAL_OBJECT_VMCORE) - builder.send(:"core#set_postexe", 0, VM_CALL_FCALL, postexe_iseq) - builder.leave - end - - builder.once(once_iseq, current_iseq.inline_storage) - builder.pop - end - - def visit_alias(node) - builder.putspecialobject(VM_SPECIAL_OBJECT_VMCORE) - builder.putspecialobject(VM_SPECIAL_OBJECT_CBASE) - visit(node.left) - visit(node.right) - builder.send(:"core#set_method_alias", 3, VM_CALL_ARGS_SIMPLE) - end - - def visit_aref(node) - visit(node.collection) - visit(node.index) - builder.send(:[], 1, VM_CALL_ARGS_SIMPLE) - end - - def visit_arg_block(node) - visit(node.value) - end - - def visit_arg_paren(node) - visit(node.arguments) - end - - def visit_arg_star(node) - visit(node.value) - builder.splatarray(false) - end - - def visit_args(node) - visit_all(node.parts) - end - - def visit_array(node) - if (compiled = RubyVisitor.compile(node)) - builder.duparray(compiled) - else - length = 0 - - node.contents.parts.each do |part| - if part.is_a?(ArgStar) - if length > 0 - builder.newarray(length) - length = 0 - end - - visit(part.value) - builder.concatarray - else - visit(part) - length += 1 - end - end - - builder.newarray(length) if length > 0 - if length > 0 && length != node.contents.parts.length - builder.concatarray - end - end - end - - def visit_assign(node) - case node.target - when ARefField - builder.putnil - visit(node.target.collection) - visit(node.target.index) - visit(node.value) - builder.setn(3) - builder.send(:[]=, 2, VM_CALL_ARGS_SIMPLE) - builder.pop - when ConstPathField - names = constant_names(node.target) - name = names.pop - - if RUBY_VERSION >= "3.2" - builder.opt_getconstant_path(names) - visit(node.value) - builder.swap - builder.topn(1) - builder.swap - builder.setconstant(name) - else - visit(node.value) - builder.dup if last_statement? - builder.opt_getconstant_path(names) - builder.setconstant(name) - end - when Field - builder.putnil - visit(node.target) - visit(node.value) - builder.setn(2) - builder.send(:"#{node.target.name.value}=", 1, VM_CALL_ARGS_SIMPLE) - builder.pop - when TopConstField - name = node.target.constant.value.to_sym - - if RUBY_VERSION >= "3.2" - builder.putobject(Object) - visit(node.value) - builder.swap - builder.topn(1) - builder.swap - builder.setconstant(name) - else - visit(node.value) - builder.dup if last_statement? - builder.putobject(Object) - builder.setconstant(name) - end - when VarField - visit(node.value) - builder.dup if last_statement? - - case node.target.value - when Const - builder.putspecialobject(VM_SPECIAL_OBJECT_CONST_BASE) - builder.setconstant(node.target.value.value.to_sym) - when CVar - builder.setclassvariable(node.target.value.value.to_sym) - when GVar - builder.setglobal(node.target.value.value.to_sym) - when Ident - local_variable = visit(node.target) - builder.setlocal(local_variable.index, local_variable.level) - when IVar - builder.setinstancevariable(node.target.value.value.to_sym) - end - end - end - - def visit_assoc(node) - visit(node.key) - visit(node.value) - end - - def visit_assoc_splat(node) - visit(node.value) - end - - def visit_backref(node) - builder.getspecial(1, 2 * node.value[1..].to_i) - end - - def visit_bare_assoc_hash(node) - if (compiled = RubyVisitor.compile(node)) - builder.duphash(compiled) - else - visit_all(node.assocs) - end - end - - def visit_binary(node) - case node.operator - when :"&&" - visit(node.left) - builder.dup - - branchunless = builder.branchunless(-1) - builder.pop - - visit(node.right) - branchunless[1] = builder.label - when :"||" - visit(node.left) - builder.dup - - branchif = builder.branchif(-1) - builder.pop - - visit(node.right) - branchif[1] = builder.label - else - visit(node.left) - visit(node.right) - builder.send(node.operator, 1, VM_CALL_ARGS_SIMPLE) - end - end - - def visit_block(node) - with_instruction_sequence( - :block, - "block in #{current_iseq.name}", - current_iseq, - node - ) do - builder.event(:RUBY_EVENT_B_CALL) - visit(node.block_var) - visit(node.bodystmt) - builder.event(:RUBY_EVENT_B_RETURN) - builder.leave - end - end - - def visit_block_var(node) - params = node.params - - if params.requireds.length == 1 && params.optionals.empty? && - !params.rest && params.posts.empty? && params.keywords.empty? && - !params.keyword_rest && !params.block - current_iseq.argument_options[:ambiguous_param0] = true - end - - visit(node.params) - - node.locals.each do |local| - current_iseq.local_table.plain(local.value.to_sym) - end - end - - def visit_blockarg(node) - current_iseq.argument_options[:block_start] = current_iseq.argument_size - current_iseq.local_table.block(node.name.value.to_sym) - current_iseq.argument_size += 1 - end - - def visit_bodystmt(node) - visit(node.statements) - end - - def visit_call(node) - if node.is_a?(CallNode) - return( - visit_call( - CommandCall.new( - receiver: node.receiver, - operator: node.operator, - message: node.message, - arguments: node.arguments, - block: nil, - location: node.location - ) - ) - ) - end - - arg_parts = argument_parts(node.arguments) - argc = arg_parts.length - - # First we're going to check if we're calling a method on an array - # literal without any arguments. In that case there are some - # specializations we might be able to perform. - if argc == 0 && (node.message.is_a?(Ident) || node.message.is_a?(Op)) - case node.receiver - when ArrayLiteral - parts = node.receiver.contents&.parts || [] - - if parts.none? { |part| part.is_a?(ArgStar) } && - RubyVisitor.compile(node.receiver).nil? - case node.message.value - when "max" - visit(node.receiver.contents) - builder.opt_newarray_max(parts.length) - return - when "min" - visit(node.receiver.contents) - builder.opt_newarray_min(parts.length) - return - end - end - when StringLiteral - if RubyVisitor.compile(node.receiver).nil? - case node.message.value - when "-@" - builder.opt_str_uminus(node.receiver.parts.first.value) - return - when "freeze" - builder.opt_str_freeze(node.receiver.parts.first.value) - return - end - end - end - end - - if node.receiver - if node.receiver.is_a?(VarRef) && - ( - lookup = - current_iseq.local_variable(node.receiver.value.value.to_sym) - ) && lookup.local.is_a?(LocalTable::BlockLocal) - builder.getblockparamproxy(lookup.index, lookup.level) - else - visit(node.receiver) - end - else - builder.putself - end - - branchnil = - if node.operator&.value == "&." - builder.dup - builder.branchnil(-1) - end - - flag = 0 - - arg_parts.each do |arg_part| - case arg_part - when ArgBlock - argc -= 1 - flag |= VM_CALL_ARGS_BLOCKARG - visit(arg_part) - when ArgStar - flag |= VM_CALL_ARGS_SPLAT - visit(arg_part) - when ArgsForward - flag |= VM_CALL_ARGS_SPLAT | VM_CALL_ARGS_BLOCKARG - - lookup = current_iseq.local_table.find(:*, 0) - builder.getlocal(lookup.index, lookup.level) - builder.splatarray(arg_parts.length != 1) - - lookup = current_iseq.local_table.find(:&, 0) - builder.getblockparamproxy(lookup.index, lookup.level) - when BareAssocHash - flag |= VM_CALL_KW_SPLAT - visit(arg_part) - else - visit(arg_part) - end - end - - block_iseq = visit(node.block) if node.block - flag |= VM_CALL_ARGS_SIMPLE if block_iseq.nil? && flag == 0 - flag |= VM_CALL_FCALL if node.receiver.nil? - - builder.send(node.message.value.to_sym, argc, flag, block_iseq) - branchnil[1] = builder.label if branchnil - end - - def visit_case(node) - visit(node.value) if node.value - - clauses = [] - else_clause = nil - - current = node.consequent - - while current - clauses << current - - if (current = current.consequent).is_a?(Else) - else_clause = current - break - end - end - - branches = - clauses.map do |clause| - visit(clause.arguments) - builder.topn(1) - builder.send(:===, 1, VM_CALL_FCALL | VM_CALL_ARGS_SIMPLE) - [clause, builder.branchif(:label_00)] - end - - builder.pop - - else_clause ? visit(else_clause) : builder.putnil - - builder.leave - - branches.each_with_index do |(clause, branchif), index| - builder.leave if index != 0 - branchif[1] = builder.label - builder.pop - visit(clause) - end - end - - def visit_class(node) - name = node.constant.constant.value.to_sym - class_iseq = - with_instruction_sequence( - :class, - "", - current_iseq, - node - ) do - builder.event(:RUBY_EVENT_CLASS) - visit(node.bodystmt) - builder.event(:RUBY_EVENT_END) - builder.leave - end - - flags = VM_DEFINECLASS_TYPE_CLASS - - case node.constant - when ConstPathRef - flags |= VM_DEFINECLASS_FLAG_SCOPED - visit(node.constant.parent) - when ConstRef - builder.putspecialobject(VM_SPECIAL_OBJECT_CONST_BASE) - when TopConstRef - flags |= VM_DEFINECLASS_FLAG_SCOPED - builder.putobject(Object) - end - - if node.superclass - flags |= VM_DEFINECLASS_FLAG_HAS_SUPERCLASS - visit(node.superclass) - else - builder.putnil - end - - builder.defineclass(name, class_iseq, flags) - end - - def visit_command(node) - visit_call( - CommandCall.new( - receiver: nil, - operator: nil, - message: node.message, - arguments: node.arguments, - block: node.block, - location: node.location - ) - ) - end - - def visit_command_call(node) - visit_call( - CommandCall.new( - receiver: node.receiver, - operator: node.operator, - message: node.message, - arguments: node.arguments, - block: node.block, - location: node.location - ) - ) - end - - def visit_const_path_field(node) - visit(node.parent) - end - - def visit_const_path_ref(node) - names = constant_names(node) - builder.opt_getconstant_path(names) - end - - def visit_def(node) - method_iseq = - with_instruction_sequence( - :method, - node.name.value, - current_iseq, - node - ) do - visit(node.params) if node.params - builder.event(:RUBY_EVENT_CALL) - visit(node.bodystmt) - builder.event(:RUBY_EVENT_RETURN) - builder.leave - end - - name = node.name.value.to_sym - - if node.target - visit(node.target) - builder.definesmethod(name, method_iseq) - else - builder.definemethod(name, method_iseq) - end - - builder.putobject(name) - end - - def visit_defined(node) - case node.value - when Assign - # If we're assigning to a local variable, then we need to make sure - # that we put it into the local table. - if node.value.target.is_a?(VarField) && - node.value.target.value.is_a?(Ident) - current_iseq.local_table.plain(node.value.target.value.value.to_sym) - end - - builder.putobject("assignment") - when VarRef - value = node.value.value - name = value.value.to_sym - - case value - when Const - builder.putnil - builder.defined(DEFINED_CONST, name, "constant") - when CVar - builder.putnil - builder.defined(DEFINED_CVAR, name, "class variable") - when GVar - builder.putnil - builder.defined(DEFINED_GVAR, name, "global-variable") - when Ident - builder.putobject("local-variable") - when IVar - builder.putnil - builder.defined(DEFINED_IVAR, name, "instance-variable") - when Kw - case name - when :false - builder.putobject("false") - when :nil - builder.putobject("nil") - when :self - builder.putobject("self") - when :true - builder.putobject("true") - end - end - when VCall - builder.putself - - name = node.value.value.value.to_sym - builder.defined(DEFINED_FUNC, name, "method") - when YieldNode - builder.putnil - builder.defined(DEFINED_YIELD, false, "yield") - when ZSuper - builder.putnil - builder.defined(DEFINED_ZSUPER, false, "super") - else - builder.putobject("expression") - end - end - - def visit_dyna_symbol(node) - if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) - builder.putobject(node.parts.first.value.to_sym) - end - end - - def visit_else(node) - visit(node.statements) - builder.pop unless last_statement? - end - - def visit_elsif(node) - visit_if( - IfNode.new( - predicate: node.predicate, - statements: node.statements, - consequent: node.consequent, - location: node.location - ) - ) - end - - def visit_field(node) - visit(node.parent) - end - - def visit_float(node) - builder.putobject(node.accept(RubyVisitor.new)) - end - - def visit_for(node) - visit(node.collection) - - name = node.index.value.value.to_sym - current_iseq.local_table.plain(name) - - block_iseq = - with_instruction_sequence( - :block, - "block in #{current_iseq.name}", - current_iseq, - node.statements - ) do - current_iseq.argument_options[:lead_num] ||= 0 - current_iseq.argument_options[:lead_num] += 1 - current_iseq.argument_options[:ambiguous_param0] = true - - current_iseq.argument_size += 1 - current_iseq.local_table.plain(2) - - builder.getlocal(0, 0) - - local_variable = current_iseq.local_variable(name) - builder.setlocal(local_variable.index, local_variable.level) - - builder.event(:RUBY_EVENT_B_CALL) - builder.nop - - visit(node.statements) - builder.event(:RUBY_EVENT_B_RETURN) - builder.leave - end - - builder.send(:each, 0, 0, block_iseq) - end - - def visit_hash(node) - builder.duphash(node.accept(RubyVisitor.new)) - rescue RubyVisitor::CompilationError - visit_all(node.assocs) - builder.newhash(node.assocs.length * 2) - end - - def visit_heredoc(node) - if node.beginning.value.end_with?("`") - visit_xstring_literal(node) - elsif node.parts.length == 1 && node.parts.first.is_a?(TStringContent) - visit(node.parts.first) - else - length = visit_string_parts(node) - builder.concatstrings(length) - end - end - - def visit_if(node) - visit(node.predicate) - branchunless = builder.branchunless(-1) - visit(node.statements) - - if last_statement? - builder.leave - branchunless[1] = builder.label - - node.consequent ? visit(node.consequent) : builder.putnil - else - builder.pop - - if node.consequent - jump = builder.jump(-1) - branchunless[1] = builder.label - visit(node.consequent) - jump[1] = builder.label - else - branchunless[1] = builder.label - end - end - end - - def visit_if_op(node) - visit_if( - IfNode.new( - predicate: node.predicate, - statements: node.truthy, - consequent: - Else.new( - keyword: Kw.new(value: "else", location: Location.default), - statements: node.falsy, - location: Location.default - ), - location: Location.default - ) - ) - end - - def visit_imaginary(node) - builder.putobject(node.accept(RubyVisitor.new)) - end - - def visit_int(node) - builder.putobject(node.accept(RubyVisitor.new)) - end - - def visit_kwrest_param(node) - current_iseq.argument_options[:kwrest] = current_iseq.argument_size - current_iseq.argument_size += 1 - current_iseq.local_table.plain(node.name.value.to_sym) - end - - def visit_label(node) - builder.putobject(node.accept(RubyVisitor.new)) - end - - def visit_lambda(node) - lambda_iseq = - with_instruction_sequence( - :block, - "block in #{current_iseq.name}", - current_iseq, - node - ) do - builder.event(:RUBY_EVENT_B_CALL) - visit(node.params) - visit(node.statements) - builder.event(:RUBY_EVENT_B_RETURN) - builder.leave - end - - builder.putspecialobject(VM_SPECIAL_OBJECT_VMCORE) - builder.send(:lambda, 0, VM_CALL_FCALL, lambda_iseq) - end - - def visit_lambda_var(node) - visit_block_var(node) - end - - def visit_massign(node) - visit(node.value) - builder.dup - visit(node.target) - end - - def visit_method_add_block(node) - visit_call( - CommandCall.new( - receiver: node.call.receiver, - operator: node.call.operator, - message: node.call.message, - arguments: node.call.arguments, - block: node.block, - location: node.location - ) - ) - end - - def visit_mlhs(node) - lookups = [] - - node.parts.each do |part| - case part - when VarField - lookups << visit(part) - end - end - - builder.expandarray(lookups.length, 0) - - lookups.each { |lookup| builder.setlocal(lookup.index, lookup.level) } - end - - def visit_module(node) - name = node.constant.constant.value.to_sym - module_iseq = - with_instruction_sequence( - :class, - "", - current_iseq, - node - ) do - builder.event(:RUBY_EVENT_CLASS) - visit(node.bodystmt) - builder.event(:RUBY_EVENT_END) - builder.leave - end - - flags = VM_DEFINECLASS_TYPE_MODULE - - case node.constant - when ConstPathRef - flags |= VM_DEFINECLASS_FLAG_SCOPED - visit(node.constant.parent) - when ConstRef - builder.putspecialobject(VM_SPECIAL_OBJECT_CONST_BASE) - when TopConstRef - flags |= VM_DEFINECLASS_FLAG_SCOPED - builder.putobject(Object) - end - - builder.putnil - builder.defineclass(name, module_iseq, flags) - end - - def visit_mrhs(node) - if (compiled = RubyVisitor.compile(node)) - builder.duparray(compiled) - else - visit_all(node.parts) - builder.newarray(node.parts.length) - end - end - - def visit_not(node) - visit(node.statement) - builder.send(:!, 0, VM_CALL_ARGS_SIMPLE) - end - - def visit_opassign(node) - flag = VM_CALL_ARGS_SIMPLE - if node.target.is_a?(ConstPathField) || node.target.is_a?(TopConstField) - flag |= VM_CALL_FCALL - end - - case (operator = node.operator.value.chomp("=").to_sym) - when :"&&" - branchunless = nil - - with_opassign(node) do - builder.dup - branchunless = builder.branchunless(-1) - builder.pop - visit(node.value) - end - - case node.target - when ARefField - builder.leave - branchunless[1] = builder.label - builder.setn(3) - builder.adjuststack(3) - when ConstPathField, TopConstField - branchunless[1] = builder.label - builder.swap - builder.pop - else - branchunless[1] = builder.label - end - when :"||" - if node.target.is_a?(ConstPathField) || - node.target.is_a?(TopConstField) - opassign_defined(node) - builder.swap - builder.pop - elsif node.target.is_a?(VarField) && - [Const, CVar, GVar].include?(node.target.value.class) - opassign_defined(node) - else - branchif = nil - - with_opassign(node) do - builder.dup - branchif = builder.branchif(-1) - builder.pop - visit(node.value) - end - - if node.target.is_a?(ARefField) - builder.leave - branchif[1] = builder.label - builder.setn(3) - builder.adjuststack(3) - else - branchif[1] = builder.label - end - end - else - with_opassign(node) do - visit(node.value) - builder.send(operator, 1, flag) - end - end - end - - def visit_params(node) - argument_options = current_iseq.argument_options - - if node.requireds.any? - argument_options[:lead_num] = 0 - - node.requireds.each do |required| - current_iseq.local_table.plain(required.value.to_sym) - current_iseq.argument_size += 1 - argument_options[:lead_num] += 1 - end - end - - node.optionals.each do |(optional, value)| - index = current_iseq.local_table.size - name = optional.value.to_sym - - current_iseq.local_table.plain(name) - current_iseq.argument_size += 1 - - unless argument_options.key?(:opt) - argument_options[:opt] = [builder.label] - end - - visit(value) - builder.setlocal(index, 0) - current_iseq.argument_options[:opt] << builder.label - end - - visit(node.rest) if node.rest - - if node.posts.any? - argument_options[:post_start] = current_iseq.argument_size - argument_options[:post_num] = 0 - - node.posts.each do |post| - current_iseq.local_table.plain(post.value.to_sym) - current_iseq.argument_size += 1 - argument_options[:post_num] += 1 - end - end - - if node.keywords.any? - argument_options[:kwbits] = 0 - argument_options[:keyword] = [] - checkkeywords = [] - - node.keywords.each_with_index do |(keyword, value), keyword_index| - name = keyword.value.chomp(":").to_sym - index = current_iseq.local_table.size - - current_iseq.local_table.plain(name) - current_iseq.argument_size += 1 - argument_options[:kwbits] += 1 - - if value.nil? - argument_options[:keyword] << name - else - begin - compiled = value.accept(RubyVisitor.new) - argument_options[:keyword] << [name, compiled] - rescue RubyVisitor::CompilationError - argument_options[:keyword] << [name] - checkkeywords << builder.checkkeyword(-1, keyword_index) - branchif = builder.branchif(-1) - visit(value) - builder.setlocal(index, 0) - branchif[1] = builder.label - end - end - end - - name = node.keyword_rest ? 3 : 2 - current_iseq.argument_size += 1 - current_iseq.local_table.plain(name) - - lookup = current_iseq.local_table.find(name, 0) - checkkeywords.each { |checkkeyword| checkkeyword[1] = lookup.index } - end - - if node.keyword_rest.is_a?(ArgsForward) - current_iseq.local_table.plain(:*) - current_iseq.local_table.plain(:&) - - current_iseq.argument_options[ - :rest_start - ] = current_iseq.argument_size - current_iseq.argument_options[ - :block_start - ] = current_iseq.argument_size + 1 - - current_iseq.argument_size += 2 - elsif node.keyword_rest - visit(node.keyword_rest) - end - - visit(node.block) if node.block - end - - def visit_paren(node) - visit(node.contents) - end - - def visit_program(node) - node.statements.body.each do |statement| - break unless statement.is_a?(Comment) - - if statement.value == "# frozen_string_literal: true" - @frozen_string_literal = true - end - end - - preexes = [] - statements = [] - - node.statements.body.each do |statement| - case statement - when Comment, EmbDoc, EndContent, VoidStmt - # ignore - when BEGINBlock - preexes << statement - else - statements << statement - end - end - - with_instruction_sequence(:top, "", nil, node) do - visit_all(preexes) - - if statements.empty? - builder.putnil - else - *statements, last_statement = statements - visit_all(statements) - with_last_statement { visit(last_statement) } - end - - builder.leave - end - end - - def visit_qsymbols(node) - builder.duparray(node.accept(RubyVisitor.new)) - end - - def visit_qwords(node) - if frozen_string_literal - builder.duparray(node.accept(RubyVisitor.new)) - else - visit_all(node.elements) - builder.newarray(node.elements.length) - end - end - - def visit_range(node) - builder.putobject(node.accept(RubyVisitor.new)) - rescue RubyVisitor::CompilationError - visit(node.left) - visit(node.right) - builder.newrange(node.operator.value == ".." ? 0 : 1) - end - - def visit_rational(node) - builder.putobject(node.accept(RubyVisitor.new)) - end - - def visit_regexp_literal(node) - builder.putobject(node.accept(RubyVisitor.new)) - rescue RubyVisitor::CompilationError - flags = RubyVisitor.new.visit_regexp_literal_flags(node) - length = visit_string_parts(node) - builder.toregexp(flags, length) - end - - def visit_rest_param(node) - current_iseq.local_table.plain(node.name.value.to_sym) - current_iseq.argument_options[:rest_start] = current_iseq.argument_size - current_iseq.argument_size += 1 - end - - def visit_sclass(node) - visit(node.target) - builder.putnil - - singleton_iseq = - with_instruction_sequence( - :class, - "singleton class", - current_iseq, - node - ) do - builder.event(:RUBY_EVENT_CLASS) - visit(node.bodystmt) - builder.event(:RUBY_EVENT_END) - builder.leave - end - - builder.defineclass( - :singletonclass, - singleton_iseq, - VM_DEFINECLASS_TYPE_SINGLETON_CLASS - ) - end - - def visit_statements(node) - statements = - node.body.select do |statement| - case statement - when Comment, EmbDoc, EndContent, VoidStmt - false - else - true - end - end - - statements.empty? ? builder.putnil : visit_all(statements) - end - - def visit_string_concat(node) - value = node.left.parts.first.value + node.right.parts.first.value - content = TStringContent.new(value: value, location: node.location) - - literal = - StringLiteral.new( - parts: [content], - quote: node.left.quote, - location: node.location - ) - visit_string_literal(literal) - end - - def visit_string_embexpr(node) - visit(node.statements) - end - - def visit_string_literal(node) - if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) - visit(node.parts.first) - else - length = visit_string_parts(node) - builder.concatstrings(length) - end - end - - def visit_super(node) - builder.putself - visit(node.arguments) - builder.invokesuper( - nil, - argument_parts(node.arguments).length, - VM_CALL_FCALL | VM_CALL_ARGS_SIMPLE | VM_CALL_SUPER, - nil - ) - end - - def visit_symbol_literal(node) - builder.putobject(node.accept(RubyVisitor.new)) - end - - def visit_symbols(node) - builder.duparray(node.accept(RubyVisitor.new)) - rescue RubyVisitor::CompilationError - node.elements.each do |element| - if element.parts.length == 1 && - element.parts.first.is_a?(TStringContent) - builder.putobject(element.parts.first.value.to_sym) - else - length = visit_string_parts(element) - builder.concatstrings(length) - builder.intern - end - end - - builder.newarray(node.elements.length) - end - - def visit_top_const_ref(node) - builder.opt_getconstant_path(constant_names(node)) - end - - def visit_tstring_content(node) - if frozen_string_literal - builder.putobject(node.accept(RubyVisitor.new)) - else - builder.putstring(node.accept(RubyVisitor.new)) - end - end - - def visit_unary(node) - method_id = - case node.operator - when "+", "-" - "#{node.operator}@" - else - node.operator - end - - visit_call( - CommandCall.new( - receiver: node.statement, - operator: nil, - message: Ident.new(value: method_id, location: Location.default), - arguments: nil, - block: nil, - location: Location.default - ) - ) - end - - def visit_undef(node) - node.symbols.each_with_index do |symbol, index| - builder.pop if index != 0 - builder.putspecialobject(VM_SPECIAL_OBJECT_VMCORE) - builder.putspecialobject(VM_SPECIAL_OBJECT_CBASE) - visit(symbol) - builder.send(:"core#undef_method", 2, VM_CALL_ARGS_SIMPLE) - end - end - - def visit_unless(node) - visit(node.predicate) - branchunless = builder.branchunless(-1) - node.consequent ? visit(node.consequent) : builder.putnil - - if last_statement? - builder.leave - branchunless[1] = builder.label - - visit(node.statements) - else - builder.pop - - if node.consequent - jump = builder.jump(-1) - branchunless[1] = builder.label - visit(node.consequent) - jump[1] = builder.label - else - branchunless[1] = builder.label - end - end - end - - def visit_until(node) - jumps = [] - - jumps << builder.jump(-1) - builder.putnil - builder.pop - jumps << builder.jump(-1) - - label = builder.label - visit(node.statements) - builder.pop - jumps.each { |jump| jump[1] = builder.label } - - visit(node.predicate) - builder.branchunless(label) - builder.putnil if last_statement? - end - - def visit_var_field(node) - case node.value - when CVar, IVar - name = node.value.value.to_sym - current_iseq.inline_storage_for(name) - when Ident - name = node.value.value.to_sym - - if (local_variable = current_iseq.local_variable(name)) - local_variable - else - current_iseq.local_table.plain(name) - current_iseq.local_variable(name) - end - end - end - - def visit_var_ref(node) - case node.value - when Const - builder.opt_getconstant_path(constant_names(node)) - when CVar - name = node.value.value.to_sym - builder.getclassvariable(name) - when GVar - builder.getglobal(node.value.value.to_sym) - when Ident - lookup = current_iseq.local_variable(node.value.value.to_sym) - - case lookup.local - when LocalTable::BlockLocal - builder.getblockparam(lookup.index, lookup.level) - when LocalTable::PlainLocal - builder.getlocal(lookup.index, lookup.level) - end - when IVar - name = node.value.value.to_sym - builder.getinstancevariable(name) - when Kw - case node.value.value - when "false" - builder.putobject(false) - when "nil" - builder.putnil - when "self" - builder.putself - when "true" - builder.putobject(true) - end - end - end - - def visit_vcall(node) - builder.putself - - flag = VM_CALL_FCALL | VM_CALL_VCALL | VM_CALL_ARGS_SIMPLE - builder.send(node.value.value.to_sym, 0, flag) - end - - def visit_when(node) - visit(node.statements) - end - - def visit_while(node) - jumps = [] - - jumps << builder.jump(-1) - builder.putnil - builder.pop - jumps << builder.jump(-1) - - label = builder.label - visit(node.statements) - builder.pop - jumps.each { |jump| jump[1] = builder.label } - - visit(node.predicate) - builder.branchif(label) - builder.putnil if last_statement? - end - - def visit_word(node) - if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) - visit(node.parts.first) - else - length = visit_string_parts(node) - builder.concatstrings(length) - end - end - - def visit_words(node) - converted = nil - - if frozen_string_literal - begin - converted = node.accept(RubyVisitor.new) - rescue RubyVisitor::CompilationError - end - end - - if converted - builder.duparray(converted) - else - visit_all(node.elements) - builder.newarray(node.elements.length) - end - end - - def visit_xstring_literal(node) - builder.putself - length = visit_string_parts(node) - builder.concatstrings(node.parts.length) if length > 1 - builder.send(:`, 1, VM_CALL_FCALL | VM_CALL_ARGS_SIMPLE) - end - - def visit_yield(node) - parts = argument_parts(node.arguments) - visit_all(parts) - builder.invokeblock(nil, parts.length, VM_CALL_ARGS_SIMPLE) - end - - def visit_zsuper(_node) - builder.putself - builder.invokesuper( - nil, - 0, - VM_CALL_FCALL | VM_CALL_ARGS_SIMPLE | VM_CALL_SUPER | VM_CALL_ZSUPER, - nil - ) - end - - private - - # This is a helper that is used in places where arguments may be present - # or they may be wrapped in parentheses. It's meant to descend down the - # tree and return an array of argument nodes. - def argument_parts(node) - case node - when nil - [] - when Args - node.parts - when ArgParen - if node.arguments.is_a?(ArgsForward) - [node.arguments] - else - node.arguments.parts - end - when Paren - node.contents.parts - end - end - - # Constant names when they are being assigned or referenced come in as a - # tree, but it's more convenient to work with them as an array. This - # method converts them into that array. This is nice because it's the - # operand that goes to opt_getconstant_path in Ruby 3.2. - def constant_names(node) - current = node - names = [] - - while current.is_a?(ConstPathField) || current.is_a?(ConstPathRef) - names.unshift(current.constant.value.to_sym) - current = current.parent - end - - case current - when VarField, VarRef - names.unshift(current.value.value.to_sym) - when TopConstRef - names.unshift(current.constant.value.to_sym) - names.unshift(:"") - end - - names - end - - # For the most part when an OpAssign (operator assignment) node with a ||= - # operator is being compiled it's a matter of reading the target, checking - # if the value should be evaluated, evaluating it if so, and then writing - # the result back to the target. - # - # However, in certain kinds of assignments (X, ::X, X::Y, @@x, and $x) we - # first check if the value is defined using the defined instruction. I - # don't know why it is necessary, and suspect that it isn't. - def opassign_defined(node) - case node.target - when ConstPathField - visit(node.target.parent) - name = node.target.constant.value.to_sym - - builder.dup - builder.defined(DEFINED_CONST_FROM, name, true) - when TopConstField - name = node.target.constant.value.to_sym - - builder.putobject(Object) - builder.dup - builder.defined(DEFINED_CONST_FROM, name, true) - when VarField - name = node.target.value.value.to_sym - builder.putnil - - case node.target.value - when Const - builder.defined(DEFINED_CONST, name, true) - when CVar - builder.defined(DEFINED_CVAR, name, true) - when GVar - builder.defined(DEFINED_GVAR, name, true) - end - end - - branchunless = builder.branchunless(-1) - - case node.target - when ConstPathField, TopConstField - builder.dup - builder.putobject(true) - builder.getconstant(name) - when VarField - case node.target.value - when Const - builder.opt_getconstant_path(constant_names(node.target)) - when CVar - builder.getclassvariable(name) - when GVar - builder.getglobal(name) - end - end - - builder.dup - branchif = builder.branchif(-1) - builder.pop - - branchunless[1] = builder.label - visit(node.value) - - case node.target - when ConstPathField, TopConstField - builder.dupn(2) - builder.swap - builder.setconstant(name) - when VarField - builder.dup - - case node.target.value - when Const - builder.putspecialobject(VM_SPECIAL_OBJECT_CONST_BASE) - builder.setconstant(name) - when CVar - builder.setclassvariable(name) - when GVar - builder.setglobal(name) - end - end - - branchif[1] = builder.label - end - - # Whenever a value is interpolated into a string-like structure, these - # three instructions are pushed. - def push_interpolate - builder.dup - builder.objtostring(:to_s, 0, VM_CALL_FCALL | VM_CALL_ARGS_SIMPLE) - builder.anytostring - end - - # There are a lot of nodes in the AST that act as contains of parts of - # strings. This includes things like string literals, regular expressions, - # heredocs, etc. This method will visit all the parts of a string within - # those containers. - def visit_string_parts(node) - length = 0 - - unless node.parts.first.is_a?(TStringContent) - builder.putobject("") - length += 1 - end - - node.parts.each do |part| - case part - when StringDVar - visit(part.variable) - push_interpolate - when StringEmbExpr - visit(part) - push_interpolate - when TStringContent - builder.putobject(part.accept(RubyVisitor.new)) - end - - length += 1 - end - - length - end - - # The current instruction sequence that we're compiling is always stored - # on the compiler. When we descend into a node that has its own - # instruction sequence, this method can be called to temporarily set the - # new value of the instruction sequence, yield, and then set it back. - def with_instruction_sequence(type, name, parent_iseq, node) - previous_iseq = current_iseq - previous_builder = builder - - begin - iseq = InstructionSequence.new(type, name, parent_iseq, node.location) - - @current_iseq = iseq - @builder = - Builder.new( - iseq, - frozen_string_literal: frozen_string_literal, - operands_unification: operands_unification, - specialized_instruction: specialized_instruction - ) - - yield - iseq - ensure - @current_iseq = previous_iseq - @builder = previous_builder - end - end - - # When we're compiling the last statement of a set of statements within a - # scope, the instructions sometimes change from pops to leaves. These - # kinds of peephole optimizations can reduce the overall number of - # instructions. Therefore, we keep track of whether we're compiling the - # last statement of a scope and allow visit methods to query that - # information. - def with_last_statement - previous = @last_statement - @last_statement = true - - begin - yield - ensure - @last_statement = previous - end - end - - def last_statement? - @last_statement - end - - # OpAssign nodes can have a number of different kinds of nodes as their - # "target" (i.e., the left-hand side of the assignment). When compiling - # these nodes we typically need to first fetch the current value of the - # variable, then perform some kind of action, then store the result back - # into the variable. This method handles that by first fetching the value, - # then yielding to the block, then storing the result. - def with_opassign(node) - case node.target - when ARefField - builder.putnil - visit(node.target.collection) - visit(node.target.index) - - builder.dupn(2) - builder.send(:[], 1, VM_CALL_ARGS_SIMPLE) - - yield - - builder.setn(3) - builder.send(:[]=, 2, VM_CALL_ARGS_SIMPLE) - builder.pop - when ConstPathField - name = node.target.constant.value.to_sym - - visit(node.target.parent) - builder.dup - builder.putobject(true) - builder.getconstant(name) - - yield - - if node.operator.value == "&&=" - builder.dupn(2) - else - builder.swap - builder.topn(1) - end - - builder.swap - builder.setconstant(name) - when TopConstField - name = node.target.constant.value.to_sym - - builder.putobject(Object) - builder.dup - builder.putobject(true) - builder.getconstant(name) - - yield - - if node.operator.value == "&&=" - builder.dupn(2) - else - builder.swap - builder.topn(1) - end - - builder.swap - builder.setconstant(name) - when VarField - case node.target.value - when Const - names = constant_names(node.target) - builder.opt_getconstant_path(names) - - yield - - builder.dup - builder.putspecialobject(VM_SPECIAL_OBJECT_CONST_BASE) - builder.setconstant(names.last) - when CVar - name = node.target.value.value.to_sym - builder.getclassvariable(name) - - yield - - builder.dup - builder.setclassvariable(name) - when GVar - name = node.target.value.value.to_sym - builder.getglobal(name) - - yield - - builder.dup - builder.setglobal(name) - when Ident - local_variable = visit(node.target) - builder.getlocal(local_variable.index, local_variable.level) - - yield - - builder.dup - builder.setlocal(local_variable.index, local_variable.level) - when IVar - name = node.target.value.value.to_sym - builder.getinstancevariable(name) - - yield - - builder.dup - builder.setinstancevariable(name) - end - end - end - end - end -end diff --git a/test/compiler_test.rb b/test/compiler_test.rb index cf0667bb..cdf2860e 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -2,17 +2,9 @@ return if !defined?(RubyVM::InstructionSequence) || RUBY_VERSION < "3.1" require_relative "test_helper" -require "fiddle" module SyntaxTree class CompilerTest < Minitest::Test - ISEQ_LOAD = - Fiddle::Function.new( - Fiddle::Handle::DEFAULT["rb_iseq_load"], - [Fiddle::TYPE_VOIDP] * 3, - Fiddle::TYPE_VOIDP - ) - CASES = [ # Various literals placed on the stack "true", @@ -457,7 +449,7 @@ def serialize_iseq(iseq) when Array insn.map do |operand| if operand.is_a?(Array) && - operand[0] == Visitor::Compiler::InstructionSequence::MAGIC + operand[0] == Compiler::InstructionSequence::MAGIC serialize_iseq(operand) else operand @@ -478,20 +470,13 @@ def assert_compiles(source, **options) assert_equal( serialize_iseq(RubyVM::InstructionSequence.compile(source, **options)), - serialize_iseq(program.accept(Visitor::Compiler.new(**options))) + serialize_iseq(program.accept(Compiler.new(**options))) ) end def assert_evaluates(expected, source, **options) program = SyntaxTree.parse(source) - compiled = program.accept(Visitor::Compiler.new(**options)).to_a - - # Temporary hack until we get these working. - compiled[4][:node_id] = 11 - compiled[4][:node_ids] = [1, 0, 3, 2, 6, 7, 9, -1] - - iseq = Fiddle.dlunwrap(ISEQ_LOAD.call(Fiddle.dlwrap(compiled), 0, nil)) - assert_equal expected, iseq.eval + assert_equal expected, program.accept(Compiler.new(**options)).eval end end end From 8b836c73b7cc2c9327a7782008a301653b2848dd Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 18 Nov 2022 20:20:45 -0500 Subject: [PATCH 252/536] Split YARV out into its own file --- lib/syntax_tree.rb | 1 + lib/syntax_tree/compiler.rb | 1062 ++++------------------------------- lib/syntax_tree/yarv.rb | 838 +++++++++++++++++++++++++++ test/compiler_test.rb | 2 +- 4 files changed, 954 insertions(+), 949 deletions(-) create mode 100644 lib/syntax_tree/yarv.rb diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index c62132e6..187ff74d 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -26,6 +26,7 @@ require_relative "syntax_tree/pattern" require_relative "syntax_tree/search" +require_relative "syntax_tree/yarv" require_relative "syntax_tree/compiler" # Syntax Tree is a suite of tools built on top of the internal CRuby parser. It diff --git a/lib/syntax_tree/compiler.rb b/lib/syntax_tree/compiler.rb index d9b7e787..c936c9c1 100644 --- a/lib/syntax_tree/compiler.rb +++ b/lib/syntax_tree/compiler.rb @@ -185,839 +185,6 @@ def visit_unsupported(_node) end end - # This object is used to track the size of the stack at any given time. It - # is effectively a mini symbolic interpreter. It's necessary because when - # instruction sequences get serialized they include a :stack_max field on - # them. This field is used to determine how much stack space to allocate - # for the instruction sequence. - class Stack - attr_reader :current_size, :maximum_size - - def initialize - @current_size = 0 - @maximum_size = 0 - end - - def change_by(value) - @current_size += value - @maximum_size = @current_size if @current_size > @maximum_size - end - end - - # This represents every local variable associated with an instruction - # sequence. There are two kinds of locals: plain locals that are what you - # expect, and block proxy locals, which represent local variables - # associated with blocks that were passed into the current instruction - # sequence. - class LocalTable - # A local representing a block passed into the current instruction - # sequence. - class BlockLocal - attr_reader :name - - def initialize(name) - @name = name - end - end - - # A regular local variable. - class PlainLocal - attr_reader :name - - def initialize(name) - @name = name - end - end - - # The result of looking up a local variable in the current local table. - class Lookup - attr_reader :local, :index, :level - - def initialize(local, index, level) - @local = local - @index = index - @level = level - end - end - - attr_reader :locals - - def initialize - @locals = [] - end - - def find(name, level) - index = locals.index { |local| local.name == name } - Lookup.new(locals[index], index, level) if index - end - - def has?(name) - locals.any? { |local| local.name == name } - end - - def names - locals.map(&:name) - end - - def size - locals.length - end - - # Add a BlockLocal to the local table. - def block(name) - locals << BlockLocal.new(name) unless has?(name) - end - - # Add a PlainLocal to the local table. - def plain(name) - locals << PlainLocal.new(name) unless has?(name) - end - - # This is the offset from the top of the stack where this local variable - # lives. - def offset(index) - size - (index - 3) - 1 - end - end - - # This class is meant to mirror RubyVM::InstructionSequence. It contains a - # list of instructions along with the metadata pertaining to them. It also - # functions as a builder for the instruction sequence. - class InstructionSequence - MAGIC = "YARVInstructionSequence/SimpleDataFormat" - - # This provides a handle to the rb_iseq_load function, which allows you to - # pass a serialized iseq to Ruby and have it return a - # RubyVM::InstructionSequence object. - ISEQ_LOAD = - Fiddle::Function.new( - Fiddle::Handle::DEFAULT["rb_iseq_load"], - [Fiddle::TYPE_VOIDP] * 3, - Fiddle::TYPE_VOIDP - ) - - # The type of the instruction sequence. - attr_reader :type - - # The name of the instruction sequence. - attr_reader :name - - # The parent instruction sequence, if there is one. - attr_reader :parent_iseq - - # The location of the root node of this instruction sequence. - attr_reader :location - - # This is the list of information about the arguments to this - # instruction sequence. - attr_accessor :argument_size - attr_reader :argument_options - - # The list of instructions for this instruction sequence. - attr_reader :insns - - # The table of local variables. - attr_reader :local_table - - # The hash of names of instance and class variables pointing to the - # index of their associated inline storage. - attr_reader :inline_storages - - # The index of the next inline storage that will be created. - attr_reader :storage_index - - # An object that will track the current size of the stack and the - # maximum size of the stack for this instruction sequence. - attr_reader :stack - - def initialize(type, name, parent_iseq, location) - @type = type - @name = name - @parent_iseq = parent_iseq - @location = location - - @argument_size = 0 - @argument_options = {} - - @local_table = LocalTable.new - @inline_storages = {} - @insns = [] - @storage_index = 0 - @stack = Stack.new - end - - def local_variable(name, level = 0) - if (lookup = local_table.find(name, level)) - lookup - elsif parent_iseq - parent_iseq.local_variable(name, level + 1) - end - end - - def push(insn) - insns << insn - insn - end - - def inline_storage - storage = storage_index - @storage_index += 1 - storage - end - - def inline_storage_for(name) - unless inline_storages.key?(name) - inline_storages[name] = inline_storage - end - - inline_storages[name] - end - - def length - insns.inject(0) do |sum, insn| - insn.is_a?(Array) ? sum + insn.length : sum - end - end - - def each_child - insns.each do |insn| - insn[1..].each do |operand| - yield operand if operand.is_a?(InstructionSequence) - end - end - end - - def eval - compiled = to_a - - # Temporary hack until we get these working. - compiled[4][:node_id] = 11 - compiled[4][:node_ids] = [1, 0, 3, 2, 6, 7, 9, -1] - - Fiddle.dlunwrap(ISEQ_LOAD.call(Fiddle.dlwrap(compiled), 0, nil)).eval - end - - def to_a - versions = RUBY_VERSION.split(".").map(&:to_i) - - [ - MAGIC, - versions[0], - versions[1], - 1, - { - arg_size: argument_size, - local_size: local_table.size, - stack_max: stack.maximum_size - }, - name, - "", - "", - location.start_line, - type, - local_table.names, - argument_options, - [], - insns.map { |insn| serialize(insn) } - ] - end - - private - - def serialize(insn) - case insn[0] - when :checkkeyword, :getblockparam, :getblockparamproxy, - :getlocal_WC_0, :getlocal_WC_1, :getlocal, :setlocal_WC_0, - :setlocal_WC_1, :setlocal - iseq = self - - case insn[0] - when :getlocal_WC_1, :setlocal_WC_1 - iseq = iseq.parent_iseq - when :getblockparam, :getblockparamproxy, :getlocal, :setlocal - insn[2].times { iseq = iseq.parent_iseq } - end - - # Here we need to map the local variable index to the offset - # from the top of the stack where it will be stored. - [insn[0], iseq.local_table.offset(insn[1]), *insn[2..]] - when :defineclass - [insn[0], insn[1], insn[2].to_a, insn[3]] - when :definemethod, :definesmethod - [insn[0], insn[1], insn[2].to_a] - when :send - # For any instructions that push instruction sequences onto the - # stack, we need to call #to_a on them as well. - [insn[0], insn[1], (insn[2].to_a if insn[2])] - when :once - [insn[0], insn[1].to_a, insn[2]] - else - insn - end - end - end - - # This class serves as a layer of indirection between the instruction - # sequence and the compiler. It allows us to provide different behavior - # for certain instructions depending on the Ruby version. For example, - # class variable reads and writes gained an inline cache in Ruby 3.0. So - # we place the logic for checking the Ruby version in this class. - class Builder - attr_reader :iseq, :stack - attr_reader :frozen_string_literal, - :operands_unification, - :specialized_instruction - - def initialize( - iseq, - frozen_string_literal: false, - operands_unification: true, - specialized_instruction: true - ) - @iseq = iseq - @stack = iseq.stack - - @frozen_string_literal = frozen_string_literal - @operands_unification = operands_unification - @specialized_instruction = specialized_instruction - end - - # This creates a new label at the current length of the instruction - # sequence. It is used as the operand for jump instructions. - def label - name = :"label_#{iseq.length}" - iseq.insns.last == name ? name : event(name) - end - - def event(name) - iseq.push(name) - name - end - - def adjuststack(number) - stack.change_by(-number) - iseq.push([:adjuststack, number]) - end - - def anytostring - stack.change_by(-2 + 1) - iseq.push([:anytostring]) - end - - def branchif(index) - stack.change_by(-1) - iseq.push([:branchif, index]) - end - - def branchnil(index) - stack.change_by(-1) - iseq.push([:branchnil, index]) - end - - def branchunless(index) - stack.change_by(-1) - iseq.push([:branchunless, index]) - end - - def checkkeyword(index, keyword_index) - stack.change_by(+1) - iseq.push([:checkkeyword, index, keyword_index]) - end - - def concatarray - stack.change_by(-2 + 1) - iseq.push([:concatarray]) - end - - def concatstrings(number) - stack.change_by(-number + 1) - iseq.push([:concatstrings, number]) - end - - def defined(type, name, message) - stack.change_by(-1 + 1) - iseq.push([:defined, type, name, message]) - end - - def defineclass(name, class_iseq, flags) - stack.change_by(-2 + 1) - iseq.push([:defineclass, name, class_iseq, flags]) - end - - def definemethod(name, method_iseq) - stack.change_by(0) - iseq.push([:definemethod, name, method_iseq]) - end - - def definesmethod(name, method_iseq) - stack.change_by(-1) - iseq.push([:definesmethod, name, method_iseq]) - end - - def dup - stack.change_by(-1 + 2) - iseq.push([:dup]) - end - - def duparray(object) - stack.change_by(+1) - iseq.push([:duparray, object]) - end - - def duphash(object) - stack.change_by(+1) - iseq.push([:duphash, object]) - end - - def dupn(number) - stack.change_by(+number) - iseq.push([:dupn, number]) - end - - def expandarray(length, flag) - stack.change_by(-1 + length) - iseq.push([:expandarray, length, flag]) - end - - def getblockparam(index, level) - stack.change_by(+1) - iseq.push([:getblockparam, index, level]) - end - - def getblockparamproxy(index, level) - stack.change_by(+1) - iseq.push([:getblockparamproxy, index, level]) - end - - def getclassvariable(name) - stack.change_by(+1) - - if RUBY_VERSION >= "3.0" - iseq.push([:getclassvariable, name, iseq.inline_storage_for(name)]) - else - iseq.push([:getclassvariable, name]) - end - end - - def getconstant(name) - stack.change_by(-2 + 1) - iseq.push([:getconstant, name]) - end - - def getglobal(name) - stack.change_by(+1) - iseq.push([:getglobal, name]) - end - - def getinstancevariable(name) - stack.change_by(+1) - - if RUBY_VERSION >= "3.2" - iseq.push([:getinstancevariable, name, iseq.inline_storage]) - else - inline_storage = iseq.inline_storage_for(name) - iseq.push([:getinstancevariable, name, inline_storage]) - end - end - - def getlocal(index, level) - stack.change_by(+1) - - if operands_unification - # Specialize the getlocal instruction based on the level of the - # local variable. If it's 0 or 1, then there's a specialized - # instruction that will look at the current scope or the parent - # scope, respectively, and requires fewer operands. - case level - when 0 - iseq.push([:getlocal_WC_0, index]) - when 1 - iseq.push([:getlocal_WC_1, index]) - else - iseq.push([:getlocal, index, level]) - end - else - iseq.push([:getlocal, index, level]) - end - end - - def getspecial(key, type) - stack.change_by(-0 + 1) - iseq.push([:getspecial, key, type]) - end - - def intern - stack.change_by(-1 + 1) - iseq.push([:intern]) - end - - def invokeblock(method_id, argc, flag) - stack.change_by(-argc + 1) - iseq.push([:invokeblock, call_data(method_id, argc, flag)]) - end - - def invokesuper(method_id, argc, flag, block_iseq) - stack.change_by(-(argc + 1) + 1) - - cdata = call_data(method_id, argc, flag) - iseq.push([:invokesuper, cdata, block_iseq]) - end - - def jump(index) - stack.change_by(0) - iseq.push([:jump, index]) - end - - def leave - stack.change_by(-1) - iseq.push([:leave]) - end - - def newarray(length) - stack.change_by(-length + 1) - iseq.push([:newarray, length]) - end - - def newhash(length) - stack.change_by(-length + 1) - iseq.push([:newhash, length]) - end - - def newrange(flag) - stack.change_by(-2 + 1) - iseq.push([:newrange, flag]) - end - - def nop - stack.change_by(0) - iseq.push([:nop]) - end - - def objtostring(method_id, argc, flag) - stack.change_by(-1 + 1) - iseq.push([:objtostring, call_data(method_id, argc, flag)]) - end - - def once(postexe_iseq, inline_storage) - stack.change_by(+1) - iseq.push([:once, postexe_iseq, inline_storage]) - end - - def opt_getconstant_path(names) - if RUBY_VERSION >= "3.2" - stack.change_by(+1) - iseq.push([:opt_getconstant_path, names]) - else - inline_storage = iseq.inline_storage - getinlinecache = opt_getinlinecache(-1, inline_storage) - - if names[0] == :"" - names.shift - pop - putobject(Object) - end - - names.each_with_index do |name, index| - putobject(index == 0) - getconstant(name) - end - - opt_setinlinecache(inline_storage) - getinlinecache[1] = label - end - end - - def opt_getinlinecache(offset, inline_storage) - stack.change_by(+1) - iseq.push([:opt_getinlinecache, offset, inline_storage]) - end - - def opt_newarray_max(length) - if specialized_instruction - stack.change_by(-length + 1) - iseq.push([:opt_newarray_max, length]) - else - newarray(length) - send(:max, 0, VM_CALL_ARGS_SIMPLE) - end - end - - def opt_newarray_min(length) - if specialized_instruction - stack.change_by(-length + 1) - iseq.push([:opt_newarray_min, length]) - else - newarray(length) - send(:min, 0, VM_CALL_ARGS_SIMPLE) - end - end - - def opt_setinlinecache(inline_storage) - stack.change_by(-1 + 1) - iseq.push([:opt_setinlinecache, inline_storage]) - end - - def opt_str_freeze(value) - if specialized_instruction - stack.change_by(+1) - iseq.push( - [ - :opt_str_freeze, - value, - call_data(:freeze, 0, VM_CALL_ARGS_SIMPLE) - ] - ) - else - putstring(value) - send(:freeze, 0, VM_CALL_ARGS_SIMPLE) - end - end - - def opt_str_uminus(value) - if specialized_instruction - stack.change_by(+1) - iseq.push( - [:opt_str_uminus, value, call_data(:-@, 0, VM_CALL_ARGS_SIMPLE)] - ) - else - putstring(value) - send(:-@, 0, VM_CALL_ARGS_SIMPLE) - end - end - - def pop - stack.change_by(-1) - iseq.push([:pop]) - end - - def putnil - stack.change_by(+1) - iseq.push([:putnil]) - end - - def putobject(object) - stack.change_by(+1) - - if operands_unification - # Specialize the putobject instruction based on the value of the - # object. If it's 0 or 1, then there's a specialized instruction - # that will push the object onto the stack and requires fewer - # operands. - if object.eql?(0) - iseq.push([:putobject_INT2FIX_0_]) - elsif object.eql?(1) - iseq.push([:putobject_INT2FIX_1_]) - else - iseq.push([:putobject, object]) - end - else - iseq.push([:putobject, object]) - end - end - - def putself - stack.change_by(+1) - iseq.push([:putself]) - end - - def putspecialobject(object) - stack.change_by(+1) - iseq.push([:putspecialobject, object]) - end - - def putstring(object) - stack.change_by(+1) - iseq.push([:putstring, object]) - end - - def send(method_id, argc, flag, block_iseq = nil) - stack.change_by(-(argc + 1) + 1) - cdata = call_data(method_id, argc, flag) - - if specialized_instruction - # Specialize the send instruction. If it doesn't have a block - # attached, then we will replace it with an opt_send_without_block - # and do further specializations based on the called method and the - # number of arguments. - - # stree-ignore - if !block_iseq && (flag & VM_CALL_ARGS_BLOCKARG) == 0 - case [method_id, argc] - when [:length, 0] then iseq.push([:opt_length, cdata]) - when [:size, 0] then iseq.push([:opt_size, cdata]) - when [:empty?, 0] then iseq.push([:opt_empty_p, cdata]) - when [:nil?, 0] then iseq.push([:opt_nil_p, cdata]) - when [:succ, 0] then iseq.push([:opt_succ, cdata]) - when [:!, 0] then iseq.push([:opt_not, cdata]) - when [:+, 1] then iseq.push([:opt_plus, cdata]) - when [:-, 1] then iseq.push([:opt_minus, cdata]) - when [:*, 1] then iseq.push([:opt_mult, cdata]) - when [:/, 1] then iseq.push([:opt_div, cdata]) - when [:%, 1] then iseq.push([:opt_mod, cdata]) - when [:==, 1] then iseq.push([:opt_eq, cdata]) - when [:=~, 1] then iseq.push([:opt_regexpmatch2, cdata]) - when [:<, 1] then iseq.push([:opt_lt, cdata]) - when [:<=, 1] then iseq.push([:opt_le, cdata]) - when [:>, 1] then iseq.push([:opt_gt, cdata]) - when [:>=, 1] then iseq.push([:opt_ge, cdata]) - when [:<<, 1] then iseq.push([:opt_ltlt, cdata]) - when [:[], 1] then iseq.push([:opt_aref, cdata]) - when [:&, 1] then iseq.push([:opt_and, cdata]) - when [:|, 1] then iseq.push([:opt_or, cdata]) - when [:[]=, 2] then iseq.push([:opt_aset, cdata]) - when [:!=, 1] - eql_data = call_data(:==, 1, VM_CALL_ARGS_SIMPLE) - iseq.push([:opt_neq, eql_data, cdata]) - else - iseq.push([:opt_send_without_block, cdata]) - end - else - iseq.push([:send, cdata, block_iseq]) - end - else - iseq.push([:send, cdata, block_iseq]) - end - end - - def setclassvariable(name) - stack.change_by(-1) - - if RUBY_VERSION >= "3.0" - iseq.push([:setclassvariable, name, iseq.inline_storage_for(name)]) - else - iseq.push([:setclassvariable, name]) - end - end - - def setconstant(name) - stack.change_by(-2) - iseq.push([:setconstant, name]) - end - - def setglobal(name) - stack.change_by(-1) - iseq.push([:setglobal, name]) - end - - def setinstancevariable(name) - stack.change_by(-1) - - if RUBY_VERSION >= "3.2" - iseq.push([:setinstancevariable, name, iseq.inline_storage]) - else - inline_storage = iseq.inline_storage_for(name) - iseq.push([:setinstancevariable, name, inline_storage]) - end - end - - def setlocal(index, level) - stack.change_by(-1) - - if operands_unification - # Specialize the setlocal instruction based on the level of the - # local variable. If it's 0 or 1, then there's a specialized - # instruction that will write to the current scope or the parent - # scope, respectively, and requires fewer operands. - case level - when 0 - iseq.push([:setlocal_WC_0, index]) - when 1 - iseq.push([:setlocal_WC_1, index]) - else - iseq.push([:setlocal, index, level]) - end - else - iseq.push([:setlocal, index, level]) - end - end - - def setn(number) - stack.change_by(-1 + 1) - iseq.push([:setn, number]) - end - - def splatarray(flag) - stack.change_by(-1 + 1) - iseq.push([:splatarray, flag]) - end - - def swap - stack.change_by(-2 + 2) - iseq.push([:swap]) - end - - def topn(number) - stack.change_by(+1) - iseq.push([:topn, number]) - end - - def toregexp(options, length) - stack.change_by(-length + 1) - iseq.push([:toregexp, options, length]) - end - - private - - # This creates a call data object that is used as the operand for the - # send, invokesuper, and objtostring instructions. - def call_data(method_id, argc, flag) - { mid: method_id, flag: flag, orig_argc: argc } - end - end - - # These constants correspond to the putspecialobject instruction. They are - # used to represent special objects that are pushed onto the stack. - VM_SPECIAL_OBJECT_VMCORE = 1 - VM_SPECIAL_OBJECT_CBASE = 2 - VM_SPECIAL_OBJECT_CONST_BASE = 3 - - # These constants correspond to the flag passed as part of the call data - # structure on the send instruction. They are used to represent various - # metadata about the callsite (e.g., were keyword arguments used?, was a - # block given?, etc.). - VM_CALL_ARGS_SPLAT = 1 << 0 - VM_CALL_ARGS_BLOCKARG = 1 << 1 - VM_CALL_FCALL = 1 << 2 - VM_CALL_VCALL = 1 << 3 - VM_CALL_ARGS_SIMPLE = 1 << 4 - VM_CALL_BLOCKISEQ = 1 << 5 - VM_CALL_KWARG = 1 << 6 - VM_CALL_KW_SPLAT = 1 << 7 - VM_CALL_TAILCALL = 1 << 8 - VM_CALL_SUPER = 1 << 9 - VM_CALL_ZSUPER = 1 << 10 - VM_CALL_OPT_SEND = 1 << 11 - VM_CALL_KW_SPLAT_MUT = 1 << 12 - - # These constants correspond to the value passed as part of the defined - # instruction. It's an enum defined in the CRuby codebase that tells that - # instruction what kind of defined check to perform. - DEFINED_NIL = 1 - DEFINED_IVAR = 2 - DEFINED_LVAR = 3 - DEFINED_GVAR = 4 - DEFINED_CVAR = 5 - DEFINED_CONST = 6 - DEFINED_METHOD = 7 - DEFINED_YIELD = 8 - DEFINED_ZSUPER = 9 - DEFINED_SELF = 10 - DEFINED_TRUE = 11 - DEFINED_FALSE = 12 - DEFINED_ASGN = 13 - DEFINED_EXPR = 14 - DEFINED_REF = 15 - DEFINED_FUNC = 16 - DEFINED_CONST_FROM = 17 - - # These constants correspond to the value passed in the flags as part of - # the defineclass instruction. - VM_DEFINECLASS_TYPE_CLASS = 0 - VM_DEFINECLASS_TYPE_SINGLETON_CLASS = 1 - VM_DEFINECLASS_TYPE_MODULE = 2 - VM_DEFINECLASS_FLAG_SCOPED = 8 - VM_DEFINECLASS_FLAG_HAS_SUPERCLASS = 16 - # These options mirror the compilation options that we currently support # that can be also passed to RubyVM::InstructionSequence.compile. attr_reader :frozen_string_literal, @@ -1074,8 +241,8 @@ def visit_END(node) builder.leave end - builder.putspecialobject(VM_SPECIAL_OBJECT_VMCORE) - builder.send(:"core#set_postexe", 0, VM_CALL_FCALL, postexe_iseq) + builder.putspecialobject(YARV::VM_SPECIAL_OBJECT_VMCORE) + builder.send(:"core#set_postexe", 0, YARV::VM_CALL_FCALL, postexe_iseq) builder.leave end @@ -1084,17 +251,17 @@ def visit_END(node) end def visit_alias(node) - builder.putspecialobject(VM_SPECIAL_OBJECT_VMCORE) - builder.putspecialobject(VM_SPECIAL_OBJECT_CBASE) + builder.putspecialobject(YARV::VM_SPECIAL_OBJECT_VMCORE) + builder.putspecialobject(YARV::VM_SPECIAL_OBJECT_CBASE) visit(node.left) visit(node.right) - builder.send(:"core#set_method_alias", 3, VM_CALL_ARGS_SIMPLE) + builder.send(:"core#set_method_alias", 3, YARV::VM_CALL_ARGS_SIMPLE) end def visit_aref(node) visit(node.collection) visit(node.index) - builder.send(:[], 1, VM_CALL_ARGS_SIMPLE) + builder.send(:[], 1, YARV::VM_CALL_ARGS_SIMPLE) end def visit_arg_block(node) @@ -1150,7 +317,7 @@ def visit_assign(node) visit(node.target.index) visit(node.value) builder.setn(3) - builder.send(:[]=, 2, VM_CALL_ARGS_SIMPLE) + builder.send(:[]=, 2, YARV::VM_CALL_ARGS_SIMPLE) builder.pop when ConstPathField names = constant_names(node.target) @@ -1174,7 +341,7 @@ def visit_assign(node) visit(node.target) visit(node.value) builder.setn(2) - builder.send(:"#{node.target.name.value}=", 1, VM_CALL_ARGS_SIMPLE) + builder.send(:"#{node.target.name.value}=", 1, YARV::VM_CALL_ARGS_SIMPLE) builder.pop when TopConstField name = node.target.constant.value.to_sym @@ -1198,7 +365,7 @@ def visit_assign(node) case node.target.value when Const - builder.putspecialobject(VM_SPECIAL_OBJECT_CONST_BASE) + builder.putspecialobject(YARV::VM_SPECIAL_OBJECT_CONST_BASE) builder.setconstant(node.target.value.value.to_sym) when CVar builder.setclassvariable(node.target.value.value.to_sym) @@ -1257,7 +424,7 @@ def visit_binary(node) else visit(node.left) visit(node.right) - builder.send(node.operator, 1, VM_CALL_ARGS_SIMPLE) + builder.send(node.operator, 1, YARV::VM_CALL_ARGS_SIMPLE) end end @@ -1357,12 +524,14 @@ def visit_call(node) end if node.receiver - if node.receiver.is_a?(VarRef) && - ( - lookup = - current_iseq.local_variable(node.receiver.value.value.to_sym) - ) && lookup.local.is_a?(LocalTable::BlockLocal) - builder.getblockparamproxy(lookup.index, lookup.level) + if node.receiver.is_a?(VarRef) + lookup = current_iseq.local_variable(node.receiver.value.value.to_sym) + + if lookup.local.is_a?(YARV::LocalTable::BlockLocal) + builder.getblockparamproxy(lookup.index, lookup.level) + else + visit(node.receiver) + end else visit(node.receiver) end @@ -1382,13 +551,13 @@ def visit_call(node) case arg_part when ArgBlock argc -= 1 - flag |= VM_CALL_ARGS_BLOCKARG + flag |= YARV::VM_CALL_ARGS_BLOCKARG visit(arg_part) when ArgStar - flag |= VM_CALL_ARGS_SPLAT + flag |= YARV::VM_CALL_ARGS_SPLAT visit(arg_part) when ArgsForward - flag |= VM_CALL_ARGS_SPLAT | VM_CALL_ARGS_BLOCKARG + flag |= YARV::VM_CALL_ARGS_SPLAT | YARV::VM_CALL_ARGS_BLOCKARG lookup = current_iseq.local_table.find(:*, 0) builder.getlocal(lookup.index, lookup.level) @@ -1397,7 +566,7 @@ def visit_call(node) lookup = current_iseq.local_table.find(:&, 0) builder.getblockparamproxy(lookup.index, lookup.level) when BareAssocHash - flag |= VM_CALL_KW_SPLAT + flag |= YARV::VM_CALL_KW_SPLAT visit(arg_part) else visit(arg_part) @@ -1405,8 +574,8 @@ def visit_call(node) end block_iseq = visit(node.block) if node.block - flag |= VM_CALL_ARGS_SIMPLE if block_iseq.nil? && flag == 0 - flag |= VM_CALL_FCALL if node.receiver.nil? + flag |= YARV::VM_CALL_ARGS_SIMPLE if block_iseq.nil? && flag == 0 + flag |= YARV::VM_CALL_FCALL if node.receiver.nil? builder.send(node.message.value.to_sym, argc, flag, block_iseq) branchnil[1] = builder.label if branchnil @@ -1433,7 +602,7 @@ def visit_case(node) clauses.map do |clause| visit(clause.arguments) builder.topn(1) - builder.send(:===, 1, VM_CALL_FCALL | VM_CALL_ARGS_SIMPLE) + builder.send(:===, 1, YARV::VM_CALL_FCALL | YARV::VM_CALL_ARGS_SIMPLE) [clause, builder.branchif(:label_00)] end @@ -1466,21 +635,21 @@ def visit_class(node) builder.leave end - flags = VM_DEFINECLASS_TYPE_CLASS + flags = YARV::VM_DEFINECLASS_TYPE_CLASS case node.constant when ConstPathRef - flags |= VM_DEFINECLASS_FLAG_SCOPED + flags |= YARV::VM_DEFINECLASS_FLAG_SCOPED visit(node.constant.parent) when ConstRef - builder.putspecialobject(VM_SPECIAL_OBJECT_CONST_BASE) + builder.putspecialobject(YARV::VM_SPECIAL_OBJECT_CONST_BASE) when TopConstRef - flags |= VM_DEFINECLASS_FLAG_SCOPED + flags |= YARV::VM_DEFINECLASS_FLAG_SCOPED builder.putobject(Object) end if node.superclass - flags |= VM_DEFINECLASS_FLAG_HAS_SUPERCLASS + flags |= YARV::VM_DEFINECLASS_FLAG_HAS_SUPERCLASS visit(node.superclass) else builder.putnil @@ -1569,18 +738,18 @@ def visit_defined(node) case value when Const builder.putnil - builder.defined(DEFINED_CONST, name, "constant") + builder.defined(YARV::DEFINED_CONST, name, "constant") when CVar builder.putnil - builder.defined(DEFINED_CVAR, name, "class variable") + builder.defined(YARV::DEFINED_CVAR, name, "class variable") when GVar builder.putnil - builder.defined(DEFINED_GVAR, name, "global-variable") + builder.defined(YARV::DEFINED_GVAR, name, "global-variable") when Ident builder.putobject("local-variable") when IVar builder.putnil - builder.defined(DEFINED_IVAR, name, "instance-variable") + builder.defined(YARV::DEFINED_IVAR, name, "instance-variable") when Kw case name when :false @@ -1597,13 +766,13 @@ def visit_defined(node) builder.putself name = node.value.value.value.to_sym - builder.defined(DEFINED_FUNC, name, "method") + builder.defined(YARV::DEFINED_FUNC, name, "method") when YieldNode builder.putnil - builder.defined(DEFINED_YIELD, false, "yield") + builder.defined(YARV::DEFINED_YIELD, false, "yield") when ZSuper builder.putnil - builder.defined(DEFINED_ZSUPER, false, "super") + builder.defined(YARV::DEFINED_ZSUPER, false, "super") else builder.putobject("expression") end @@ -1676,10 +845,12 @@ def visit_for(node) end def visit_hash(node) - builder.duphash(node.accept(RubyVisitor.new)) - rescue RubyVisitor::CompilationError - visit_all(node.assocs) - builder.newhash(node.assocs.length * 2) + if (compiled = RubyVisitor.compile(node)) + builder.duphash(compiled) + else + visit_all(node.assocs) + builder.newhash(node.assocs.length * 2) + end end def visit_heredoc(node) @@ -1766,8 +937,8 @@ def visit_lambda(node) builder.leave end - builder.putspecialobject(VM_SPECIAL_OBJECT_VMCORE) - builder.send(:lambda, 0, VM_CALL_FCALL, lambda_iseq) + builder.putspecialobject(YARV::VM_SPECIAL_OBJECT_VMCORE) + builder.send(:lambda, 0, YARV::VM_CALL_FCALL, lambda_iseq) end def visit_lambda_var(node) @@ -1823,16 +994,16 @@ def visit_module(node) builder.leave end - flags = VM_DEFINECLASS_TYPE_MODULE + flags = YARV::VM_DEFINECLASS_TYPE_MODULE case node.constant when ConstPathRef - flags |= VM_DEFINECLASS_FLAG_SCOPED + flags |= YARV::VM_DEFINECLASS_FLAG_SCOPED visit(node.constant.parent) when ConstRef - builder.putspecialobject(VM_SPECIAL_OBJECT_CONST_BASE) + builder.putspecialobject(YARV::VM_SPECIAL_OBJECT_CONST_BASE) when TopConstRef - flags |= VM_DEFINECLASS_FLAG_SCOPED + flags |= YARV::VM_DEFINECLASS_FLAG_SCOPED builder.putobject(Object) end @@ -1851,13 +1022,13 @@ def visit_mrhs(node) def visit_not(node) visit(node.statement) - builder.send(:!, 0, VM_CALL_ARGS_SIMPLE) + builder.send(:!, 0, YARV::VM_CALL_ARGS_SIMPLE) end def visit_opassign(node) - flag = VM_CALL_ARGS_SIMPLE + flag = YARV::VM_CALL_ARGS_SIMPLE if node.target.is_a?(ConstPathField) || node.target.is_a?(TopConstField) - flag |= VM_CALL_FCALL + flag |= YARV::VM_CALL_FCALL end case (operator = node.operator.value.chomp("=").to_sym) @@ -1977,18 +1148,16 @@ def visit_params(node) if value.nil? argument_options[:keyword] << name + elsif (compiled = RubyVisitor.compile(value)) + compiled = value.accept(RubyVisitor.new) + argument_options[:keyword] << [name, compiled] else - begin - compiled = value.accept(RubyVisitor.new) - argument_options[:keyword] << [name, compiled] - rescue RubyVisitor::CompilationError - argument_options[:keyword] << [name] - checkkeywords << builder.checkkeyword(-1, keyword_index) - branchif = builder.branchif(-1) - visit(value) - builder.setlocal(index, 0) - branchif[1] = builder.label - end + argument_options[:keyword] << [name] + checkkeywords << builder.checkkeyword(-1, keyword_index) + branchif = builder.branchif(-1) + visit(value) + builder.setlocal(index, 0) + branchif[1] = builder.label end end @@ -2075,11 +1244,13 @@ def visit_qwords(node) end def visit_range(node) - builder.putobject(node.accept(RubyVisitor.new)) - rescue RubyVisitor::CompilationError - visit(node.left) - visit(node.right) - builder.newrange(node.operator.value == ".." ? 0 : 1) + if (compiled = RubyVisitor.compile(node)) + builder.putobject(compiled) + else + visit(node.left) + visit(node.right) + builder.newrange(node.operator.value == ".." ? 0 : 1) + end end def visit_rational(node) @@ -2087,11 +1258,13 @@ def visit_rational(node) end def visit_regexp_literal(node) - builder.putobject(node.accept(RubyVisitor.new)) - rescue RubyVisitor::CompilationError - flags = RubyVisitor.new.visit_regexp_literal_flags(node) - length = visit_string_parts(node) - builder.toregexp(flags, length) + if (compiled = RubyVisitor.compile(node)) + builder.putobject(compiled) + else + flags = RubyVisitor.new.visit_regexp_literal_flags(node) + length = visit_string_parts(node) + builder.toregexp(flags, length) + end end def visit_rest_param(node) @@ -2120,7 +1293,7 @@ def visit_sclass(node) builder.defineclass( :singletonclass, singleton_iseq, - VM_DEFINECLASS_TYPE_SINGLETON_CLASS + YARV::VM_DEFINECLASS_TYPE_SINGLETON_CLASS ) end @@ -2170,7 +1343,7 @@ def visit_super(node) builder.invokesuper( nil, argument_parts(node.arguments).length, - VM_CALL_FCALL | VM_CALL_ARGS_SIMPLE | VM_CALL_SUPER, + YARV::VM_CALL_FCALL | YARV::VM_CALL_ARGS_SIMPLE | YARV::VM_CALL_SUPER, nil ) end @@ -2180,20 +1353,22 @@ def visit_symbol_literal(node) end def visit_symbols(node) - builder.duparray(node.accept(RubyVisitor.new)) - rescue RubyVisitor::CompilationError - node.elements.each do |element| - if element.parts.length == 1 && - element.parts.first.is_a?(TStringContent) - builder.putobject(element.parts.first.value.to_sym) - else - length = visit_string_parts(element) - builder.concatstrings(length) - builder.intern + if (compiled = RubyVisitor.compile(node)) + builder.duparray(compiled) + else + node.elements.each do |element| + if element.parts.length == 1 && + element.parts.first.is_a?(TStringContent) + builder.putobject(element.parts.first.value.to_sym) + else + length = visit_string_parts(element) + builder.concatstrings(length) + builder.intern + end end - end - builder.newarray(node.elements.length) + builder.newarray(node.elements.length) + end end def visit_top_const_ref(node) @@ -2232,10 +1407,10 @@ def visit_unary(node) def visit_undef(node) node.symbols.each_with_index do |symbol, index| builder.pop if index != 0 - builder.putspecialobject(VM_SPECIAL_OBJECT_VMCORE) - builder.putspecialobject(VM_SPECIAL_OBJECT_CBASE) + builder.putspecialobject(YARV::VM_SPECIAL_OBJECT_VMCORE) + builder.putspecialobject(YARV::VM_SPECIAL_OBJECT_CBASE) visit(symbol) - builder.send(:"core#undef_method", 2, VM_CALL_ARGS_SIMPLE) + builder.send(:"core#undef_method", 2, YARV::VM_CALL_ARGS_SIMPLE) end end @@ -2311,9 +1486,9 @@ def visit_var_ref(node) lookup = current_iseq.local_variable(node.value.value.to_sym) case lookup.local - when LocalTable::BlockLocal + when YARV::LocalTable::BlockLocal builder.getblockparam(lookup.index, lookup.level) - when LocalTable::PlainLocal + when YARV::LocalTable::PlainLocal builder.getlocal(lookup.index, lookup.level) end when IVar @@ -2336,7 +1511,7 @@ def visit_var_ref(node) def visit_vcall(node) builder.putself - flag = VM_CALL_FCALL | VM_CALL_VCALL | VM_CALL_ARGS_SIMPLE + flag = YARV::VM_CALL_FCALL | YARV::VM_CALL_VCALL | YARV::VM_CALL_ARGS_SIMPLE builder.send(node.value.value.to_sym, 0, flag) end @@ -2372,17 +1547,8 @@ def visit_word(node) end def visit_words(node) - converted = nil - - if frozen_string_literal - begin - converted = node.accept(RubyVisitor.new) - rescue RubyVisitor::CompilationError - end - end - - if converted - builder.duparray(converted) + if frozen_string_literal && (compiled = RubyVisitor.compile(node)) + builder.duparray(compiled) else visit_all(node.elements) builder.newarray(node.elements.length) @@ -2393,13 +1559,13 @@ def visit_xstring_literal(node) builder.putself length = visit_string_parts(node) builder.concatstrings(node.parts.length) if length > 1 - builder.send(:`, 1, VM_CALL_FCALL | VM_CALL_ARGS_SIMPLE) + builder.send(:`, 1, YARV::VM_CALL_FCALL | YARV::VM_CALL_ARGS_SIMPLE) end def visit_yield(node) parts = argument_parts(node.arguments) visit_all(parts) - builder.invokeblock(nil, parts.length, VM_CALL_ARGS_SIMPLE) + builder.invokeblock(nil, parts.length, YARV::VM_CALL_ARGS_SIMPLE) end def visit_zsuper(_node) @@ -2407,7 +1573,7 @@ def visit_zsuper(_node) builder.invokesuper( nil, 0, - VM_CALL_FCALL | VM_CALL_ARGS_SIMPLE | VM_CALL_SUPER | VM_CALL_ZSUPER, + YARV::VM_CALL_FCALL | YARV::VM_CALL_ARGS_SIMPLE | YARV::VM_CALL_SUPER | YARV::VM_CALL_ZSUPER, nil ) end @@ -2473,24 +1639,24 @@ def opassign_defined(node) name = node.target.constant.value.to_sym builder.dup - builder.defined(DEFINED_CONST_FROM, name, true) + builder.defined(YARV::DEFINED_CONST_FROM, name, true) when TopConstField name = node.target.constant.value.to_sym builder.putobject(Object) builder.dup - builder.defined(DEFINED_CONST_FROM, name, true) + builder.defined(YARV::DEFINED_CONST_FROM, name, true) when VarField name = node.target.value.value.to_sym builder.putnil case node.target.value when Const - builder.defined(DEFINED_CONST, name, true) + builder.defined(YARV::DEFINED_CONST, name, true) when CVar - builder.defined(DEFINED_CVAR, name, true) + builder.defined(YARV::DEFINED_CVAR, name, true) when GVar - builder.defined(DEFINED_GVAR, name, true) + builder.defined(YARV::DEFINED_GVAR, name, true) end end @@ -2529,7 +1695,7 @@ def opassign_defined(node) case node.target.value when Const - builder.putspecialobject(VM_SPECIAL_OBJECT_CONST_BASE) + builder.putspecialobject(YARV::VM_SPECIAL_OBJECT_CONST_BASE) builder.setconstant(name) when CVar builder.setclassvariable(name) @@ -2545,7 +1711,7 @@ def opassign_defined(node) # three instructions are pushed. def push_interpolate builder.dup - builder.objtostring(:to_s, 0, VM_CALL_FCALL | VM_CALL_ARGS_SIMPLE) + builder.objtostring(:to_s, 0, YARV::VM_CALL_FCALL | YARV::VM_CALL_ARGS_SIMPLE) builder.anytostring end @@ -2588,11 +1754,11 @@ def with_instruction_sequence(type, name, parent_iseq, node) previous_builder = builder begin - iseq = InstructionSequence.new(type, name, parent_iseq, node.location) + iseq = YARV::InstructionSequence.new(type, name, parent_iseq, node.location) @current_iseq = iseq @builder = - Builder.new( + YARV::Builder.new( iseq, frozen_string_literal: frozen_string_literal, operands_unification: operands_unification, @@ -2642,12 +1808,12 @@ def with_opassign(node) visit(node.target.index) builder.dupn(2) - builder.send(:[], 1, VM_CALL_ARGS_SIMPLE) + builder.send(:[], 1, YARV::VM_CALL_ARGS_SIMPLE) yield builder.setn(3) - builder.send(:[]=, 2, VM_CALL_ARGS_SIMPLE) + builder.send(:[]=, 2, YARV::VM_CALL_ARGS_SIMPLE) builder.pop when ConstPathField name = node.target.constant.value.to_sym @@ -2696,7 +1862,7 @@ def with_opassign(node) yield builder.dup - builder.putspecialobject(VM_SPECIAL_OBJECT_CONST_BASE) + builder.putspecialobject(YARV::VM_SPECIAL_OBJECT_CONST_BASE) builder.setconstant(names.last) when CVar name = node.target.value.value.to_sym diff --git a/lib/syntax_tree/yarv.rb b/lib/syntax_tree/yarv.rb new file mode 100644 index 00000000..42faa66b --- /dev/null +++ b/lib/syntax_tree/yarv.rb @@ -0,0 +1,838 @@ +# frozen_string_literal: true + +module SyntaxTree + module YARV + # This object is used to track the size of the stack at any given time. It + # is effectively a mini symbolic interpreter. It's necessary because when + # instruction sequences get serialized they include a :stack_max field on + # them. This field is used to determine how much stack space to allocate + # for the instruction sequence. + class Stack + attr_reader :current_size, :maximum_size + + def initialize + @current_size = 0 + @maximum_size = 0 + end + + def change_by(value) + @current_size += value + @maximum_size = @current_size if @current_size > @maximum_size + end + end + + # This represents every local variable associated with an instruction + # sequence. There are two kinds of locals: plain locals that are what you + # expect, and block proxy locals, which represent local variables + # associated with blocks that were passed into the current instruction + # sequence. + class LocalTable + # A local representing a block passed into the current instruction + # sequence. + class BlockLocal + attr_reader :name + + def initialize(name) + @name = name + end + end + + # A regular local variable. + class PlainLocal + attr_reader :name + + def initialize(name) + @name = name + end + end + + # The result of looking up a local variable in the current local table. + class Lookup + attr_reader :local, :index, :level + + def initialize(local, index, level) + @local = local + @index = index + @level = level + end + end + + attr_reader :locals + + def initialize + @locals = [] + end + + def find(name, level) + index = locals.index { |local| local.name == name } + Lookup.new(locals[index], index, level) if index + end + + def has?(name) + locals.any? { |local| local.name == name } + end + + def names + locals.map(&:name) + end + + def size + locals.length + end + + # Add a BlockLocal to the local table. + def block(name) + locals << BlockLocal.new(name) unless has?(name) + end + + # Add a PlainLocal to the local table. + def plain(name) + locals << PlainLocal.new(name) unless has?(name) + end + + # This is the offset from the top of the stack where this local variable + # lives. + def offset(index) + size - (index - 3) - 1 + end + end + + # This class is meant to mirror RubyVM::InstructionSequence. It contains a + # list of instructions along with the metadata pertaining to them. It also + # functions as a builder for the instruction sequence. + class InstructionSequence + MAGIC = "YARVInstructionSequence/SimpleDataFormat" + + # This provides a handle to the rb_iseq_load function, which allows you to + # pass a serialized iseq to Ruby and have it return a + # RubyVM::InstructionSequence object. + ISEQ_LOAD = + Fiddle::Function.new( + Fiddle::Handle::DEFAULT["rb_iseq_load"], + [Fiddle::TYPE_VOIDP] * 3, + Fiddle::TYPE_VOIDP + ) + + # The type of the instruction sequence. + attr_reader :type + + # The name of the instruction sequence. + attr_reader :name + + # The parent instruction sequence, if there is one. + attr_reader :parent_iseq + + # The location of the root node of this instruction sequence. + attr_reader :location + + # This is the list of information about the arguments to this + # instruction sequence. + attr_accessor :argument_size + attr_reader :argument_options + + # The list of instructions for this instruction sequence. + attr_reader :insns + + # The table of local variables. + attr_reader :local_table + + # The hash of names of instance and class variables pointing to the + # index of their associated inline storage. + attr_reader :inline_storages + + # The index of the next inline storage that will be created. + attr_reader :storage_index + + # An object that will track the current size of the stack and the + # maximum size of the stack for this instruction sequence. + attr_reader :stack + + def initialize(type, name, parent_iseq, location) + @type = type + @name = name + @parent_iseq = parent_iseq + @location = location + + @argument_size = 0 + @argument_options = {} + + @local_table = LocalTable.new + @inline_storages = {} + @insns = [] + @storage_index = 0 + @stack = Stack.new + end + + def local_variable(name, level = 0) + if (lookup = local_table.find(name, level)) + lookup + elsif parent_iseq + parent_iseq.local_variable(name, level + 1) + end + end + + def push(insn) + insns << insn + insn + end + + def inline_storage + storage = storage_index + @storage_index += 1 + storage + end + + def inline_storage_for(name) + unless inline_storages.key?(name) + inline_storages[name] = inline_storage + end + + inline_storages[name] + end + + def length + insns.inject(0) do |sum, insn| + insn.is_a?(Array) ? sum + insn.length : sum + end + end + + def each_child + insns.each do |insn| + insn[1..].each do |operand| + yield operand if operand.is_a?(InstructionSequence) + end + end + end + + def eval + compiled = to_a + + # Temporary hack until we get these working. + compiled[4][:node_id] = 11 + compiled[4][:node_ids] = [1, 0, 3, 2, 6, 7, 9, -1] + + Fiddle.dlunwrap(ISEQ_LOAD.call(Fiddle.dlwrap(compiled), 0, nil)).eval + end + + def to_a + versions = RUBY_VERSION.split(".").map(&:to_i) + + [ + MAGIC, + versions[0], + versions[1], + 1, + { + arg_size: argument_size, + local_size: local_table.size, + stack_max: stack.maximum_size + }, + name, + "", + "", + location.start_line, + type, + local_table.names, + argument_options, + [], + insns.map { |insn| serialize(insn) } + ] + end + + private + + def serialize(insn) + case insn[0] + when :checkkeyword, :getblockparam, :getblockparamproxy, + :getlocal_WC_0, :getlocal_WC_1, :getlocal, :setlocal_WC_0, + :setlocal_WC_1, :setlocal + iseq = self + + case insn[0] + when :getlocal_WC_1, :setlocal_WC_1 + iseq = iseq.parent_iseq + when :getblockparam, :getblockparamproxy, :getlocal, :setlocal + insn[2].times { iseq = iseq.parent_iseq } + end + + # Here we need to map the local variable index to the offset + # from the top of the stack where it will be stored. + [insn[0], iseq.local_table.offset(insn[1]), *insn[2..]] + when :defineclass + [insn[0], insn[1], insn[2].to_a, insn[3]] + when :definemethod, :definesmethod + [insn[0], insn[1], insn[2].to_a] + when :send + # For any instructions that push instruction sequences onto the + # stack, we need to call #to_a on them as well. + [insn[0], insn[1], (insn[2].to_a if insn[2])] + when :once + [insn[0], insn[1].to_a, insn[2]] + else + insn + end + end + end + + # This class serves as a layer of indirection between the instruction + # sequence and the compiler. It allows us to provide different behavior + # for certain instructions depending on the Ruby version. For example, + # class variable reads and writes gained an inline cache in Ruby 3.0. So + # we place the logic for checking the Ruby version in this class. + class Builder + attr_reader :iseq, :stack + attr_reader :frozen_string_literal, + :operands_unification, + :specialized_instruction + + def initialize( + iseq, + frozen_string_literal: false, + operands_unification: true, + specialized_instruction: true + ) + @iseq = iseq + @stack = iseq.stack + + @frozen_string_literal = frozen_string_literal + @operands_unification = operands_unification + @specialized_instruction = specialized_instruction + end + + # This creates a new label at the current length of the instruction + # sequence. It is used as the operand for jump instructions. + def label + name = :"label_#{iseq.length}" + iseq.insns.last == name ? name : event(name) + end + + def event(name) + iseq.push(name) + name + end + + def adjuststack(number) + stack.change_by(-number) + iseq.push([:adjuststack, number]) + end + + def anytostring + stack.change_by(-2 + 1) + iseq.push([:anytostring]) + end + + def branchif(index) + stack.change_by(-1) + iseq.push([:branchif, index]) + end + + def branchnil(index) + stack.change_by(-1) + iseq.push([:branchnil, index]) + end + + def branchunless(index) + stack.change_by(-1) + iseq.push([:branchunless, index]) + end + + def checkkeyword(index, keyword_index) + stack.change_by(+1) + iseq.push([:checkkeyword, index, keyword_index]) + end + + def concatarray + stack.change_by(-2 + 1) + iseq.push([:concatarray]) + end + + def concatstrings(number) + stack.change_by(-number + 1) + iseq.push([:concatstrings, number]) + end + + def defined(type, name, message) + stack.change_by(-1 + 1) + iseq.push([:defined, type, name, message]) + end + + def defineclass(name, class_iseq, flags) + stack.change_by(-2 + 1) + iseq.push([:defineclass, name, class_iseq, flags]) + end + + def definemethod(name, method_iseq) + stack.change_by(0) + iseq.push([:definemethod, name, method_iseq]) + end + + def definesmethod(name, method_iseq) + stack.change_by(-1) + iseq.push([:definesmethod, name, method_iseq]) + end + + def dup + stack.change_by(-1 + 2) + iseq.push([:dup]) + end + + def duparray(object) + stack.change_by(+1) + iseq.push([:duparray, object]) + end + + def duphash(object) + stack.change_by(+1) + iseq.push([:duphash, object]) + end + + def dupn(number) + stack.change_by(+number) + iseq.push([:dupn, number]) + end + + def expandarray(length, flag) + stack.change_by(-1 + length) + iseq.push([:expandarray, length, flag]) + end + + def getblockparam(index, level) + stack.change_by(+1) + iseq.push([:getblockparam, index, level]) + end + + def getblockparamproxy(index, level) + stack.change_by(+1) + iseq.push([:getblockparamproxy, index, level]) + end + + def getclassvariable(name) + stack.change_by(+1) + + if RUBY_VERSION >= "3.0" + iseq.push([:getclassvariable, name, iseq.inline_storage_for(name)]) + else + iseq.push([:getclassvariable, name]) + end + end + + def getconstant(name) + stack.change_by(-2 + 1) + iseq.push([:getconstant, name]) + end + + def getglobal(name) + stack.change_by(+1) + iseq.push([:getglobal, name]) + end + + def getinstancevariable(name) + stack.change_by(+1) + + if RUBY_VERSION >= "3.2" + iseq.push([:getinstancevariable, name, iseq.inline_storage]) + else + inline_storage = iseq.inline_storage_for(name) + iseq.push([:getinstancevariable, name, inline_storage]) + end + end + + def getlocal(index, level) + stack.change_by(+1) + + if operands_unification + # Specialize the getlocal instruction based on the level of the + # local variable. If it's 0 or 1, then there's a specialized + # instruction that will look at the current scope or the parent + # scope, respectively, and requires fewer operands. + case level + when 0 + iseq.push([:getlocal_WC_0, index]) + when 1 + iseq.push([:getlocal_WC_1, index]) + else + iseq.push([:getlocal, index, level]) + end + else + iseq.push([:getlocal, index, level]) + end + end + + def getspecial(key, type) + stack.change_by(-0 + 1) + iseq.push([:getspecial, key, type]) + end + + def intern + stack.change_by(-1 + 1) + iseq.push([:intern]) + end + + def invokeblock(method_id, argc, flag) + stack.change_by(-argc + 1) + iseq.push([:invokeblock, call_data(method_id, argc, flag)]) + end + + def invokesuper(method_id, argc, flag, block_iseq) + stack.change_by(-(argc + 1) + 1) + + cdata = call_data(method_id, argc, flag) + iseq.push([:invokesuper, cdata, block_iseq]) + end + + def jump(index) + stack.change_by(0) + iseq.push([:jump, index]) + end + + def leave + stack.change_by(-1) + iseq.push([:leave]) + end + + def newarray(length) + stack.change_by(-length + 1) + iseq.push([:newarray, length]) + end + + def newhash(length) + stack.change_by(-length + 1) + iseq.push([:newhash, length]) + end + + def newrange(flag) + stack.change_by(-2 + 1) + iseq.push([:newrange, flag]) + end + + def nop + stack.change_by(0) + iseq.push([:nop]) + end + + def objtostring(method_id, argc, flag) + stack.change_by(-1 + 1) + iseq.push([:objtostring, call_data(method_id, argc, flag)]) + end + + def once(postexe_iseq, inline_storage) + stack.change_by(+1) + iseq.push([:once, postexe_iseq, inline_storage]) + end + + def opt_getconstant_path(names) + if RUBY_VERSION >= "3.2" + stack.change_by(+1) + iseq.push([:opt_getconstant_path, names]) + else + inline_storage = iseq.inline_storage + getinlinecache = opt_getinlinecache(-1, inline_storage) + + if names[0] == :"" + names.shift + pop + putobject(Object) + end + + names.each_with_index do |name, index| + putobject(index == 0) + getconstant(name) + end + + opt_setinlinecache(inline_storage) + getinlinecache[1] = label + end + end + + def opt_getinlinecache(offset, inline_storage) + stack.change_by(+1) + iseq.push([:opt_getinlinecache, offset, inline_storage]) + end + + def opt_newarray_max(length) + if specialized_instruction + stack.change_by(-length + 1) + iseq.push([:opt_newarray_max, length]) + else + newarray(length) + send(:max, 0, VM_CALL_ARGS_SIMPLE) + end + end + + def opt_newarray_min(length) + if specialized_instruction + stack.change_by(-length + 1) + iseq.push([:opt_newarray_min, length]) + else + newarray(length) + send(:min, 0, VM_CALL_ARGS_SIMPLE) + end + end + + def opt_setinlinecache(inline_storage) + stack.change_by(-1 + 1) + iseq.push([:opt_setinlinecache, inline_storage]) + end + + def opt_str_freeze(value) + if specialized_instruction + stack.change_by(+1) + iseq.push( + [ + :opt_str_freeze, + value, + call_data(:freeze, 0, VM_CALL_ARGS_SIMPLE) + ] + ) + else + putstring(value) + send(:freeze, 0, VM_CALL_ARGS_SIMPLE) + end + end + + def opt_str_uminus(value) + if specialized_instruction + stack.change_by(+1) + iseq.push( + [:opt_str_uminus, value, call_data(:-@, 0, VM_CALL_ARGS_SIMPLE)] + ) + else + putstring(value) + send(:-@, 0, VM_CALL_ARGS_SIMPLE) + end + end + + def pop + stack.change_by(-1) + iseq.push([:pop]) + end + + def putnil + stack.change_by(+1) + iseq.push([:putnil]) + end + + def putobject(object) + stack.change_by(+1) + + if operands_unification + # Specialize the putobject instruction based on the value of the + # object. If it's 0 or 1, then there's a specialized instruction + # that will push the object onto the stack and requires fewer + # operands. + if object.eql?(0) + iseq.push([:putobject_INT2FIX_0_]) + elsif object.eql?(1) + iseq.push([:putobject_INT2FIX_1_]) + else + iseq.push([:putobject, object]) + end + else + iseq.push([:putobject, object]) + end + end + + def putself + stack.change_by(+1) + iseq.push([:putself]) + end + + def putspecialobject(object) + stack.change_by(+1) + iseq.push([:putspecialobject, object]) + end + + def putstring(object) + stack.change_by(+1) + iseq.push([:putstring, object]) + end + + def send(method_id, argc, flag, block_iseq = nil) + stack.change_by(-(argc + 1) + 1) + cdata = call_data(method_id, argc, flag) + + if specialized_instruction + # Specialize the send instruction. If it doesn't have a block + # attached, then we will replace it with an opt_send_without_block + # and do further specializations based on the called method and the + # number of arguments. + + # stree-ignore + if !block_iseq && (flag & VM_CALL_ARGS_BLOCKARG) == 0 + case [method_id, argc] + when [:length, 0] then iseq.push([:opt_length, cdata]) + when [:size, 0] then iseq.push([:opt_size, cdata]) + when [:empty?, 0] then iseq.push([:opt_empty_p, cdata]) + when [:nil?, 0] then iseq.push([:opt_nil_p, cdata]) + when [:succ, 0] then iseq.push([:opt_succ, cdata]) + when [:!, 0] then iseq.push([:opt_not, cdata]) + when [:+, 1] then iseq.push([:opt_plus, cdata]) + when [:-, 1] then iseq.push([:opt_minus, cdata]) + when [:*, 1] then iseq.push([:opt_mult, cdata]) + when [:/, 1] then iseq.push([:opt_div, cdata]) + when [:%, 1] then iseq.push([:opt_mod, cdata]) + when [:==, 1] then iseq.push([:opt_eq, cdata]) + when [:=~, 1] then iseq.push([:opt_regexpmatch2, cdata]) + when [:<, 1] then iseq.push([:opt_lt, cdata]) + when [:<=, 1] then iseq.push([:opt_le, cdata]) + when [:>, 1] then iseq.push([:opt_gt, cdata]) + when [:>=, 1] then iseq.push([:opt_ge, cdata]) + when [:<<, 1] then iseq.push([:opt_ltlt, cdata]) + when [:[], 1] then iseq.push([:opt_aref, cdata]) + when [:&, 1] then iseq.push([:opt_and, cdata]) + when [:|, 1] then iseq.push([:opt_or, cdata]) + when [:[]=, 2] then iseq.push([:opt_aset, cdata]) + when [:!=, 1] + eql_data = call_data(:==, 1, VM_CALL_ARGS_SIMPLE) + iseq.push([:opt_neq, eql_data, cdata]) + else + iseq.push([:opt_send_without_block, cdata]) + end + else + iseq.push([:send, cdata, block_iseq]) + end + else + iseq.push([:send, cdata, block_iseq]) + end + end + + def setclassvariable(name) + stack.change_by(-1) + + if RUBY_VERSION >= "3.0" + iseq.push([:setclassvariable, name, iseq.inline_storage_for(name)]) + else + iseq.push([:setclassvariable, name]) + end + end + + def setconstant(name) + stack.change_by(-2) + iseq.push([:setconstant, name]) + end + + def setglobal(name) + stack.change_by(-1) + iseq.push([:setglobal, name]) + end + + def setinstancevariable(name) + stack.change_by(-1) + + if RUBY_VERSION >= "3.2" + iseq.push([:setinstancevariable, name, iseq.inline_storage]) + else + inline_storage = iseq.inline_storage_for(name) + iseq.push([:setinstancevariable, name, inline_storage]) + end + end + + def setlocal(index, level) + stack.change_by(-1) + + if operands_unification + # Specialize the setlocal instruction based on the level of the + # local variable. If it's 0 or 1, then there's a specialized + # instruction that will write to the current scope or the parent + # scope, respectively, and requires fewer operands. + case level + when 0 + iseq.push([:setlocal_WC_0, index]) + when 1 + iseq.push([:setlocal_WC_1, index]) + else + iseq.push([:setlocal, index, level]) + end + else + iseq.push([:setlocal, index, level]) + end + end + + def setn(number) + stack.change_by(-1 + 1) + iseq.push([:setn, number]) + end + + def splatarray(flag) + stack.change_by(-1 + 1) + iseq.push([:splatarray, flag]) + end + + def swap + stack.change_by(-2 + 2) + iseq.push([:swap]) + end + + def topn(number) + stack.change_by(+1) + iseq.push([:topn, number]) + end + + def toregexp(options, length) + stack.change_by(-length + 1) + iseq.push([:toregexp, options, length]) + end + + private + + # This creates a call data object that is used as the operand for the + # send, invokesuper, and objtostring instructions. + def call_data(method_id, argc, flag) + { mid: method_id, flag: flag, orig_argc: argc } + end + end + + # These constants correspond to the putspecialobject instruction. They are + # used to represent special objects that are pushed onto the stack. + VM_SPECIAL_OBJECT_VMCORE = 1 + VM_SPECIAL_OBJECT_CBASE = 2 + VM_SPECIAL_OBJECT_CONST_BASE = 3 + + # These constants correspond to the flag passed as part of the call data + # structure on the send instruction. They are used to represent various + # metadata about the callsite (e.g., were keyword arguments used?, was a + # block given?, etc.). + VM_CALL_ARGS_SPLAT = 1 << 0 + VM_CALL_ARGS_BLOCKARG = 1 << 1 + VM_CALL_FCALL = 1 << 2 + VM_CALL_VCALL = 1 << 3 + VM_CALL_ARGS_SIMPLE = 1 << 4 + VM_CALL_BLOCKISEQ = 1 << 5 + VM_CALL_KWARG = 1 << 6 + VM_CALL_KW_SPLAT = 1 << 7 + VM_CALL_TAILCALL = 1 << 8 + VM_CALL_SUPER = 1 << 9 + VM_CALL_ZSUPER = 1 << 10 + VM_CALL_OPT_SEND = 1 << 11 + VM_CALL_KW_SPLAT_MUT = 1 << 12 + + # These constants correspond to the value passed as part of the defined + # instruction. It's an enum defined in the CRuby codebase that tells that + # instruction what kind of defined check to perform. + DEFINED_NIL = 1 + DEFINED_IVAR = 2 + DEFINED_LVAR = 3 + DEFINED_GVAR = 4 + DEFINED_CVAR = 5 + DEFINED_CONST = 6 + DEFINED_METHOD = 7 + DEFINED_YIELD = 8 + DEFINED_ZSUPER = 9 + DEFINED_SELF = 10 + DEFINED_TRUE = 11 + DEFINED_FALSE = 12 + DEFINED_ASGN = 13 + DEFINED_EXPR = 14 + DEFINED_REF = 15 + DEFINED_FUNC = 16 + DEFINED_CONST_FROM = 17 + + # These constants correspond to the value passed in the flags as part of + # the defineclass instruction. + VM_DEFINECLASS_TYPE_CLASS = 0 + VM_DEFINECLASS_TYPE_SINGLETON_CLASS = 1 + VM_DEFINECLASS_TYPE_MODULE = 2 + VM_DEFINECLASS_FLAG_SCOPED = 8 + VM_DEFINECLASS_FLAG_HAS_SUPERCLASS = 16 + end +end diff --git a/test/compiler_test.rb b/test/compiler_test.rb index cdf2860e..3b8c0ea2 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -449,7 +449,7 @@ def serialize_iseq(iseq) when Array insn.map do |operand| if operand.is_a?(Array) && - operand[0] == Compiler::InstructionSequence::MAGIC + operand[0] == YARV::InstructionSequence::MAGIC serialize_iseq(operand) else operand From 6c6b88b1f4eeb5f43164d6eb81c5c8272dbd4315 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 18 Nov 2022 20:30:07 -0500 Subject: [PATCH 253/536] Start the disassembler --- lib/syntax_tree/yarv.rb | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/lib/syntax_tree/yarv.rb b/lib/syntax_tree/yarv.rb index 42faa66b..e3780a0c 100644 --- a/lib/syntax_tree/yarv.rb +++ b/lib/syntax_tree/yarv.rb @@ -274,6 +274,43 @@ def serialize(insn) end end + # This class is responsible for taking a compiled instruction sequence and + # walking through it to generate equivalent Ruby code. + class Disassembler + attr_reader :iseq + + def initialize(iseq) + @iseq = iseq + end + + def to_ruby + stack = [] + + iseq.insns.each do |insn| + case insn[0] + when :leave + stack << ReturnNode.new(arguments: Args.new(parts: [stack.pop], location: Location.default), location: Location.default) + when :opt_plus + left, right = stack.pop(2) + stack << Binary.new(left: left, operator: :+, right: right, location: Location.default) + when :putobject + case insn[1] + when Integer + stack << Int.new(value: insn[1].inspect, location: Location.default) + else + raise "Unknown object type: #{insn[1].class.name}" + end + when :putobject_INT2FIX_1_ + stack << Int.new(value: "1", location: Location.default) + else + raise "Unknown instruction #{insn[0]}" + end + end + + Statements.new(nil, body: stack, location: Location.default) + end + end + # This class serves as a layer of indirection between the instruction # sequence and the compiler. It allows us to provide different behavior # for certain instructions depending on the Ruby version. For example, From c9db96bc925c10d80e530a3238ce50980aa57f3f Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 18 Nov 2022 20:32:52 -0500 Subject: [PATCH 254/536] opt_mult, Float, and Rational --- lib/syntax_tree/yarv.rb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/syntax_tree/yarv.rb b/lib/syntax_tree/yarv.rb index e3780a0c..cbb91f1e 100644 --- a/lib/syntax_tree/yarv.rb +++ b/lib/syntax_tree/yarv.rb @@ -290,13 +290,20 @@ def to_ruby case insn[0] when :leave stack << ReturnNode.new(arguments: Args.new(parts: [stack.pop], location: Location.default), location: Location.default) + when :opt_mult + left, right = stack.pop(2) + stack << Binary.new(left: left, operator: :*, right: right, location: Location.default) when :opt_plus left, right = stack.pop(2) stack << Binary.new(left: left, operator: :+, right: right, location: Location.default) when :putobject case insn[1] + when Float + stack << FloatLiteral.new(value: insn[1].inspect, location: Location.default) when Integer stack << Int.new(value: insn[1].inspect, location: Location.default) + when Rational + stack << RationalLiteral.new(value: insn[1].inspect, location: Location.default) else raise "Unknown object type: #{insn[1].class.name}" end From 8ad799ad2dfb73ec90b8a8def55b7c088fc45bed Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 18 Nov 2022 20:37:27 -0500 Subject: [PATCH 255/536] Local variables and assignments --- lib/syntax_tree/yarv.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/syntax_tree/yarv.rb b/lib/syntax_tree/yarv.rb index cbb91f1e..7290d87f 100644 --- a/lib/syntax_tree/yarv.rb +++ b/lib/syntax_tree/yarv.rb @@ -288,6 +288,9 @@ def to_ruby iseq.insns.each do |insn| case insn[0] + when :getlocal_WC_0 + value = iseq.local_table.locals[insn[1]].name.to_s + stack << VarRef.new(value: Ident.new(value: value, location: Location.default), location: Location.default) when :leave stack << ReturnNode.new(arguments: Args.new(parts: [stack.pop], location: Location.default), location: Location.default) when :opt_mult @@ -309,6 +312,9 @@ def to_ruby end when :putobject_INT2FIX_1_ stack << Int.new(value: "1", location: Location.default) + when :setlocal_WC_0 + target = VarField.new(value: Ident.new(value: iseq.local_table.locals[insn[1]].name.to_s, location: Location.default), location: Location.default) + stack << Assign.new(target: target, value: stack.pop, location: Location.default) else raise "Unknown instruction #{insn[0]}" end From 0047065d4227b141e0d9d17542696b5adb75e12b Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Sat, 19 Nov 2022 14:48:55 -0500 Subject: [PATCH 256/536] Inline builder into ISeq --- lib/syntax_tree/compiler.rb | 912 +++++++++++++++++------------------- lib/syntax_tree/yarv.rb | 485 ++++++++++--------- 2 files changed, 687 insertions(+), 710 deletions(-) diff --git a/lib/syntax_tree/compiler.rb b/lib/syntax_tree/compiler.rb index c936c9c1..424a9cf5 100644 --- a/lib/syntax_tree/compiler.rb +++ b/lib/syntax_tree/compiler.rb @@ -192,11 +192,7 @@ def visit_unsupported(_node) :specialized_instruction # The current instruction sequence that is being compiled. - attr_reader :current_iseq - - # This is the current builder that is being used to construct the current - # instruction sequence. - attr_reader :builder + attr_reader :iseq # A boolean to track if we're currently compiling the last statement # within a set of statements. This information is necessary to determine @@ -212,8 +208,7 @@ def initialize( @operands_unification = operands_unification @specialized_instruction = specialized_instruction - @current_iseq = nil - @builder = nil + @iseq = nil @last_statement = false end @@ -223,45 +218,45 @@ def visit_BEGIN(node) def visit_CHAR(node) if frozen_string_literal - builder.putobject(node.value[1..]) + iseq.putobject(node.value[1..]) else - builder.putstring(node.value[1..]) + iseq.putstring(node.value[1..]) end end def visit_END(node) - name = "block in #{current_iseq.name}" + name = "block in #{iseq.name}" once_iseq = - with_instruction_sequence(:block, name, current_iseq, node) do + with_instruction_sequence(:block, name, node) do postexe_iseq = - with_instruction_sequence(:block, name, current_iseq, node) do + with_instruction_sequence(:block, name, node) do *statements, last_statement = node.statements.body visit_all(statements) with_last_statement { visit(last_statement) } - builder.leave + iseq.leave end - builder.putspecialobject(YARV::VM_SPECIAL_OBJECT_VMCORE) - builder.send(:"core#set_postexe", 0, YARV::VM_CALL_FCALL, postexe_iseq) - builder.leave + iseq.putspecialobject(YARV::VM_SPECIAL_OBJECT_VMCORE) + iseq.send(:"core#set_postexe", 0, YARV::VM_CALL_FCALL, postexe_iseq) + iseq.leave end - builder.once(once_iseq, current_iseq.inline_storage) - builder.pop + iseq.once(once_iseq, iseq.inline_storage) + iseq.pop end def visit_alias(node) - builder.putspecialobject(YARV::VM_SPECIAL_OBJECT_VMCORE) - builder.putspecialobject(YARV::VM_SPECIAL_OBJECT_CBASE) + iseq.putspecialobject(YARV::VM_SPECIAL_OBJECT_VMCORE) + iseq.putspecialobject(YARV::VM_SPECIAL_OBJECT_CBASE) visit(node.left) visit(node.right) - builder.send(:"core#set_method_alias", 3, YARV::VM_CALL_ARGS_SIMPLE) + iseq.send(:"core#set_method_alias", 3, YARV::VM_CALL_ARGS_SIMPLE) end def visit_aref(node) visit(node.collection) visit(node.index) - builder.send(:[], 1, YARV::VM_CALL_ARGS_SIMPLE) + iseq.send(:[], 1, YARV::VM_CALL_ARGS_SIMPLE) end def visit_arg_block(node) @@ -274,7 +269,7 @@ def visit_arg_paren(node) def visit_arg_star(node) visit(node.value) - builder.splatarray(false) + iseq.splatarray(false) end def visit_args(node) @@ -283,99 +278,97 @@ def visit_args(node) def visit_array(node) if (compiled = RubyVisitor.compile(node)) - builder.duparray(compiled) + iseq.duparray(compiled) else length = 0 node.contents.parts.each do |part| if part.is_a?(ArgStar) if length > 0 - builder.newarray(length) + iseq.newarray(length) length = 0 end visit(part.value) - builder.concatarray + iseq.concatarray else visit(part) length += 1 end end - builder.newarray(length) if length > 0 - if length > 0 && length != node.contents.parts.length - builder.concatarray - end + iseq.newarray(length) if length > 0 + iseq.concatarray if length > 0 && length != node.contents.parts.length end end def visit_assign(node) case node.target when ARefField - builder.putnil + iseq.putnil visit(node.target.collection) visit(node.target.index) visit(node.value) - builder.setn(3) - builder.send(:[]=, 2, YARV::VM_CALL_ARGS_SIMPLE) - builder.pop + iseq.setn(3) + iseq.send(:[]=, 2, YARV::VM_CALL_ARGS_SIMPLE) + iseq.pop when ConstPathField names = constant_names(node.target) name = names.pop if RUBY_VERSION >= "3.2" - builder.opt_getconstant_path(names) + iseq.opt_getconstant_path(names) visit(node.value) - builder.swap - builder.topn(1) - builder.swap - builder.setconstant(name) + iseq.swap + iseq.topn(1) + iseq.swap + iseq.setconstant(name) else visit(node.value) - builder.dup if last_statement? - builder.opt_getconstant_path(names) - builder.setconstant(name) + iseq.dup if last_statement? + iseq.opt_getconstant_path(names) + iseq.setconstant(name) end when Field - builder.putnil + iseq.putnil visit(node.target) visit(node.value) - builder.setn(2) - builder.send(:"#{node.target.name.value}=", 1, YARV::VM_CALL_ARGS_SIMPLE) - builder.pop + iseq.setn(2) + iseq.send(:"#{node.target.name.value}=", 1, YARV::VM_CALL_ARGS_SIMPLE) + iseq.pop when TopConstField name = node.target.constant.value.to_sym if RUBY_VERSION >= "3.2" - builder.putobject(Object) + iseq.putobject(Object) visit(node.value) - builder.swap - builder.topn(1) - builder.swap - builder.setconstant(name) + iseq.swap + iseq.topn(1) + iseq.swap + iseq.setconstant(name) else visit(node.value) - builder.dup if last_statement? - builder.putobject(Object) - builder.setconstant(name) + iseq.dup if last_statement? + iseq.putobject(Object) + iseq.setconstant(name) end when VarField visit(node.value) - builder.dup if last_statement? + iseq.dup if last_statement? case node.target.value when Const - builder.putspecialobject(YARV::VM_SPECIAL_OBJECT_CONST_BASE) - builder.setconstant(node.target.value.value.to_sym) + iseq.putspecialobject(YARV::VM_SPECIAL_OBJECT_CONST_BASE) + iseq.setconstant(node.target.value.value.to_sym) when CVar - builder.setclassvariable(node.target.value.value.to_sym) + iseq.setclassvariable(node.target.value.value.to_sym) when GVar - builder.setglobal(node.target.value.value.to_sym) + iseq.setglobal(node.target.value.value.to_sym) when Ident local_variable = visit(node.target) - builder.setlocal(local_variable.index, local_variable.level) + iseq.setlocal(local_variable.index, local_variable.level) when IVar - builder.setinstancevariable(node.target.value.value.to_sym) + iseq.setinstancevariable(node.target.value.value.to_sym) end end end @@ -390,12 +383,12 @@ def visit_assoc_splat(node) end def visit_backref(node) - builder.getspecial(1, 2 * node.value[1..].to_i) + iseq.getspecial(1, 2 * node.value[1..].to_i) end def visit_bare_assoc_hash(node) if (compiled = RubyVisitor.compile(node)) - builder.duphash(compiled) + iseq.duphash(compiled) else visit_all(node.assocs) end @@ -405,41 +398,36 @@ def visit_binary(node) case node.operator when :"&&" visit(node.left) - builder.dup + iseq.dup - branchunless = builder.branchunless(-1) - builder.pop + branchunless = iseq.branchunless(-1) + iseq.pop visit(node.right) - branchunless[1] = builder.label + branchunless[1] = iseq.label when :"||" visit(node.left) - builder.dup + iseq.dup - branchif = builder.branchif(-1) - builder.pop + branchif = iseq.branchif(-1) + iseq.pop visit(node.right) - branchif[1] = builder.label + branchif[1] = iseq.label else visit(node.left) visit(node.right) - builder.send(node.operator, 1, YARV::VM_CALL_ARGS_SIMPLE) + iseq.send(node.operator, 1, YARV::VM_CALL_ARGS_SIMPLE) end end def visit_block(node) - with_instruction_sequence( - :block, - "block in #{current_iseq.name}", - current_iseq, - node - ) do - builder.event(:RUBY_EVENT_B_CALL) + with_instruction_sequence(:block, "block in #{iseq.name}", node) do + iseq.event(:RUBY_EVENT_B_CALL) visit(node.block_var) visit(node.bodystmt) - builder.event(:RUBY_EVENT_B_RETURN) - builder.leave + iseq.event(:RUBY_EVENT_B_RETURN) + iseq.leave end end @@ -447,22 +435,20 @@ def visit_block_var(node) params = node.params if params.requireds.length == 1 && params.optionals.empty? && - !params.rest && params.posts.empty? && params.keywords.empty? && - !params.keyword_rest && !params.block - current_iseq.argument_options[:ambiguous_param0] = true + !params.rest && params.posts.empty? && params.keywords.empty? && + !params.keyword_rest && !params.block + iseq.argument_options[:ambiguous_param0] = true end visit(node.params) - node.locals.each do |local| - current_iseq.local_table.plain(local.value.to_sym) - end + node.locals.each { |local| iseq.local_table.plain(local.value.to_sym) } end def visit_blockarg(node) - current_iseq.argument_options[:block_start] = current_iseq.argument_size - current_iseq.local_table.block(node.name.value.to_sym) - current_iseq.argument_size += 1 + iseq.argument_options[:block_start] = iseq.argument_size + iseq.local_table.block(node.name.value.to_sym) + iseq.argument_size += 1 end def visit_bodystmt(node) @@ -497,15 +483,15 @@ def visit_call(node) parts = node.receiver.contents&.parts || [] if parts.none? { |part| part.is_a?(ArgStar) } && - RubyVisitor.compile(node.receiver).nil? + RubyVisitor.compile(node.receiver).nil? case node.message.value when "max" visit(node.receiver.contents) - builder.opt_newarray_max(parts.length) + iseq.opt_newarray_max(parts.length) return when "min" visit(node.receiver.contents) - builder.opt_newarray_min(parts.length) + iseq.opt_newarray_min(parts.length) return end end @@ -513,10 +499,10 @@ def visit_call(node) if RubyVisitor.compile(node.receiver).nil? case node.message.value when "-@" - builder.opt_str_uminus(node.receiver.parts.first.value) + iseq.opt_str_uminus(node.receiver.parts.first.value) return when "freeze" - builder.opt_str_freeze(node.receiver.parts.first.value) + iseq.opt_str_freeze(node.receiver.parts.first.value) return end end @@ -525,10 +511,10 @@ def visit_call(node) if node.receiver if node.receiver.is_a?(VarRef) - lookup = current_iseq.local_variable(node.receiver.value.value.to_sym) + lookup = iseq.local_variable(node.receiver.value.value.to_sym) if lookup.local.is_a?(YARV::LocalTable::BlockLocal) - builder.getblockparamproxy(lookup.index, lookup.level) + iseq.getblockparamproxy(lookup.index, lookup.level) else visit(node.receiver) end @@ -536,13 +522,13 @@ def visit_call(node) visit(node.receiver) end else - builder.putself + iseq.putself end branchnil = if node.operator&.value == "&." - builder.dup - builder.branchnil(-1) + iseq.dup + iseq.branchnil(-1) end flag = 0 @@ -559,12 +545,12 @@ def visit_call(node) when ArgsForward flag |= YARV::VM_CALL_ARGS_SPLAT | YARV::VM_CALL_ARGS_BLOCKARG - lookup = current_iseq.local_table.find(:*, 0) - builder.getlocal(lookup.index, lookup.level) - builder.splatarray(arg_parts.length != 1) + lookup = iseq.local_table.find(:*, 0) + iseq.getlocal(lookup.index, lookup.level) + iseq.splatarray(arg_parts.length != 1) - lookup = current_iseq.local_table.find(:&, 0) - builder.getblockparamproxy(lookup.index, lookup.level) + lookup = iseq.local_table.find(:&, 0) + iseq.getblockparamproxy(lookup.index, lookup.level) when BareAssocHash flag |= YARV::VM_CALL_KW_SPLAT visit(arg_part) @@ -577,8 +563,8 @@ def visit_call(node) flag |= YARV::VM_CALL_ARGS_SIMPLE if block_iseq.nil? && flag == 0 flag |= YARV::VM_CALL_FCALL if node.receiver.nil? - builder.send(node.message.value.to_sym, argc, flag, block_iseq) - branchnil[1] = builder.label if branchnil + iseq.send(node.message.value.to_sym, argc, flag, block_iseq) + branchnil[1] = iseq.label if branchnil end def visit_case(node) @@ -586,7 +572,6 @@ def visit_case(node) clauses = [] else_clause = nil - current = node.consequent while current @@ -601,21 +586,19 @@ def visit_case(node) branches = clauses.map do |clause| visit(clause.arguments) - builder.topn(1) - builder.send(:===, 1, YARV::VM_CALL_FCALL | YARV::VM_CALL_ARGS_SIMPLE) - [clause, builder.branchif(:label_00)] + iseq.topn(1) + iseq.send(:===, 1, YARV::VM_CALL_FCALL | YARV::VM_CALL_ARGS_SIMPLE) + [clause, iseq.branchif(:label_00)] end - builder.pop - - else_clause ? visit(else_clause) : builder.putnil - - builder.leave + iseq.pop + else_clause ? visit(else_clause) : iseq.putnil + iseq.leave branches.each_with_index do |(clause, branchif), index| - builder.leave if index != 0 - branchif[1] = builder.label - builder.pop + iseq.leave if index != 0 + branchif[1] = iseq.label + iseq.pop visit(clause) end end @@ -623,16 +606,11 @@ def visit_case(node) def visit_class(node) name = node.constant.constant.value.to_sym class_iseq = - with_instruction_sequence( - :class, - "", - current_iseq, - node - ) do - builder.event(:RUBY_EVENT_CLASS) + with_instruction_sequence(:class, "", node) do + iseq.event(:RUBY_EVENT_CLASS) visit(node.bodystmt) - builder.event(:RUBY_EVENT_END) - builder.leave + iseq.event(:RUBY_EVENT_END) + iseq.leave end flags = YARV::VM_DEFINECLASS_TYPE_CLASS @@ -642,20 +620,20 @@ def visit_class(node) flags |= YARV::VM_DEFINECLASS_FLAG_SCOPED visit(node.constant.parent) when ConstRef - builder.putspecialobject(YARV::VM_SPECIAL_OBJECT_CONST_BASE) + iseq.putspecialobject(YARV::VM_SPECIAL_OBJECT_CONST_BASE) when TopConstRef flags |= YARV::VM_DEFINECLASS_FLAG_SCOPED - builder.putobject(Object) + iseq.putobject(Object) end if node.superclass flags |= YARV::VM_DEFINECLASS_FLAG_HAS_SUPERCLASS visit(node.superclass) else - builder.putnil + iseq.putnil end - builder.defineclass(name, class_iseq, flags) + iseq.defineclass(name, class_iseq, flags) end def visit_command(node) @@ -690,34 +668,29 @@ def visit_const_path_field(node) def visit_const_path_ref(node) names = constant_names(node) - builder.opt_getconstant_path(names) + iseq.opt_getconstant_path(names) end def visit_def(node) method_iseq = - with_instruction_sequence( - :method, - node.name.value, - current_iseq, - node - ) do + with_instruction_sequence(:method, node.name.value, node) do visit(node.params) if node.params - builder.event(:RUBY_EVENT_CALL) + iseq.event(:RUBY_EVENT_CALL) visit(node.bodystmt) - builder.event(:RUBY_EVENT_RETURN) - builder.leave + iseq.event(:RUBY_EVENT_RETURN) + iseq.leave end name = node.name.value.to_sym if node.target visit(node.target) - builder.definesmethod(name, method_iseq) + iseq.definesmethod(name, method_iseq) else - builder.definemethod(name, method_iseq) + iseq.definemethod(name, method_iseq) end - builder.putobject(name) + iseq.putobject(name) end def visit_defined(node) @@ -726,67 +699,67 @@ def visit_defined(node) # If we're assigning to a local variable, then we need to make sure # that we put it into the local table. if node.value.target.is_a?(VarField) && - node.value.target.value.is_a?(Ident) - current_iseq.local_table.plain(node.value.target.value.value.to_sym) + node.value.target.value.is_a?(Ident) + iseq.local_table.plain(node.value.target.value.value.to_sym) end - builder.putobject("assignment") + iseq.putobject("assignment") when VarRef value = node.value.value name = value.value.to_sym case value when Const - builder.putnil - builder.defined(YARV::DEFINED_CONST, name, "constant") + iseq.putnil + iseq.defined(YARV::DEFINED_CONST, name, "constant") when CVar - builder.putnil - builder.defined(YARV::DEFINED_CVAR, name, "class variable") + iseq.putnil + iseq.defined(YARV::DEFINED_CVAR, name, "class variable") when GVar - builder.putnil - builder.defined(YARV::DEFINED_GVAR, name, "global-variable") + iseq.putnil + iseq.defined(YARV::DEFINED_GVAR, name, "global-variable") when Ident - builder.putobject("local-variable") + iseq.putobject("local-variable") when IVar - builder.putnil - builder.defined(YARV::DEFINED_IVAR, name, "instance-variable") + iseq.putnil + iseq.defined(YARV::DEFINED_IVAR, name, "instance-variable") when Kw case name when :false - builder.putobject("false") + iseq.putobject("false") when :nil - builder.putobject("nil") + iseq.putobject("nil") when :self - builder.putobject("self") + iseq.putobject("self") when :true - builder.putobject("true") + iseq.putobject("true") end end when VCall - builder.putself + iseq.putself name = node.value.value.value.to_sym - builder.defined(YARV::DEFINED_FUNC, name, "method") + iseq.defined(YARV::DEFINED_FUNC, name, "method") when YieldNode - builder.putnil - builder.defined(YARV::DEFINED_YIELD, false, "yield") + iseq.putnil + iseq.defined(YARV::DEFINED_YIELD, false, "yield") when ZSuper - builder.putnil - builder.defined(YARV::DEFINED_ZSUPER, false, "super") + iseq.putnil + iseq.defined(YARV::DEFINED_ZSUPER, false, "super") else - builder.putobject("expression") + iseq.putobject("expression") end end def visit_dyna_symbol(node) if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) - builder.putobject(node.parts.first.value.to_sym) + iseq.putobject(node.parts.first.value.to_sym) end end def visit_else(node) visit(node.statements) - builder.pop unless last_statement? + iseq.pop unless last_statement? end def visit_elsif(node) @@ -805,51 +778,50 @@ def visit_field(node) end def visit_float(node) - builder.putobject(node.accept(RubyVisitor.new)) + iseq.putobject(node.accept(RubyVisitor.new)) end def visit_for(node) visit(node.collection) name = node.index.value.value.to_sym - current_iseq.local_table.plain(name) + iseq.local_table.plain(name) block_iseq = with_instruction_sequence( :block, - "block in #{current_iseq.name}", - current_iseq, + "block in #{iseq.name}", node.statements ) do - current_iseq.argument_options[:lead_num] ||= 0 - current_iseq.argument_options[:lead_num] += 1 - current_iseq.argument_options[:ambiguous_param0] = true + iseq.argument_options[:lead_num] ||= 0 + iseq.argument_options[:lead_num] += 1 + iseq.argument_options[:ambiguous_param0] = true - current_iseq.argument_size += 1 - current_iseq.local_table.plain(2) + iseq.argument_size += 1 + iseq.local_table.plain(2) - builder.getlocal(0, 0) + iseq.getlocal(0, 0) - local_variable = current_iseq.local_variable(name) - builder.setlocal(local_variable.index, local_variable.level) + local_variable = iseq.local_variable(name) + iseq.setlocal(local_variable.index, local_variable.level) - builder.event(:RUBY_EVENT_B_CALL) - builder.nop + iseq.event(:RUBY_EVENT_B_CALL) + iseq.nop visit(node.statements) - builder.event(:RUBY_EVENT_B_RETURN) - builder.leave + iseq.event(:RUBY_EVENT_B_RETURN) + iseq.leave end - builder.send(:each, 0, 0, block_iseq) + iseq.send(:each, 0, 0, block_iseq) end def visit_hash(node) if (compiled = RubyVisitor.compile(node)) - builder.duphash(compiled) + iseq.duphash(compiled) else visit_all(node.assocs) - builder.newhash(node.assocs.length * 2) + iseq.newhash(node.assocs.length * 2) end end @@ -860,30 +832,30 @@ def visit_heredoc(node) visit(node.parts.first) else length = visit_string_parts(node) - builder.concatstrings(length) + iseq.concatstrings(length) end end def visit_if(node) visit(node.predicate) - branchunless = builder.branchunless(-1) + branchunless = iseq.branchunless(-1) visit(node.statements) if last_statement? - builder.leave - branchunless[1] = builder.label + iseq.leave + branchunless[1] = iseq.label - node.consequent ? visit(node.consequent) : builder.putnil + node.consequent ? visit(node.consequent) : iseq.putnil else - builder.pop + iseq.pop if node.consequent - jump = builder.jump(-1) - branchunless[1] = builder.label + jump = iseq.jump(-1) + branchunless[1] = iseq.label visit(node.consequent) - jump[1] = builder.label + jump[1] = iseq.label else - branchunless[1] = builder.label + branchunless[1] = iseq.label end end end @@ -905,40 +877,35 @@ def visit_if_op(node) end def visit_imaginary(node) - builder.putobject(node.accept(RubyVisitor.new)) + iseq.putobject(node.accept(RubyVisitor.new)) end def visit_int(node) - builder.putobject(node.accept(RubyVisitor.new)) + iseq.putobject(node.accept(RubyVisitor.new)) end def visit_kwrest_param(node) - current_iseq.argument_options[:kwrest] = current_iseq.argument_size - current_iseq.argument_size += 1 - current_iseq.local_table.plain(node.name.value.to_sym) + iseq.argument_options[:kwrest] = iseq.argument_size + iseq.argument_size += 1 + iseq.local_table.plain(node.name.value.to_sym) end def visit_label(node) - builder.putobject(node.accept(RubyVisitor.new)) + iseq.putobject(node.accept(RubyVisitor.new)) end def visit_lambda(node) lambda_iseq = - with_instruction_sequence( - :block, - "block in #{current_iseq.name}", - current_iseq, - node - ) do - builder.event(:RUBY_EVENT_B_CALL) + with_instruction_sequence(:block, "block in #{iseq.name}", node) do + iseq.event(:RUBY_EVENT_B_CALL) visit(node.params) visit(node.statements) - builder.event(:RUBY_EVENT_B_RETURN) - builder.leave + iseq.event(:RUBY_EVENT_B_RETURN) + iseq.leave end - builder.putspecialobject(YARV::VM_SPECIAL_OBJECT_VMCORE) - builder.send(:lambda, 0, YARV::VM_CALL_FCALL, lambda_iseq) + iseq.putspecialobject(YARV::VM_SPECIAL_OBJECT_VMCORE) + iseq.send(:lambda, 0, YARV::VM_CALL_FCALL, lambda_iseq) end def visit_lambda_var(node) @@ -947,7 +914,7 @@ def visit_lambda_var(node) def visit_massign(node) visit(node.value) - builder.dup + iseq.dup visit(node.target) end @@ -966,7 +933,6 @@ def visit_method_add_block(node) def visit_mlhs(node) lookups = [] - node.parts.each do |part| case part when VarField @@ -974,24 +940,18 @@ def visit_mlhs(node) end end - builder.expandarray(lookups.length, 0) - - lookups.each { |lookup| builder.setlocal(lookup.index, lookup.level) } + iseq.expandarray(lookups.length, 0) + lookups.each { |lookup| iseq.setlocal(lookup.index, lookup.level) } end def visit_module(node) name = node.constant.constant.value.to_sym module_iseq = - with_instruction_sequence( - :class, - "", - current_iseq, - node - ) do - builder.event(:RUBY_EVENT_CLASS) + with_instruction_sequence(:class, "", node) do + iseq.event(:RUBY_EVENT_CLASS) visit(node.bodystmt) - builder.event(:RUBY_EVENT_END) - builder.leave + iseq.event(:RUBY_EVENT_END) + iseq.leave end flags = YARV::VM_DEFINECLASS_TYPE_MODULE @@ -1001,28 +961,28 @@ def visit_module(node) flags |= YARV::VM_DEFINECLASS_FLAG_SCOPED visit(node.constant.parent) when ConstRef - builder.putspecialobject(YARV::VM_SPECIAL_OBJECT_CONST_BASE) + iseq.putspecialobject(YARV::VM_SPECIAL_OBJECT_CONST_BASE) when TopConstRef flags |= YARV::VM_DEFINECLASS_FLAG_SCOPED - builder.putobject(Object) + iseq.putobject(Object) end - builder.putnil - builder.defineclass(name, module_iseq, flags) + iseq.putnil + iseq.defineclass(name, module_iseq, flags) end def visit_mrhs(node) if (compiled = RubyVisitor.compile(node)) - builder.duparray(compiled) + iseq.duparray(compiled) else visit_all(node.parts) - builder.newarray(node.parts.length) + iseq.newarray(node.parts.length) end end def visit_not(node) visit(node.statement) - builder.send(:!, 0, YARV::VM_CALL_ARGS_SIMPLE) + iseq.send(:!, 0, YARV::VM_CALL_ARGS_SIMPLE) end def visit_opassign(node) @@ -1036,31 +996,30 @@ def visit_opassign(node) branchunless = nil with_opassign(node) do - builder.dup - branchunless = builder.branchunless(-1) - builder.pop + iseq.dup + branchunless = iseq.branchunless(-1) + iseq.pop visit(node.value) end case node.target when ARefField - builder.leave - branchunless[1] = builder.label - builder.setn(3) - builder.adjuststack(3) + iseq.leave + branchunless[1] = iseq.label + iseq.setn(3) + iseq.adjuststack(3) when ConstPathField, TopConstField - branchunless[1] = builder.label - builder.swap - builder.pop + branchunless[1] = iseq.label + iseq.swap + iseq.pop else - branchunless[1] = builder.label + branchunless[1] = iseq.label end when :"||" - if node.target.is_a?(ConstPathField) || - node.target.is_a?(TopConstField) + if node.target.is_a?(ConstPathField) || node.target.is_a?(TopConstField) opassign_defined(node) - builder.swap - builder.pop + iseq.swap + iseq.pop elsif node.target.is_a?(VarField) && [Const, CVar, GVar].include?(node.target.value.class) opassign_defined(node) @@ -1068,67 +1027,65 @@ def visit_opassign(node) branchif = nil with_opassign(node) do - builder.dup - branchif = builder.branchif(-1) - builder.pop + iseq.dup + branchif = iseq.branchif(-1) + iseq.pop visit(node.value) end if node.target.is_a?(ARefField) - builder.leave - branchif[1] = builder.label - builder.setn(3) - builder.adjuststack(3) + iseq.leave + branchif[1] = iseq.label + iseq.setn(3) + iseq.adjuststack(3) else - branchif[1] = builder.label + branchif[1] = iseq.label end end else with_opassign(node) do visit(node.value) - builder.send(operator, 1, flag) + iseq.send(operator, 1, flag) end end end def visit_params(node) - argument_options = current_iseq.argument_options + argument_options = iseq.argument_options if node.requireds.any? argument_options[:lead_num] = 0 node.requireds.each do |required| - current_iseq.local_table.plain(required.value.to_sym) - current_iseq.argument_size += 1 + iseq.local_table.plain(required.value.to_sym) + iseq.argument_size += 1 argument_options[:lead_num] += 1 end end node.optionals.each do |(optional, value)| - index = current_iseq.local_table.size + index = iseq.local_table.size name = optional.value.to_sym - current_iseq.local_table.plain(name) - current_iseq.argument_size += 1 + iseq.local_table.plain(name) + iseq.argument_size += 1 - unless argument_options.key?(:opt) - argument_options[:opt] = [builder.label] - end + argument_options[:opt] = [iseq.label] unless argument_options.key?(:opt) visit(value) - builder.setlocal(index, 0) - current_iseq.argument_options[:opt] << builder.label + iseq.setlocal(index, 0) + iseq.argument_options[:opt] << iseq.label end visit(node.rest) if node.rest if node.posts.any? - argument_options[:post_start] = current_iseq.argument_size + argument_options[:post_start] = iseq.argument_size argument_options[:post_num] = 0 node.posts.each do |post| - current_iseq.local_table.plain(post.value.to_sym) - current_iseq.argument_size += 1 + iseq.local_table.plain(post.value.to_sym) + iseq.argument_size += 1 argument_options[:post_num] += 1 end end @@ -1140,10 +1097,10 @@ def visit_params(node) node.keywords.each_with_index do |(keyword, value), keyword_index| name = keyword.value.chomp(":").to_sym - index = current_iseq.local_table.size + index = iseq.local_table.size - current_iseq.local_table.plain(name) - current_iseq.argument_size += 1 + iseq.local_table.plain(name) + iseq.argument_size += 1 argument_options[:kwbits] += 1 if value.nil? @@ -1153,34 +1110,30 @@ def visit_params(node) argument_options[:keyword] << [name, compiled] else argument_options[:keyword] << [name] - checkkeywords << builder.checkkeyword(-1, keyword_index) - branchif = builder.branchif(-1) + checkkeywords << iseq.checkkeyword(-1, keyword_index) + branchif = iseq.branchif(-1) visit(value) - builder.setlocal(index, 0) - branchif[1] = builder.label + iseq.setlocal(index, 0) + branchif[1] = iseq.label end end name = node.keyword_rest ? 3 : 2 - current_iseq.argument_size += 1 - current_iseq.local_table.plain(name) + iseq.argument_size += 1 + iseq.local_table.plain(name) - lookup = current_iseq.local_table.find(name, 0) + lookup = iseq.local_table.find(name, 0) checkkeywords.each { |checkkeyword| checkkeyword[1] = lookup.index } end if node.keyword_rest.is_a?(ArgsForward) - current_iseq.local_table.plain(:*) - current_iseq.local_table.plain(:&) + iseq.local_table.plain(:*) + iseq.local_table.plain(:&) - current_iseq.argument_options[ - :rest_start - ] = current_iseq.argument_size - current_iseq.argument_options[ - :block_start - ] = current_iseq.argument_size + 1 + iseq.argument_options[:rest_start] = iseq.argument_size + iseq.argument_options[:block_start] = iseq.argument_size + 1 - current_iseq.argument_size += 2 + iseq.argument_size += 2 elsif node.keyword_rest visit(node.keyword_rest) end @@ -1215,82 +1168,77 @@ def visit_program(node) end end - with_instruction_sequence(:top, "", nil, node) do + with_instruction_sequence(:top, "", node) do visit_all(preexes) if statements.empty? - builder.putnil + iseq.putnil else *statements, last_statement = statements visit_all(statements) with_last_statement { visit(last_statement) } end - builder.leave + iseq.leave end end def visit_qsymbols(node) - builder.duparray(node.accept(RubyVisitor.new)) + iseq.duparray(node.accept(RubyVisitor.new)) end def visit_qwords(node) if frozen_string_literal - builder.duparray(node.accept(RubyVisitor.new)) + iseq.duparray(node.accept(RubyVisitor.new)) else visit_all(node.elements) - builder.newarray(node.elements.length) + iseq.newarray(node.elements.length) end end def visit_range(node) if (compiled = RubyVisitor.compile(node)) - builder.putobject(compiled) + iseq.putobject(compiled) else visit(node.left) visit(node.right) - builder.newrange(node.operator.value == ".." ? 0 : 1) + iseq.newrange(node.operator.value == ".." ? 0 : 1) end end def visit_rational(node) - builder.putobject(node.accept(RubyVisitor.new)) + iseq.putobject(node.accept(RubyVisitor.new)) end def visit_regexp_literal(node) if (compiled = RubyVisitor.compile(node)) - builder.putobject(compiled) + iseq.putobject(compiled) else flags = RubyVisitor.new.visit_regexp_literal_flags(node) length = visit_string_parts(node) - builder.toregexp(flags, length) + iseq.toregexp(flags, length) end end def visit_rest_param(node) - current_iseq.local_table.plain(node.name.value.to_sym) - current_iseq.argument_options[:rest_start] = current_iseq.argument_size - current_iseq.argument_size += 1 + iseq.local_table.plain(node.name.value.to_sym) + iseq.argument_options[:rest_start] = iseq.argument_size + iseq.argument_size += 1 end def visit_sclass(node) visit(node.target) - builder.putnil + iseq.putnil singleton_iseq = - with_instruction_sequence( - :class, - "singleton class", - current_iseq, - node - ) do - builder.event(:RUBY_EVENT_CLASS) + with_instruction_sequence(:class, "singleton class", node) do + iseq.event(:RUBY_EVENT_CLASS) visit(node.bodystmt) - builder.event(:RUBY_EVENT_END) - builder.leave + iseq.event(:RUBY_EVENT_END) + iseq.leave end - builder.defineclass( + iseq.defineclass( :singletonclass, singleton_iseq, YARV::VM_DEFINECLASS_TYPE_SINGLETON_CLASS @@ -1308,20 +1256,19 @@ def visit_statements(node) end end - statements.empty? ? builder.putnil : visit_all(statements) + statements.empty? ? iseq.putnil : visit_all(statements) end def visit_string_concat(node) value = node.left.parts.first.value + node.right.parts.first.value - content = TStringContent.new(value: value, location: node.location) - literal = + visit_string_literal( StringLiteral.new( - parts: [content], + parts: [TStringContent.new(value: value, location: node.location)], quote: node.left.quote, location: node.location ) - visit_string_literal(literal) + ) end def visit_string_embexpr(node) @@ -1333,14 +1280,14 @@ def visit_string_literal(node) visit(node.parts.first) else length = visit_string_parts(node) - builder.concatstrings(length) + iseq.concatstrings(length) end end def visit_super(node) - builder.putself + iseq.putself visit(node.arguments) - builder.invokesuper( + iseq.invokesuper( nil, argument_parts(node.arguments).length, YARV::VM_CALL_FCALL | YARV::VM_CALL_ARGS_SIMPLE | YARV::VM_CALL_SUPER, @@ -1349,37 +1296,37 @@ def visit_super(node) end def visit_symbol_literal(node) - builder.putobject(node.accept(RubyVisitor.new)) + iseq.putobject(node.accept(RubyVisitor.new)) end def visit_symbols(node) if (compiled = RubyVisitor.compile(node)) - builder.duparray(compiled) + iseq.duparray(compiled) else node.elements.each do |element| if element.parts.length == 1 && - element.parts.first.is_a?(TStringContent) - builder.putobject(element.parts.first.value.to_sym) + element.parts.first.is_a?(TStringContent) + iseq.putobject(element.parts.first.value.to_sym) else length = visit_string_parts(element) - builder.concatstrings(length) - builder.intern + iseq.concatstrings(length) + iseq.intern end end - builder.newarray(node.elements.length) + iseq.newarray(node.elements.length) end end def visit_top_const_ref(node) - builder.opt_getconstant_path(constant_names(node)) + iseq.opt_getconstant_path(constant_names(node)) end def visit_tstring_content(node) if frozen_string_literal - builder.putobject(node.accept(RubyVisitor.new)) + iseq.putobject(node.accept(RubyVisitor.new)) else - builder.putstring(node.accept(RubyVisitor.new)) + iseq.putstring(node.accept(RubyVisitor.new)) end end @@ -1406,34 +1353,34 @@ def visit_unary(node) def visit_undef(node) node.symbols.each_with_index do |symbol, index| - builder.pop if index != 0 - builder.putspecialobject(YARV::VM_SPECIAL_OBJECT_VMCORE) - builder.putspecialobject(YARV::VM_SPECIAL_OBJECT_CBASE) + iseq.pop if index != 0 + iseq.putspecialobject(YARV::VM_SPECIAL_OBJECT_VMCORE) + iseq.putspecialobject(YARV::VM_SPECIAL_OBJECT_CBASE) visit(symbol) - builder.send(:"core#undef_method", 2, YARV::VM_CALL_ARGS_SIMPLE) + iseq.send(:"core#undef_method", 2, YARV::VM_CALL_ARGS_SIMPLE) end end def visit_unless(node) visit(node.predicate) - branchunless = builder.branchunless(-1) - node.consequent ? visit(node.consequent) : builder.putnil + branchunless = iseq.branchunless(-1) + node.consequent ? visit(node.consequent) : iseq.putnil if last_statement? - builder.leave - branchunless[1] = builder.label + iseq.leave + branchunless[1] = iseq.label visit(node.statements) else - builder.pop + iseq.pop if node.consequent - jump = builder.jump(-1) - branchunless[1] = builder.label + jump = iseq.jump(-1) + branchunless[1] = iseq.label visit(node.consequent) - jump[1] = builder.label + jump[1] = iseq.label else - branchunless[1] = builder.label + branchunless[1] = iseq.label end end end @@ -1441,34 +1388,34 @@ def visit_unless(node) def visit_until(node) jumps = [] - jumps << builder.jump(-1) - builder.putnil - builder.pop - jumps << builder.jump(-1) + jumps << iseq.jump(-1) + iseq.putnil + iseq.pop + jumps << iseq.jump(-1) - label = builder.label + label = iseq.label visit(node.statements) - builder.pop - jumps.each { |jump| jump[1] = builder.label } + iseq.pop + jumps.each { |jump| jump[1] = iseq.label } visit(node.predicate) - builder.branchunless(label) - builder.putnil if last_statement? + iseq.branchunless(label) + iseq.putnil if last_statement? end def visit_var_field(node) case node.value when CVar, IVar name = node.value.value.to_sym - current_iseq.inline_storage_for(name) + iseq.inline_storage_for(name) when Ident name = node.value.value.to_sym - if (local_variable = current_iseq.local_variable(name)) + if (local_variable = iseq.local_variable(name)) local_variable else - current_iseq.local_table.plain(name) - current_iseq.local_variable(name) + iseq.local_table.plain(name) + iseq.local_variable(name) end end end @@ -1476,43 +1423,44 @@ def visit_var_field(node) def visit_var_ref(node) case node.value when Const - builder.opt_getconstant_path(constant_names(node)) + iseq.opt_getconstant_path(constant_names(node)) when CVar name = node.value.value.to_sym - builder.getclassvariable(name) + iseq.getclassvariable(name) when GVar - builder.getglobal(node.value.value.to_sym) + iseq.getglobal(node.value.value.to_sym) when Ident - lookup = current_iseq.local_variable(node.value.value.to_sym) + lookup = iseq.local_variable(node.value.value.to_sym) case lookup.local when YARV::LocalTable::BlockLocal - builder.getblockparam(lookup.index, lookup.level) + iseq.getblockparam(lookup.index, lookup.level) when YARV::LocalTable::PlainLocal - builder.getlocal(lookup.index, lookup.level) + iseq.getlocal(lookup.index, lookup.level) end when IVar name = node.value.value.to_sym - builder.getinstancevariable(name) + iseq.getinstancevariable(name) when Kw case node.value.value when "false" - builder.putobject(false) + iseq.putobject(false) when "nil" - builder.putnil + iseq.putnil when "self" - builder.putself + iseq.putself when "true" - builder.putobject(true) + iseq.putobject(true) end end end def visit_vcall(node) - builder.putself + iseq.putself - flag = YARV::VM_CALL_FCALL | YARV::VM_CALL_VCALL | YARV::VM_CALL_ARGS_SIMPLE - builder.send(node.value.value.to_sym, 0, flag) + flag = + YARV::VM_CALL_FCALL | YARV::VM_CALL_VCALL | YARV::VM_CALL_ARGS_SIMPLE + iseq.send(node.value.value.to_sym, 0, flag) end def visit_when(node) @@ -1522,19 +1470,19 @@ def visit_when(node) def visit_while(node) jumps = [] - jumps << builder.jump(-1) - builder.putnil - builder.pop - jumps << builder.jump(-1) + jumps << iseq.jump(-1) + iseq.putnil + iseq.pop + jumps << iseq.jump(-1) - label = builder.label + label = iseq.label visit(node.statements) - builder.pop - jumps.each { |jump| jump[1] = builder.label } + iseq.pop + jumps.each { |jump| jump[1] = iseq.label } visit(node.predicate) - builder.branchif(label) - builder.putnil if last_statement? + iseq.branchif(label) + iseq.putnil if last_statement? end def visit_word(node) @@ -1542,38 +1490,39 @@ def visit_word(node) visit(node.parts.first) else length = visit_string_parts(node) - builder.concatstrings(length) + iseq.concatstrings(length) end end def visit_words(node) if frozen_string_literal && (compiled = RubyVisitor.compile(node)) - builder.duparray(compiled) + iseq.duparray(compiled) else visit_all(node.elements) - builder.newarray(node.elements.length) + iseq.newarray(node.elements.length) end end def visit_xstring_literal(node) - builder.putself + iseq.putself length = visit_string_parts(node) - builder.concatstrings(node.parts.length) if length > 1 - builder.send(:`, 1, YARV::VM_CALL_FCALL | YARV::VM_CALL_ARGS_SIMPLE) + iseq.concatstrings(node.parts.length) if length > 1 + iseq.send(:`, 1, YARV::VM_CALL_FCALL | YARV::VM_CALL_ARGS_SIMPLE) end def visit_yield(node) parts = argument_parts(node.arguments) visit_all(parts) - builder.invokeblock(nil, parts.length, YARV::VM_CALL_ARGS_SIMPLE) + iseq.invokeblock(nil, parts.length, YARV::VM_CALL_ARGS_SIMPLE) end def visit_zsuper(_node) - builder.putself - builder.invokesuper( + iseq.putself + iseq.invokesuper( nil, 0, - YARV::VM_CALL_FCALL | YARV::VM_CALL_ARGS_SIMPLE | YARV::VM_CALL_SUPER | YARV::VM_CALL_ZSUPER, + YARV::VM_CALL_FCALL | YARV::VM_CALL_ARGS_SIMPLE | YARV::VM_CALL_SUPER | + YARV::VM_CALL_ZSUPER, nil ) end @@ -1638,81 +1587,85 @@ def opassign_defined(node) visit(node.target.parent) name = node.target.constant.value.to_sym - builder.dup - builder.defined(YARV::DEFINED_CONST_FROM, name, true) + iseq.dup + iseq.defined(YARV::DEFINED_CONST_FROM, name, true) when TopConstField name = node.target.constant.value.to_sym - builder.putobject(Object) - builder.dup - builder.defined(YARV::DEFINED_CONST_FROM, name, true) + iseq.putobject(Object) + iseq.dup + iseq.defined(YARV::DEFINED_CONST_FROM, name, true) when VarField name = node.target.value.value.to_sym - builder.putnil + iseq.putnil case node.target.value when Const - builder.defined(YARV::DEFINED_CONST, name, true) + iseq.defined(YARV::DEFINED_CONST, name, true) when CVar - builder.defined(YARV::DEFINED_CVAR, name, true) + iseq.defined(YARV::DEFINED_CVAR, name, true) when GVar - builder.defined(YARV::DEFINED_GVAR, name, true) + iseq.defined(YARV::DEFINED_GVAR, name, true) end end - branchunless = builder.branchunless(-1) + branchunless = iseq.branchunless(-1) case node.target when ConstPathField, TopConstField - builder.dup - builder.putobject(true) - builder.getconstant(name) + iseq.dup + iseq.putobject(true) + iseq.getconstant(name) when VarField case node.target.value when Const - builder.opt_getconstant_path(constant_names(node.target)) + iseq.opt_getconstant_path(constant_names(node.target)) when CVar - builder.getclassvariable(name) + iseq.getclassvariable(name) when GVar - builder.getglobal(name) + iseq.getglobal(name) end end - builder.dup - branchif = builder.branchif(-1) - builder.pop + iseq.dup + branchif = iseq.branchif(-1) + iseq.pop - branchunless[1] = builder.label + branchunless[1] = iseq.label visit(node.value) case node.target when ConstPathField, TopConstField - builder.dupn(2) - builder.swap - builder.setconstant(name) + iseq.dupn(2) + iseq.swap + iseq.setconstant(name) when VarField - builder.dup + iseq.dup case node.target.value when Const - builder.putspecialobject(YARV::VM_SPECIAL_OBJECT_CONST_BASE) - builder.setconstant(name) + iseq.putspecialobject(YARV::VM_SPECIAL_OBJECT_CONST_BASE) + iseq.setconstant(name) when CVar - builder.setclassvariable(name) + iseq.setclassvariable(name) when GVar - builder.setglobal(name) + iseq.setglobal(name) end end - branchif[1] = builder.label + branchif[1] = iseq.label end # Whenever a value is interpolated into a string-like structure, these # three instructions are pushed. def push_interpolate - builder.dup - builder.objtostring(:to_s, 0, YARV::VM_CALL_FCALL | YARV::VM_CALL_ARGS_SIMPLE) - builder.anytostring + iseq.dup + iseq.objtostring( + :to_s, + 0, + YARV::VM_CALL_FCALL | YARV::VM_CALL_ARGS_SIMPLE + ) + iseq.anytostring end # There are a lot of nodes in the AST that act as contains of parts of @@ -1723,7 +1676,7 @@ def visit_string_parts(node) length = 0 unless node.parts.first.is_a?(TStringContent) - builder.putobject("") + iseq.putobject("") length += 1 end @@ -1736,7 +1689,7 @@ def visit_string_parts(node) visit(part) push_interpolate when TStringContent - builder.putobject(part.accept(RubyVisitor.new)) + iseq.putobject(part.accept(RubyVisitor.new)) end length += 1 @@ -1749,27 +1702,26 @@ def visit_string_parts(node) # on the compiler. When we descend into a node that has its own # instruction sequence, this method can be called to temporarily set the # new value of the instruction sequence, yield, and then set it back. - def with_instruction_sequence(type, name, parent_iseq, node) - previous_iseq = current_iseq - previous_builder = builder + def with_instruction_sequence(type, name, node) + parent_iseq = iseq begin - iseq = YARV::InstructionSequence.new(type, name, parent_iseq, node.location) - - @current_iseq = iseq - @builder = - YARV::Builder.new( - iseq, + iseq = + YARV::InstructionSequence.new( + type, + name, + parent_iseq, + node.location, frozen_string_literal: frozen_string_literal, operands_unification: operands_unification, specialized_instruction: specialized_instruction ) + @iseq = iseq yield iseq ensure - @current_iseq = previous_iseq - @builder = previous_builder + @iseq = parent_iseq end end @@ -1803,99 +1755,99 @@ def last_statement? def with_opassign(node) case node.target when ARefField - builder.putnil + iseq.putnil visit(node.target.collection) visit(node.target.index) - builder.dupn(2) - builder.send(:[], 1, YARV::VM_CALL_ARGS_SIMPLE) + iseq.dupn(2) + iseq.send(:[], 1, YARV::VM_CALL_ARGS_SIMPLE) yield - builder.setn(3) - builder.send(:[]=, 2, YARV::VM_CALL_ARGS_SIMPLE) - builder.pop + iseq.setn(3) + iseq.send(:[]=, 2, YARV::VM_CALL_ARGS_SIMPLE) + iseq.pop when ConstPathField name = node.target.constant.value.to_sym visit(node.target.parent) - builder.dup - builder.putobject(true) - builder.getconstant(name) + iseq.dup + iseq.putobject(true) + iseq.getconstant(name) yield if node.operator.value == "&&=" - builder.dupn(2) + iseq.dupn(2) else - builder.swap - builder.topn(1) + iseq.swap + iseq.topn(1) end - builder.swap - builder.setconstant(name) + iseq.swap + iseq.setconstant(name) when TopConstField name = node.target.constant.value.to_sym - builder.putobject(Object) - builder.dup - builder.putobject(true) - builder.getconstant(name) + iseq.putobject(Object) + iseq.dup + iseq.putobject(true) + iseq.getconstant(name) yield if node.operator.value == "&&=" - builder.dupn(2) + iseq.dupn(2) else - builder.swap - builder.topn(1) + iseq.swap + iseq.topn(1) end - builder.swap - builder.setconstant(name) + iseq.swap + iseq.setconstant(name) when VarField case node.target.value when Const names = constant_names(node.target) - builder.opt_getconstant_path(names) + iseq.opt_getconstant_path(names) yield - builder.dup - builder.putspecialobject(YARV::VM_SPECIAL_OBJECT_CONST_BASE) - builder.setconstant(names.last) + iseq.dup + iseq.putspecialobject(YARV::VM_SPECIAL_OBJECT_CONST_BASE) + iseq.setconstant(names.last) when CVar name = node.target.value.value.to_sym - builder.getclassvariable(name) + iseq.getclassvariable(name) yield - builder.dup - builder.setclassvariable(name) + iseq.dup + iseq.setclassvariable(name) when GVar name = node.target.value.value.to_sym - builder.getglobal(name) + iseq.getglobal(name) yield - builder.dup - builder.setglobal(name) + iseq.dup + iseq.setglobal(name) when Ident local_variable = visit(node.target) - builder.getlocal(local_variable.index, local_variable.level) + iseq.getlocal(local_variable.index, local_variable.level) yield - builder.dup - builder.setlocal(local_variable.index, local_variable.level) + iseq.dup + iseq.setlocal(local_variable.index, local_variable.level) when IVar name = node.target.value.value.to_sym - builder.getinstancevariable(name) + iseq.getinstancevariable(name) yield - builder.dup - builder.setinstancevariable(name) + iseq.dup + iseq.setinstancevariable(name) end end end diff --git a/lib/syntax_tree/yarv.rb b/lib/syntax_tree/yarv.rb index 7290d87f..b6c3468c 100644 --- a/lib/syntax_tree/yarv.rb +++ b/lib/syntax_tree/yarv.rb @@ -147,7 +147,20 @@ class InstructionSequence # maximum size of the stack for this instruction sequence. attr_reader :stack - def initialize(type, name, parent_iseq, location) + # These are various compilation options provided. + attr_reader :frozen_string_literal, + :operands_unification, + :specialized_instruction + + def initialize( + type, + name, + parent_iseq, + location, + frozen_string_literal: false, + operands_unification: true, + specialized_instruction: true + ) @type = type @name = name @parent_iseq = parent_iseq @@ -161,8 +174,16 @@ def initialize(type, name, parent_iseq, location) @insns = [] @storage_index = 0 @stack = Stack.new + + @frozen_string_literal = frozen_string_literal + @operands_unification = operands_unification + @specialized_instruction = specialized_instruction end + ########################################################################## + # Query methods + ########################################################################## + def local_variable(name, level = 0) if (lookup = local_table.find(name, level)) lookup @@ -171,11 +192,6 @@ def local_variable(name, level = 0) end end - def push(insn) - insns << insn - insn - end - def inline_storage storage = storage_index @storage_index += 1 @@ -183,9 +199,7 @@ def inline_storage end def inline_storage_for(name) - unless inline_storages.key?(name) - inline_storages[name] = inline_storage - end + inline_storages[name] = inline_storage unless inline_storages.key?(name) inline_storages[name] end @@ -239,251 +253,149 @@ def to_a ] end - private - - def serialize(insn) - case insn[0] - when :checkkeyword, :getblockparam, :getblockparamproxy, - :getlocal_WC_0, :getlocal_WC_1, :getlocal, :setlocal_WC_0, - :setlocal_WC_1, :setlocal - iseq = self - - case insn[0] - when :getlocal_WC_1, :setlocal_WC_1 - iseq = iseq.parent_iseq - when :getblockparam, :getblockparamproxy, :getlocal, :setlocal - insn[2].times { iseq = iseq.parent_iseq } - end - - # Here we need to map the local variable index to the offset - # from the top of the stack where it will be stored. - [insn[0], iseq.local_table.offset(insn[1]), *insn[2..]] - when :defineclass - [insn[0], insn[1], insn[2].to_a, insn[3]] - when :definemethod, :definesmethod - [insn[0], insn[1], insn[2].to_a] - when :send - # For any instructions that push instruction sequences onto the - # stack, we need to call #to_a on them as well. - [insn[0], insn[1], (insn[2].to_a if insn[2])] - when :once - [insn[0], insn[1].to_a, insn[2]] - else - insn - end - end - end - - # This class is responsible for taking a compiled instruction sequence and - # walking through it to generate equivalent Ruby code. - class Disassembler - attr_reader :iseq - - def initialize(iseq) - @iseq = iseq - end - - def to_ruby - stack = [] - - iseq.insns.each do |insn| - case insn[0] - when :getlocal_WC_0 - value = iseq.local_table.locals[insn[1]].name.to_s - stack << VarRef.new(value: Ident.new(value: value, location: Location.default), location: Location.default) - when :leave - stack << ReturnNode.new(arguments: Args.new(parts: [stack.pop], location: Location.default), location: Location.default) - when :opt_mult - left, right = stack.pop(2) - stack << Binary.new(left: left, operator: :*, right: right, location: Location.default) - when :opt_plus - left, right = stack.pop(2) - stack << Binary.new(left: left, operator: :+, right: right, location: Location.default) - when :putobject - case insn[1] - when Float - stack << FloatLiteral.new(value: insn[1].inspect, location: Location.default) - when Integer - stack << Int.new(value: insn[1].inspect, location: Location.default) - when Rational - stack << RationalLiteral.new(value: insn[1].inspect, location: Location.default) - else - raise "Unknown object type: #{insn[1].class.name}" - end - when :putobject_INT2FIX_1_ - stack << Int.new(value: "1", location: Location.default) - when :setlocal_WC_0 - target = VarField.new(value: Ident.new(value: iseq.local_table.locals[insn[1]].name.to_s, location: Location.default), location: Location.default) - stack << Assign.new(target: target, value: stack.pop, location: Location.default) - else - raise "Unknown instruction #{insn[0]}" - end - end - - Statements.new(nil, body: stack, location: Location.default) - end - end - - # This class serves as a layer of indirection between the instruction - # sequence and the compiler. It allows us to provide different behavior - # for certain instructions depending on the Ruby version. For example, - # class variable reads and writes gained an inline cache in Ruby 3.0. So - # we place the logic for checking the Ruby version in this class. - class Builder - attr_reader :iseq, :stack - attr_reader :frozen_string_literal, - :operands_unification, - :specialized_instruction + ########################################################################## + # Instruction push methods + ########################################################################## - def initialize( - iseq, - frozen_string_literal: false, - operands_unification: true, - specialized_instruction: true - ) - @iseq = iseq - @stack = iseq.stack - - @frozen_string_literal = frozen_string_literal - @operands_unification = operands_unification - @specialized_instruction = specialized_instruction + def push(insn) + insns << insn + insn end # This creates a new label at the current length of the instruction # sequence. It is used as the operand for jump instructions. def label - name = :"label_#{iseq.length}" - iseq.insns.last == name ? name : event(name) + name = :"label_#{length}" + insns.last == name ? name : event(name) end def event(name) - iseq.push(name) - name + push(name) end def adjuststack(number) stack.change_by(-number) - iseq.push([:adjuststack, number]) + push([:adjuststack, number]) end def anytostring stack.change_by(-2 + 1) - iseq.push([:anytostring]) + push([:anytostring]) end def branchif(index) stack.change_by(-1) - iseq.push([:branchif, index]) + push([:branchif, index]) end def branchnil(index) stack.change_by(-1) - iseq.push([:branchnil, index]) + push([:branchnil, index]) end def branchunless(index) stack.change_by(-1) - iseq.push([:branchunless, index]) + push([:branchunless, index]) end def checkkeyword(index, keyword_index) stack.change_by(+1) - iseq.push([:checkkeyword, index, keyword_index]) + push([:checkkeyword, index, keyword_index]) end def concatarray stack.change_by(-2 + 1) - iseq.push([:concatarray]) + push([:concatarray]) end def concatstrings(number) stack.change_by(-number + 1) - iseq.push([:concatstrings, number]) + push([:concatstrings, number]) end def defined(type, name, message) stack.change_by(-1 + 1) - iseq.push([:defined, type, name, message]) + push([:defined, type, name, message]) end def defineclass(name, class_iseq, flags) stack.change_by(-2 + 1) - iseq.push([:defineclass, name, class_iseq, flags]) + push([:defineclass, name, class_iseq, flags]) end def definemethod(name, method_iseq) stack.change_by(0) - iseq.push([:definemethod, name, method_iseq]) + push([:definemethod, name, method_iseq]) end def definesmethod(name, method_iseq) stack.change_by(-1) - iseq.push([:definesmethod, name, method_iseq]) + push([:definesmethod, name, method_iseq]) end def dup stack.change_by(-1 + 2) - iseq.push([:dup]) + push([:dup]) end def duparray(object) stack.change_by(+1) - iseq.push([:duparray, object]) + push([:duparray, object]) end def duphash(object) stack.change_by(+1) - iseq.push([:duphash, object]) + push([:duphash, object]) end def dupn(number) stack.change_by(+number) - iseq.push([:dupn, number]) + push([:dupn, number]) end def expandarray(length, flag) stack.change_by(-1 + length) - iseq.push([:expandarray, length, flag]) + push([:expandarray, length, flag]) end def getblockparam(index, level) stack.change_by(+1) - iseq.push([:getblockparam, index, level]) + push([:getblockparam, index, level]) end def getblockparamproxy(index, level) stack.change_by(+1) - iseq.push([:getblockparamproxy, index, level]) + push([:getblockparamproxy, index, level]) end def getclassvariable(name) stack.change_by(+1) if RUBY_VERSION >= "3.0" - iseq.push([:getclassvariable, name, iseq.inline_storage_for(name)]) + push([:getclassvariable, name, inline_storage_for(name)]) else - iseq.push([:getclassvariable, name]) + push([:getclassvariable, name]) end end def getconstant(name) stack.change_by(-2 + 1) - iseq.push([:getconstant, name]) + push([:getconstant, name]) end def getglobal(name) stack.change_by(+1) - iseq.push([:getglobal, name]) + push([:getglobal, name]) end def getinstancevariable(name) stack.change_by(+1) if RUBY_VERSION >= "3.2" - iseq.push([:getinstancevariable, name, iseq.inline_storage]) + push([:getinstancevariable, name, inline_storage]) else - inline_storage = iseq.inline_storage_for(name) - iseq.push([:getinstancevariable, name, inline_storage]) + inline_storage = inline_storage_for(name) + push([:getinstancevariable, name, inline_storage]) end end @@ -497,86 +409,86 @@ def getlocal(index, level) # scope, respectively, and requires fewer operands. case level when 0 - iseq.push([:getlocal_WC_0, index]) + push([:getlocal_WC_0, index]) when 1 - iseq.push([:getlocal_WC_1, index]) + push([:getlocal_WC_1, index]) else - iseq.push([:getlocal, index, level]) + push([:getlocal, index, level]) end else - iseq.push([:getlocal, index, level]) + push([:getlocal, index, level]) end end def getspecial(key, type) stack.change_by(-0 + 1) - iseq.push([:getspecial, key, type]) + push([:getspecial, key, type]) end def intern stack.change_by(-1 + 1) - iseq.push([:intern]) + push([:intern]) end def invokeblock(method_id, argc, flag) stack.change_by(-argc + 1) - iseq.push([:invokeblock, call_data(method_id, argc, flag)]) + push([:invokeblock, call_data(method_id, argc, flag)]) end def invokesuper(method_id, argc, flag, block_iseq) stack.change_by(-(argc + 1) + 1) cdata = call_data(method_id, argc, flag) - iseq.push([:invokesuper, cdata, block_iseq]) + push([:invokesuper, cdata, block_iseq]) end def jump(index) stack.change_by(0) - iseq.push([:jump, index]) + push([:jump, index]) end def leave stack.change_by(-1) - iseq.push([:leave]) + push([:leave]) end def newarray(length) stack.change_by(-length + 1) - iseq.push([:newarray, length]) + push([:newarray, length]) end def newhash(length) stack.change_by(-length + 1) - iseq.push([:newhash, length]) + push([:newhash, length]) end def newrange(flag) stack.change_by(-2 + 1) - iseq.push([:newrange, flag]) + push([:newrange, flag]) end def nop stack.change_by(0) - iseq.push([:nop]) + push([:nop]) end def objtostring(method_id, argc, flag) stack.change_by(-1 + 1) - iseq.push([:objtostring, call_data(method_id, argc, flag)]) + push([:objtostring, call_data(method_id, argc, flag)]) end def once(postexe_iseq, inline_storage) stack.change_by(+1) - iseq.push([:once, postexe_iseq, inline_storage]) + push([:once, postexe_iseq, inline_storage]) end def opt_getconstant_path(names) if RUBY_VERSION >= "3.2" stack.change_by(+1) - iseq.push([:opt_getconstant_path, names]) + push([:opt_getconstant_path, names]) else - inline_storage = iseq.inline_storage - getinlinecache = opt_getinlinecache(-1, inline_storage) + const_inline_storage = inline_storage + getinlinecache = opt_getinlinecache(-1, const_inline_storage) if names[0] == :"" names.shift @@ -589,20 +501,20 @@ def opt_getconstant_path(names) getconstant(name) end - opt_setinlinecache(inline_storage) + opt_setinlinecache(const_inline_storage) getinlinecache[1] = label end end def opt_getinlinecache(offset, inline_storage) stack.change_by(+1) - iseq.push([:opt_getinlinecache, offset, inline_storage]) + push([:opt_getinlinecache, offset, inline_storage]) end def opt_newarray_max(length) if specialized_instruction stack.change_by(-length + 1) - iseq.push([:opt_newarray_max, length]) + push([:opt_newarray_max, length]) else newarray(length) send(:max, 0, VM_CALL_ARGS_SIMPLE) @@ -612,7 +524,7 @@ def opt_newarray_max(length) def opt_newarray_min(length) if specialized_instruction stack.change_by(-length + 1) - iseq.push([:opt_newarray_min, length]) + push([:opt_newarray_min, length]) else newarray(length) send(:min, 0, VM_CALL_ARGS_SIMPLE) @@ -621,18 +533,14 @@ def opt_newarray_min(length) def opt_setinlinecache(inline_storage) stack.change_by(-1 + 1) - iseq.push([:opt_setinlinecache, inline_storage]) + push([:opt_setinlinecache, inline_storage]) end def opt_str_freeze(value) if specialized_instruction stack.change_by(+1) - iseq.push( - [ - :opt_str_freeze, - value, - call_data(:freeze, 0, VM_CALL_ARGS_SIMPLE) - ] + push( + [:opt_str_freeze, value, call_data(:freeze, 0, VM_CALL_ARGS_SIMPLE)] ) else putstring(value) @@ -643,9 +551,7 @@ def opt_str_freeze(value) def opt_str_uminus(value) if specialized_instruction stack.change_by(+1) - iseq.push( - [:opt_str_uminus, value, call_data(:-@, 0, VM_CALL_ARGS_SIMPLE)] - ) + push([:opt_str_uminus, value, call_data(:-@, 0, VM_CALL_ARGS_SIMPLE)]) else putstring(value) send(:-@, 0, VM_CALL_ARGS_SIMPLE) @@ -654,12 +560,12 @@ def opt_str_uminus(value) def pop stack.change_by(-1) - iseq.push([:pop]) + push([:pop]) end def putnil stack.change_by(+1) - iseq.push([:putnil]) + push([:putnil]) end def putobject(object) @@ -671,30 +577,30 @@ def putobject(object) # that will push the object onto the stack and requires fewer # operands. if object.eql?(0) - iseq.push([:putobject_INT2FIX_0_]) + push([:putobject_INT2FIX_0_]) elsif object.eql?(1) - iseq.push([:putobject_INT2FIX_1_]) + push([:putobject_INT2FIX_1_]) else - iseq.push([:putobject, object]) + push([:putobject, object]) end else - iseq.push([:putobject, object]) + push([:putobject, object]) end end def putself stack.change_by(+1) - iseq.push([:putself]) + push([:putself]) end def putspecialobject(object) stack.change_by(+1) - iseq.push([:putspecialobject, object]) + push([:putspecialobject, object]) end def putstring(object) stack.change_by(+1) - iseq.push([:putstring, object]) + push([:putstring, object]) end def send(method_id, argc, flag, block_iseq = nil) @@ -710,39 +616,39 @@ def send(method_id, argc, flag, block_iseq = nil) # stree-ignore if !block_iseq && (flag & VM_CALL_ARGS_BLOCKARG) == 0 case [method_id, argc] - when [:length, 0] then iseq.push([:opt_length, cdata]) - when [:size, 0] then iseq.push([:opt_size, cdata]) - when [:empty?, 0] then iseq.push([:opt_empty_p, cdata]) - when [:nil?, 0] then iseq.push([:opt_nil_p, cdata]) - when [:succ, 0] then iseq.push([:opt_succ, cdata]) - when [:!, 0] then iseq.push([:opt_not, cdata]) - when [:+, 1] then iseq.push([:opt_plus, cdata]) - when [:-, 1] then iseq.push([:opt_minus, cdata]) - when [:*, 1] then iseq.push([:opt_mult, cdata]) - when [:/, 1] then iseq.push([:opt_div, cdata]) - when [:%, 1] then iseq.push([:opt_mod, cdata]) - when [:==, 1] then iseq.push([:opt_eq, cdata]) - when [:=~, 1] then iseq.push([:opt_regexpmatch2, cdata]) - when [:<, 1] then iseq.push([:opt_lt, cdata]) - when [:<=, 1] then iseq.push([:opt_le, cdata]) - when [:>, 1] then iseq.push([:opt_gt, cdata]) - when [:>=, 1] then iseq.push([:opt_ge, cdata]) - when [:<<, 1] then iseq.push([:opt_ltlt, cdata]) - when [:[], 1] then iseq.push([:opt_aref, cdata]) - when [:&, 1] then iseq.push([:opt_and, cdata]) - when [:|, 1] then iseq.push([:opt_or, cdata]) - when [:[]=, 2] then iseq.push([:opt_aset, cdata]) + when [:length, 0] then push([:opt_length, cdata]) + when [:size, 0] then push([:opt_size, cdata]) + when [:empty?, 0] then push([:opt_empty_p, cdata]) + when [:nil?, 0] then push([:opt_nil_p, cdata]) + when [:succ, 0] then push([:opt_succ, cdata]) + when [:!, 0] then push([:opt_not, cdata]) + when [:+, 1] then push([:opt_plus, cdata]) + when [:-, 1] then push([:opt_minus, cdata]) + when [:*, 1] then push([:opt_mult, cdata]) + when [:/, 1] then push([:opt_div, cdata]) + when [:%, 1] then push([:opt_mod, cdata]) + when [:==, 1] then push([:opt_eq, cdata]) + when [:=~, 1] then push([:opt_regexpmatch2, cdata]) + when [:<, 1] then push([:opt_lt, cdata]) + when [:<=, 1] then push([:opt_le, cdata]) + when [:>, 1] then push([:opt_gt, cdata]) + when [:>=, 1] then push([:opt_ge, cdata]) + when [:<<, 1] then push([:opt_ltlt, cdata]) + when [:[], 1] then push([:opt_aref, cdata]) + when [:&, 1] then push([:opt_and, cdata]) + when [:|, 1] then push([:opt_or, cdata]) + when [:[]=, 2] then push([:opt_aset, cdata]) when [:!=, 1] eql_data = call_data(:==, 1, VM_CALL_ARGS_SIMPLE) - iseq.push([:opt_neq, eql_data, cdata]) + push([:opt_neq, eql_data, cdata]) else - iseq.push([:opt_send_without_block, cdata]) + push([:opt_send_without_block, cdata]) end else - iseq.push([:send, cdata, block_iseq]) + push([:send, cdata, block_iseq]) end else - iseq.push([:send, cdata, block_iseq]) + push([:send, cdata, block_iseq]) end end @@ -750,30 +656,29 @@ def setclassvariable(name) stack.change_by(-1) if RUBY_VERSION >= "3.0" - iseq.push([:setclassvariable, name, iseq.inline_storage_for(name)]) + push([:setclassvariable, name, inline_storage_for(name)]) else - iseq.push([:setclassvariable, name]) + push([:setclassvariable, name]) end end def setconstant(name) stack.change_by(-2) - iseq.push([:setconstant, name]) + push([:setconstant, name]) end def setglobal(name) stack.change_by(-1) - iseq.push([:setglobal, name]) + push([:setglobal, name]) end def setinstancevariable(name) stack.change_by(-1) if RUBY_VERSION >= "3.2" - iseq.push([:setinstancevariable, name, iseq.inline_storage]) + push([:setinstancevariable, name, inline_storage]) else - inline_storage = iseq.inline_storage_for(name) - iseq.push([:setinstancevariable, name, inline_storage]) + push([:setinstancevariable, name, inline_storage_for(name)]) end end @@ -787,40 +692,40 @@ def setlocal(index, level) # scope, respectively, and requires fewer operands. case level when 0 - iseq.push([:setlocal_WC_0, index]) + push([:setlocal_WC_0, index]) when 1 - iseq.push([:setlocal_WC_1, index]) + push([:setlocal_WC_1, index]) else - iseq.push([:setlocal, index, level]) + push([:setlocal, index, level]) end else - iseq.push([:setlocal, index, level]) + push([:setlocal, index, level]) end end def setn(number) stack.change_by(-1 + 1) - iseq.push([:setn, number]) + push([:setn, number]) end def splatarray(flag) stack.change_by(-1 + 1) - iseq.push([:splatarray, flag]) + push([:splatarray, flag]) end def swap stack.change_by(-2 + 2) - iseq.push([:swap]) + push([:swap]) end def topn(number) stack.change_by(+1) - iseq.push([:topn, number]) + push([:topn, number]) end def toregexp(options, length) stack.change_by(-length + 1) - iseq.push([:toregexp, options, length]) + push([:toregexp, options, length]) end private @@ -830,6 +735,126 @@ def toregexp(options, length) def call_data(method_id, argc, flag) { mid: method_id, flag: flag, orig_argc: argc } end + + def serialize(insn) + case insn[0] + when :checkkeyword, :getblockparam, :getblockparamproxy, :getlocal_WC_0, + :getlocal_WC_1, :getlocal, :setlocal_WC_0, :setlocal_WC_1, + :setlocal + iseq = self + + case insn[0] + when :getlocal_WC_1, :setlocal_WC_1 + iseq = iseq.parent_iseq + when :getblockparam, :getblockparamproxy, :getlocal, :setlocal + insn[2].times { iseq = iseq.parent_iseq } + end + + # Here we need to map the local variable index to the offset + # from the top of the stack where it will be stored. + [insn[0], iseq.local_table.offset(insn[1]), *insn[2..]] + when :defineclass + [insn[0], insn[1], insn[2].to_a, insn[3]] + when :definemethod, :definesmethod + [insn[0], insn[1], insn[2].to_a] + when :send + # For any instructions that push instruction sequences onto the + # stack, we need to call #to_a on them as well. + [insn[0], insn[1], (insn[2].to_a if insn[2])] + when :once + [insn[0], insn[1].to_a, insn[2]] + else + insn + end + end + end + + # This class is responsible for taking a compiled instruction sequence and + # walking through it to generate equivalent Ruby code. + class Disassembler + attr_reader :iseq + + def initialize(iseq) + @iseq = iseq + end + + def to_ruby + stack = [] + + iseq.insns.each do |insn| + case insn[0] + when :getlocal_WC_0 + value = iseq.local_table.locals[insn[1]].name.to_s + stack << VarRef.new( + value: Ident.new(value: value, location: Location.default), + location: Location.default + ) + when :leave + stack << ReturnNode.new( + arguments: + Args.new(parts: [stack.pop], location: Location.default), + location: Location.default + ) + when :opt_mult + left, right = stack.pop(2) + stack << Binary.new( + left: left, + operator: :*, + right: right, + location: Location.default + ) + when :opt_plus + left, right = stack.pop(2) + stack << Binary.new( + left: left, + operator: :+, + right: right, + location: Location.default + ) + when :putobject + case insn[1] + when Float + stack << FloatLiteral.new( + value: insn[1].inspect, + location: Location.default + ) + when Integer + stack << Int.new( + value: insn[1].inspect, + location: Location.default + ) + when Rational + stack << RationalLiteral.new( + value: insn[1].inspect, + location: Location.default + ) + else + raise "Unknown object type: #{insn[1].class.name}" + end + when :putobject_INT2FIX_1_ + stack << Int.new(value: "1", location: Location.default) + when :setlocal_WC_0 + target = + VarField.new( + value: + Ident.new( + value: iseq.local_table.locals[insn[1]].name.to_s, + location: Location.default + ), + location: Location.default + ) + stack << Assign.new( + target: target, + value: stack.pop, + location: Location.default + ) + else + raise "Unknown instruction #{insn[0]}" + end + end + + Statements.new(nil, body: stack, location: Location.default) + end end # These constants correspond to the putspecialobject instruction. They are From 154e75f9fe4f831237206fff080b03ad22d59d32 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 21 Nov 2022 10:02:31 -0500 Subject: [PATCH 257/536] Put child iseq methods on iseq --- lib/syntax_tree/compiler.rb | 57 ++++++++++++++++++------------------- lib/syntax_tree/yarv.rb | 46 ++++++++++++++++++++++++------ test/compiler_test.rb | 3 ++ 3 files changed, 69 insertions(+), 37 deletions(-) diff --git a/lib/syntax_tree/compiler.rb b/lib/syntax_tree/compiler.rb index 424a9cf5..926661cc 100644 --- a/lib/syntax_tree/compiler.rb +++ b/lib/syntax_tree/compiler.rb @@ -225,14 +225,17 @@ def visit_CHAR(node) end def visit_END(node) - name = "block in #{iseq.name}" once_iseq = - with_instruction_sequence(:block, name, node) do + with_child_iseq(iseq.block_child_iseq(node.location)) do postexe_iseq = - with_instruction_sequence(:block, name, node) do + with_child_iseq(iseq.block_child_iseq(node.location)) do + iseq.event(:RUBY_EVENT_B_CALL) + *statements, last_statement = node.statements.body visit_all(statements) with_last_statement { visit(last_statement) } + + iseq.event(:RUBY_EVENT_B_RETURN) iseq.leave end @@ -422,7 +425,7 @@ def visit_binary(node) end def visit_block(node) - with_instruction_sequence(:block, "block in #{iseq.name}", node) do + with_child_iseq(iseq.block_child_iseq(node.location)) do iseq.event(:RUBY_EVENT_B_CALL) visit(node.block_var) visit(node.bodystmt) @@ -606,7 +609,7 @@ def visit_case(node) def visit_class(node) name = node.constant.constant.value.to_sym class_iseq = - with_instruction_sequence(:class, "", node) do + with_child_iseq(iseq.class_child_iseq(name, node.location)) do iseq.event(:RUBY_EVENT_CLASS) visit(node.bodystmt) iseq.event(:RUBY_EVENT_END) @@ -673,7 +676,7 @@ def visit_const_path_ref(node) def visit_def(node) method_iseq = - with_instruction_sequence(:method, node.name.value, node) do + with_child_iseq(iseq.method_child_iseq(node.name.value, node.location)) do visit(node.params) if node.params iseq.event(:RUBY_EVENT_CALL) visit(node.bodystmt) @@ -788,11 +791,7 @@ def visit_for(node) iseq.local_table.plain(name) block_iseq = - with_instruction_sequence( - :block, - "block in #{iseq.name}", - node.statements - ) do + with_child_iseq(iseq.block_child_iseq(node.statements.location)) do iseq.argument_options[:lead_num] ||= 0 iseq.argument_options[:lead_num] += 1 iseq.argument_options[:ambiguous_param0] = true @@ -896,7 +895,7 @@ def visit_label(node) def visit_lambda(node) lambda_iseq = - with_instruction_sequence(:block, "block in #{iseq.name}", node) do + with_child_iseq(iseq.block_child_iseq(node.location)) do iseq.event(:RUBY_EVENT_B_CALL) visit(node.params) visit(node.statements) @@ -947,7 +946,7 @@ def visit_mlhs(node) def visit_module(node) name = node.constant.constant.value.to_sym module_iseq = - with_instruction_sequence(:class, "", node) do + with_child_iseq(iseq.module_child_iseq(name, node.location)) do iseq.event(:RUBY_EVENT_CLASS) visit(node.bodystmt) iseq.event(:RUBY_EVENT_END) @@ -1168,7 +1167,18 @@ def visit_program(node) end end - with_instruction_sequence(:top, "", node) do + top_iseq = + YARV::InstructionSequence.new( + :top, + "", + nil, + node.location, + frozen_string_literal: frozen_string_literal, + operands_unification: operands_unification, + specialized_instruction: specialized_instruction + ) + + with_child_iseq(top_iseq) do visit_all(preexes) if statements.empty? @@ -1231,7 +1241,7 @@ def visit_sclass(node) iseq.putnil singleton_iseq = - with_instruction_sequence(:class, "singleton class", node) do + with_child_iseq(iseq.singleton_class_child_iseq(node.location)) do iseq.event(:RUBY_EVENT_CLASS) visit(node.bodystmt) iseq.event(:RUBY_EVENT_END) @@ -1702,24 +1712,13 @@ def visit_string_parts(node) # on the compiler. When we descend into a node that has its own # instruction sequence, this method can be called to temporarily set the # new value of the instruction sequence, yield, and then set it back. - def with_instruction_sequence(type, name, node) + def with_child_iseq(child_iseq) parent_iseq = iseq begin - iseq = - YARV::InstructionSequence.new( - type, - name, - parent_iseq, - node.location, - frozen_string_literal: frozen_string_literal, - operands_unification: operands_unification, - specialized_instruction: specialized_instruction - ) - - @iseq = iseq + @iseq = child_iseq yield - iseq + child_iseq ensure @iseq = parent_iseq end diff --git a/lib/syntax_tree/yarv.rb b/lib/syntax_tree/yarv.rb index b6c3468c..12d1dba2 100644 --- a/lib/syntax_tree/yarv.rb +++ b/lib/syntax_tree/yarv.rb @@ -210,14 +210,6 @@ def length end end - def each_child - insns.each do |insn| - insn[1..].each do |operand| - yield operand if operand.is_a?(InstructionSequence) - end - end - end - def eval compiled = to_a @@ -253,6 +245,44 @@ def to_a ] end + ########################################################################## + # Child instruction sequence methods + ########################################################################## + + def child_iseq(type, name, location) + InstructionSequence.new( + type, + name, + self, + location, + frozen_string_literal: frozen_string_literal, + operands_unification: operands_unification, + specialized_instruction: specialized_instruction + ) + end + + def block_child_iseq(location) + current = self + current = current.parent_iseq while current.type == :block + child_iseq(:block, "block in #{current.name}", location) + end + + def class_child_iseq(name, location) + child_iseq(:class, "", location) + end + + def method_child_iseq(name, location) + child_iseq(:method, name, location) + end + + def module_child_iseq(name, location) + child_iseq(:class, "", location) + end + + def singleton_class_child_iseq(location) + child_iseq(:class, "singleton class", location) + end + ########################################################################## # Instruction push methods ########################################################################## diff --git a/test/compiler_test.rb b/test/compiler_test.rb index 3b8c0ea2..27bf993d 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -6,6 +6,9 @@ module SyntaxTree class CompilerTest < Minitest::Test CASES = [ + # Hooks + "BEGIN { a = 1 }", + "a = 1; END { a = 1 }; a", # Various literals placed on the stack "true", "false", From df9f6220c009126f0a5b02c4a618ec54548d6e43 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 21 Nov 2022 10:19:30 -0500 Subject: [PATCH 258/536] Test out disassembler --- lib/syntax_tree/yarv.rb | 171 ++++++++++++++++++++++++++++------------ test/yarv_test.rb | 46 +++++++++++ 2 files changed, 166 insertions(+), 51 deletions(-) create mode 100644 test/yarv_test.rb diff --git a/lib/syntax_tree/yarv.rb b/lib/syntax_tree/yarv.rb index 12d1dba2..93f2ac06 100644 --- a/lib/syntax_tree/yarv.rb +++ b/lib/syntax_tree/yarv.rb @@ -802,6 +802,65 @@ def serialize(insn) # This class is responsible for taking a compiled instruction sequence and # walking through it to generate equivalent Ruby code. class Disassembler + module DSL + def Args(parts) + Args.new(parts: parts, location: Location.default) + end + + def ArgParen(arguments) + ArgParen.new(arguments: arguments, location: Location.default) + end + + def Assign(target, value) + Assign.new(target: target, value: value, location: Location.default) + end + + def Binary(left, operator, right) + Binary.new(left: left, operator: operator, right: right, location: Location.default) + end + + def CallNode(receiver, operator, message, arguments) + CallNode.new(receiver: receiver, operator: operator, message: message, arguments: arguments, location: Location.default) + end + + def FloatLiteral(value) + FloatLiteral.new(value: value, location: Location.default) + end + + def Ident(value) + Ident.new(value: value, location: Location.default) + end + + def Int(value) + Int.new(value: value, location: Location.default) + end + + def Period(value) + Period.new(value: value, location: Location.default) + end + + def Program(statements) + Program.new(statements: statements, location: Location.default) + end + + def ReturnNode(arguments) + ReturnNode.new(arguments: arguments, location: Location.default) + end + + def Statements(body) + Statements.new(nil, body: body, location: Location.default) + end + + def VarField(value) + VarField.new(value: value, location: Location.default) + end + + def VarRef(value) + VarRef.new(value: value, location: Location.default) + end + end + + include DSL attr_reader :iseq def initialize(iseq) @@ -812,78 +871,88 @@ def to_ruby stack = [] iseq.insns.each do |insn| + # skip line numbers and events + next unless insn.is_a?(Array) + case insn[0] when :getlocal_WC_0 - value = iseq.local_table.locals[insn[1]].name.to_s - stack << VarRef.new( - value: Ident.new(value: value, location: Location.default), - location: Location.default - ) + stack << VarRef(Ident(local_name(insn[1], 0))) when :leave - stack << ReturnNode.new( - arguments: - Args.new(parts: [stack.pop], location: Location.default), - location: Location.default - ) + stack << ReturnNode(Args([stack.pop])) + when :opt_and + left, right = stack.pop(2) + stack << Binary(left, :&, right) + when :opt_div + left, right = stack.pop(2) + stack << Binary(left, :/, right) + when :opt_eq + left, right = stack.pop(2) + stack << Binary(left, :==, right) + when :opt_ge + left, right = stack.pop(2) + stack << Binary(left, :>=, right) + when :opt_gt + left, right = stack.pop(2) + stack << Binary(left, :>, right) + when :opt_le + left, right = stack.pop(2) + stack << Binary(left, :<=, right) + when :opt_lt + left, right = stack.pop(2) + stack << Binary(left, :<, right) + when :opt_ltlt + left, right = stack.pop(2) + stack << Binary(left, :<<, right) + when :opt_minus + left, right = stack.pop(2) + stack << Binary(left, :-, right) + when :opt_mod + left, right = stack.pop(2) + stack << Binary(left, :%, right) when :opt_mult left, right = stack.pop(2) - stack << Binary.new( - left: left, - operator: :*, - right: right, - location: Location.default - ) + stack << Binary(left, :*, right) + when :opt_neq + left, right = stack.pop(2) + stack << Binary(left, :"!=", right) + when :opt_or + left, right = stack.pop(2) + stack << Binary(left, :|, right) when :opt_plus left, right = stack.pop(2) - stack << Binary.new( - left: left, - operator: :+, - right: right, - location: Location.default - ) + stack << Binary(left, :+, right) + when :opt_send_without_block + receiver, *arguments = stack.pop(insn[1][:orig_argc] + 1) + stack << CallNode(receiver, Period("."), Ident(insn[1][:mid]), ArgParen(Args(arguments))) when :putobject case insn[1] when Float - stack << FloatLiteral.new( - value: insn[1].inspect, - location: Location.default - ) + stack << FloatLiteral(insn[1].inspect) when Integer - stack << Int.new( - value: insn[1].inspect, - location: Location.default - ) - when Rational - stack << RationalLiteral.new( - value: insn[1].inspect, - location: Location.default - ) + stack << Int(insn[1].inspect) else raise "Unknown object type: #{insn[1].class.name}" end + when :putobject_INT2FIX_0_ + stack << Int("0") when :putobject_INT2FIX_1_ - stack << Int.new(value: "1", location: Location.default) + stack << Int("1") when :setlocal_WC_0 - target = - VarField.new( - value: - Ident.new( - value: iseq.local_table.locals[insn[1]].name.to_s, - location: Location.default - ), - location: Location.default - ) - stack << Assign.new( - target: target, - value: stack.pop, - location: Location.default - ) + stack << Assign(VarField(Ident(local_name(insn[1], 0))), stack.pop) else raise "Unknown instruction #{insn[0]}" end end - Statements.new(nil, body: stack, location: Location.default) + Program(Statements(stack)) + end + + private + + def local_name(index, level) + current = iseq + level.times { current = current.parent_iseq } + current.local_table.locals[index].name.to_s end end diff --git a/test/yarv_test.rb b/test/yarv_test.rb new file mode 100644 index 00000000..57371ba3 --- /dev/null +++ b/test/yarv_test.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +return if !defined?(RubyVM::InstructionSequence) || RUBY_VERSION < "3.1" +require_relative "test_helper" + +module SyntaxTree + class YARVTest < Minitest::Test + CASES = { + "0" => "return 0\n", + "1" => "return 1\n", + "2" => "return 2\n", + "1.0" => "return 1.0\n", + "1 + 2" => "return 1 + 2\n", + "1 - 2" => "return 1 - 2\n", + "1 * 2" => "return 1 * 2\n", + "1 / 2" => "return 1 / 2\n", + "1 % 2" => "return 1 % 2\n", + "1 < 2" => "return 1 < 2\n", + "1 <= 2" => "return 1 <= 2\n", + "1 > 2" => "return 1 > 2\n", + "1 >= 2" => "return 1 >= 2\n", + "1 == 2" => "return 1 == 2\n", + "1 != 2" => "return 1 != 2\n", + "1 & 2" => "return 1 & 2\n", + "1 | 2" => "return 1 | 2\n", + "1 << 2" => "return 1 << 2\n", + "1 >> 2" => "return 1.>>(2)\n", + "1 ** 2" => "return 1.**(2)\n", + "a = 1; a" => "a = 1\nreturn a\n", + }.freeze + + CASES.each do |source, expected| + define_method("test_disassemble_#{source}") do + assert_disassembles(expected, source) + end + end + + private + + def assert_disassembles(expected, source) + iseq = SyntaxTree.parse(source).accept(Compiler.new) + actual = Formatter.format(source, YARV::Disassembler.new(iseq).to_ruby) + assert_equal expected, actual + end + end +end From 6c6b4376b88b27d911c577ab8c90de9c9cc47f95 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 21 Nov 2022 12:23:22 -0500 Subject: [PATCH 259/536] Add BF compiler --- lib/syntax_tree.rb | 3 + lib/syntax_tree/dsl.rb | 129 ++++++++ lib/syntax_tree/yarv.rb | 157 --------- lib/syntax_tree/yarv/bf.rb | 466 +++++++++++++++++++++++++++ lib/syntax_tree/yarv/disassembler.rb | 209 ++++++++++++ 5 files changed, 807 insertions(+), 157 deletions(-) create mode 100644 lib/syntax_tree/dsl.rb create mode 100644 lib/syntax_tree/yarv/bf.rb create mode 100644 lib/syntax_tree/yarv/disassembler.rb diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index 187ff74d..2cbfa2e4 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -26,8 +26,11 @@ require_relative "syntax_tree/pattern" require_relative "syntax_tree/search" +require_relative "syntax_tree/dsl" require_relative "syntax_tree/yarv" require_relative "syntax_tree/compiler" +require_relative "syntax_tree/yarv/bf" +require_relative "syntax_tree/yarv/disassembler" # Syntax Tree is a suite of tools built on top of the internal CRuby parser. It # provides the ability to generate a syntax tree from source, as well as the diff --git a/lib/syntax_tree/dsl.rb b/lib/syntax_tree/dsl.rb new file mode 100644 index 00000000..05911ee3 --- /dev/null +++ b/lib/syntax_tree/dsl.rb @@ -0,0 +1,129 @@ +# frozen_string_literal: true + +module SyntaxTree + module DSL + def ARef(collection, index) + ARef.new(collection: collection, index: index, location: Location.default) + end + + def ARefField(collection, index) + ARefField.new(collection: collection, index: index, location: Location.default) + end + + def Args(parts) + Args.new(parts: parts, location: Location.default) + end + + def ArgParen(arguments) + ArgParen.new(arguments: arguments, location: Location.default) + end + + def Assign(target, value) + Assign.new(target: target, value: value, location: Location.default) + end + + def Assoc(key, value) + Assoc.new(key: key, value: value, location: Location.default) + end + + def Binary(left, operator, right) + Binary.new(left: left, operator: operator, right: right, location: Location.default) + end + + def BlockNode(opening, block_var, bodystmt) + BlockNode.new(opening: opening, block_var: block_var, bodystmt: bodystmt, location: Location.default) + end + + def BodyStmt(statements, rescue_clause, else_keyword, else_clause, ensure_clause) + BodyStmt.new(statements: statements, rescue_clause: rescue_clause, else_keyword: else_keyword, else_clause: else_clause, ensure_clause: ensure_clause, location: Location.default) + end + + def CallNode(receiver, operator, message, arguments) + CallNode.new(receiver: receiver, operator: operator, message: message, arguments: arguments, location: Location.default) + end + + def Case(keyword, value, consequent) + Case.new(keyword: keyword, value: value, consequent: consequent, location: Location.default) + end + + def FloatLiteral(value) + FloatLiteral.new(value: value, location: Location.default) + end + + def GVar(value) + GVar.new(value: value, location: Location.default) + end + + def HashLiteral(lbrace, assocs) + HashLiteral.new(lbrace: lbrace, assocs: assocs, location: Location.default) + end + + def Ident(value) + Ident.new(value: value, location: Location.default) + end + + def IfNode(predicate, statements, consequent) + IfNode.new(predicate: predicate, statements: statements, consequent: consequent, location: Location.default) + end + + def Int(value) + Int.new(value: value, location: Location.default) + end + + def Kw(value) + Kw.new(value: value, location: Location.default) + end + + def LBrace(value) + LBrace.new(value: value, location: Location.default) + end + + def MethodAddBlock(call, block) + MethodAddBlock.new(call: call, block: block, location: Location.default) + end + + def Next(arguments) + Next.new(arguments: arguments, location: Location.default) + end + + def Op(value) + Op.new(value: value, location: Location.default) + end + + def OpAssign(target, operator, value) + OpAssign.new(target: target, operator: operator, value: value, location: Location.default) + end + + def Period(value) + Period.new(value: value, location: Location.default) + end + + def Program(statements) + Program.new(statements: statements, location: Location.default) + end + + def ReturnNode(arguments) + ReturnNode.new(arguments: arguments, location: Location.default) + end + + def Statements(body) + Statements.new(nil, body: body, location: Location.default) + end + + def SymbolLiteral(value) + SymbolLiteral.new(value: value, location: Location.default) + end + + def VarField(value) + VarField.new(value: value, location: Location.default) + end + + def VarRef(value) + VarRef.new(value: value, location: Location.default) + end + + def When(arguments, statements, consequent) + When.new(arguments: arguments, statements: statements, consequent: consequent, location: Location.default) + end + end +end diff --git a/lib/syntax_tree/yarv.rb b/lib/syntax_tree/yarv.rb index 93f2ac06..2224792a 100644 --- a/lib/syntax_tree/yarv.rb +++ b/lib/syntax_tree/yarv.rb @@ -799,163 +799,6 @@ def serialize(insn) end end - # This class is responsible for taking a compiled instruction sequence and - # walking through it to generate equivalent Ruby code. - class Disassembler - module DSL - def Args(parts) - Args.new(parts: parts, location: Location.default) - end - - def ArgParen(arguments) - ArgParen.new(arguments: arguments, location: Location.default) - end - - def Assign(target, value) - Assign.new(target: target, value: value, location: Location.default) - end - - def Binary(left, operator, right) - Binary.new(left: left, operator: operator, right: right, location: Location.default) - end - - def CallNode(receiver, operator, message, arguments) - CallNode.new(receiver: receiver, operator: operator, message: message, arguments: arguments, location: Location.default) - end - - def FloatLiteral(value) - FloatLiteral.new(value: value, location: Location.default) - end - - def Ident(value) - Ident.new(value: value, location: Location.default) - end - - def Int(value) - Int.new(value: value, location: Location.default) - end - - def Period(value) - Period.new(value: value, location: Location.default) - end - - def Program(statements) - Program.new(statements: statements, location: Location.default) - end - - def ReturnNode(arguments) - ReturnNode.new(arguments: arguments, location: Location.default) - end - - def Statements(body) - Statements.new(nil, body: body, location: Location.default) - end - - def VarField(value) - VarField.new(value: value, location: Location.default) - end - - def VarRef(value) - VarRef.new(value: value, location: Location.default) - end - end - - include DSL - attr_reader :iseq - - def initialize(iseq) - @iseq = iseq - end - - def to_ruby - stack = [] - - iseq.insns.each do |insn| - # skip line numbers and events - next unless insn.is_a?(Array) - - case insn[0] - when :getlocal_WC_0 - stack << VarRef(Ident(local_name(insn[1], 0))) - when :leave - stack << ReturnNode(Args([stack.pop])) - when :opt_and - left, right = stack.pop(2) - stack << Binary(left, :&, right) - when :opt_div - left, right = stack.pop(2) - stack << Binary(left, :/, right) - when :opt_eq - left, right = stack.pop(2) - stack << Binary(left, :==, right) - when :opt_ge - left, right = stack.pop(2) - stack << Binary(left, :>=, right) - when :opt_gt - left, right = stack.pop(2) - stack << Binary(left, :>, right) - when :opt_le - left, right = stack.pop(2) - stack << Binary(left, :<=, right) - when :opt_lt - left, right = stack.pop(2) - stack << Binary(left, :<, right) - when :opt_ltlt - left, right = stack.pop(2) - stack << Binary(left, :<<, right) - when :opt_minus - left, right = stack.pop(2) - stack << Binary(left, :-, right) - when :opt_mod - left, right = stack.pop(2) - stack << Binary(left, :%, right) - when :opt_mult - left, right = stack.pop(2) - stack << Binary(left, :*, right) - when :opt_neq - left, right = stack.pop(2) - stack << Binary(left, :"!=", right) - when :opt_or - left, right = stack.pop(2) - stack << Binary(left, :|, right) - when :opt_plus - left, right = stack.pop(2) - stack << Binary(left, :+, right) - when :opt_send_without_block - receiver, *arguments = stack.pop(insn[1][:orig_argc] + 1) - stack << CallNode(receiver, Period("."), Ident(insn[1][:mid]), ArgParen(Args(arguments))) - when :putobject - case insn[1] - when Float - stack << FloatLiteral(insn[1].inspect) - when Integer - stack << Int(insn[1].inspect) - else - raise "Unknown object type: #{insn[1].class.name}" - end - when :putobject_INT2FIX_0_ - stack << Int("0") - when :putobject_INT2FIX_1_ - stack << Int("1") - when :setlocal_WC_0 - stack << Assign(VarField(Ident(local_name(insn[1], 0))), stack.pop) - else - raise "Unknown instruction #{insn[0]}" - end - end - - Program(Statements(stack)) - end - - private - - def local_name(index, level) - current = iseq - level.times { current = current.parent_iseq } - current.local_table.locals[index].name.to_s - end - end - # These constants correspond to the putspecialobject instruction. They are # used to represent special objects that are pushed onto the stack. VM_SPECIAL_OBJECT_VMCORE = 1 diff --git a/lib/syntax_tree/yarv/bf.rb b/lib/syntax_tree/yarv/bf.rb new file mode 100644 index 00000000..b826ebf2 --- /dev/null +++ b/lib/syntax_tree/yarv/bf.rb @@ -0,0 +1,466 @@ +# frozen_string_literal: true + +module SyntaxTree + module YARV + # Parses the given source code into a syntax tree, compiles that syntax tree + # into YARV bytecode. + class Bf + class Node + def format(q) + Format.new(q).visit(self) + end + + def pretty_print(q) + PrettyPrint.new(q).visit(self) + end + end + + # The root node of the syntax tree. + class Root < Node + attr_reader :nodes, :location + + def initialize(nodes:, location:) + @nodes = nodes + @location = location + end + + def accept(visitor) + visitor.visit_root(self) + end + + def child_nodes + nodes + end + + alias deconstruct child_nodes + + def deconstruct_keys(keys) + { nodes: nodes, location: location } + end + end + + # [ ... ] + class Loop < Node + attr_reader :nodes, :location + + def initialize(nodes:, location:) + @nodes = nodes + @location = location + end + + def accept(visitor) + visitor.visit_loop(self) + end + + def child_nodes + nodes + end + + alias deconstruct child_nodes + + def deconstruct_keys(keys) + { nodes: nodes, location: location } + end + end + + # + + class Increment < Node + attr_reader :location + + def initialize(location:) + @location = location + end + + def accept(visitor) + visitor.visit_increment(self) + end + + def child_nodes + [] + end + + alias deconstruct child_nodes + + def deconstruct_keys(keys) + { value: "+", location: location } + end + end + + # - + class Decrement < Node + attr_reader :location + + def initialize(location:) + @location = location + end + + def accept(visitor) + visitor.visit_decrement(self) + end + + def child_nodes + [] + end + + alias deconstruct child_nodes + + def deconstruct_keys(keys) + { value: "-", location: location } + end + end + + # > + class ShiftRight < Node + attr_reader :location + + def initialize(location:) + @location = location + end + + def accept(visitor) + visitor.visit_shift_right(self) + end + + def child_nodes + [] + end + + alias deconstruct child_nodes + + def deconstruct_keys(keys) + { value: ">", location: location } + end + end + + # < + class ShiftLeft < Node + attr_reader :location + + def initialize(location:) + @location = location + end + + def accept(visitor) + visitor.visit_shift_left(self) + end + + def child_nodes + [] + end + + alias deconstruct child_nodes + + def deconstruct_keys(keys) + { value: "<", location: location } + end + end + + # , + class Input < Node + attr_reader :location + + def initialize(location:) + @location = location + end + + def accept(visitor) + visitor.visit_input(self) + end + + def child_nodes + [] + end + + alias deconstruct child_nodes + + def deconstruct_keys(keys) + { value: ",", location: location } + end + end + + # . + class Output < Node + attr_reader :location + + def initialize(location:) + @location = location + end + + def accept(visitor) + visitor.visit_output(self) + end + + def child_nodes + [] + end + + alias deconstruct child_nodes + + def deconstruct_keys(keys) + { value: ".", location: location } + end + end + + # Allows visiting the syntax tree recursively. + class Visitor + def visit(node) + node.accept(self) + end + + def visit_all(nodes) + nodes.map { |node| visit(node) } + end + + def visit_child_nodes(node) + visit_all(node.child_nodes) + end + + # Visit a Root node. + alias visit_root visit_child_nodes + + # Visit a Loop node. + alias visit_loop visit_child_nodes + + # Visit an Increment node. + alias visit_increment visit_child_nodes + + # Visit a Decrement node. + alias visit_decrement visit_child_nodes + + # Visit a ShiftRight node. + alias visit_shift_right visit_child_nodes + + # Visit a ShiftLeft node. + alias visit_shift_left visit_child_nodes + + # Visit an Input node. + alias visit_input visit_child_nodes + + # Visit an Output node. + alias visit_output visit_child_nodes + end + + # Compiles the syntax tree into YARV bytecode. + class Compiler < Visitor + attr_reader :iseq + + def initialize + @iseq = InstructionSequence.new(:top, "", nil, Location.default) + end + + def visit_decrement(node) + change_by(-1) + end + + def visit_increment(node) + change_by(1) + end + + def visit_input(node) + iseq.getglobal(:$tape) + iseq.getglobal(:$cursor) + iseq.getglobal(:$stdin) + iseq.send(:getc, 0, VM_CALL_ARGS_SIMPLE) + iseq.send(:ord, 0, VM_CALL_ARGS_SIMPLE) + iseq.send(:[]=, 2, VM_CALL_ARGS_SIMPLE) + end + + def visit_loop(node) + start_label = iseq.label + + # First, we're going to compare the value at the current cursor to 0. + # If it's 0, then we'll jump past the loop. Otherwise we'll execute + # the loop. + iseq.getglobal(:$tape) + iseq.getglobal(:$cursor) + iseq.send(:[], 1, VM_CALL_ARGS_SIMPLE) + iseq.putobject(0) + iseq.send(:==, 1, VM_CALL_ARGS_SIMPLE) + branchunless = iseq.branchunless(-1) + + # Otherwise, here we'll execute the loop. + visit_nodes(node.nodes) + + # Now that we've visited all of the child nodes, we need to jump back + # to the start of the loop. + iseq.jump(start_label) + + # Now that we have all of the instructions in place, we can patch the + # branchunless to point to the next instruction for skipping the loop. + branchunless[1] = iseq.label + end + + def visit_output(node) + iseq.getglobal(:$stdout) + iseq.getglobal(:$tape) + iseq.getglobal(:$cursor) + iseq.send(:[], 1, VM_CALL_ARGS_SIMPLE) + iseq.send(:chr, 0, VM_CALL_ARGS_SIMPLE) + iseq.send(:putc, 1, VM_CALL_ARGS_SIMPLE) + end + + def visit_root(node) + iseq.duphash({ 0 => 0 }) + iseq.setglobal(:$tape) + iseq.getglobal(:$tape) + iseq.putobject(0) + iseq.send(:default=, 1, VM_CALL_ARGS_SIMPLE) + + iseq.putobject(0) + iseq.setglobal(:$cursor) + + visit_nodes(node.nodes) + + iseq.putself + iseq.send(:exit, 0, VM_CALL_ARGS_SIMPLE) + iseq + end + + def visit_shift_left(node) + shift_by(-1) + end + + def visit_shift_right(node) + shift_by(1) + end + + private + + def change_by(value) + iseq.getglobal(:$tape) + iseq.getglobal(:$cursor) + iseq.getglobal(:$tape) + iseq.getglobal(:$cursor) + iseq.send(:[], 1, VM_CALL_ARGS_SIMPLE) + + if value < 0 + iseq.putobject(-value) + iseq.send(:-, 1, VM_CALL_ARGS_SIMPLE) + else + iseq.putobject(value) + iseq.send(:+, 1, VM_CALL_ARGS_SIMPLE) + end + + iseq.send(:[]=, 2, VM_CALL_ARGS_SIMPLE) + end + + def shift_by(value) + iseq.getglobal(:$cursor) + + if value < 0 + iseq.putobject(-value) + iseq.send(:-, 1, VM_CALL_ARGS_SIMPLE) + else + iseq.putobject(value) + iseq.send(:+, 1, VM_CALL_ARGS_SIMPLE) + end + + iseq.setglobal(:$cursor) + end + + def visit_nodes(nodes) + nodes + .chunk do |child| + case child + when Increment, Decrement + :change + when ShiftLeft, ShiftRight + :shift + else + :default + end + end + .each do |type, children| + case type + when :change + value = 0 + children.each { |child| value += child.is_a?(Increment) ? 1 : -1 } + change_by(value) + when :shift + value = 0 + children.each { |child| value += child.is_a?(ShiftRight) ? 1 : -1 } + shift_by(value) + else + visit_all(children) + end + end + end + end + + class Error < StandardError + end + + attr_reader :source + + def initialize(source) + @source = source + end + + def compile + Root.new(nodes: parse_segment(source, 0), location: 0...source.length).accept(Compiler.new) + end + + private + + def parse_segment(segment, offset) + index = 0 + nodes = [] + + while index < segment.length + location = offset + index + + case segment[index] + when "+" + nodes << Increment.new(location: location...(location + 1)) + index += 1 + when "-" + nodes << Decrement.new(location: location...(location + 1)) + index += 1 + when ">" + nodes << ShiftRight.new(location: location...(location + 1)) + index += 1 + when "<" + nodes << ShiftLeft.new(location: location...(location + 1)) + index += 1 + when "." + nodes << Output.new(location: location...(location + 1)) + index += 1 + when "," + nodes << Input.new(location: location...(location + 1)) + index += 1 + when "[" + matched = 1 + end_index = index + 1 + + while matched != 0 && end_index < segment.length + case segment[end_index] + when "[" + matched += 1 + when "]" + matched -= 1 + end + + end_index += 1 + end + + raise Error, "Unmatched start loop" if matched != 0 + + content = segment[(index + 1)...(end_index - 1)] + nodes << Loop.new( + nodes: parse_segment(content, offset + index + 1), + location: location...(offset + end_index) + ) + + index = end_index + when "]" + raise Error, "Unmatched end loop" + else + index += 1 + end + end + + nodes + end + end + end +end diff --git a/lib/syntax_tree/yarv/disassembler.rb b/lib/syntax_tree/yarv/disassembler.rb new file mode 100644 index 00000000..51d6fc08 --- /dev/null +++ b/lib/syntax_tree/yarv/disassembler.rb @@ -0,0 +1,209 @@ +# frozen_string_literal: true + +module SyntaxTree + module YARV + # This class is responsible for taking a compiled instruction sequence and + # walking through it to generate equivalent Ruby code. + class Disassembler + include DSL + attr_reader :iseq, :label_name, :label_field, :label_ref + + def initialize(iseq) + @iseq = iseq + + @label_name = "__disasm_label" + @label_field = VarField(Ident(label_name)) + @label_ref = VarRef(Ident(label_name)) + end + + def to_ruby + Program(Statements(disassemble(iseq.insns))) + end + + private + + def node_for(value) + case value + when Integer + Int(value.to_s) + when Symbol + SymbolLiteral(Ident(value.to_s)) + end + end + + def disassemble(insns) + label = :label_0 + clauses = {} + clause = [] + + insns.each do |insn| + if insn.is_a?(Symbol) && insn.start_with?("label_") + clause << Assign(label_field, node_for(insn)) unless clause.last.is_a?(Next) + clauses[label] = clause + clause = [] + label = insn + next + end + + case insn[0] + when :branchunless + clause << IfNode(clause.pop, Statements([Assign(label_field, node_for(insn[1])), Next(Args([]))]), nil) + when :dup + clause << clause.last + when :duphash + assocs = insn[1].map { |key, value| Assoc(node_for(key), node_for(value)) } + clause << HashLiteral(LBrace("{"), assocs) + when :getglobal + clause << VarRef(GVar(insn[1].to_s)) + when :getlocal_WC_0 + clause << VarRef(Ident(local_name(insn[1], 0))) + when :jump + clause << Assign(label_field, node_for(insn[1])) + clause << Next(Args([])) + when :leave + clause << ReturnNode(Args([clause.pop])) + when :opt_and + left, right = clause.pop(2) + clause << Binary(left, :&, right) + when :opt_aref + collection, arg = clause.pop(2) + clause << ARef(collection, Args([arg])) + when :opt_aset + collection, arg, value = clause.pop(3) + + if value.is_a?(Binary) && value.left.is_a?(ARef) && collection === value.left.collection && arg === value.left.index.parts[0] + clause << OpAssign(ARefField(collection, Args([arg])), Op("#{value.operator}="), value.right) + else + clause << Assign(ARefField(collection, Args([arg])), value) + end + when :opt_div + left, right = clause.pop(2) + clause << Binary(left, :/, right) + when :opt_eq + left, right = clause.pop(2) + clause << Binary(left, :==, right) + when :opt_ge + left, right = clause.pop(2) + clause << Binary(left, :>=, right) + when :opt_gt + left, right = clause.pop(2) + clause << Binary(left, :>, right) + when :opt_le + left, right = clause.pop(2) + clause << Binary(left, :<=, right) + when :opt_lt + left, right = clause.pop(2) + clause << Binary(left, :<, right) + when :opt_ltlt + left, right = clause.pop(2) + clause << Binary(left, :<<, right) + when :opt_minus + left, right = clause.pop(2) + clause << Binary(left, :-, right) + when :opt_mod + left, right = clause.pop(2) + clause << Binary(left, :%, right) + when :opt_mult + left, right = clause.pop(2) + clause << Binary(left, :*, right) + when :opt_neq + left, right = clause.pop(2) + clause << Binary(left, :"!=", right) + when :opt_or + left, right = clause.pop(2) + clause << Binary(left, :|, right) + when :opt_plus + left, right = clause.pop(2) + clause << Binary(left, :+, right) + when :opt_send_without_block + if insn[1][:orig_argc] == 0 + clause << CallNode(clause.pop, Period("."), Ident(insn[1][:mid]), nil) + elsif insn[1][:orig_argc] == 1 && insn[1][:mid].end_with?("=") + receiver, argument = clause.pop(2) + clause << Assign(CallNode(receiver, Period("."), Ident(insn[1][:mid][0..-2]), nil), argument) + else + receiver, *arguments = clause.pop(insn[1][:orig_argc] + 1) + clause << CallNode(receiver, Period("."), Ident(insn[1][:mid]), ArgParen(Args(arguments))) + end + when :putobject + case insn[1] + when Float + clause << FloatLiteral(insn[1].inspect) + when Integer + clause << Int(insn[1].inspect) + else + raise "Unknown object type: #{insn[1].class.name}" + end + when :putobject_INT2FIX_0_ + clause << Int("0") + when :putobject_INT2FIX_1_ + clause << Int("1") + when :putself + clause << VarRef(Kw("self")) + when :setglobal + target = GVar(insn[1].to_s) + value = clause.pop + + if value.is_a?(Binary) && VarRef(target) === value.left + clause << OpAssign(VarField(target), Op("#{value.operator}="), value.right) + else + clause << Assign(VarField(target), value) + end + when :setlocal_WC_0 + target = Ident(local_name(insn[1], 0)) + value = clause.pop + + if value.is_a?(Binary) && VarRef(target) === value.left + clause << OpAssign(VarField(target), Op("#{value.operator}="), value.right) + else + clause << Assign(VarField(target), value) + end + else + raise "Unknown instruction #{insn[0]}" + end + end + + # If there's only one clause, then we don't need a case statement, and + # we can just disassemble the first clause. + clauses[label] = clause + return clauses.values.first if clauses.size == 1 + + # Here we're going to build up a big case statement that will handle all + # of the different labels. + current = nil + clauses.reverse_each do |label, clause| + current = When(Args([node_for(label)]), Statements(clause), current) + end + switch = Case(Kw("case"), label_ref, current) + + # Here we're going to make sure that any locals that were established in + # the label_0 block are initialized so that scoping rules work + # correctly. + stack = [] + locals = [label_name] + + clauses[:label_0].each do |node| + if node.is_a?(Assign) && node.target.is_a?(VarField) && node.target.value.is_a?(Ident) + value = node.target.value.value + next if locals.include?(value) + + stack << Assign(node.target, VarRef(Kw("nil"))) + locals << value + end + end + + # Finally, we'll set up the initial label and loop the entire case + # statement. + stack << Assign(label_field, node_for(:label_0)) + stack << MethodAddBlock(CallNode(nil, nil, Ident("loop"), Args([])), BlockNode(Kw("do"), nil, BodyStmt(Statements([switch]), nil, nil, nil, nil))) + stack + end + + def local_name(index, level) + current = iseq + level.times { current = current.parent_iseq } + current.local_table.locals[index].name.to_s + end + end + end +end From a1236fd6c4e4a22292e2a1d52facb95ecdc7a208 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 21 Nov 2022 13:25:14 -0500 Subject: [PATCH 260/536] Default to VM_CALL_ARGS_SIMPLE --- lib/syntax_tree/compiler.rb | 20 ++++++------- lib/syntax_tree/dsl.rb | 4 +++ lib/syntax_tree/yarv.rb | 23 +++++++-------- lib/syntax_tree/yarv/bf.rb | 33 +++++++++++----------- lib/syntax_tree/yarv/disassembler.rb | 40 +++++++++++++++++--------- test/yarv_test.rb | 42 ++++++++++++++-------------- 6 files changed, 88 insertions(+), 74 deletions(-) diff --git a/lib/syntax_tree/compiler.rb b/lib/syntax_tree/compiler.rb index 926661cc..32b5f089 100644 --- a/lib/syntax_tree/compiler.rb +++ b/lib/syntax_tree/compiler.rb @@ -253,13 +253,13 @@ def visit_alias(node) iseq.putspecialobject(YARV::VM_SPECIAL_OBJECT_CBASE) visit(node.left) visit(node.right) - iseq.send(:"core#set_method_alias", 3, YARV::VM_CALL_ARGS_SIMPLE) + iseq.send(:"core#set_method_alias", 3) end def visit_aref(node) visit(node.collection) visit(node.index) - iseq.send(:[], 1, YARV::VM_CALL_ARGS_SIMPLE) + iseq.send(:[], 1) end def visit_arg_block(node) @@ -313,7 +313,7 @@ def visit_assign(node) visit(node.target.index) visit(node.value) iseq.setn(3) - iseq.send(:[]=, 2, YARV::VM_CALL_ARGS_SIMPLE) + iseq.send(:[]=, 2) iseq.pop when ConstPathField names = constant_names(node.target) @@ -337,7 +337,7 @@ def visit_assign(node) visit(node.target) visit(node.value) iseq.setn(2) - iseq.send(:"#{node.target.name.value}=", 1, YARV::VM_CALL_ARGS_SIMPLE) + iseq.send(:"#{node.target.name.value}=", 1) iseq.pop when TopConstField name = node.target.constant.value.to_sym @@ -420,7 +420,7 @@ def visit_binary(node) else visit(node.left) visit(node.right) - iseq.send(node.operator, 1, YARV::VM_CALL_ARGS_SIMPLE) + iseq.send(node.operator, 1) end end @@ -981,7 +981,7 @@ def visit_mrhs(node) def visit_not(node) visit(node.statement) - iseq.send(:!, 0, YARV::VM_CALL_ARGS_SIMPLE) + iseq.send(:!, 0) end def visit_opassign(node) @@ -1367,7 +1367,7 @@ def visit_undef(node) iseq.putspecialobject(YARV::VM_SPECIAL_OBJECT_VMCORE) iseq.putspecialobject(YARV::VM_SPECIAL_OBJECT_CBASE) visit(symbol) - iseq.send(:"core#undef_method", 2, YARV::VM_CALL_ARGS_SIMPLE) + iseq.send(:"core#undef_method", 2) end end @@ -1523,7 +1523,7 @@ def visit_xstring_literal(node) def visit_yield(node) parts = argument_parts(node.arguments) visit_all(parts) - iseq.invokeblock(nil, parts.length, YARV::VM_CALL_ARGS_SIMPLE) + iseq.invokeblock(nil, parts.length) end def visit_zsuper(_node) @@ -1759,12 +1759,12 @@ def with_opassign(node) visit(node.target.index) iseq.dupn(2) - iseq.send(:[], 1, YARV::VM_CALL_ARGS_SIMPLE) + iseq.send(:[], 1) yield iseq.setn(3) - iseq.send(:[]=, 2, YARV::VM_CALL_ARGS_SIMPLE) + iseq.send(:[]=, 2) iseq.pop when ConstPathField name = node.target.constant.value.to_sym diff --git a/lib/syntax_tree/dsl.rb b/lib/syntax_tree/dsl.rb index 05911ee3..1d1324df 100644 --- a/lib/syntax_tree/dsl.rb +++ b/lib/syntax_tree/dsl.rb @@ -38,6 +38,10 @@ def BodyStmt(statements, rescue_clause, else_keyword, else_clause, ensure_clause BodyStmt.new(statements: statements, rescue_clause: rescue_clause, else_keyword: else_keyword, else_clause: else_clause, ensure_clause: ensure_clause, location: Location.default) end + def Break(arguments) + Break.new(arguments: arguments, location: Location.default) + end + def CallNode(receiver, operator, message, arguments) CallNode.new(receiver: receiver, operator: operator, message: message, arguments: arguments, location: Location.default) end diff --git a/lib/syntax_tree/yarv.rb b/lib/syntax_tree/yarv.rb index 2224792a..822844fb 100644 --- a/lib/syntax_tree/yarv.rb +++ b/lib/syntax_tree/yarv.rb @@ -460,7 +460,7 @@ def intern push([:intern]) end - def invokeblock(method_id, argc, flag) + def invokeblock(method_id, argc, flag = VM_CALL_ARGS_SIMPLE) stack.change_by(-argc + 1) push([:invokeblock, call_data(method_id, argc, flag)]) end @@ -547,7 +547,7 @@ def opt_newarray_max(length) push([:opt_newarray_max, length]) else newarray(length) - send(:max, 0, VM_CALL_ARGS_SIMPLE) + send(:max, 0) end end @@ -557,7 +557,7 @@ def opt_newarray_min(length) push([:opt_newarray_min, length]) else newarray(length) - send(:min, 0, VM_CALL_ARGS_SIMPLE) + send(:min, 0) end end @@ -569,22 +569,20 @@ def opt_setinlinecache(inline_storage) def opt_str_freeze(value) if specialized_instruction stack.change_by(+1) - push( - [:opt_str_freeze, value, call_data(:freeze, 0, VM_CALL_ARGS_SIMPLE)] - ) + push([:opt_str_freeze, value, call_data(:freeze, 0)]) else putstring(value) - send(:freeze, 0, VM_CALL_ARGS_SIMPLE) + send(:freeze, 0) end end def opt_str_uminus(value) if specialized_instruction stack.change_by(+1) - push([:opt_str_uminus, value, call_data(:-@, 0, VM_CALL_ARGS_SIMPLE)]) + push([:opt_str_uminus, value, call_data(:-@, 0)]) else putstring(value) - send(:-@, 0, VM_CALL_ARGS_SIMPLE) + send(:-@, 0) end end @@ -633,7 +631,7 @@ def putstring(object) push([:putstring, object]) end - def send(method_id, argc, flag, block_iseq = nil) + def send(method_id, argc, flag = VM_CALL_ARGS_SIMPLE, block_iseq = nil) stack.change_by(-(argc + 1) + 1) cdata = call_data(method_id, argc, flag) @@ -669,8 +667,7 @@ def send(method_id, argc, flag, block_iseq = nil) when [:|, 1] then push([:opt_or, cdata]) when [:[]=, 2] then push([:opt_aset, cdata]) when [:!=, 1] - eql_data = call_data(:==, 1, VM_CALL_ARGS_SIMPLE) - push([:opt_neq, eql_data, cdata]) + push([:opt_neq, call_data(:==, 1), cdata]) else push([:opt_send_without_block, cdata]) end @@ -762,7 +759,7 @@ def toregexp(options, length) # This creates a call data object that is used as the operand for the # send, invokesuper, and objtostring instructions. - def call_data(method_id, argc, flag) + def call_data(method_id, argc, flag = VM_CALL_ARGS_SIMPLE) { mid: method_id, flag: flag, orig_argc: argc } end diff --git a/lib/syntax_tree/yarv/bf.rb b/lib/syntax_tree/yarv/bf.rb index b826ebf2..16098190 100644 --- a/lib/syntax_tree/yarv/bf.rb +++ b/lib/syntax_tree/yarv/bf.rb @@ -260,9 +260,9 @@ def visit_input(node) iseq.getglobal(:$tape) iseq.getglobal(:$cursor) iseq.getglobal(:$stdin) - iseq.send(:getc, 0, VM_CALL_ARGS_SIMPLE) - iseq.send(:ord, 0, VM_CALL_ARGS_SIMPLE) - iseq.send(:[]=, 2, VM_CALL_ARGS_SIMPLE) + iseq.send(:getc, 0) + iseq.send(:ord, 0) + iseq.send(:[]=, 2) end def visit_loop(node) @@ -273,9 +273,9 @@ def visit_loop(node) # the loop. iseq.getglobal(:$tape) iseq.getglobal(:$cursor) - iseq.send(:[], 1, VM_CALL_ARGS_SIMPLE) + iseq.send(:[], 1) iseq.putobject(0) - iseq.send(:==, 1, VM_CALL_ARGS_SIMPLE) + iseq.send(:==, 1) branchunless = iseq.branchunless(-1) # Otherwise, here we'll execute the loop. @@ -294,9 +294,9 @@ def visit_output(node) iseq.getglobal(:$stdout) iseq.getglobal(:$tape) iseq.getglobal(:$cursor) - iseq.send(:[], 1, VM_CALL_ARGS_SIMPLE) - iseq.send(:chr, 0, VM_CALL_ARGS_SIMPLE) - iseq.send(:putc, 1, VM_CALL_ARGS_SIMPLE) + iseq.send(:[], 1) + iseq.send(:chr, 0) + iseq.send(:putc, 1) end def visit_root(node) @@ -304,15 +304,14 @@ def visit_root(node) iseq.setglobal(:$tape) iseq.getglobal(:$tape) iseq.putobject(0) - iseq.send(:default=, 1, VM_CALL_ARGS_SIMPLE) + iseq.send(:default=, 1) iseq.putobject(0) iseq.setglobal(:$cursor) visit_nodes(node.nodes) - iseq.putself - iseq.send(:exit, 0, VM_CALL_ARGS_SIMPLE) + iseq.leave iseq end @@ -331,17 +330,17 @@ def change_by(value) iseq.getglobal(:$cursor) iseq.getglobal(:$tape) iseq.getglobal(:$cursor) - iseq.send(:[], 1, VM_CALL_ARGS_SIMPLE) + iseq.send(:[], 1) if value < 0 iseq.putobject(-value) - iseq.send(:-, 1, VM_CALL_ARGS_SIMPLE) + iseq.send(:-, 1) else iseq.putobject(value) - iseq.send(:+, 1, VM_CALL_ARGS_SIMPLE) + iseq.send(:+, 1) end - iseq.send(:[]=, 2, VM_CALL_ARGS_SIMPLE) + iseq.send(:[]=, 2) end def shift_by(value) @@ -349,10 +348,10 @@ def shift_by(value) if value < 0 iseq.putobject(-value) - iseq.send(:-, 1, VM_CALL_ARGS_SIMPLE) + iseq.send(:-, 1) else iseq.putobject(value) - iseq.send(:+, 1, VM_CALL_ARGS_SIMPLE) + iseq.send(:+, 1) end iseq.setglobal(:$cursor) diff --git a/lib/syntax_tree/yarv/disassembler.rb b/lib/syntax_tree/yarv/disassembler.rb index 51d6fc08..566ed984 100644 --- a/lib/syntax_tree/yarv/disassembler.rb +++ b/lib/syntax_tree/yarv/disassembler.rb @@ -17,7 +17,7 @@ def initialize(iseq) end def to_ruby - Program(Statements(disassemble(iseq.insns))) + Program(disassemble(iseq)) end private @@ -31,12 +31,12 @@ def node_for(value) end end - def disassemble(insns) + def disassemble(iseq) label = :label_0 clauses = {} clause = [] - insns.each do |insn| + iseq.insns.each do |insn| if insn.is_a?(Symbol) && insn.start_with?("label_") clause << Assign(label_field, node_for(insn)) unless clause.last.is_a?(Next) clauses[label] = clause @@ -61,7 +61,8 @@ def disassemble(insns) clause << Assign(label_field, node_for(insn[1])) clause << Next(Args([])) when :leave - clause << ReturnNode(Args([clause.pop])) + value = Args([clause.pop]) + clause << (iseq.type == :top ? Break(value) : ReturnNode(value)) when :opt_and left, right = clause.pop(2) clause << Binary(left, :&, right) @@ -116,14 +117,27 @@ def disassemble(insns) left, right = clause.pop(2) clause << Binary(left, :+, right) when :opt_send_without_block - if insn[1][:orig_argc] == 0 - clause << CallNode(clause.pop, Period("."), Ident(insn[1][:mid]), nil) - elsif insn[1][:orig_argc] == 1 && insn[1][:mid].end_with?("=") - receiver, argument = clause.pop(2) - clause << Assign(CallNode(receiver, Period("."), Ident(insn[1][:mid][0..-2]), nil), argument) + if insn[1][:flag] & VM_CALL_FCALL > 0 + if insn[1][:orig_argc] == 0 + clause.pop + clause << CallNode(nil, nil, Ident(insn[1][:mid]), Args([])) + elsif insn[1][:orig_argc] == 1 && insn[1][:mid].end_with?("=") + _receiver, argument = clause.pop(2) + clause << Assign(CallNode(nil, nil, Ident(insn[1][:mid][0..-2]), nil), argument) + else + _receiver, *arguments = clause.pop(insn[1][:orig_argc] + 1) + clause << CallNode(nil, nil, Ident(insn[1][:mid]), ArgParen(Args(arguments))) + end else - receiver, *arguments = clause.pop(insn[1][:orig_argc] + 1) - clause << CallNode(receiver, Period("."), Ident(insn[1][:mid]), ArgParen(Args(arguments))) + if insn[1][:orig_argc] == 0 + clause << CallNode(clause.pop, Period("."), Ident(insn[1][:mid]), nil) + elsif insn[1][:orig_argc] == 1 && insn[1][:mid].end_with?("=") + receiver, argument = clause.pop(2) + clause << Assign(CallNode(receiver, Period("."), Ident(insn[1][:mid][0..-2]), nil), argument) + else + receiver, *arguments = clause.pop(insn[1][:orig_argc] + 1) + clause << CallNode(receiver, Period("."), Ident(insn[1][:mid]), ArgParen(Args(arguments))) + end end when :putobject case insn[1] @@ -166,7 +180,7 @@ def disassemble(insns) # If there's only one clause, then we don't need a case statement, and # we can just disassemble the first clause. clauses[label] = clause - return clauses.values.first if clauses.size == 1 + return Statements(clauses.values.first) if clauses.size == 1 # Here we're going to build up a big case statement that will handle all # of the different labels. @@ -196,7 +210,7 @@ def disassemble(insns) # statement. stack << Assign(label_field, node_for(:label_0)) stack << MethodAddBlock(CallNode(nil, nil, Ident("loop"), Args([])), BlockNode(Kw("do"), nil, BodyStmt(Statements([switch]), nil, nil, nil, nil))) - stack + Statements(stack) end def local_name(index, level) diff --git a/test/yarv_test.rb b/test/yarv_test.rb index 57371ba3..da348224 100644 --- a/test/yarv_test.rb +++ b/test/yarv_test.rb @@ -6,27 +6,27 @@ module SyntaxTree class YARVTest < Minitest::Test CASES = { - "0" => "return 0\n", - "1" => "return 1\n", - "2" => "return 2\n", - "1.0" => "return 1.0\n", - "1 + 2" => "return 1 + 2\n", - "1 - 2" => "return 1 - 2\n", - "1 * 2" => "return 1 * 2\n", - "1 / 2" => "return 1 / 2\n", - "1 % 2" => "return 1 % 2\n", - "1 < 2" => "return 1 < 2\n", - "1 <= 2" => "return 1 <= 2\n", - "1 > 2" => "return 1 > 2\n", - "1 >= 2" => "return 1 >= 2\n", - "1 == 2" => "return 1 == 2\n", - "1 != 2" => "return 1 != 2\n", - "1 & 2" => "return 1 & 2\n", - "1 | 2" => "return 1 | 2\n", - "1 << 2" => "return 1 << 2\n", - "1 >> 2" => "return 1.>>(2)\n", - "1 ** 2" => "return 1.**(2)\n", - "a = 1; a" => "a = 1\nreturn a\n", + "0" => "break 0\n", + "1" => "break 1\n", + "2" => "break 2\n", + "1.0" => "break 1.0\n", + "1 + 2" => "break 1 + 2\n", + "1 - 2" => "break 1 - 2\n", + "1 * 2" => "break 1 * 2\n", + "1 / 2" => "break 1 / 2\n", + "1 % 2" => "break 1 % 2\n", + "1 < 2" => "break 1 < 2\n", + "1 <= 2" => "break 1 <= 2\n", + "1 > 2" => "break 1 > 2\n", + "1 >= 2" => "break 1 >= 2\n", + "1 == 2" => "break 1 == 2\n", + "1 != 2" => "break 1 != 2\n", + "1 & 2" => "break 1 & 2\n", + "1 | 2" => "break 1 | 2\n", + "1 << 2" => "break 1 << 2\n", + "1 >> 2" => "break 1.>>(2)\n", + "1 ** 2" => "break 1.**(2)\n", + "a = 1; a" => "a = 1\nbreak a\n", }.freeze CASES.each do |source, expected| From d8815de6b2c00ae2001980d557cc62302e029123 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 21 Nov 2022 13:40:46 -0500 Subject: [PATCH 261/536] Add objects to wrap instructions --- .rubocop.yml | 3 + lib/syntax_tree.rb | 1 + lib/syntax_tree/compiler.rb | 109 ++- lib/syntax_tree/dsl.rb | 905 +++++++++++++++++++++- lib/syntax_tree/yarv.rb | 210 ++--- lib/syntax_tree/yarv/bf.rb | 553 ++++--------- lib/syntax_tree/yarv/disassembler.rb | 366 +++++---- lib/syntax_tree/yarv/instructions.rb | 1071 ++++++++++++++++++++++++++ test/yarv_test.rb | 11 +- 9 files changed, 2466 insertions(+), 763 deletions(-) create mode 100644 lib/syntax_tree/yarv/instructions.rb diff --git a/.rubocop.yml b/.rubocop.yml index d0bf0830..134a75dc 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -94,6 +94,9 @@ Style/MutableConstant: Style/NegatedIfElseCondition: Enabled: false +Style/Next: + Enabled: false + Style/NumericPredicate: Enabled: false diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index 2cbfa2e4..792ba20c 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -31,6 +31,7 @@ require_relative "syntax_tree/compiler" require_relative "syntax_tree/yarv/bf" require_relative "syntax_tree/yarv/disassembler" +require_relative "syntax_tree/yarv/instructions" # Syntax Tree is a suite of tools built on top of the internal CRuby parser. It # provides the ability to generate a syntax tree from source, as well as the diff --git a/lib/syntax_tree/compiler.rb b/lib/syntax_tree/compiler.rb index 32b5f089..8327a080 100644 --- a/lib/syntax_tree/compiler.rb +++ b/lib/syntax_tree/compiler.rb @@ -407,7 +407,7 @@ def visit_binary(node) iseq.pop visit(node.right) - branchunless[1] = iseq.label + branchunless.patch!(iseq) when :"||" visit(node.left) iseq.dup @@ -416,7 +416,7 @@ def visit_binary(node) iseq.pop visit(node.right) - branchif[1] = iseq.label + branchif.patch!(iseq) else visit(node.left) visit(node.right) @@ -567,7 +567,7 @@ def visit_call(node) flag |= YARV::VM_CALL_FCALL if node.receiver.nil? iseq.send(node.message.value.to_sym, argc, flag, block_iseq) - branchnil[1] = iseq.label if branchnil + branchnil.patch!(iseq) if branchnil end def visit_case(node) @@ -600,7 +600,7 @@ def visit_case(node) branches.each_with_index do |(clause, branchif), index| iseq.leave if index != 0 - branchif[1] = iseq.label + branchif.patch!(iseq) iseq.pop visit(clause) end @@ -616,21 +616,21 @@ def visit_class(node) iseq.leave end - flags = YARV::VM_DEFINECLASS_TYPE_CLASS + flags = YARV::DefineClass::TYPE_CLASS case node.constant when ConstPathRef - flags |= YARV::VM_DEFINECLASS_FLAG_SCOPED + flags |= YARV::DefineClass::FLAG_SCOPED visit(node.constant.parent) when ConstRef iseq.putspecialobject(YARV::VM_SPECIAL_OBJECT_CONST_BASE) when TopConstRef - flags |= YARV::VM_DEFINECLASS_FLAG_SCOPED + flags |= YARV::DefineClass::FLAG_SCOPED iseq.putobject(Object) end if node.superclass - flags |= YARV::VM_DEFINECLASS_FLAG_HAS_SUPERCLASS + flags |= YARV::DefineClass::FLAG_HAS_SUPERCLASS visit(node.superclass) else iseq.putnil @@ -675,16 +675,16 @@ def visit_const_path_ref(node) end def visit_def(node) - method_iseq = - with_child_iseq(iseq.method_child_iseq(node.name.value, node.location)) do - visit(node.params) if node.params - iseq.event(:RUBY_EVENT_CALL) - visit(node.bodystmt) - iseq.event(:RUBY_EVENT_RETURN) - iseq.leave - end - name = node.name.value.to_sym + method_iseq = iseq.method_child_iseq(name.to_s, node.location) + + with_child_iseq(method_iseq) do + visit(node.params) if node.params + iseq.event(:RUBY_EVENT_CALL) + visit(node.bodystmt) + iseq.event(:RUBY_EVENT_RETURN) + iseq.leave + end if node.target visit(node.target) @@ -714,18 +714,18 @@ def visit_defined(node) case value when Const iseq.putnil - iseq.defined(YARV::DEFINED_CONST, name, "constant") + iseq.defined(YARV::Defined::CONST, name, "constant") when CVar iseq.putnil - iseq.defined(YARV::DEFINED_CVAR, name, "class variable") + iseq.defined(YARV::Defined::CVAR, name, "class variable") when GVar iseq.putnil - iseq.defined(YARV::DEFINED_GVAR, name, "global-variable") + iseq.defined(YARV::Defined::GVAR, name, "global-variable") when Ident iseq.putobject("local-variable") when IVar iseq.putnil - iseq.defined(YARV::DEFINED_IVAR, name, "instance-variable") + iseq.defined(YARV::Defined::IVAR, name, "instance-variable") when Kw case name when :false @@ -742,13 +742,13 @@ def visit_defined(node) iseq.putself name = node.value.value.value.to_sym - iseq.defined(YARV::DEFINED_FUNC, name, "method") + iseq.defined(YARV::Defined::FUNC, name, "method") when YieldNode iseq.putnil - iseq.defined(YARV::DEFINED_YIELD, false, "yield") + iseq.defined(YARV::Defined::YIELD, false, "yield") when ZSuper iseq.putnil - iseq.defined(YARV::DEFINED_ZSUPER, false, "super") + iseq.defined(YARV::Defined::ZSUPER, false, "super") else iseq.putobject("expression") end @@ -842,7 +842,7 @@ def visit_if(node) if last_statement? iseq.leave - branchunless[1] = iseq.label + branchunless.patch!(iseq) node.consequent ? visit(node.consequent) : iseq.putnil else @@ -850,11 +850,11 @@ def visit_if(node) if node.consequent jump = iseq.jump(-1) - branchunless[1] = iseq.label + branchunless.patch!(iseq) visit(node.consequent) jump[1] = iseq.label else - branchunless[1] = iseq.label + branchunless.patch!(iseq) end end end @@ -953,16 +953,16 @@ def visit_module(node) iseq.leave end - flags = YARV::VM_DEFINECLASS_TYPE_MODULE + flags = YARV::DefineClass::TYPE_MODULE case node.constant when ConstPathRef - flags |= YARV::VM_DEFINECLASS_FLAG_SCOPED + flags |= YARV::DefineClass::FLAG_SCOPED visit(node.constant.parent) when ConstRef iseq.putspecialobject(YARV::VM_SPECIAL_OBJECT_CONST_BASE) when TopConstRef - flags |= YARV::VM_DEFINECLASS_FLAG_SCOPED + flags |= YARV::DefineClass::FLAG_SCOPED iseq.putobject(Object) end @@ -1004,15 +1004,15 @@ def visit_opassign(node) case node.target when ARefField iseq.leave - branchunless[1] = iseq.label + branchunless.patch!(iseq) iseq.setn(3) iseq.adjuststack(3) when ConstPathField, TopConstField - branchunless[1] = iseq.label + branchunless.patch!(iseq) iseq.swap iseq.pop else - branchunless[1] = iseq.label + branchunless.patch!(iseq) end when :"||" if node.target.is_a?(ConstPathField) || node.target.is_a?(TopConstField) @@ -1034,11 +1034,11 @@ def visit_opassign(node) if node.target.is_a?(ARefField) iseq.leave - branchif[1] = iseq.label + branchif.patch!(iseq) iseq.setn(3) iseq.adjuststack(3) else - branchif[1] = iseq.label + branchif.patch!(iseq) end end else @@ -1092,7 +1092,10 @@ def visit_params(node) if node.keywords.any? argument_options[:kwbits] = 0 argument_options[:keyword] = [] - checkkeywords = [] + + keyword_bits_name = node.keyword_rest ? 3 : 2 + iseq.argument_size += 1 + keyword_bits_index = iseq.local_table.locals.size + node.keywords.size node.keywords.each_with_index do |(keyword, value), keyword_index| name = keyword.value.chomp(":").to_sym @@ -1105,24 +1108,18 @@ def visit_params(node) if value.nil? argument_options[:keyword] << name elsif (compiled = RubyVisitor.compile(value)) - compiled = value.accept(RubyVisitor.new) argument_options[:keyword] << [name, compiled] else argument_options[:keyword] << [name] - checkkeywords << iseq.checkkeyword(-1, keyword_index) + iseq.checkkeyword(keyword_bits_index, keyword_index) branchif = iseq.branchif(-1) visit(value) iseq.setlocal(index, 0) - branchif[1] = iseq.label + branchif.patch!(iseq) end end - name = node.keyword_rest ? 3 : 2 - iseq.argument_size += 1 - iseq.local_table.plain(name) - - lookup = iseq.local_table.find(name, 0) - checkkeywords.each { |checkkeyword| checkkeyword[1] = lookup.index } + iseq.local_table.plain(keyword_bits_name) end if node.keyword_rest.is_a?(ArgsForward) @@ -1251,7 +1248,7 @@ def visit_sclass(node) iseq.defineclass( :singletonclass, singleton_iseq, - YARV::VM_DEFINECLASS_TYPE_SINGLETON_CLASS + YARV::DefineClass::TYPE_SINGLETON_CLASS ) end @@ -1378,7 +1375,7 @@ def visit_unless(node) if last_statement? iseq.leave - branchunless[1] = iseq.label + branchunless.patch!(iseq) visit(node.statements) else @@ -1386,11 +1383,11 @@ def visit_unless(node) if node.consequent jump = iseq.jump(-1) - branchunless[1] = iseq.label + branchunless.patch!(iseq) visit(node.consequent) jump[1] = iseq.label else - branchunless[1] = iseq.label + branchunless.patch!(iseq) end end end @@ -1598,24 +1595,24 @@ def opassign_defined(node) name = node.target.constant.value.to_sym iseq.dup - iseq.defined(YARV::DEFINED_CONST_FROM, name, true) + iseq.defined(YARV::Defined::CONST_FROM, name, true) when TopConstField name = node.target.constant.value.to_sym iseq.putobject(Object) iseq.dup - iseq.defined(YARV::DEFINED_CONST_FROM, name, true) + iseq.defined(YARV::Defined::CONST_FROM, name, true) when VarField name = node.target.value.value.to_sym iseq.putnil case node.target.value when Const - iseq.defined(YARV::DEFINED_CONST, name, true) + iseq.defined(YARV::Defined::CONST, name, true) when CVar - iseq.defined(YARV::DEFINED_CVAR, name, true) + iseq.defined(YARV::Defined::CVAR, name, true) when GVar - iseq.defined(YARV::DEFINED_GVAR, name, true) + iseq.defined(YARV::Defined::GVAR, name, true) end end @@ -1641,7 +1638,7 @@ def opassign_defined(node) branchif = iseq.branchif(-1) iseq.pop - branchunless[1] = iseq.label + branchunless.patch!(iseq) visit(node.value) case node.target @@ -1663,7 +1660,7 @@ def opassign_defined(node) end end - branchif[1] = iseq.label + branchif.patch!(iseq) end # Whenever a value is interpolated into a string-like structure, these diff --git a/lib/syntax_tree/dsl.rb b/lib/syntax_tree/dsl.rb index 1d1324df..860a1fe5 100644 --- a/lib/syntax_tree/dsl.rb +++ b/lib/syntax_tree/dsl.rb @@ -1,133 +1,1004 @@ # frozen_string_literal: true module SyntaxTree + # This module provides shortcuts for creating AST nodes. module DSL + # Create a new BEGINBlock node. + def BEGINBlock(lbrace, statements) + BEGINBlock.new( + lbrace: lbrace, + statements: statements, + location: Location.default + ) + end + + # Create a new CHAR node. + def CHAR(value) + CHAR.new(value: value, location: Location.default) + end + + # Create a new ENDBlock node. + def ENDBlock(lbrace, statements) + ENDBlock.new( + lbrace: lbrace, + statements: statements, + location: Location.default + ) + end + + # Create a new EndContent node. + def EndContent(value) + EndContent.new(value: value, location: Location.default) + end + + # Create a new AliasNode node. + def AliasNode(left, right) + AliasNode.new(left: left, right: right, location: Location.default) + end + + # Create a new ARef node. def ARef(collection, index) ARef.new(collection: collection, index: index, location: Location.default) end + # Create a new ARefField node. def ARefField(collection, index) - ARefField.new(collection: collection, index: index, location: Location.default) + ARefField.new( + collection: collection, + index: index, + location: Location.default + ) end + # Create a new ArgParen node. + def ArgParen(arguments) + ArgParen.new(arguments: arguments, location: Location.default) + end + + # Create a new Args node. def Args(parts) Args.new(parts: parts, location: Location.default) end - def ArgParen(arguments) - ArgParen.new(arguments: arguments, location: Location.default) + # Create a new ArgBlock node. + def ArgBlock(value) + ArgBlock.new(value: value, location: Location.default) + end + + # Create a new ArgStar node. + def ArgStar(value) + ArgStar.new(value: value, location: Location.default) + end + + # Create a new ArgsForward node. + def ArgsForward + ArgsForward.new(location: Location.default) + end + + # Create a new ArrayLiteral node. + def ArrayLiteral(lbracket, contents) + ArrayLiteral.new( + lbracket: lbracket, + contents: contents, + location: Location.default + ) end + # Create a new AryPtn node. + def AryPtn(constant, requireds, rest, posts) + AryPtn.new( + constant: constant, + requireds: requireds, + rest: rest, + posts: posts, + location: Location.default + ) + end + + # Create a new Assign node. def Assign(target, value) Assign.new(target: target, value: value, location: Location.default) end + # Create a new Assoc node. def Assoc(key, value) Assoc.new(key: key, value: value, location: Location.default) end - def Binary(left, operator, right) - Binary.new(left: left, operator: operator, right: right, location: Location.default) + # Create a new AssocSplat node. + def AssocSplat(value) + AssocSplat.new(value: value, location: Location.default) end - def BlockNode(opening, block_var, bodystmt) - BlockNode.new(opening: opening, block_var: block_var, bodystmt: bodystmt, location: Location.default) + # Create a new Backref node. + def Backref(value) + Backref.new(value: value, location: Location.default) + end + + # Create a new Backtick node. + def Backtick(value) + Backtick.new(value: value, location: Location.default) + end + + # Create a new BareAssocHash node. + def BareAssocHash(assocs) + BareAssocHash.new(assocs: assocs, location: Location.default) end - def BodyStmt(statements, rescue_clause, else_keyword, else_clause, ensure_clause) - BodyStmt.new(statements: statements, rescue_clause: rescue_clause, else_keyword: else_keyword, else_clause: else_clause, ensure_clause: ensure_clause, location: Location.default) + # Create a new Begin node. + def Begin(bodystmt) + Begin.new(bodystmt: bodystmt, location: Location.default) end + # Create a new PinnedBegin node. + def PinnedBegin(statement) + PinnedBegin.new(statement: statement, location: Location.default) + end + + # Create a new Binary node. + def Binary(left, operator, right) + Binary.new( + left: left, + operator: operator, + right: right, + location: Location.default + ) + end + + # Create a new BlockVar node. + def BlockVar(params, locals) + BlockVar.new(params: params, locals: locals, location: Location.default) + end + + # Create a new BlockArg node. + def BlockArg(name) + BlockArg.new(name: name, location: Location.default) + end + + # Create a new BodyStmt node. + def BodyStmt( + statements, + rescue_clause, + else_keyword, + else_clause, + ensure_clause + ) + BodyStmt.new( + statements: statements, + rescue_clause: rescue_clause, + else_keyword: else_keyword, + else_clause: else_clause, + ensure_clause: ensure_clause, + location: Location.default + ) + end + + # Create a new Break node. def Break(arguments) Break.new(arguments: arguments, location: Location.default) end + # Create a new CallNode node. def CallNode(receiver, operator, message, arguments) - CallNode.new(receiver: receiver, operator: operator, message: message, arguments: arguments, location: Location.default) + CallNode.new( + receiver: receiver, + operator: operator, + message: message, + arguments: arguments, + location: Location.default + ) end + # Create a new Case node. def Case(keyword, value, consequent) - Case.new(keyword: keyword, value: value, consequent: consequent, location: Location.default) + Case.new( + keyword: keyword, + value: value, + consequent: consequent, + location: Location.default + ) + end + + # Create a new RAssign node. + def RAssign(value, operator, pattern) + RAssign.new( + value: value, + operator: operator, + pattern: pattern, + location: Location.default + ) + end + + # Create a new ClassDeclaration node. + def ClassDeclaration(constant, superclass, bodystmt) + ClassDeclaration.new( + constant: constant, + superclass: superclass, + bodystmt: bodystmt, + location: Location.default + ) + end + + # Create a new Comma node. + def Comma(value) + Comma.new(value: value, location: Location.default) end + # Create a new Command node. + def Command(message, arguments, block) + Command.new( + message: message, + arguments: arguments, + block: block, + location: Location.default + ) + end + + # Create a new CommandCall node. + def CommandCall(receiver, operator, message, arguments, block) + CommandCall.new( + receiver: receiver, + operator: operator, + message: message, + arguments: arguments, + block: block, + location: Location.default + ) + end + + # Create a new Comment node. + def Comment(value, inline) + Comment.new(value: value, inline: inline, location: Location.default) + end + + # Create a new Const node. + def Const(value) + Const.new(value: value, location: Location.default) + end + + # Create a new ConstPathField node. + def ConstPathField(parent, constant) + ConstPathField.new( + parent: parent, + constant: constant, + location: Location.default + ) + end + + # Create a new ConstPathRef node. + def ConstPathRef(parent, constant) + ConstPathRef.new( + parent: parent, + constant: constant, + location: Location.default + ) + end + + # Create a new ConstRef node. + def ConstRef(constant) + ConstRef.new(constant: constant, location: Location.default) + end + + # Create a new CVar node. + def CVar(value) + CVar.new(value: value, location: Location.default) + end + + # Create a new DefNode node. + def DefNode(target, operator, name, params, bodystmt) + DefNode.new( + target: target, + operator: operator, + name: name, + params: params, + bodystmt: bodystmt, + location: Location.default + ) + end + + # Create a new Defined node. + def Defined(value) + Defined.new(value: value, location: Location.default) + end + + # Create a new BlockNode node. + def BlockNode(opening, block_var, bodystmt) + BlockNode.new( + opening: opening, + block_var: block_var, + bodystmt: bodystmt, + location: Location.default + ) + end + + # Create a new RangeNode node. + def RangeNode(left, operator, right) + RangeNode.new( + left: left, + operator: operator, + right: right, + location: Location.default + ) + end + + # Create a new DynaSymbol node. + def DynaSymbol(parts, quote) + DynaSymbol.new(parts: parts, quote: quote, location: Location.default) + end + + # Create a new Else node. + def Else(keyword, statements) + Else.new( + keyword: keyword, + statements: statements, + location: Location.default + ) + end + + # Create a new Elsif node. + def Elsif(predicate, statements, consequent) + Elsif.new( + predicate: predicate, + statements: statements, + consequent: consequent, + location: Location.default + ) + end + + # Create a new EmbDoc node. + def EmbDoc(value) + EmbDoc.new(value: value, location: Location.default) + end + + # Create a new EmbExprBeg node. + def EmbExprBeg(value) + EmbExprBeg.new(value: value, location: Location.default) + end + + # Create a new EmbExprEnd node. + def EmbExprEnd(value) + EmbExprEnd.new(value: value, location: Location.default) + end + + # Create a new EmbVar node. + def EmbVar(value) + EmbVar.new(value: value, location: Location.default) + end + + # Create a new Ensure node. + def Ensure(keyword, statements) + Ensure.new( + keyword: keyword, + statements: statements, + location: Location.default + ) + end + + # Create a new ExcessedComma node. + def ExcessedComma(value) + ExcessedComma.new(value: value, location: Location.default) + end + + # Create a new Field node. + def Field(parent, operator, name) + Field.new( + parent: parent, + operator: operator, + name: name, + location: Location.default + ) + end + + # Create a new FloatLiteral node. def FloatLiteral(value) FloatLiteral.new(value: value, location: Location.default) end + # Create a new FndPtn node. + def FndPtn(constant, left, values, right) + FndPtn.new( + constant: constant, + left: left, + values: values, + right: right, + location: Location.default + ) + end + + # Create a new For node. + def For(index, collection, statements) + For.new( + index: index, + collection: collection, + statements: statements, + location: Location.default + ) + end + + # Create a new GVar node. def GVar(value) GVar.new(value: value, location: Location.default) end + # Create a new HashLiteral node. def HashLiteral(lbrace, assocs) - HashLiteral.new(lbrace: lbrace, assocs: assocs, location: Location.default) + HashLiteral.new( + lbrace: lbrace, + assocs: assocs, + location: Location.default + ) + end + + # Create a new Heredoc node. + def Heredoc(beginning, ending, dedent, parts) + Heredoc.new( + beginning: beginning, + ending: ending, + dedent: dedent, + parts: parts, + location: Location.default + ) + end + + # Create a new HeredocBeg node. + def HeredocBeg(value) + HeredocBeg.new(value: value, location: Location.default) + end + + # Create a new HeredocEnd node. + def HeredocEnd(value) + HeredocEnd.new(value: value, location: Location.default) + end + + # Create a new HshPtn node. + def HshPtn(constant, keywords, keyword_rest) + HshPtn.new( + constant: constant, + keywords: keywords, + keyword_rest: keyword_rest, + location: Location.default + ) end + # Create a new Ident node. def Ident(value) Ident.new(value: value, location: Location.default) end + # Create a new IfNode node. def IfNode(predicate, statements, consequent) - IfNode.new(predicate: predicate, statements: statements, consequent: consequent, location: Location.default) + IfNode.new( + predicate: predicate, + statements: statements, + consequent: consequent, + location: Location.default + ) end + # Create a new IfOp node. + def IfOp(predicate, truthy, falsy) + IfOp.new( + predicate: predicate, + truthy: truthy, + falsy: falsy, + location: Location.default + ) + end + + # Create a new Imaginary node. + def Imaginary(value) + Imaginary.new(value: value, location: Location.default) + end + + # Create a new In node. + def In(pattern, statements, consequent) + In.new( + pattern: pattern, + statements: statements, + consequent: consequent, + location: Location.default + ) + end + + # Create a new Int node. def Int(value) Int.new(value: value, location: Location.default) end + # Create a new IVar node. + def IVar(value) + IVar.new(value: value, location: Location.default) + end + + # Create a new Kw node. def Kw(value) Kw.new(value: value, location: Location.default) end + # Create a new KwRestParam node. + def KwRestParam(name) + KwRestParam.new(name: name, location: Location.default) + end + + # Create a new Label node. + def Label(value) + Label.new(value: value, location: Location.default) + end + + # Create a new LabelEnd node. + def LabelEnd(value) + LabelEnd.new(value: value, location: Location.default) + end + + # Create a new Lambda node. + def Lambda(params, statements) + Lambda.new( + params: params, + statements: statements, + location: Location.default + ) + end + + # Create a new LambdaVar node. + def LambdaVar(params, locals) + LambdaVar.new(params: params, locals: locals, location: Location.default) + end + + # Create a new LBrace node. def LBrace(value) LBrace.new(value: value, location: Location.default) end + # Create a new LBracket node. + def LBracket(value) + LBracket.new(value: value, location: Location.default) + end + + # Create a new LParen node. + def LParen(value) + LParen.new(value: value, location: Location.default) + end + + # Create a new MAssign node. + def MAssign(target, value) + MAssign.new(target: target, value: value, location: Location.default) + end + + # Create a new MethodAddBlock node. def MethodAddBlock(call, block) MethodAddBlock.new(call: call, block: block, location: Location.default) end + # Create a new MLHS node. + def MLHS(parts, comma) + MLHS.new(parts: parts, comma: comma, location: Location.default) + end + + # Create a new MLHSParen node. + def MLHSParen(contents, comma) + MLHSParen.new( + contents: contents, + comma: comma, + location: Location.default + ) + end + + # Create a new ModuleDeclaration node. + def ModuleDeclaration(constant, bodystmt) + ModuleDeclaration.new( + constant: constant, + bodystmt: bodystmt, + location: Location.default + ) + end + + # Create a new MRHS node. + def MRHS(parts) + MRHS.new(parts: parts, location: Location.default) + end + + # Create a new Next node. def Next(arguments) Next.new(arguments: arguments, location: Location.default) end + # Create a new Op node. def Op(value) Op.new(value: value, location: Location.default) end + # Create a new OpAssign node. def OpAssign(target, operator, value) - OpAssign.new(target: target, operator: operator, value: value, location: Location.default) - end - + OpAssign.new( + target: target, + operator: operator, + value: value, + location: Location.default + ) + end + + # Create a new Params node. + def Params(requireds, optionals, rest, posts, keywords, keyword_rest, block) + Params.new( + requireds: requireds, + optionals: optionals, + rest: rest, + posts: posts, + keywords: keywords, + keyword_rest: keyword_rest, + block: block, + location: Location.default + ) + end + + # Create a new Paren node. + def Paren(lparen, contents) + Paren.new(lparen: lparen, contents: contents, location: Location.default) + end + + # Create a new Period node. def Period(value) Period.new(value: value, location: Location.default) end + # Create a new Program node. def Program(statements) Program.new(statements: statements, location: Location.default) end + # Create a new QSymbols node. + def QSymbols(beginning, elements) + QSymbols.new( + beginning: beginning, + elements: elements, + location: Location.default + ) + end + + # Create a new QSymbolsBeg node. + def QSymbolsBeg(value) + QSymbolsBeg.new(value: value, location: Location.default) + end + + # Create a new QWords node. + def QWords(beginning, elements) + QWords.new( + beginning: beginning, + elements: elements, + location: Location.default + ) + end + + # Create a new QWordsBeg node. + def QWordsBeg(value) + QWordsBeg.new(value: value, location: Location.default) + end + + # Create a new RationalLiteral node. + def RationalLiteral(value) + RationalLiteral.new(value: value, location: Location.default) + end + + # Create a new RBrace node. + def RBrace(value) + RBrace.new(value: value, location: Location.default) + end + + # Create a new RBracket node. + def RBracket(value) + RBracket.new(value: value, location: Location.default) + end + + # Create a new Redo node. + def Redo + Redo.new(location: Location.default) + end + + # Create a new RegexpContent node. + def RegexpContent(beginning, parts) + RegexpContent.new( + beginning: beginning, + parts: parts, + location: Location.default + ) + end + + # Create a new RegexpBeg node. + def RegexpBeg(value) + RegexpBeg.new(value: value, location: Location.default) + end + + # Create a new RegexpEnd node. + def RegexpEnd(value) + RegexpEnd.new(value: value, location: Location.default) + end + + # Create a new RegexpLiteral node. + def RegexpLiteral(beginning, ending, parts) + RegexpLiteral.new( + beginning: beginning, + ending: ending, + parts: parts, + location: Location.default + ) + end + + # Create a new RescueEx node. + def RescueEx(exceptions, variable) + RescueEx.new( + exceptions: exceptions, + variable: variable, + location: Location.default + ) + end + + # Create a new Rescue node. + def Rescue(keyword, exception, statements, consequent) + Rescue.new( + keyword: keyword, + exception: exception, + statements: statements, + consequent: consequent, + location: Location.default + ) + end + + # Create a new RescueMod node. + def RescueMod(statement, value) + RescueMod.new( + statement: statement, + value: value, + location: Location.default + ) + end + + # Create a new RestParam node. + def RestParam(name) + RestParam.new(name: name, location: Location.default) + end + + # Create a new Retry node. + def Retry + Retry.new(location: Location.default) + end + + # Create a new ReturnNode node. def ReturnNode(arguments) ReturnNode.new(arguments: arguments, location: Location.default) end + # Create a new RParen node. + def RParen(value) + RParen.new(value: value, location: Location.default) + end + + # Create a new SClass node. + def SClass(target, bodystmt) + SClass.new(target: target, bodystmt: bodystmt, location: Location.default) + end + + # Create a new Statements node. def Statements(body) Statements.new(nil, body: body, location: Location.default) end + # Create a new StringContent node. + def StringContent(parts) + StringContent.new(parts: parts, location: Location.default) + end + + # Create a new StringConcat node. + def StringConcat(left, right) + StringConcat.new(left: left, right: right, location: Location.default) + end + + # Create a new StringDVar node. + def StringDVar(variable) + StringDVar.new(variable: variable, location: Location.default) + end + + # Create a new StringEmbExpr node. + def StringEmbExpr(statements) + StringEmbExpr.new(statements: statements, location: Location.default) + end + + # Create a new StringLiteral node. + def StringLiteral(parts, quote) + StringLiteral.new(parts: parts, quote: quote, location: Location.default) + end + + # Create a new Super node. + def Super(arguments) + Super.new(arguments: arguments, location: Location.default) + end + + # Create a new SymBeg node. + def SymBeg(value) + SymBeg.new(value: value, location: Location.default) + end + + # Create a new SymbolContent node. + def SymbolContent(value) + SymbolContent.new(value: value, location: Location.default) + end + + # Create a new SymbolLiteral node. def SymbolLiteral(value) SymbolLiteral.new(value: value, location: Location.default) end + # Create a new Symbols node. + def Symbols(beginning, elements) + Symbols.new( + beginning: beginning, + elements: elements, + location: Location.default + ) + end + + # Create a new SymbolsBeg node. + def SymbolsBeg(value) + SymbolsBeg.new(value: value, location: Location.default) + end + + # Create a new TLambda node. + def TLambda(value) + TLambda.new(value: value, location: Location.default) + end + + # Create a new TLamBeg node. + def TLamBeg(value) + TLamBeg.new(value: value, location: Location.default) + end + + # Create a new TopConstField node. + def TopConstField(constant) + TopConstField.new(constant: constant, location: Location.default) + end + + # Create a new TopConstRef node. + def TopConstRef(constant) + TopConstRef.new(constant: constant, location: Location.default) + end + + # Create a new TStringBeg node. + def TStringBeg(value) + TStringBeg.new(value: value, location: Location.default) + end + + # Create a new TStringContent node. + def TStringContent(value) + TStringContent.new(value: value, location: Location.default) + end + + # Create a new TStringEnd node. + def TStringEnd(value) + TStringEnd.new(value: value, location: Location.default) + end + + # Create a new Not node. + def Not(statement, parentheses) + Not.new( + statement: statement, + parentheses: parentheses, + location: Location.default + ) + end + + # Create a new Unary node. + def Unary(operator, statement) + Unary.new( + operator: operator, + statement: statement, + location: Location.default + ) + end + + # Create a new Undef node. + def Undef(symbols) + Undef.new(symbols: symbols, location: Location.default) + end + + # Create a new UnlessNode node. + def UnlessNode(predicate, statements, consequent) + UnlessNode.new( + predicate: predicate, + statements: statements, + consequent: consequent, + location: Location.default + ) + end + + # Create a new UntilNode node. + def UntilNode(predicate, statements) + UntilNode.new( + predicate: predicate, + statements: statements, + location: Location.default + ) + end + + # Create a new VarField node. def VarField(value) VarField.new(value: value, location: Location.default) end + # Create a new VarRef node. def VarRef(value) VarRef.new(value: value, location: Location.default) end + # Create a new PinnedVarRef node. + def PinnedVarRef(value) + PinnedVarRef.new(value: value, location: Location.default) + end + + # Create a new VCall node. + def VCall(value) + VCall.new(value: value, location: Location.default) + end + + # Create a new VoidStmt node. + def VoidStmt + VoidStmt.new(location: Location.default) + end + + # Create a new When node. def When(arguments, statements, consequent) - When.new(arguments: arguments, statements: statements, consequent: consequent, location: Location.default) + When.new( + arguments: arguments, + statements: statements, + consequent: consequent, + location: Location.default + ) + end + + # Create a new WhileNode node. + def WhileNode(predicate, statements) + WhileNode.new( + predicate: predicate, + statements: statements, + location: Location.default + ) + end + + # Create a new Word node. + def Word(parts) + Word.new(parts: parts, location: Location.default) + end + + # Create a new Words node. + def Words(beginning, elements) + Words.new( + beginning: beginning, + elements: elements, + location: Location.default + ) + end + + # Create a new WordsBeg node. + def WordsBeg(value) + WordsBeg.new(value: value, location: Location.default) + end + + # Create a new XString node. + def XString(parts) + XString.new(parts: parts, location: Location.default) + end + + # Create a new XStringLiteral node. + def XStringLiteral(parts) + XStringLiteral.new(parts: parts, location: Location.default) + end + + # Create a new YieldNode node. + def YieldNode(arguments) + YieldNode.new(arguments: arguments, location: Location.default) + end + + # Create a new ZSuper node. + def ZSuper + ZSuper.new(location: Location.default) end end end diff --git a/lib/syntax_tree/yarv.rb b/lib/syntax_tree/yarv.rb index 822844fb..a29714a5 100644 --- a/lib/syntax_tree/yarv.rb +++ b/lib/syntax_tree/yarv.rb @@ -206,7 +206,12 @@ def inline_storage_for(name) def length insns.inject(0) do |sum, insn| - insn.is_a?(Array) ? sum + insn.length : sum + case insn + when Integer, Symbol + sum + else + sum + insn.length + end end end @@ -241,7 +246,38 @@ def to_a local_table.names, argument_options, [], - insns.map { |insn| serialize(insn) } + insns.map do |insn| + case insn + when Integer, Symbol + insn + when Array + case insn[0] + when :setlocal_WC_0, :setlocal_WC_1, :setlocal + iseq = self + + case insn[0] + when :setlocal_WC_1 + iseq = iseq.parent_iseq + when :setlocal + insn[2].times { iseq = iseq.parent_iseq } + end + + # Here we need to map the local variable index to the offset + # from the top of the stack where it will be stored. + [insn[0], iseq.local_table.offset(insn[1]), *insn[2..]] + when :send + # For any instructions that push instruction sequences onto the + # stack, we need to call #to_a on them as well. + [insn[0], insn[1], (insn[2].to_a if insn[2])] + when :once + [insn[0], insn[1].to_a, insn[2]] + else + insn + end + else + insn.to_a(self) + end + end ] end @@ -289,7 +325,14 @@ def singleton_class_child_iseq(location) def push(insn) insns << insn - insn + + case insn + when Integer, Symbol, Array + insn + else + stack.change_by(-insn.pops + insn.pushes) + insn + end end # This creates a new label at the current length of the instruction @@ -304,134 +347,106 @@ def event(name) end def adjuststack(number) - stack.change_by(-number) - push([:adjuststack, number]) + push(AdjustStack.new(number)) end def anytostring - stack.change_by(-2 + 1) - push([:anytostring]) + push(AnyToString.new) end - def branchif(index) - stack.change_by(-1) - push([:branchif, index]) + def branchif(label) + push(BranchIf.new(label)) end - def branchnil(index) - stack.change_by(-1) - push([:branchnil, index]) + def branchnil(label) + push(BranchNil.new(label)) end - def branchunless(index) - stack.change_by(-1) - push([:branchunless, index]) + def branchunless(label) + push(BranchUnless.new(label)) end - def checkkeyword(index, keyword_index) - stack.change_by(+1) - push([:checkkeyword, index, keyword_index]) + def checkkeyword(keyword_bits_index, keyword_index) + push(CheckKeyword.new(keyword_bits_index, keyword_index)) end def concatarray - stack.change_by(-2 + 1) - push([:concatarray]) + push(ConcatArray.new) end def concatstrings(number) - stack.change_by(-number + 1) - push([:concatstrings, number]) + push(ConcatStrings.new(number)) end def defined(type, name, message) - stack.change_by(-1 + 1) - push([:defined, type, name, message]) + push(Defined.new(type, name, message)) end def defineclass(name, class_iseq, flags) - stack.change_by(-2 + 1) - push([:defineclass, name, class_iseq, flags]) + push(DefineClass.new(name, class_iseq, flags)) end def definemethod(name, method_iseq) - stack.change_by(0) - push([:definemethod, name, method_iseq]) + push(DefineMethod.new(name, method_iseq)) end def definesmethod(name, method_iseq) - stack.change_by(-1) - push([:definesmethod, name, method_iseq]) + push(DefineSMethod.new(name, method_iseq)) end def dup - stack.change_by(-1 + 2) - push([:dup]) + push(Dup.new) end def duparray(object) - stack.change_by(+1) - push([:duparray, object]) + push(DupArray.new(object)) end def duphash(object) - stack.change_by(+1) - push([:duphash, object]) + push(DupHash.new(object)) end def dupn(number) - stack.change_by(+number) - push([:dupn, number]) + push(DupN.new(number)) end - def expandarray(length, flag) - stack.change_by(-1 + length) - push([:expandarray, length, flag]) + def expandarray(length, flags) + push(ExpandArray.new(length, flags)) end def getblockparam(index, level) - stack.change_by(+1) - push([:getblockparam, index, level]) + push(GetBlockParam.new(index, level)) end def getblockparamproxy(index, level) - stack.change_by(+1) - push([:getblockparamproxy, index, level]) + push(GetBlockParamProxy.new(index, level)) end def getclassvariable(name) - stack.change_by(+1) - - if RUBY_VERSION >= "3.0" - push([:getclassvariable, name, inline_storage_for(name)]) + if RUBY_VERSION < "3.0" + push(GetClassVariableUncached.new(name)) else - push([:getclassvariable, name]) + push(GetClassVariable.new(name, inline_storage_for(name))) end end def getconstant(name) - stack.change_by(-2 + 1) - push([:getconstant, name]) + push(GetConstant.new(name)) end def getglobal(name) - stack.change_by(+1) - push([:getglobal, name]) + push(GetGlobal.new(name)) end def getinstancevariable(name) - stack.change_by(+1) - - if RUBY_VERSION >= "3.2" - push([:getinstancevariable, name, inline_storage]) + if RUBY_VERSION < "3.2" + push(GetInstanceVariable.new(name, inline_storage_for(name))) else - inline_storage = inline_storage_for(name) - push([:getinstancevariable, name, inline_storage]) + push(GetInstanceVariable.new(name, inline_storage)) end end def getlocal(index, level) - stack.change_by(+1) - if operands_unification # Specialize the getlocal instruction based on the level of the # local variable. If it's 0 or 1, then there's a specialized @@ -439,14 +454,14 @@ def getlocal(index, level) # scope, respectively, and requires fewer operands. case level when 0 - push([:getlocal_WC_0, index]) + push(GetLocalWC0.new(index)) when 1 - push([:getlocal_WC_1, index]) + push(GetLocalWC1.new(index)) else - push([:getlocal, index, level]) + push(GetLocal.new(index, level)) end else - push([:getlocal, index, level]) + push(GetLocal.new(index, level)) end end @@ -762,38 +777,6 @@ def toregexp(options, length) def call_data(method_id, argc, flag = VM_CALL_ARGS_SIMPLE) { mid: method_id, flag: flag, orig_argc: argc } end - - def serialize(insn) - case insn[0] - when :checkkeyword, :getblockparam, :getblockparamproxy, :getlocal_WC_0, - :getlocal_WC_1, :getlocal, :setlocal_WC_0, :setlocal_WC_1, - :setlocal - iseq = self - - case insn[0] - when :getlocal_WC_1, :setlocal_WC_1 - iseq = iseq.parent_iseq - when :getblockparam, :getblockparamproxy, :getlocal, :setlocal - insn[2].times { iseq = iseq.parent_iseq } - end - - # Here we need to map the local variable index to the offset - # from the top of the stack where it will be stored. - [insn[0], iseq.local_table.offset(insn[1]), *insn[2..]] - when :defineclass - [insn[0], insn[1], insn[2].to_a, insn[3]] - when :definemethod, :definesmethod - [insn[0], insn[1], insn[2].to_a] - when :send - # For any instructions that push instruction sequences onto the - # stack, we need to call #to_a on them as well. - [insn[0], insn[1], (insn[2].to_a if insn[2])] - when :once - [insn[0], insn[1].to_a, insn[2]] - else - insn - end - end end # These constants correspond to the putspecialobject instruction. They are @@ -819,34 +802,5 @@ def serialize(insn) VM_CALL_ZSUPER = 1 << 10 VM_CALL_OPT_SEND = 1 << 11 VM_CALL_KW_SPLAT_MUT = 1 << 12 - - # These constants correspond to the value passed as part of the defined - # instruction. It's an enum defined in the CRuby codebase that tells that - # instruction what kind of defined check to perform. - DEFINED_NIL = 1 - DEFINED_IVAR = 2 - DEFINED_LVAR = 3 - DEFINED_GVAR = 4 - DEFINED_CVAR = 5 - DEFINED_CONST = 6 - DEFINED_METHOD = 7 - DEFINED_YIELD = 8 - DEFINED_ZSUPER = 9 - DEFINED_SELF = 10 - DEFINED_TRUE = 11 - DEFINED_FALSE = 12 - DEFINED_ASGN = 13 - DEFINED_EXPR = 14 - DEFINED_REF = 15 - DEFINED_FUNC = 16 - DEFINED_CONST_FROM = 17 - - # These constants correspond to the value passed in the flags as part of - # the defineclass instruction. - VM_DEFINECLASS_TYPE_CLASS = 0 - VM_DEFINECLASS_TYPE_SINGLETON_CLASS = 1 - VM_DEFINECLASS_TYPE_MODULE = 2 - VM_DEFINECLASS_FLAG_SCOPED = 8 - VM_DEFINECLASS_FLAG_HAS_SUPERCLASS = 16 end end diff --git a/lib/syntax_tree/yarv/bf.rb b/lib/syntax_tree/yarv/bf.rb index 16098190..05c05705 100644 --- a/lib/syntax_tree/yarv/bf.rb +++ b/lib/syntax_tree/yarv/bf.rb @@ -5,460 +5,171 @@ module YARV # Parses the given source code into a syntax tree, compiles that syntax tree # into YARV bytecode. class Bf - class Node - def format(q) - Format.new(q).visit(self) - end - - def pretty_print(q) - PrettyPrint.new(q).visit(self) - end - end - - # The root node of the syntax tree. - class Root < Node - attr_reader :nodes, :location - - def initialize(nodes:, location:) - @nodes = nodes - @location = location - end - - def accept(visitor) - visitor.visit_root(self) - end - - def child_nodes - nodes - end - - alias deconstruct child_nodes - - def deconstruct_keys(keys) - { nodes: nodes, location: location } - end - end - - # [ ... ] - class Loop < Node - attr_reader :nodes, :location - - def initialize(nodes:, location:) - @nodes = nodes - @location = location - end - - def accept(visitor) - visitor.visit_loop(self) - end - - def child_nodes - nodes - end - - alias deconstruct child_nodes - - def deconstruct_keys(keys) - { nodes: nodes, location: location } - end - end - - # + - class Increment < Node - attr_reader :location - - def initialize(location:) - @location = location - end - - def accept(visitor) - visitor.visit_increment(self) - end - - def child_nodes - [] - end - - alias deconstruct child_nodes + attr_reader :source - def deconstruct_keys(keys) - { value: "+", location: location } - end + def initialize(source) + @source = source end - # - - class Decrement < Node - attr_reader :location - - def initialize(location:) - @location = location - end - - def accept(visitor) - visitor.visit_decrement(self) - end - - def child_nodes - [] - end - - alias deconstruct child_nodes + def compile + # Set up the top-level instruction sequence that will be returned. + iseq = InstructionSequence.new(:top, "", nil, location) + + # Set up the $tape global variable that will hold our state. + iseq.duphash({ 0 => 0 }) + iseq.setglobal(:$tape) + iseq.getglobal(:$tape) + iseq.putobject(0) + iseq.send(:default=, 1) + + # Set up the $cursor global variable that will hold the current position + # in the tape. + iseq.putobject(0) + iseq.setglobal(:$cursor) + + stack = [] + source + .each_char + .chunk do |char| + # For each character, we're going to assign a type to it. This + # allows a couple of optimizations to be made by combining multiple + # instructions into single instructions, e.g., +++ becomes a single + # change_by(3) instruction. + case char + when "+", "-" + :change + when ">", "<" + :shift + when "." + :output + when "," + :input + when "[", "]" + :loop + else + :ignored + end + end + .each do |type, chunk| + # For each chunk, we're going to emit the appropriate instruction. + case type + when :change + change_by(iseq, chunk.count("+") - chunk.count("-")) + when :shift + shift_by(iseq, chunk.count(">") - chunk.count("<")) + when :output + chunk.length.times { output_char(iseq) } + when :input + chunk.length.times { input_char(iseq) } + when :loop + chunk.each do |char| + case char + when "[" + stack << loop_start(iseq) + when "]" + loop_end(iseq, *stack.pop) + end + end + end + end - def deconstruct_keys(keys) - { value: "-", location: location } - end + iseq.leave + iseq end - # > - class ShiftRight < Node - attr_reader :location - - def initialize(location:) - @location = location - end - - def accept(visitor) - visitor.visit_shift_right(self) - end - - def child_nodes - [] - end - - alias deconstruct child_nodes + private - def deconstruct_keys(keys) - { value: ">", location: location } - end + # This is the location of the top instruction sequence, derived from the + # source string. + def location + Location.new( + start_line: 1, + start_char: 0, + start_column: 0, + end_line: source.count("\n") + 1, + end_char: source.size, + end_column: source.size - (source.rindex("\n") || 0) - 1 + ) end - # < - class ShiftLeft < Node - attr_reader :location - - def initialize(location:) - @location = location - end + # $tape[$cursor] += value + def change_by(iseq, value) + iseq.getglobal(:$tape) + iseq.getglobal(:$cursor) - def accept(visitor) - visitor.visit_shift_left(self) - end + iseq.getglobal(:$tape) + iseq.getglobal(:$cursor) + iseq.send(:[], 1) - def child_nodes - [] + if value < 0 + iseq.putobject(-value) + iseq.send(:-, 1) + else + iseq.putobject(value) + iseq.send(:+, 1) end - alias deconstruct child_nodes - - def deconstruct_keys(keys) - { value: "<", location: location } - end + iseq.send(:[]=, 2) end - # , - class Input < Node - attr_reader :location - - def initialize(location:) - @location = location - end + # $cursor += value + def shift_by(iseq, value) + iseq.getglobal(:$cursor) - def accept(visitor) - visitor.visit_input(self) + if value < 0 + iseq.putobject(-value) + iseq.send(:-, 1) + else + iseq.putobject(value) + iseq.send(:+, 1) end - def child_nodes - [] - end - - alias deconstruct child_nodes - - def deconstruct_keys(keys) - { value: ",", location: location } - end + iseq.setglobal(:$cursor) end - # . - class Output < Node - attr_reader :location + # $stdout.putc($tape[$cursor].chr) + def output_char(iseq) + iseq.getglobal(:$stdout) - def initialize(location:) - @location = location - end - - def accept(visitor) - visitor.visit_output(self) - end - - def child_nodes - [] - end - - alias deconstruct child_nodes - - def deconstruct_keys(keys) - { value: ".", location: location } - end - end + iseq.getglobal(:$tape) + iseq.getglobal(:$cursor) + iseq.send(:[], 1) + iseq.send(:chr, 0) - # Allows visiting the syntax tree recursively. - class Visitor - def visit(node) - node.accept(self) - end - - def visit_all(nodes) - nodes.map { |node| visit(node) } - end - - def visit_child_nodes(node) - visit_all(node.child_nodes) - end - - # Visit a Root node. - alias visit_root visit_child_nodes - - # Visit a Loop node. - alias visit_loop visit_child_nodes - - # Visit an Increment node. - alias visit_increment visit_child_nodes - - # Visit a Decrement node. - alias visit_decrement visit_child_nodes - - # Visit a ShiftRight node. - alias visit_shift_right visit_child_nodes - - # Visit a ShiftLeft node. - alias visit_shift_left visit_child_nodes - - # Visit an Input node. - alias visit_input visit_child_nodes - - # Visit an Output node. - alias visit_output visit_child_nodes + iseq.send(:putc, 1) end - # Compiles the syntax tree into YARV bytecode. - class Compiler < Visitor - attr_reader :iseq - - def initialize - @iseq = InstructionSequence.new(:top, "", nil, Location.default) - end - - def visit_decrement(node) - change_by(-1) - end - - def visit_increment(node) - change_by(1) - end - - def visit_input(node) - iseq.getglobal(:$tape) - iseq.getglobal(:$cursor) - iseq.getglobal(:$stdin) - iseq.send(:getc, 0) - iseq.send(:ord, 0) - iseq.send(:[]=, 2) - end - - def visit_loop(node) - start_label = iseq.label - - # First, we're going to compare the value at the current cursor to 0. - # If it's 0, then we'll jump past the loop. Otherwise we'll execute - # the loop. - iseq.getglobal(:$tape) - iseq.getglobal(:$cursor) - iseq.send(:[], 1) - iseq.putobject(0) - iseq.send(:==, 1) - branchunless = iseq.branchunless(-1) - - # Otherwise, here we'll execute the loop. - visit_nodes(node.nodes) - - # Now that we've visited all of the child nodes, we need to jump back - # to the start of the loop. - iseq.jump(start_label) - - # Now that we have all of the instructions in place, we can patch the - # branchunless to point to the next instruction for skipping the loop. - branchunless[1] = iseq.label - end - - def visit_output(node) - iseq.getglobal(:$stdout) - iseq.getglobal(:$tape) - iseq.getglobal(:$cursor) - iseq.send(:[], 1) - iseq.send(:chr, 0) - iseq.send(:putc, 1) - end - - def visit_root(node) - iseq.duphash({ 0 => 0 }) - iseq.setglobal(:$tape) - iseq.getglobal(:$tape) - iseq.putobject(0) - iseq.send(:default=, 1) - - iseq.putobject(0) - iseq.setglobal(:$cursor) - - visit_nodes(node.nodes) - - iseq.leave - iseq - end - - def visit_shift_left(node) - shift_by(-1) - end - - def visit_shift_right(node) - shift_by(1) - end - - private - - def change_by(value) - iseq.getglobal(:$tape) - iseq.getglobal(:$cursor) - iseq.getglobal(:$tape) - iseq.getglobal(:$cursor) - iseq.send(:[], 1) - - if value < 0 - iseq.putobject(-value) - iseq.send(:-, 1) - else - iseq.putobject(value) - iseq.send(:+, 1) - end - - iseq.send(:[]=, 2) - end - - def shift_by(value) - iseq.getglobal(:$cursor) - - if value < 0 - iseq.putobject(-value) - iseq.send(:-, 1) - else - iseq.putobject(value) - iseq.send(:+, 1) - end + # $tape[$cursor] = $stdin.getc.ord + def input_char(iseq) + iseq.getglobal(:$tape) + iseq.getglobal(:$cursor) - iseq.setglobal(:$cursor) - end + iseq.getglobal(:$stdin) + iseq.send(:getc, 0) + iseq.send(:ord, 0) - def visit_nodes(nodes) - nodes - .chunk do |child| - case child - when Increment, Decrement - :change - when ShiftLeft, ShiftRight - :shift - else - :default - end - end - .each do |type, children| - case type - when :change - value = 0 - children.each { |child| value += child.is_a?(Increment) ? 1 : -1 } - change_by(value) - when :shift - value = 0 - children.each { |child| value += child.is_a?(ShiftRight) ? 1 : -1 } - shift_by(value) - else - visit_all(children) - end - end - end + iseq.send(:[]=, 2) end - class Error < StandardError - end + # unless $tape[$cursor] == 0 + def loop_start(iseq) + start_label = iseq.label - attr_reader :source + iseq.getglobal(:$tape) + iseq.getglobal(:$cursor) + iseq.send(:[], 1) - def initialize(source) - @source = source - end + iseq.putobject(0) + iseq.send(:==, 1) - def compile - Root.new(nodes: parse_segment(source, 0), location: 0...source.length).accept(Compiler.new) + branchunless = iseq.branchunless(-1) + [start_label, branchunless] end - private - - def parse_segment(segment, offset) - index = 0 - nodes = [] - - while index < segment.length - location = offset + index - - case segment[index] - when "+" - nodes << Increment.new(location: location...(location + 1)) - index += 1 - when "-" - nodes << Decrement.new(location: location...(location + 1)) - index += 1 - when ">" - nodes << ShiftRight.new(location: location...(location + 1)) - index += 1 - when "<" - nodes << ShiftLeft.new(location: location...(location + 1)) - index += 1 - when "." - nodes << Output.new(location: location...(location + 1)) - index += 1 - when "," - nodes << Input.new(location: location...(location + 1)) - index += 1 - when "[" - matched = 1 - end_index = index + 1 - - while matched != 0 && end_index < segment.length - case segment[end_index] - when "[" - matched += 1 - when "]" - matched -= 1 - end - - end_index += 1 - end - - raise Error, "Unmatched start loop" if matched != 0 - - content = segment[(index + 1)...(end_index - 1)] - nodes << Loop.new( - nodes: parse_segment(content, offset + index + 1), - location: location...(offset + end_index) - ) - - index = end_index - when "]" - raise Error, "Unmatched end loop" - else - index += 1 - end - end - - nodes + # Jump back to the start of the loop. + def loop_end(iseq, start_label, branchunless) + iseq.jump(start_label) + branchunless.patch!(iseq) end end end diff --git a/lib/syntax_tree/yarv/disassembler.rb b/lib/syntax_tree/yarv/disassembler.rb index 566ed984..7a6e8893 100644 --- a/lib/syntax_tree/yarv/disassembler.rb +++ b/lib/syntax_tree/yarv/disassembler.rb @@ -5,15 +5,33 @@ module YARV # This class is responsible for taking a compiled instruction sequence and # walking through it to generate equivalent Ruby code. class Disassembler + # When we're disassmebling, we use a looped case statement to emulate + # jumping around in the same way the virtual machine would. This class + # provides convenience methods for generating the AST nodes that have to + # do with that label. + class DisasmLabel + include DSL + attr_reader :name + + def initialize(name) + @name = name + end + + def field + VarField(Ident(name)) + end + + def ref + VarRef(Ident(name)) + end + end + include DSL - attr_reader :iseq, :label_name, :label_field, :label_ref + attr_reader :iseq, :disasm_label def initialize(iseq) @iseq = iseq - - @label_name = "__disasm_label" - @label_field = VarField(Ident(label_name)) - @label_ref = VarRef(Ident(label_name)) + @disasm_label = DisasmLabel.new("__disasm_label") end def to_ruby @@ -37,143 +55,198 @@ def disassemble(iseq) clause = [] iseq.insns.each do |insn| - if insn.is_a?(Symbol) && insn.start_with?("label_") - clause << Assign(label_field, node_for(insn)) unless clause.last.is_a?(Next) - clauses[label] = clause - clause = [] - label = insn - next - end + case insn + when Symbol + if insn.start_with?("label_") + unless clause.last.is_a?(Next) + clause << Assign(disasm_label.field, node_for(insn)) + end + + clauses[label] = clause + clause = [] + label = insn + end + when BranchUnless + body = [ + Assign(disasm_label.field, node_for(insn.label)), + Next(Args([])) + ] - case insn[0] - when :branchunless - clause << IfNode(clause.pop, Statements([Assign(label_field, node_for(insn[1])), Next(Args([]))]), nil) - when :dup + clause << IfNode(clause.pop, Statements(body), nil) + when Dup clause << clause.last - when :duphash - assocs = insn[1].map { |key, value| Assoc(node_for(key), node_for(value)) } + when DupHash + assocs = + insn.object.map do |key, value| + Assoc(node_for(key), node_for(value)) + end + clause << HashLiteral(LBrace("{"), assocs) - when :getglobal - clause << VarRef(GVar(insn[1].to_s)) - when :getlocal_WC_0 - clause << VarRef(Ident(local_name(insn[1], 0))) - when :jump - clause << Assign(label_field, node_for(insn[1])) - clause << Next(Args([])) - when :leave - value = Args([clause.pop]) - clause << (iseq.type == :top ? Break(value) : ReturnNode(value)) - when :opt_and - left, right = clause.pop(2) - clause << Binary(left, :&, right) - when :opt_aref - collection, arg = clause.pop(2) - clause << ARef(collection, Args([arg])) - when :opt_aset - collection, arg, value = clause.pop(3) - - if value.is_a?(Binary) && value.left.is_a?(ARef) && collection === value.left.collection && arg === value.left.index.parts[0] - clause << OpAssign(ARefField(collection, Args([arg])), Op("#{value.operator}="), value.right) - else - clause << Assign(ARefField(collection, Args([arg])), value) - end - when :opt_div - left, right = clause.pop(2) - clause << Binary(left, :/, right) - when :opt_eq - left, right = clause.pop(2) - clause << Binary(left, :==, right) - when :opt_ge - left, right = clause.pop(2) - clause << Binary(left, :>=, right) - when :opt_gt - left, right = clause.pop(2) - clause << Binary(left, :>, right) - when :opt_le - left, right = clause.pop(2) - clause << Binary(left, :<=, right) - when :opt_lt - left, right = clause.pop(2) - clause << Binary(left, :<, right) - when :opt_ltlt - left, right = clause.pop(2) - clause << Binary(left, :<<, right) - when :opt_minus - left, right = clause.pop(2) - clause << Binary(left, :-, right) - when :opt_mod - left, right = clause.pop(2) - clause << Binary(left, :%, right) - when :opt_mult - left, right = clause.pop(2) - clause << Binary(left, :*, right) - when :opt_neq - left, right = clause.pop(2) - clause << Binary(left, :"!=", right) - when :opt_or - left, right = clause.pop(2) - clause << Binary(left, :|, right) - when :opt_plus - left, right = clause.pop(2) - clause << Binary(left, :+, right) - when :opt_send_without_block - if insn[1][:flag] & VM_CALL_FCALL > 0 - if insn[1][:orig_argc] == 0 - clause.pop - clause << CallNode(nil, nil, Ident(insn[1][:mid]), Args([])) - elsif insn[1][:orig_argc] == 1 && insn[1][:mid].end_with?("=") - _receiver, argument = clause.pop(2) - clause << Assign(CallNode(nil, nil, Ident(insn[1][:mid][0..-2]), nil), argument) + when GetGlobal + clause << VarRef(GVar(insn.name.to_s)) + when GetLocalWC0 + local = iseq.local_table.locals[insn.index] + clause << VarRef(Ident(local.name.to_s)) + when Array + case insn[0] + when :jump + clause << Assign(disasm_label.field, node_for(insn[1])) + clause << Next(Args([])) + when :leave + value = Args([clause.pop]) + clause << (iseq.type == :top ? Break(value) : ReturnNode(value)) + when :opt_and + left, right = clause.pop(2) + clause << Binary(left, :&, right) + when :opt_aref + collection, arg = clause.pop(2) + clause << ARef(collection, Args([arg])) + when :opt_aset + collection, arg, value = clause.pop(3) + + clause << if value.is_a?(Binary) && value.left.is_a?(ARef) && + collection === value.left.collection && + arg === value.left.index.parts[0] + OpAssign( + ARefField(collection, Args([arg])), + Op("#{value.operator}="), + value.right + ) else - _receiver, *arguments = clause.pop(insn[1][:orig_argc] + 1) - clause << CallNode(nil, nil, Ident(insn[1][:mid]), ArgParen(Args(arguments))) + Assign(ARefField(collection, Args([arg])), value) end - else - if insn[1][:orig_argc] == 0 - clause << CallNode(clause.pop, Period("."), Ident(insn[1][:mid]), nil) - elsif insn[1][:orig_argc] == 1 && insn[1][:mid].end_with?("=") - receiver, argument = clause.pop(2) - clause << Assign(CallNode(receiver, Period("."), Ident(insn[1][:mid][0..-2]), nil), argument) + when :opt_div + left, right = clause.pop(2) + clause << Binary(left, :/, right) + when :opt_eq + left, right = clause.pop(2) + clause << Binary(left, :==, right) + when :opt_ge + left, right = clause.pop(2) + clause << Binary(left, :>=, right) + when :opt_gt + left, right = clause.pop(2) + clause << Binary(left, :>, right) + when :opt_le + left, right = clause.pop(2) + clause << Binary(left, :<=, right) + when :opt_lt + left, right = clause.pop(2) + clause << Binary(left, :<, right) + when :opt_ltlt + left, right = clause.pop(2) + clause << Binary(left, :<<, right) + when :opt_minus + left, right = clause.pop(2) + clause << Binary(left, :-, right) + when :opt_mod + left, right = clause.pop(2) + clause << Binary(left, :%, right) + when :opt_mult + left, right = clause.pop(2) + clause << Binary(left, :*, right) + when :opt_neq + left, right = clause.pop(2) + clause << Binary(left, :"!=", right) + when :opt_or + left, right = clause.pop(2) + clause << Binary(left, :|, right) + when :opt_plus + left, right = clause.pop(2) + clause << Binary(left, :+, right) + when :opt_send_without_block + if insn[1][:flag] & VM_CALL_FCALL > 0 + if insn[1][:orig_argc] == 0 + clause.pop + clause << CallNode(nil, nil, Ident(insn[1][:mid]), Args([])) + elsif insn[1][:orig_argc] == 1 && insn[1][:mid].end_with?("=") + _receiver, argument = clause.pop(2) + clause << Assign( + CallNode(nil, nil, Ident(insn[1][:mid][0..-2]), nil), + argument + ) + else + _receiver, *arguments = clause.pop(insn[1][:orig_argc] + 1) + clause << CallNode( + nil, + nil, + Ident(insn[1][:mid]), + ArgParen(Args(arguments)) + ) + end else - receiver, *arguments = clause.pop(insn[1][:orig_argc] + 1) - clause << CallNode(receiver, Period("."), Ident(insn[1][:mid]), ArgParen(Args(arguments))) + if insn[1][:orig_argc] == 0 + clause << CallNode( + clause.pop, + Period("."), + Ident(insn[1][:mid]), + nil + ) + elsif insn[1][:orig_argc] == 1 && insn[1][:mid].end_with?("=") + receiver, argument = clause.pop(2) + clause << Assign( + CallNode( + receiver, + Period("."), + Ident(insn[1][:mid][0..-2]), + nil + ), + argument + ) + else + receiver, *arguments = clause.pop(insn[1][:orig_argc] + 1) + clause << CallNode( + receiver, + Period("."), + Ident(insn[1][:mid]), + ArgParen(Args(arguments)) + ) + end end - end - when :putobject - case insn[1] - when Float - clause << FloatLiteral(insn[1].inspect) - when Integer - clause << Int(insn[1].inspect) - else - raise "Unknown object type: #{insn[1].class.name}" - end - when :putobject_INT2FIX_0_ - clause << Int("0") - when :putobject_INT2FIX_1_ - clause << Int("1") - when :putself - clause << VarRef(Kw("self")) - when :setglobal - target = GVar(insn[1].to_s) - value = clause.pop - - if value.is_a?(Binary) && VarRef(target) === value.left - clause << OpAssign(VarField(target), Op("#{value.operator}="), value.right) - else - clause << Assign(VarField(target), value) - end - when :setlocal_WC_0 - target = Ident(local_name(insn[1], 0)) - value = clause.pop + when :putobject + case insn[1] + when Float + clause << FloatLiteral(insn[1].inspect) + when Integer + clause << Int(insn[1].inspect) + else + raise "Unknown object type: #{insn[1].class.name}" + end + when :putobject_INT2FIX_0_ + clause << Int("0") + when :putobject_INT2FIX_1_ + clause << Int("1") + when :putself + clause << VarRef(Kw("self")) + when :setglobal + target = GVar(insn[1].to_s) + value = clause.pop - if value.is_a?(Binary) && VarRef(target) === value.left - clause << OpAssign(VarField(target), Op("#{value.operator}="), value.right) + clause << if value.is_a?(Binary) && VarRef(target) === value.left + OpAssign( + VarField(target), + Op("#{value.operator}="), + value.right + ) + else + Assign(VarField(target), value) + end + when :setlocal_WC_0 + target = Ident(local_name(insn[1], 0)) + value = clause.pop + + clause << if value.is_a?(Binary) && VarRef(target) === value.left + OpAssign( + VarField(target), + Op("#{value.operator}="), + value.right + ) + else + Assign(VarField(target), value) + end else - clause << Assign(VarField(target), value) + raise "Unknown instruction #{insn[0]}" end - else - raise "Unknown instruction #{insn[0]}" end end @@ -185,31 +258,44 @@ def disassemble(iseq) # Here we're going to build up a big case statement that will handle all # of the different labels. current = nil - clauses.reverse_each do |label, clause| - current = When(Args([node_for(label)]), Statements(clause), current) + clauses.reverse_each do |current_label, current_clause| + current = + When( + Args([node_for(current_label)]), + Statements(current_clause), + current + ) end - switch = Case(Kw("case"), label_ref, current) + switch = Case(Kw("case"), disasm_label.ref, current) # Here we're going to make sure that any locals that were established in # the label_0 block are initialized so that scoping rules work # correctly. stack = [] - locals = [label_name] + locals = [disasm_label.name] clauses[:label_0].each do |node| - if node.is_a?(Assign) && node.target.is_a?(VarField) && node.target.value.is_a?(Ident) + if node.is_a?(Assign) && node.target.is_a?(VarField) && + node.target.value.is_a?(Ident) value = node.target.value.value next if locals.include?(value) stack << Assign(node.target, VarRef(Kw("nil"))) - locals << value + locals << value end end # Finally, we'll set up the initial label and loop the entire case # statement. - stack << Assign(label_field, node_for(:label_0)) - stack << MethodAddBlock(CallNode(nil, nil, Ident("loop"), Args([])), BlockNode(Kw("do"), nil, BodyStmt(Statements([switch]), nil, nil, nil, nil))) + stack << Assign(disasm_label.field, node_for(:label_0)) + stack << MethodAddBlock( + CallNode(nil, nil, Ident("loop"), Args([])), + BlockNode( + Kw("do"), + nil, + BodyStmt(Statements([switch]), nil, nil, nil, nil) + ) + ) Statements(stack) end diff --git a/lib/syntax_tree/yarv/instructions.rb b/lib/syntax_tree/yarv/instructions.rb new file mode 100644 index 00000000..c50c5c84 --- /dev/null +++ b/lib/syntax_tree/yarv/instructions.rb @@ -0,0 +1,1071 @@ +# frozen_string_literal: true + +module SyntaxTree + module YARV + # ### Summary + # + # `adjuststack` accepts a single integer argument and removes that many + # elements from the top of the stack. + # + # ### Usage + # + # ~~~ruby + # x = [true] + # x[0] ||= nil + # x[0] + # ~~~ + # + class AdjustStack + attr_reader :number + + def initialize(number) + @number = number + end + + def to_a(_iseq) + [:adjuststack, number] + end + + def length + 2 + end + + def pops + number + end + + def pushes + 0 + end + end + + # ### Summary + # + # `anytostring` ensures that the value on top of the stack is a string. + # + # It pops two values off the stack. If the first value is a string it + # pushes it back on the stack. If the first value is not a string, it uses + # Ruby's built in string coercion to coerce the second value to a string + # and then pushes that back on the stack. + # + # This is used in conjunction with `objtostring` as a fallback for when an + # object's `to_s` method does not return a string. + # + # ### Usage + # + # ~~~ruby + # "#{5}" + # ~~~ + # + class AnyToString + def to_a(_iseq) + [:anytostring] + end + + def length + 1 + end + + def pops + 2 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `branchif` has one argument: the jump index. It pops one value off the + # stack: the jump condition. + # + # If the value popped off the stack is true, `branchif` jumps to + # the jump index and continues executing there. + # + # ### Usage + # + # ~~~ruby + # x = true + # x ||= "foo" + # puts x + # ~~~ + # + class BranchIf + attr_reader :label + + def initialize(label) + @label = label + end + + def patch!(iseq) + @label = iseq.label + end + + def to_a(_iseq) + [:branchif, label] + end + + def length + 2 + end + + def pops + 1 + end + + def pushes + 0 + end + end + + # ### Summary + # + # `branchnil` has one argument: the jump index. It pops one value off the + # stack: the jump condition. + # + # If the value popped off the stack is nil, `branchnil` jumps to + # the jump index and continues executing there. + # + # ### Usage + # + # ~~~ruby + # x = nil + # if x&.to_s + # puts "hi" + # end + # ~~~ + # + class BranchNil + attr_reader :label + + def initialize(label) + @label = label + end + + def patch!(iseq) + @label = iseq.label + end + + def to_a(_iseq) + [:branchnil, label] + end + + def length + 2 + end + + def pops + 1 + end + + def pushes + 0 + end + end + + # ### Summary + # + # `branchunless` has one argument: the jump index. It pops one value off + # the stack: the jump condition. + # + # If the value popped off the stack is false or nil, `branchunless` jumps + # to the jump index and continues executing there. + # + # ### Usage + # + # ~~~ruby + # if 2 + 3 + # puts "foo" + # end + # ~~~ + # + class BranchUnless + attr_reader :label + + def initialize(label) + @label = label + end + + def patch!(iseq) + @label = iseq.label + end + + def to_a(_iseq) + [:branchunless, label] + end + + def length + 2 + end + + def pops + 1 + end + + def pushes + 0 + end + end + + # ### Summary + # + # `checkkeyword` checks if a keyword was passed at the callsite that + # called into the method represented by the instruction sequence. It has + # two arguments: the index of the local variable that stores the keywords + # metadata and the index of the keyword within that metadata. It pushes + # a boolean onto the stack indicating whether or not the keyword was + # given. + # + # ### Usage + # + # ~~~ruby + # def evaluate(value: rand) + # value + # end + # + # evaluate(value: 3) + # ~~~ + # + class CheckKeyword + attr_reader :keyword_bits_index, :keyword_index + + def initialize(keyword_bits_index, keyword_index) + @keyword_bits_index = keyword_bits_index + @keyword_index = keyword_index + end + + def patch!(iseq) + @label = iseq.label + end + + def to_a(iseq) + [ + :checkkeyword, + iseq.local_table.offset(keyword_bits_index), + keyword_index + ] + end + + def length + 3 + end + + def pops + 0 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `concatarray` concatenates the two Arrays on top of the stack. + # + # It coerces the two objects at the top of the stack into Arrays by + # calling `to_a` if necessary, and makes sure to `dup` the first Array if + # it was already an Array, to avoid mutating it when concatenating. + # + # ### Usage + # + # ~~~ruby + # [1, *2] + # ~~~ + # + class ConcatArray + def to_a(_iseq) + [:concatarray] + end + + def length + 1 + end + + def pops + 2 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `concatstrings` pops a number of strings from the stack joins them + # together into a single string and pushes that string back on the stack. + # + # This does no coercion and so is always used in conjunction with + # `objtostring` and `anytostring` to ensure the stack contents are always + # strings. + # + # ### Usage + # + # ~~~ruby + # "#{5}" + # ~~~ + # + class ConcatStrings + attr_reader :number + + def initialize(number) + @number = number + end + + def to_a(_iseq) + [:concatstrings, number] + end + + def length + 2 + end + + def pops + number + end + + def pushes + 1 + end + end + + # ### Summary + # + # `defined` checks if the top value of the stack is defined. If it is, it + # pushes its value onto the stack. Otherwise it pushes `nil`. + # + # ### Usage + # + # ~~~ruby + # defined?(x) + # ~~~ + # + class Defined + NIL = 1 + IVAR = 2 + LVAR = 3 + GVAR = 4 + CVAR = 5 + CONST = 6 + METHOD = 7 + YIELD = 8 + ZSUPER = 9 + SELF = 10 + TRUE = 11 + FALSE = 12 + ASGN = 13 + EXPR = 14 + REF = 15 + FUNC = 16 + CONST_FROM = 17 + + attr_reader :type, :name, :message + + def initialize(type, name, message) + @type = type + @name = name + @message = message + end + + def to_a(_iseq) + [:defined, type, name, message] + end + + def length + 4 + end + + def pops + 1 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `defineclass` defines a class. First it pops the superclass off the + # stack, then it pops the object off the stack that the class should be + # defined under. It has three arguments: the name of the constant, the + # instruction sequence associated with the class, and various flags that + # indicate if it is a singleton class, a module, or a regular class. + # + # ### Usage + # + # ~~~ruby + # class Foo + # end + # ~~~ + # + class DefineClass + TYPE_CLASS = 0 + TYPE_SINGLETON_CLASS = 1 + TYPE_MODULE = 2 + FLAG_SCOPED = 8 + FLAG_HAS_SUPERCLASS = 16 + + attr_reader :name, :class_iseq, :flags + + def initialize(name, class_iseq, flags) + @name = name + @class_iseq = class_iseq + @flags = flags + end + + def to_a(_iseq) + [:defineclass, name, class_iseq.to_a, flags] + end + + def length + 4 + end + + def pops + 2 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `definemethod` defines a method on the class of the current value of + # `self`. It accepts two arguments. The first is the name of the method + # being defined. The second is the instruction sequence representing the + # body of the method. + # + # ### Usage + # + # ~~~ruby + # def value = "value" + # ~~~ + # + class DefineMethod + attr_reader :name, :method_iseq + + def initialize(name, method_iseq) + @name = name + @method_iseq = method_iseq + end + + def to_a(_iseq) + [:definemethod, name, method_iseq.to_a] + end + + def length + 3 + end + + def pops + 0 + end + + def pushes + 0 + end + end + + # ### Summary + # + # `definesmethod` defines a method on the singleton class of the current + # value of `self`. It accepts two arguments. The first is the name of the + # method being defined. The second is the instruction sequence representing + # the body of the method. It pops the object off the stack that the method + # should be defined on. + # + # ### Usage + # + # ~~~ruby + # def self.value = "value" + # ~~~ + # + class DefineSMethod + attr_reader :name, :method_iseq + + def initialize(name, method_iseq) + @name = name + @method_iseq = method_iseq + end + + def to_a(_iseq) + [:definesmethod, name, method_iseq.to_a] + end + + def length + 3 + end + + def pops + 1 + end + + def pushes + 0 + end + end + + # ### Summary + # + # `dup` copies the top value of the stack and pushes it onto the stack. + # + # ### Usage + # + # ~~~ruby + # $global = 5 + # ~~~ + # + class Dup + def to_a(_iseq) + [:dup] + end + + def length + 1 + end + + def pops + 1 + end + + def pushes + 2 + end + end + + # ### Summary + # + # `duparray` dups an Array literal and pushes it onto the stack. + # + # ### Usage + # + # ~~~ruby + # [true] + # ~~~ + # + class DupArray + attr_reader :object + + def initialize(object) + @object = object + end + + def to_a(_iseq) + [:duparray, object] + end + + def length + 2 + end + + def pops + 0 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `duphash` dups a Hash literal and pushes it onto the stack. + # + # ### Usage + # + # ~~~ruby + # { a: 1 } + # ~~~ + # + class DupHash + attr_reader :object + + def initialize(object) + @object = object + end + + def to_a(_iseq) + [:duphash, object] + end + + def length + 2 + end + + def pops + 0 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `dupn` duplicates the top `n` stack elements. + # + # ### Usage + # + # ~~~ruby + # Object::X ||= true + # ~~~ + # + class DupN + attr_reader :number + + def initialize(number) + @number = number + end + + def to_a(_iseq) + [:dupn, number] + end + + def length + 2 + end + + def pops + number + end + + def pushes + number * 2 + end + end + + # ### Summary + # + # `expandarray` looks at the top of the stack, and if the value is an array + # it replaces it on the stack with `number` elements of the array, or `nil` + # if the elements are missing. + # + # ### Usage + # + # ~~~ruby + # x, = [true, false, nil] + # ~~~ + # + class ExpandArray + attr_reader :number, :flags + + def initialize(number, flags) + @number = number + @flags = flags + end + + def to_a(_iseq) + [:expandarray, number, flags] + end + + def length + 3 + end + + def pops + 1 + end + + def pushes + number + end + end + + # ### Summary + # + # `getblockparam` is a similar instruction to `getlocal` in that it looks + # for a local variable in the current instruction sequence's local table and + # walks recursively up the parent instruction sequences until it finds it. + # The local it retrieves, however, is a special block local that was passed + # to the current method. It pushes the value of the block local onto the + # stack. + # + # ### Usage + # + # ~~~ruby + # def foo(&block) + # block + # end + # ~~~ + # + class GetBlockParam + attr_reader :index, :level + + def initialize(index, level) + @index = index + @level = level + end + + def to_a(iseq) + current = iseq + level.times { current = iseq.parent_iseq } + [:getblockparam, current.local_table.offset(index), level] + end + + def length + 3 + end + + def pops + 0 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `getblockparamproxy` is almost the same as `getblockparam` except that it + # pushes a proxy object onto the stack instead of the actual value of the + # block local. This is used when a method is being called on the block + # local. + # + # ### Usage + # + # ~~~ruby + # def foo(&block) + # block.call + # end + # ~~~ + # + class GetBlockParamProxy + attr_reader :index, :level + + def initialize(index, level) + @index = index + @level = level + end + + def to_a(iseq) + current = iseq + level.times { current = iseq.parent_iseq } + [:getblockparamproxy, current.local_table.offset(index), level] + end + + def length + 3 + end + + def pops + 0 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `getclassvariable` looks for a class variable in the current class and + # pushes its value onto the stack. It uses an inline cache to reduce the + # need to lookup the class variable in the class hierarchy every time. + # + # ### Usage + # + # ~~~ruby + # @@class_variable + # ~~~ + # + class GetClassVariable + attr_reader :name, :cache + + def initialize(name, cache) + @name = name + @cache = cache + end + + def to_a(_iseq) + [:getclassvariable, name, cache] + end + + def length + 3 + end + + def pops + 0 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `getclassvariable` looks for a class variable in the current class and + # pushes its value onto the stack. + # + # This version of the `getclassvariable` instruction is no longer used since + # in Ruby 3.0 it gained an inline cache.` + # + # ### Usage + # + # ~~~ruby + # @@class_variable + # ~~~ + # + class GetClassVariableUncached + attr_reader :name + + def initialize(name) + @name = name + end + + def to_a(_iseq) + [:getclassvariable, name] + end + + def length + 2 + end + + def pops + 0 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `getconstant` performs a constant lookup and pushes the value of the + # constant onto the stack. It pops both the class it should look in and + # whether or not it should look globally as well. + # + # This instruction is no longer used since in Ruby 3.2 it was replaced by + # the consolidated `opt_getconstant_path` instruction. + # + # ### Usage + # + # ~~~ruby + # Constant + # ~~~ + # + class GetConstant + attr_reader :name + + def initialize(name) + @name = name + end + + def to_a(_iseq) + [:getconstant, name] + end + + def length + 2 + end + + def pops + 2 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `getglobal` pushes the value of a global variables onto the stack. + # + # ### Usage + # + # ~~~ruby + # $$ + # ~~~ + # + class GetGlobal + attr_reader :name + + def initialize(name) + @name = name + end + + def to_a(_iseq) + [:getglobal, name] + end + + def length + 2 + end + + def pops + 0 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `getinstancevariable` pushes the value of an instance variable onto the + # stack. It uses an inline cache to avoid having to look up the instance + # variable in the class hierarchy every time. + # + # This instruction has two forms, but both have the same structure. Before + # Ruby 3.2, the inline cache corresponded to both the get and set + # instructions and could be shared. Since Ruby 3.2, it uses object shapes + # instead so the caches are unique per instruction. + # + # ### Usage + # + # ~~~ruby + # @instance_variable + # ~~~ + # + class GetInstanceVariable + attr_reader :name, :cache + + def initialize(name, cache) + @name = name + @cache = cache + end + + def to_a(_iseq) + [:getinstancevariable, name, cache] + end + + def length + 3 + end + + def pops + 0 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `getlocal_WC_0` is a specialized version of the `getlocal` instruction. It + # fetches the value of a local variable from the current frame determined by + # the index given as its only argument. + # + # ### Usage + # + # ~~~ruby + # value = 5 + # value + # ~~~ + # + class GetLocalWC0 + attr_reader :index + + def initialize(index) + @index = index + end + + def to_a(iseq) + [:getlocal_WC_0, iseq.local_table.offset(index)] + end + + def length + 2 + end + + def pops + 0 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `getlocal_WC_1` is a specialized version of the `getlocal` instruction. It + # fetches the value of a local variable from the parent frame determined by + # the index given as its only argument. + # + # ### Usage + # + # ~~~ruby + # value = 5 + # self.then { value } + # ~~~ + # + class GetLocalWC1 + attr_reader :index + + def initialize(index) + @index = index + end + + def to_a(iseq) + [:getlocal_WC_1, iseq.parent_iseq.local_table.offset(index)] + end + + def length + 2 + end + + def pops + 0 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `getlocal` fetches the value of a local variable from a frame determined + # by the level and index arguments. The level is the number of frames back + # to look and the index is the index in the local table. It pushes the value + # it finds onto the stack. + # + # ### Usage + # + # ~~~ruby + # value = 5 + # tap { tap { value } } + # ~~~ + # + class GetLocal + attr_reader :index, :level + + def initialize(index, level) + @index = index + @level = level + end + + def to_a(iseq) + current = iseq + level.times { current = current.parent_iseq } + [:getlocal, current.local_table.offset(index), level] + end + + def length + 3 + end + + def pops + 0 + end + + def pushes + 1 + end + end + end +end diff --git a/test/yarv_test.rb b/test/yarv_test.rb index da348224..55cdb657 100644 --- a/test/yarv_test.rb +++ b/test/yarv_test.rb @@ -26,7 +26,7 @@ class YARVTest < Minitest::Test "1 << 2" => "break 1 << 2\n", "1 >> 2" => "break 1.>>(2)\n", "1 ** 2" => "break 1.**(2)\n", - "a = 1; a" => "a = 1\nbreak a\n", + "a = 1; a" => "a = 1\nbreak a\n" }.freeze CASES.each do |source, expected| @@ -35,6 +35,15 @@ class YARVTest < Minitest::Test end end + def test_bf + hello_world = + "++++++++[>++++[>++>+++>+++>+<<<<-]>+>+>->>+[<]<-]" \ + ">>.>---.+++++++..+++.>>.<-.<.+++.------.--------.>>+.>++." + + iseq = YARV::Bf.new(hello_world).compile + Formatter.format(hello_world, YARV::Disassembler.new(iseq).to_ruby) + end + private def assert_disassembles(expected, source) From 441bc01d9f68e07c3acd891c915f950652f70176 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 21 Nov 2022 22:00:09 -0500 Subject: [PATCH 262/536] opt_aref_with --- lib/syntax_tree/compiler.rb | 29 +++++++++++++++++++++++++++++ lib/syntax_tree/yarv.rb | 5 +++++ test/compiler_test.rb | 1 + 3 files changed, 35 insertions(+) diff --git a/lib/syntax_tree/compiler.rb b/lib/syntax_tree/compiler.rb index 8327a080..106c3ca3 100644 --- a/lib/syntax_tree/compiler.rb +++ b/lib/syntax_tree/compiler.rb @@ -158,6 +158,21 @@ def visit_tstring_content(node) node.value end + def visit_var_ref(node) + raise CompilationError unless node.value.is_a?(Kw) + + case node.value.value + when "nil" + nil + when "true" + true + when "false" + false + else + raise CompilationError + end + end + def visit_word(node) if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) node.parts.first.value @@ -258,6 +273,20 @@ def visit_alias(node) def visit_aref(node) visit(node.collection) + + if !frozen_string_literal && specialized_instruction && (node.index.parts.length == 1) + arg = node.index.parts.first + + if arg.is_a?(StringLiteral) && (arg.parts.length == 1) + string_part = arg.parts.first + + if string_part.is_a?(TStringContent) + iseq.opt_aref_with(string_part.value, :[], 1) + return + end + end + end + visit(node.index) iseq.send(:[], 1) end diff --git a/lib/syntax_tree/yarv.rb b/lib/syntax_tree/yarv.rb index a29714a5..57a21f2c 100644 --- a/lib/syntax_tree/yarv.rb +++ b/lib/syntax_tree/yarv.rb @@ -527,6 +527,11 @@ def once(postexe_iseq, inline_storage) push([:once, postexe_iseq, inline_storage]) end + def opt_aref_with(object, method_id, argc, flag = VM_CALL_ARGS_SIMPLE) + stack.change_by(-1 + 1) + push([:opt_aref_with, object, call_data(method_id, argc, flag)]) + end + def opt_getconstant_path(names) if RUBY_VERSION >= "3.2" stack.change_by(+1) diff --git a/test/compiler_test.rb b/test/compiler_test.rb index 27bf993d..485e92fc 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -201,6 +201,7 @@ class CompilerTest < Minitest::Test "foo[bar] ||= 1", "foo[bar] <<= 1", "foo[bar] ^= 1", + "foo['true']", # Constants (single) "Foo", "Foo = 1", From cc24d7f4198beb08cb3c37e244535afee013554b Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 21 Nov 2022 22:04:03 -0500 Subject: [PATCH 263/536] opt_aset_with --- lib/syntax_tree/compiler.rb | 18 ++++++++++++++++++ lib/syntax_tree/yarv.rb | 5 +++++ test/compiler_test.rb | 1 + 3 files changed, 24 insertions(+) diff --git a/lib/syntax_tree/compiler.rb b/lib/syntax_tree/compiler.rb index 106c3ca3..91ec3d30 100644 --- a/lib/syntax_tree/compiler.rb +++ b/lib/syntax_tree/compiler.rb @@ -337,6 +337,24 @@ def visit_array(node) def visit_assign(node) case node.target when ARefField + if !frozen_string_literal && specialized_instruction && (node.target.index.parts.length == 1) + arg = node.target.index.parts.first + + if arg.is_a?(StringLiteral) && (arg.parts.length == 1) + string_part = arg.parts.first + + if string_part.is_a?(TStringContent) + visit(node.target.collection) + visit(node.value) + iseq.swap + iseq.topn(1) + iseq.opt_aset_with(string_part.value, :[]=, 2) + iseq.pop + return + end + end + end + iseq.putnil visit(node.target.collection) visit(node.target.index) diff --git a/lib/syntax_tree/yarv.rb b/lib/syntax_tree/yarv.rb index 57a21f2c..0c4c3fc9 100644 --- a/lib/syntax_tree/yarv.rb +++ b/lib/syntax_tree/yarv.rb @@ -532,6 +532,11 @@ def opt_aref_with(object, method_id, argc, flag = VM_CALL_ARGS_SIMPLE) push([:opt_aref_with, object, call_data(method_id, argc, flag)]) end + def opt_aset_with(object, method_id, argc, flag = VM_CALL_ARGS_SIMPLE) + stack.change_by(-2 + 1) + push([:opt_aset_with, object, call_data(method_id, argc, flag)]) + end + def opt_getconstant_path(names) if RUBY_VERSION >= "3.2" stack.change_by(+1) diff --git a/test/compiler_test.rb b/test/compiler_test.rb index 485e92fc..98559664 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -202,6 +202,7 @@ class CompilerTest < Minitest::Test "foo[bar] <<= 1", "foo[bar] ^= 1", "foo['true']", + "foo['true'] = 1", # Constants (single) "Foo", "Foo = 1", From 5bd3463db4f0c4b24fb7068c73be802c7b49e9fe Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 21 Nov 2022 22:08:38 -0500 Subject: [PATCH 264/536] setblockparam --- lib/syntax_tree/compiler.rb | 9 +++++++-- lib/syntax_tree/yarv.rb | 9 +++++++-- test/compiler_test.rb | 1 + 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/lib/syntax_tree/compiler.rb b/lib/syntax_tree/compiler.rb index 91ec3d30..8e1a0eaf 100644 --- a/lib/syntax_tree/compiler.rb +++ b/lib/syntax_tree/compiler.rb @@ -415,8 +415,13 @@ def visit_assign(node) when GVar iseq.setglobal(node.target.value.value.to_sym) when Ident - local_variable = visit(node.target) - iseq.setlocal(local_variable.index, local_variable.level) + lookup = visit(node.target) + + if lookup.local.is_a?(YARV::LocalTable::BlockLocal) + iseq.setblockparam(lookup.index, lookup.level) + else + iseq.setlocal(lookup.index, lookup.level) + end when IVar iseq.setinstancevariable(node.target.value.value.to_sym) end diff --git a/lib/syntax_tree/yarv.rb b/lib/syntax_tree/yarv.rb index 0c4c3fc9..a204989e 100644 --- a/lib/syntax_tree/yarv.rb +++ b/lib/syntax_tree/yarv.rb @@ -252,13 +252,13 @@ def to_a insn when Array case insn[0] - when :setlocal_WC_0, :setlocal_WC_1, :setlocal + when :setlocal_WC_0, :setlocal_WC_1, :setlocal, :setblockparam iseq = self case insn[0] when :setlocal_WC_1 iseq = iseq.parent_iseq - when :setlocal + when :setlocal, :setblockparam insn[2].times { iseq = iseq.parent_iseq } end @@ -704,6 +704,11 @@ def send(method_id, argc, flag = VM_CALL_ARGS_SIMPLE, block_iseq = nil) end end + def setblockparam(index, level) + stack.change_by(-1) + push([:setblockparam, index, level]) + end + def setclassvariable(name) stack.change_by(-1) diff --git a/test/compiler_test.rb b/test/compiler_test.rb index 98559664..56e38577 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -361,6 +361,7 @@ class CompilerTest < Minitest::Test "def foo(bar, *baz, &qux); end", "def foo(&qux); qux; end", "def foo(&qux); qux.call; end", + "def foo(&qux); qux = bar; end", "def foo(bar:); end", "def foo(bar:, baz:); end", "def foo(bar: 1); end", From f35c452221590d1f3dcea49e99d2992d674952e6 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 21 Nov 2022 22:21:32 -0500 Subject: [PATCH 265/536] setspecial --- lib/syntax_tree/compiler.rb | 52 +++++++++++++++++++++++++++---------- lib/syntax_tree/yarv.rb | 10 +++++++ test/compiler_test.rb | 1 + 3 files changed, 49 insertions(+), 14 deletions(-) diff --git a/lib/syntax_tree/compiler.rb b/lib/syntax_tree/compiler.rb index 8e1a0eaf..3a4af3da 100644 --- a/lib/syntax_tree/compiler.rb +++ b/lib/syntax_tree/compiler.rb @@ -438,7 +438,7 @@ def visit_assoc_splat(node) end def visit_backref(node) - iseq.getspecial(1, 2 * node.value[1..].to_i) + iseq.getspecial(YARV::VM_SVAR_BACKREF, 2 * node.value[1..].to_i) end def visit_bare_assoc_hash(node) @@ -888,25 +888,49 @@ def visit_heredoc(node) end def visit_if(node) - visit(node.predicate) - branchunless = iseq.branchunless(-1) - visit(node.statements) + if node.predicate.is_a?(RangeNode) + iseq.getspecial(YARV::VM_SVAR_FLIPFLOP_START, 0) + branchif = iseq.branchif(-1) - if last_statement? - iseq.leave - branchunless.patch!(iseq) + visit(node.predicate.left) + branchunless_true = iseq.branchunless(-1) - node.consequent ? visit(node.consequent) : iseq.putnil + iseq.putobject(true) + iseq.setspecial(YARV::VM_SVAR_FLIPFLOP_START) + branchif.patch!(iseq) + + visit(node.predicate.right) + branchunless_false = iseq.branchunless(-1) + + iseq.putobject(false) + iseq.setspecial(YARV::VM_SVAR_FLIPFLOP_START) + branchunless_false.patch!(iseq) + + visit(node.statements) + iseq.leave + branchunless_true.patch!(iseq) + iseq.putnil else - iseq.pop + visit(node.predicate) + branchunless = iseq.branchunless(-1) + visit(node.statements) - if node.consequent - jump = iseq.jump(-1) + if last_statement? + iseq.leave branchunless.patch!(iseq) - visit(node.consequent) - jump[1] = iseq.label + + node.consequent ? visit(node.consequent) : iseq.putnil else - branchunless.patch!(iseq) + iseq.pop + + if node.consequent + jump = iseq.jump(-1) + branchunless.patch!(iseq) + visit(node.consequent) + jump[1] = iseq.label + else + branchunless.patch!(iseq) + end end end end diff --git a/lib/syntax_tree/yarv.rb b/lib/syntax_tree/yarv.rb index a204989e..6056fded 100644 --- a/lib/syntax_tree/yarv.rb +++ b/lib/syntax_tree/yarv.rb @@ -765,6 +765,11 @@ def setn(number) push([:setn, number]) end + def setspecial(key) + stack.change_by(-1) + push([:setspecial, key]) + end + def splatarray(flag) stack.change_by(-1 + 1) push([:splatarray, flag]) @@ -817,5 +822,10 @@ def call_data(method_id, argc, flag = VM_CALL_ARGS_SIMPLE) VM_CALL_ZSUPER = 1 << 10 VM_CALL_OPT_SEND = 1 << 11 VM_CALL_KW_SPLAT_MUT = 1 << 12 + + # These constants correspond to the setspecial instruction. + VM_SVAR_LASTLINE = 0 # $_ + VM_SVAR_BACKREF = 1 # $~ + VM_SVAR_FLIPFLOP_START = 2 # flipflop end end diff --git a/test/compiler_test.rb b/test/compiler_test.rb index 56e38577..c1dab39c 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -285,6 +285,7 @@ class CompilerTest < Minitest::Test "foo ? bar : baz", "case foo when bar then 1 end", "case foo when bar then 1 else 2 end", + "baz if (foo == 1) .. (bar == 1)", # Constructed values "foo..bar", "foo...bar", From 1262b52c781d35df4c911d87ed47be2322812b0d Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 21 Nov 2022 22:33:43 -0500 Subject: [PATCH 266/536] newarraykwsplat --- lib/syntax_tree/compiler.rb | 9 +++++++++ lib/syntax_tree/yarv.rb | 5 +++++ test/compiler_test.rb | 1 + 3 files changed, 15 insertions(+) diff --git a/lib/syntax_tree/compiler.rb b/lib/syntax_tree/compiler.rb index 3a4af3da..1b2c5987 100644 --- a/lib/syntax_tree/compiler.rb +++ b/lib/syntax_tree/compiler.rb @@ -311,6 +311,15 @@ def visit_args(node) def visit_array(node) if (compiled = RubyVisitor.compile(node)) iseq.duparray(compiled) + elsif node.contents && node.contents.parts.length == 1 && + node.contents.parts.first.is_a?(BareAssocHash) && + node.contents.parts.first.assocs.length == 1 && + node.contents.parts.first.assocs.first.is_a?(AssocSplat) + iseq.putspecialobject(YARV::VM_SPECIAL_OBJECT_VMCORE) + iseq.newhash(0) + visit(node.contents.parts.first) + iseq.send(:"core#hash_merge_kwd", 2) + iseq.newarraykwsplat(1) else length = 0 diff --git a/lib/syntax_tree/yarv.rb b/lib/syntax_tree/yarv.rb index 6056fded..b168a135 100644 --- a/lib/syntax_tree/yarv.rb +++ b/lib/syntax_tree/yarv.rb @@ -502,6 +502,11 @@ def newarray(length) push([:newarray, length]) end + def newarraykwsplat(length) + stack.change_by(-length + 1) + push([:newarraykwsplat, length]) + end + def newhash(length) stack.change_by(-length + 1) push([:newhash, length]) diff --git a/test/compiler_test.rb b/test/compiler_test.rb index c1dab39c..d44eef50 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -311,6 +311,7 @@ class CompilerTest < Minitest::Test "[1, 2, 3].min", "[foo, bar, baz].min", "[foo, bar, baz].min(1)", + "[**{ x: true }][0][:x]", # Core method calls "alias foo bar", "alias :foo :bar", From d4d7f0b4a65e94dc98b434ceec2c805fe62e8f1c Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 22 Nov 2022 10:38:43 -0500 Subject: [PATCH 267/536] Pattern match for arrays --- lib/syntax_tree/compiler.rb | 122 +++++++++++++++++++++++++++ lib/syntax_tree/yarv.rb | 18 ++++ lib/syntax_tree/yarv/instructions.rb | 4 +- test/compiler_test.rb | 8 +- 4 files changed, 149 insertions(+), 3 deletions(-) diff --git a/lib/syntax_tree/compiler.rb b/lib/syntax_tree/compiler.rb index 1b2c5987..ac49f7e0 100644 --- a/lib/syntax_tree/compiler.rb +++ b/lib/syntax_tree/compiler.rb @@ -343,6 +343,101 @@ def visit_array(node) end end + def visit_aryptn(node) + match_failures = [] + jumps_to_exit = [] + + # If there's a constant, then check if we match against that constant or + # not first. Branch to failure if we don't. + if node.constant + iseq.dup + visit(node.constant) + iseq.checkmatch(YARV::VM_CHECKMATCH_TYPE_CASE) + match_failures << iseq.branchunless(-1) + end + + # First, check if the #deconstruct cache is nil. If it is, we're going to + # call #deconstruct on the object and cache the result. + iseq.topn(2) + branchnil = iseq.branchnil(-1) + + # Next, ensure that the cached value was cached correctly, otherwise fail + # the match. + iseq.topn(2) + match_failures << iseq.branchunless(-1) + + # Since we have a valid cached value, we can skip past the part where we + # call #deconstruct on the object. + iseq.pop + iseq.topn(1) + jump = iseq.jump(-1) + + # Check if the object responds to #deconstruct, fail the match otherwise. + branchnil.patch!(iseq) + iseq.dup + iseq.putobject(:deconstruct) + iseq.send(:respond_to?, 1) + iseq.setn(3) + match_failures << iseq.branchunless(-1) + + # Call #deconstruct and ensure that it's an array, raise an error + # otherwise. + iseq.send(:deconstruct, 0) + iseq.setn(2) + iseq.dup + iseq.checktype(YARV::VM_CHECKTYPE_ARRAY) + match_error = iseq.branchunless(-1) + + # Ensure that the deconstructed array has the correct size, fail the match + # otherwise. + jump[1] = iseq.label + iseq.dup + iseq.send(:length, 0) + iseq.putobject(node.requireds.length) + iseq.send(:==, 1) + match_failures << iseq.branchunless(-1) + + # For each required element, check if the deconstructed array contains the + # element, otherwise jump out to the top-level match failure. + iseq.dup + node.requireds.each_with_index do |required, index| + iseq.putobject(index) + iseq.send(:[], 1) + + case required + when VarField + lookup = visit(required) + iseq.setlocal(lookup.index, lookup.level) + else + visit(required) + iseq.checkmatch(YARV::VM_CHECKMATCH_TYPE_CASE) + match_failures << iseq.branchunless(-1) + end + + if index < node.requireds.length - 1 + iseq.dup + else + iseq.pop + jumps_to_exit << iseq.jump(-1) + end + end + + # Set up the routine here to raise an error to indicate that the type of + # the deconstructed array was incorrect. + match_error.patch!(iseq) + iseq.putspecialobject(YARV::VM_SPECIAL_OBJECT_VMCORE) + iseq.putobject(TypeError) + iseq.putobject("deconstruct must return Array") + iseq.send(:"core#raise", 2) + iseq.pop + + # Patch all of the match failures to jump here so that we pop a final + # value before returning to the parent node. + match_failures.each { |match_failure| match_failure.patch!(iseq) } + iseq.pop + jumps_to_exit + end + def visit_assign(node) case node.target when ARefField @@ -1298,6 +1393,33 @@ def visit_range(node) end end + def visit_rassign(node) + if node.operator.is_a?(Kw) + iseq.putnil + visit(node.value) + iseq.dup + jumps = [] + + case node.pattern + when VarField + lookup = visit(node.pattern) + iseq.setlocal(lookup.index, lookup.level) + jumps << iseq.jump(-1) + else + jumps.concat(visit(node.pattern)) + end + + iseq.pop + iseq.pop + iseq.putobject(false) + iseq.leave + + jumps.each { |jump| jump[1] = iseq.label } + iseq.adjuststack(2) + iseq.putobject(true) + end + end + def visit_rational(node) iseq.putobject(node.accept(RubyVisitor.new)) end diff --git a/lib/syntax_tree/yarv.rb b/lib/syntax_tree/yarv.rb index b168a135..2ca29de7 100644 --- a/lib/syntax_tree/yarv.rb +++ b/lib/syntax_tree/yarv.rb @@ -370,6 +370,16 @@ def checkkeyword(keyword_bits_index, keyword_index) push(CheckKeyword.new(keyword_bits_index, keyword_index)) end + def checkmatch(flag) + stack.change_by(-2 + 1) + push([:checkmatch, flag]) + end + + def checktype(type) + stack.change_by(-1 + 2) + push([:checktype, type]) + end + def concatarray push(ConcatArray.new) end @@ -832,5 +842,13 @@ def call_data(method_id, argc, flag = VM_CALL_ARGS_SIMPLE) VM_SVAR_LASTLINE = 0 # $_ VM_SVAR_BACKREF = 1 # $~ VM_SVAR_FLIPFLOP_START = 2 # flipflop + + # These constants correspond to the checktype instruction. + VM_CHECKTYPE_ARRAY = 7 + + # These constants correspond to the checkmatch instruction. + VM_CHECKMATCH_TYPE_WHEN = 1 + VM_CHECKMATCH_TYPE_CASE = 2 + VM_CHECKMATCH_TYPE_RESCUE = 3 end end diff --git a/lib/syntax_tree/yarv/instructions.rb b/lib/syntax_tree/yarv/instructions.rb index c50c5c84..ccb7a345 100644 --- a/lib/syntax_tree/yarv/instructions.rb +++ b/lib/syntax_tree/yarv/instructions.rb @@ -632,11 +632,11 @@ def length end def pops - number + 0 end def pushes - number * 2 + number end end diff --git a/test/compiler_test.rb b/test/compiler_test.rb index d44eef50..4f4fa9f3 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -416,7 +416,13 @@ class CompilerTest < Minitest::Test "-> {}", "-> (bar) do end", "-> (bar) {}", - "-> (bar; baz) { }" + "-> (bar; baz) { }", + # Pattern matching + "foo in bar", + "foo in [bar]", + "foo in [bar, baz]", + "foo in [1, 2, 3, bar, 4, 5, 6, baz]", + "foo in Foo[1, 2, 3, bar, 4, 5, 6, baz]", ] # These are the combinations of instructions that we're going to test. From 5abcb5a646fc3d4a9f22c2de085dc162e53b8ebd Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 22 Nov 2022 11:01:49 -0500 Subject: [PATCH 268/536] Handle => operator for rightward assignment --- lib/syntax_tree/compiler.rb | 77 ++++++++++++++++++++++++++++++++++++- test/compiler_test.rb | 1 + 2 files changed, 76 insertions(+), 2 deletions(-) diff --git a/lib/syntax_tree/compiler.rb b/lib/syntax_tree/compiler.rb index ac49f7e0..4050f4c9 100644 --- a/lib/syntax_tree/compiler.rb +++ b/lib/syntax_tree/compiler.rb @@ -1394,11 +1394,13 @@ def visit_range(node) end def visit_rassign(node) + iseq.putnil + if node.operator.is_a?(Kw) - iseq.putnil + jumps = [] + visit(node.value) iseq.dup - jumps = [] case node.pattern when VarField @@ -1417,6 +1419,77 @@ def visit_rassign(node) jumps.each { |jump| jump[1] = iseq.label } iseq.adjuststack(2) iseq.putobject(true) + else + jumps_to_match = [] + + iseq.putnil + iseq.putobject(false) + iseq.putnil + iseq.putnil + visit(node.value) + iseq.dup + + # Visit the pattern. If it matches, + case node.pattern + when VarField + lookup = visit(node.pattern) + iseq.setlocal(lookup.index, lookup.level) + jumps_to_match << iseq.jump(-1) + else + jumps_to_match.concat(visit(node.pattern)) + end + + # First we're going to push the core onto the stack, then we'll check if + # the value to match is truthy. If it is, we'll jump down to raise + # NoMatchingPatternKeyError. Otherwise we'll raise + # NoMatchingPatternError. + iseq.putspecialobject(YARV::VM_SPECIAL_OBJECT_VMCORE) + iseq.topn(4) + branchif_no_key = iseq.branchif(-1) + + # Here we're going to raise NoMatchingPatternError. + iseq.putobject(NoMatchingPatternError) + iseq.putspecialobject(YARV::VM_SPECIAL_OBJECT_VMCORE) + iseq.putobject("%p: %s") + iseq.topn(4) + iseq.topn(7) + iseq.send(:"core#sprintf", 3) + iseq.send(:"core#raise", 2) + jump_to_exit = iseq.jump(-1) + + # Here we're going to raise NoMatchingPatternKeyError. + branchif_no_key.patch!(iseq) + iseq.putobject(NoMatchingPatternKeyError) + iseq.putspecialobject(YARV::VM_SPECIAL_OBJECT_VMCORE) + iseq.putobject("%p: %s") + iseq.topn(4) + iseq.topn(7) + iseq.send(:"core#sprintf", 3) + iseq.topn(7) + iseq.topn(9) + + # Super special behavior here because of the weird kw_arg handling. + iseq.stack.change_by(-(1 + 1) + 1) + call_data = { mid: :new, flag: YARV::VM_CALL_KWARG, orig_argc: 1, kw_arg: [:matchee, :key] } + + if specialized_instruction + iseq.push([:opt_send_without_block, call_data]) + else + iseq.push([:send, call_data, nil]) + end + + iseq.send(:"core#raise", 1) + + # This runs when the pattern fails to match. + jump_to_exit[1] = iseq.label + iseq.adjuststack(7) + iseq.putnil + iseq.leave + + # This runs when the pattern matches successfully. + jumps_to_match.each { |jump| jump[1] = iseq.label } + iseq.adjuststack(6) + iseq.putnil end end diff --git a/test/compiler_test.rb b/test/compiler_test.rb index 4f4fa9f3..c2472432 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -423,6 +423,7 @@ class CompilerTest < Minitest::Test "foo in [bar, baz]", "foo in [1, 2, 3, bar, 4, 5, 6, baz]", "foo in Foo[1, 2, 3, bar, 4, 5, 6, baz]", + "foo => bar" ] # These are the combinations of instructions that we're going to test. From 8a0f1ecc1eae2943d50a3a86473ffc2c329e27be Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 22 Nov 2022 11:41:06 -0500 Subject: [PATCH 269/536] Create Legacy module for legacy YARV instructions --- lib/syntax_tree/compiler.rb | 24 +-- lib/syntax_tree/yarv.rb | 4 +- lib/syntax_tree/yarv/instructions.rb | 250 ++++++++++++++------------- 3 files changed, 141 insertions(+), 137 deletions(-) diff --git a/lib/syntax_tree/compiler.rb b/lib/syntax_tree/compiler.rb index 4050f4c9..c4eb5194 100644 --- a/lib/syntax_tree/compiler.rb +++ b/lib/syntax_tree/compiler.rb @@ -870,18 +870,18 @@ def visit_defined(node) case value when Const iseq.putnil - iseq.defined(YARV::Defined::CONST, name, "constant") + iseq.defined(YARV::Defined::TYPE_CONST, name, "constant") when CVar iseq.putnil - iseq.defined(YARV::Defined::CVAR, name, "class variable") + iseq.defined(YARV::Defined::TYPE_CVAR, name, "class variable") when GVar iseq.putnil - iseq.defined(YARV::Defined::GVAR, name, "global-variable") + iseq.defined(YARV::Defined::TYPE_GVAR, name, "global-variable") when Ident iseq.putobject("local-variable") when IVar iseq.putnil - iseq.defined(YARV::Defined::IVAR, name, "instance-variable") + iseq.defined(YARV::Defined::TYPE_IVAR, name, "instance-variable") when Kw case name when :false @@ -898,13 +898,13 @@ def visit_defined(node) iseq.putself name = node.value.value.value.to_sym - iseq.defined(YARV::Defined::FUNC, name, "method") + iseq.defined(YARV::Defined::TYPE_FUNC, name, "method") when YieldNode iseq.putnil - iseq.defined(YARV::Defined::YIELD, false, "yield") + iseq.defined(YARV::Defined::TYPE_YIELD, false, "yield") when ZSuper iseq.putnil - iseq.defined(YARV::Defined::ZSUPER, false, "super") + iseq.defined(YARV::Defined::TYPE_ZSUPER, false, "super") else iseq.putobject("expression") end @@ -1875,24 +1875,24 @@ def opassign_defined(node) name = node.target.constant.value.to_sym iseq.dup - iseq.defined(YARV::Defined::CONST_FROM, name, true) + iseq.defined(YARV::Defined::TYPE_CONST_FROM, name, true) when TopConstField name = node.target.constant.value.to_sym iseq.putobject(Object) iseq.dup - iseq.defined(YARV::Defined::CONST_FROM, name, true) + iseq.defined(YARV::Defined::TYPE_CONST_FROM, name, true) when VarField name = node.target.value.value.to_sym iseq.putnil case node.target.value when Const - iseq.defined(YARV::Defined::CONST, name, true) + iseq.defined(YARV::Defined::TYPE_CONST, name, true) when CVar - iseq.defined(YARV::Defined::CVAR, name, true) + iseq.defined(YARV::Defined::TYPE_CVAR, name, true) when GVar - iseq.defined(YARV::Defined::GVAR, name, true) + iseq.defined(YARV::Defined::TYPE_GVAR, name, true) end end diff --git a/lib/syntax_tree/yarv.rb b/lib/syntax_tree/yarv.rb index 2ca29de7..89920c6a 100644 --- a/lib/syntax_tree/yarv.rb +++ b/lib/syntax_tree/yarv.rb @@ -434,14 +434,14 @@ def getblockparamproxy(index, level) def getclassvariable(name) if RUBY_VERSION < "3.0" - push(GetClassVariableUncached.new(name)) + push(Legacy::GetClassVariable.new(name)) else push(GetClassVariable.new(name, inline_storage_for(name))) end end def getconstant(name) - push(GetConstant.new(name)) + push(Legacy::GetConstant.new(name)) end def getglobal(name) diff --git a/lib/syntax_tree/yarv/instructions.rb b/lib/syntax_tree/yarv/instructions.rb index ccb7a345..e6853a87 100644 --- a/lib/syntax_tree/yarv/instructions.rb +++ b/lib/syntax_tree/yarv/instructions.rb @@ -333,44 +333,36 @@ def pushes # ### Summary # - # `defined` checks if the top value of the stack is defined. If it is, it - # pushes its value onto the stack. Otherwise it pushes `nil`. + # `defineclass` defines a class. First it pops the superclass off the + # stack, then it pops the object off the stack that the class should be + # defined under. It has three arguments: the name of the constant, the + # instruction sequence associated with the class, and various flags that + # indicate if it is a singleton class, a module, or a regular class. # # ### Usage # # ~~~ruby - # defined?(x) + # class Foo + # end # ~~~ # - class Defined - NIL = 1 - IVAR = 2 - LVAR = 3 - GVAR = 4 - CVAR = 5 - CONST = 6 - METHOD = 7 - YIELD = 8 - ZSUPER = 9 - SELF = 10 - TRUE = 11 - FALSE = 12 - ASGN = 13 - EXPR = 14 - REF = 15 - FUNC = 16 - CONST_FROM = 17 + class DefineClass + TYPE_CLASS = 0 + TYPE_SINGLETON_CLASS = 1 + TYPE_MODULE = 2 + FLAG_SCOPED = 8 + FLAG_HAS_SUPERCLASS = 16 - attr_reader :type, :name, :message + attr_reader :name, :class_iseq, :flags - def initialize(type, name, message) - @type = type + def initialize(name, class_iseq, flags) @name = name - @message = message + @class_iseq = class_iseq + @flags = flags end def to_a(_iseq) - [:defined, type, name, message] + [:defineclass, name, class_iseq.to_a, flags] end def length @@ -378,7 +370,7 @@ def length end def pops - 1 + 2 end def pushes @@ -388,36 +380,44 @@ def pushes # ### Summary # - # `defineclass` defines a class. First it pops the superclass off the - # stack, then it pops the object off the stack that the class should be - # defined under. It has three arguments: the name of the constant, the - # instruction sequence associated with the class, and various flags that - # indicate if it is a singleton class, a module, or a regular class. + # `defined` checks if the top value of the stack is defined. If it is, it + # pushes its value onto the stack. Otherwise it pushes `nil`. # # ### Usage # # ~~~ruby - # class Foo - # end + # defined?(x) # ~~~ # - class DefineClass - TYPE_CLASS = 0 - TYPE_SINGLETON_CLASS = 1 - TYPE_MODULE = 2 - FLAG_SCOPED = 8 - FLAG_HAS_SUPERCLASS = 16 + class Defined + TYPE_NIL = 1 + TYPE_IVAR = 2 + TYPE_LVAR = 3 + TYPE_GVAR = 4 + TYPE_CVAR = 5 + TYPE_CONST = 6 + TYPE_METHOD = 7 + TYPE_YIELD = 8 + TYPE_ZSUPER = 9 + TYPE_SELF = 10 + TYPE_TRUE = 11 + TYPE_FALSE = 12 + TYPE_ASGN = 13 + TYPE_EXPR = 14 + TYPE_REF = 15 + TYPE_FUNC = 16 + TYPE_CONST_FROM = 17 - attr_reader :name, :class_iseq, :flags + attr_reader :type, :name, :message - def initialize(name, class_iseq, flags) + def initialize(type, name, message) + @type = type @name = name - @class_iseq = class_iseq - @flags = flags + @message = message end def to_a(_iseq) - [:defineclass, name, class_iseq.to_a, flags] + [:defined, type, name, message] end def length @@ -425,7 +425,7 @@ def length end def pops - 2 + 1 end def pushes @@ -800,83 +800,6 @@ def pushes end end - # ### Summary - # - # `getclassvariable` looks for a class variable in the current class and - # pushes its value onto the stack. - # - # This version of the `getclassvariable` instruction is no longer used since - # in Ruby 3.0 it gained an inline cache.` - # - # ### Usage - # - # ~~~ruby - # @@class_variable - # ~~~ - # - class GetClassVariableUncached - attr_reader :name - - def initialize(name) - @name = name - end - - def to_a(_iseq) - [:getclassvariable, name] - end - - def length - 2 - end - - def pops - 0 - end - - def pushes - 1 - end - end - - # ### Summary - # - # `getconstant` performs a constant lookup and pushes the value of the - # constant onto the stack. It pops both the class it should look in and - # whether or not it should look globally as well. - # - # This instruction is no longer used since in Ruby 3.2 it was replaced by - # the consolidated `opt_getconstant_path` instruction. - # - # ### Usage - # - # ~~~ruby - # Constant - # ~~~ - # - class GetConstant - attr_reader :name - - def initialize(name) - @name = name - end - - def to_a(_iseq) - [:getconstant, name] - end - - def length - 2 - end - - def pops - 2 - end - - def pushes - 1 - end - end - # ### Summary # # `getglobal` pushes the value of a global variables onto the stack. @@ -1067,5 +990,86 @@ def pushes 1 end end + + # This module contains the instructions that used to be a part of YARV but + # have been replaced or removed in more recent versions. + module Legacy + # ### Summary + # + # `getclassvariable` looks for a class variable in the current class and + # pushes its value onto the stack. + # + # This version of the `getclassvariable` instruction is no longer used + # since in Ruby 3.0 it gained an inline cache.` + # + # ### Usage + # + # ~~~ruby + # @@class_variable + # ~~~ + # + class GetClassVariable + attr_reader :name + + def initialize(name) + @name = name + end + + def to_a(_iseq) + [:getclassvariable, name] + end + + def length + 2 + end + + def pops + 0 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `getconstant` performs a constant lookup and pushes the value of the + # constant onto the stack. It pops both the class it should look in and + # whether or not it should look globally as well. + # + # This instruction is no longer used since in Ruby 3.2 it was replaced by + # the consolidated `opt_getconstant_path` instruction. + # + # ### Usage + # + # ~~~ruby + # Constant + # ~~~ + # + class GetConstant + attr_reader :name + + def initialize(name) + @name = name + end + + def to_a(_iseq) + [:getconstant, name] + end + + def length + 2 + end + + def pops + 2 + end + + def pushes + 1 + end + end + end end end From ba8cad0d1485b5e039e669decc0d2f6dbb61fa07 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 22 Nov 2022 11:45:28 -0500 Subject: [PATCH 270/536] More instructions to classes --- .rubocop.yml | 3 + lib/syntax_tree.rb | 7 +- lib/syntax_tree/compiler.rb | 2131 -------------- lib/syntax_tree/yarv.rb | 851 +----- lib/syntax_tree/yarv/bf.rb | 30 +- lib/syntax_tree/yarv/compiler.rb | 2164 ++++++++++++++ lib/syntax_tree/yarv/disassembler.rb | 247 +- lib/syntax_tree/yarv/instruction_sequence.rb | 671 +++++ lib/syntax_tree/yarv/instructions.rb | 2688 +++++++++++++++++- lib/syntax_tree/yarv/legacy.rb | 169 ++ lib/syntax_tree/yarv/local_table.rb | 81 + test/compiler_test.rb | 5 +- test/yarv_test.rb | 4 +- 13 files changed, 5823 insertions(+), 3228 deletions(-) delete mode 100644 lib/syntax_tree/compiler.rb create mode 100644 lib/syntax_tree/yarv/compiler.rb create mode 100644 lib/syntax_tree/yarv/instruction_sequence.rb create mode 100644 lib/syntax_tree/yarv/legacy.rb create mode 100644 lib/syntax_tree/yarv/local_table.rb diff --git a/.rubocop.yml b/.rubocop.yml index 134a75dc..b7ba43e8 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -64,6 +64,9 @@ Style/CaseEquality: Style/CaseLikeIf: Enabled: false +Style/Documentation: + Enabled: false + Style/ExplicitBlockArgument: Enabled: false diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index 792ba20c..b2ff8414 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -10,6 +10,7 @@ require_relative "syntax_tree/formatter" require_relative "syntax_tree/node" +require_relative "syntax_tree/dsl" require_relative "syntax_tree/version" require_relative "syntax_tree/basic_visitor" @@ -26,12 +27,14 @@ require_relative "syntax_tree/pattern" require_relative "syntax_tree/search" -require_relative "syntax_tree/dsl" require_relative "syntax_tree/yarv" -require_relative "syntax_tree/compiler" require_relative "syntax_tree/yarv/bf" +require_relative "syntax_tree/yarv/compiler" require_relative "syntax_tree/yarv/disassembler" +require_relative "syntax_tree/yarv/instruction_sequence" require_relative "syntax_tree/yarv/instructions" +require_relative "syntax_tree/yarv/legacy" +require_relative "syntax_tree/yarv/local_table" # Syntax Tree is a suite of tools built on top of the internal CRuby parser. It # provides the ability to generate a syntax tree from source, as well as the diff --git a/lib/syntax_tree/compiler.rb b/lib/syntax_tree/compiler.rb deleted file mode 100644 index c4eb5194..00000000 --- a/lib/syntax_tree/compiler.rb +++ /dev/null @@ -1,2131 +0,0 @@ -# frozen_string_literal: true - -module SyntaxTree - # This class is an experiment in transforming Syntax Tree nodes into their - # corresponding YARV instruction sequences. It attempts to mirror the - # behavior of RubyVM::InstructionSequence.compile. - # - # You use this as with any other visitor. First you parse code into a tree, - # then you visit it with this compiler. Visiting the root node of the tree - # will return a SyntaxTree::Visitor::Compiler::InstructionSequence object. - # With that object you can call #to_a on it, which will return a serialized - # form of the instruction sequence as an array. This array _should_ mirror - # the array given by RubyVM::InstructionSequence#to_a. - # - # As an example, here is how you would compile a single expression: - # - # program = SyntaxTree.parse("1 + 2") - # program.accept(SyntaxTree::Visitor::Compiler.new).to_a - # - # [ - # "YARVInstructionSequence/SimpleDataFormat", - # 3, - # 1, - # 1, - # {:arg_size=>0, :local_size=>0, :stack_max=>2}, - # "", - # "", - # "", - # 1, - # :top, - # [], - # {}, - # [], - # [ - # [:putobject_INT2FIX_1_], - # [:putobject, 2], - # [:opt_plus, {:mid=>:+, :flag=>16, :orig_argc=>1}], - # [:leave] - # ] - # ] - # - # Note that this is the same output as calling: - # - # RubyVM::InstructionSequence.compile("1 + 2").to_a - # - class Compiler < BasicVisitor - # This visitor is responsible for converting Syntax Tree nodes into their - # corresponding Ruby structures. This is used to convert the operands of - # some instructions like putobject that push a Ruby object directly onto - # the stack. It is only used when the entire structure can be represented - # at compile-time, as opposed to constructed at run-time. - class RubyVisitor < BasicVisitor - # This error is raised whenever a node cannot be converted into a Ruby - # object at compile-time. - class CompilationError < StandardError - end - - # This will attempt to compile the given node. If it's possible, then - # it will return the compiled object. Otherwise it will return nil. - def self.compile(node) - node.accept(new) - rescue CompilationError - end - - def visit_array(node) - visit_all(node.contents.parts) - end - - def visit_bare_assoc_hash(node) - node.assocs.to_h do |assoc| - # We can only convert regular key-value pairs. A double splat ** - # operator means it has to be converted at run-time. - raise CompilationError unless assoc.is_a?(Assoc) - [visit(assoc.key), visit(assoc.value)] - end - end - - def visit_float(node) - node.value.to_f - end - - alias visit_hash visit_bare_assoc_hash - - def visit_imaginary(node) - node.value.to_c - end - - def visit_int(node) - node.value.to_i - end - - def visit_label(node) - node.value.chomp(":").to_sym - end - - def visit_mrhs(node) - visit_all(node.parts) - end - - def visit_qsymbols(node) - node.elements.map { |element| visit(element).to_sym } - end - - def visit_qwords(node) - visit_all(node.elements) - end - - def visit_range(node) - left, right = [visit(node.left), visit(node.right)] - node.operator.value === ".." ? left..right : left...right - end - - def visit_rational(node) - node.value.to_r - end - - def visit_regexp_literal(node) - if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) - Regexp.new(node.parts.first.value, visit_regexp_literal_flags(node)) - else - # Any interpolation of expressions or variables will result in the - # regular expression being constructed at run-time. - raise CompilationError - end - end - - # This isn't actually a visit method, though maybe it should be. It is - # responsible for converting the set of string options on a regular - # expression into its equivalent integer. - def visit_regexp_literal_flags(node) - node - .options - .chars - .inject(0) do |accum, option| - accum | - case option - when "i" - Regexp::IGNORECASE - when "x" - Regexp::EXTENDED - when "m" - Regexp::MULTILINE - else - raise "Unknown regexp option: #{option}" - end - end - end - - def visit_symbol_literal(node) - node.value.value.to_sym - end - - def visit_symbols(node) - node.elements.map { |element| visit(element).to_sym } - end - - def visit_tstring_content(node) - node.value - end - - def visit_var_ref(node) - raise CompilationError unless node.value.is_a?(Kw) - - case node.value.value - when "nil" - nil - when "true" - true - when "false" - false - else - raise CompilationError - end - end - - def visit_word(node) - if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) - node.parts.first.value - else - # Any interpolation of expressions or variables will result in the - # string being constructed at run-time. - raise CompilationError - end - end - - def visit_words(node) - visit_all(node.elements) - end - - def visit_unsupported(_node) - raise CompilationError - end - - # Please forgive the metaprogramming here. This is used to create visit - # methods for every node that we did not explicitly handle. By default - # each of these methods will raise a CompilationError. - handled = instance_methods(false) - (Visitor.instance_methods(false) - handled).each do |method| - alias_method method, :visit_unsupported - end - end - - # These options mirror the compilation options that we currently support - # that can be also passed to RubyVM::InstructionSequence.compile. - attr_reader :frozen_string_literal, - :operands_unification, - :specialized_instruction - - # The current instruction sequence that is being compiled. - attr_reader :iseq - - # A boolean to track if we're currently compiling the last statement - # within a set of statements. This information is necessary to determine - # if we need to return the value of the last statement. - attr_reader :last_statement - - def initialize( - frozen_string_literal: false, - operands_unification: true, - specialized_instruction: true - ) - @frozen_string_literal = frozen_string_literal - @operands_unification = operands_unification - @specialized_instruction = specialized_instruction - - @iseq = nil - @last_statement = false - end - - def visit_BEGIN(node) - visit(node.statements) - end - - def visit_CHAR(node) - if frozen_string_literal - iseq.putobject(node.value[1..]) - else - iseq.putstring(node.value[1..]) - end - end - - def visit_END(node) - once_iseq = - with_child_iseq(iseq.block_child_iseq(node.location)) do - postexe_iseq = - with_child_iseq(iseq.block_child_iseq(node.location)) do - iseq.event(:RUBY_EVENT_B_CALL) - - *statements, last_statement = node.statements.body - visit_all(statements) - with_last_statement { visit(last_statement) } - - iseq.event(:RUBY_EVENT_B_RETURN) - iseq.leave - end - - iseq.putspecialobject(YARV::VM_SPECIAL_OBJECT_VMCORE) - iseq.send(:"core#set_postexe", 0, YARV::VM_CALL_FCALL, postexe_iseq) - iseq.leave - end - - iseq.once(once_iseq, iseq.inline_storage) - iseq.pop - end - - def visit_alias(node) - iseq.putspecialobject(YARV::VM_SPECIAL_OBJECT_VMCORE) - iseq.putspecialobject(YARV::VM_SPECIAL_OBJECT_CBASE) - visit(node.left) - visit(node.right) - iseq.send(:"core#set_method_alias", 3) - end - - def visit_aref(node) - visit(node.collection) - - if !frozen_string_literal && specialized_instruction && (node.index.parts.length == 1) - arg = node.index.parts.first - - if arg.is_a?(StringLiteral) && (arg.parts.length == 1) - string_part = arg.parts.first - - if string_part.is_a?(TStringContent) - iseq.opt_aref_with(string_part.value, :[], 1) - return - end - end - end - - visit(node.index) - iseq.send(:[], 1) - end - - def visit_arg_block(node) - visit(node.value) - end - - def visit_arg_paren(node) - visit(node.arguments) - end - - def visit_arg_star(node) - visit(node.value) - iseq.splatarray(false) - end - - def visit_args(node) - visit_all(node.parts) - end - - def visit_array(node) - if (compiled = RubyVisitor.compile(node)) - iseq.duparray(compiled) - elsif node.contents && node.contents.parts.length == 1 && - node.contents.parts.first.is_a?(BareAssocHash) && - node.contents.parts.first.assocs.length == 1 && - node.contents.parts.first.assocs.first.is_a?(AssocSplat) - iseq.putspecialobject(YARV::VM_SPECIAL_OBJECT_VMCORE) - iseq.newhash(0) - visit(node.contents.parts.first) - iseq.send(:"core#hash_merge_kwd", 2) - iseq.newarraykwsplat(1) - else - length = 0 - - node.contents.parts.each do |part| - if part.is_a?(ArgStar) - if length > 0 - iseq.newarray(length) - length = 0 - end - - visit(part.value) - iseq.concatarray - else - visit(part) - length += 1 - end - end - - iseq.newarray(length) if length > 0 - iseq.concatarray if length > 0 && length != node.contents.parts.length - end - end - - def visit_aryptn(node) - match_failures = [] - jumps_to_exit = [] - - # If there's a constant, then check if we match against that constant or - # not first. Branch to failure if we don't. - if node.constant - iseq.dup - visit(node.constant) - iseq.checkmatch(YARV::VM_CHECKMATCH_TYPE_CASE) - match_failures << iseq.branchunless(-1) - end - - # First, check if the #deconstruct cache is nil. If it is, we're going to - # call #deconstruct on the object and cache the result. - iseq.topn(2) - branchnil = iseq.branchnil(-1) - - # Next, ensure that the cached value was cached correctly, otherwise fail - # the match. - iseq.topn(2) - match_failures << iseq.branchunless(-1) - - # Since we have a valid cached value, we can skip past the part where we - # call #deconstruct on the object. - iseq.pop - iseq.topn(1) - jump = iseq.jump(-1) - - # Check if the object responds to #deconstruct, fail the match otherwise. - branchnil.patch!(iseq) - iseq.dup - iseq.putobject(:deconstruct) - iseq.send(:respond_to?, 1) - iseq.setn(3) - match_failures << iseq.branchunless(-1) - - # Call #deconstruct and ensure that it's an array, raise an error - # otherwise. - iseq.send(:deconstruct, 0) - iseq.setn(2) - iseq.dup - iseq.checktype(YARV::VM_CHECKTYPE_ARRAY) - match_error = iseq.branchunless(-1) - - # Ensure that the deconstructed array has the correct size, fail the match - # otherwise. - jump[1] = iseq.label - iseq.dup - iseq.send(:length, 0) - iseq.putobject(node.requireds.length) - iseq.send(:==, 1) - match_failures << iseq.branchunless(-1) - - # For each required element, check if the deconstructed array contains the - # element, otherwise jump out to the top-level match failure. - iseq.dup - node.requireds.each_with_index do |required, index| - iseq.putobject(index) - iseq.send(:[], 1) - - case required - when VarField - lookup = visit(required) - iseq.setlocal(lookup.index, lookup.level) - else - visit(required) - iseq.checkmatch(YARV::VM_CHECKMATCH_TYPE_CASE) - match_failures << iseq.branchunless(-1) - end - - if index < node.requireds.length - 1 - iseq.dup - else - iseq.pop - jumps_to_exit << iseq.jump(-1) - end - end - - # Set up the routine here to raise an error to indicate that the type of - # the deconstructed array was incorrect. - match_error.patch!(iseq) - iseq.putspecialobject(YARV::VM_SPECIAL_OBJECT_VMCORE) - iseq.putobject(TypeError) - iseq.putobject("deconstruct must return Array") - iseq.send(:"core#raise", 2) - iseq.pop - - # Patch all of the match failures to jump here so that we pop a final - # value before returning to the parent node. - match_failures.each { |match_failure| match_failure.patch!(iseq) } - iseq.pop - jumps_to_exit - end - - def visit_assign(node) - case node.target - when ARefField - if !frozen_string_literal && specialized_instruction && (node.target.index.parts.length == 1) - arg = node.target.index.parts.first - - if arg.is_a?(StringLiteral) && (arg.parts.length == 1) - string_part = arg.parts.first - - if string_part.is_a?(TStringContent) - visit(node.target.collection) - visit(node.value) - iseq.swap - iseq.topn(1) - iseq.opt_aset_with(string_part.value, :[]=, 2) - iseq.pop - return - end - end - end - - iseq.putnil - visit(node.target.collection) - visit(node.target.index) - visit(node.value) - iseq.setn(3) - iseq.send(:[]=, 2) - iseq.pop - when ConstPathField - names = constant_names(node.target) - name = names.pop - - if RUBY_VERSION >= "3.2" - iseq.opt_getconstant_path(names) - visit(node.value) - iseq.swap - iseq.topn(1) - iseq.swap - iseq.setconstant(name) - else - visit(node.value) - iseq.dup if last_statement? - iseq.opt_getconstant_path(names) - iseq.setconstant(name) - end - when Field - iseq.putnil - visit(node.target) - visit(node.value) - iseq.setn(2) - iseq.send(:"#{node.target.name.value}=", 1) - iseq.pop - when TopConstField - name = node.target.constant.value.to_sym - - if RUBY_VERSION >= "3.2" - iseq.putobject(Object) - visit(node.value) - iseq.swap - iseq.topn(1) - iseq.swap - iseq.setconstant(name) - else - visit(node.value) - iseq.dup if last_statement? - iseq.putobject(Object) - iseq.setconstant(name) - end - when VarField - visit(node.value) - iseq.dup if last_statement? - - case node.target.value - when Const - iseq.putspecialobject(YARV::VM_SPECIAL_OBJECT_CONST_BASE) - iseq.setconstant(node.target.value.value.to_sym) - when CVar - iseq.setclassvariable(node.target.value.value.to_sym) - when GVar - iseq.setglobal(node.target.value.value.to_sym) - when Ident - lookup = visit(node.target) - - if lookup.local.is_a?(YARV::LocalTable::BlockLocal) - iseq.setblockparam(lookup.index, lookup.level) - else - iseq.setlocal(lookup.index, lookup.level) - end - when IVar - iseq.setinstancevariable(node.target.value.value.to_sym) - end - end - end - - def visit_assoc(node) - visit(node.key) - visit(node.value) - end - - def visit_assoc_splat(node) - visit(node.value) - end - - def visit_backref(node) - iseq.getspecial(YARV::VM_SVAR_BACKREF, 2 * node.value[1..].to_i) - end - - def visit_bare_assoc_hash(node) - if (compiled = RubyVisitor.compile(node)) - iseq.duphash(compiled) - else - visit_all(node.assocs) - end - end - - def visit_binary(node) - case node.operator - when :"&&" - visit(node.left) - iseq.dup - - branchunless = iseq.branchunless(-1) - iseq.pop - - visit(node.right) - branchunless.patch!(iseq) - when :"||" - visit(node.left) - iseq.dup - - branchif = iseq.branchif(-1) - iseq.pop - - visit(node.right) - branchif.patch!(iseq) - else - visit(node.left) - visit(node.right) - iseq.send(node.operator, 1) - end - end - - def visit_block(node) - with_child_iseq(iseq.block_child_iseq(node.location)) do - iseq.event(:RUBY_EVENT_B_CALL) - visit(node.block_var) - visit(node.bodystmt) - iseq.event(:RUBY_EVENT_B_RETURN) - iseq.leave - end - end - - def visit_block_var(node) - params = node.params - - if params.requireds.length == 1 && params.optionals.empty? && - !params.rest && params.posts.empty? && params.keywords.empty? && - !params.keyword_rest && !params.block - iseq.argument_options[:ambiguous_param0] = true - end - - visit(node.params) - - node.locals.each { |local| iseq.local_table.plain(local.value.to_sym) } - end - - def visit_blockarg(node) - iseq.argument_options[:block_start] = iseq.argument_size - iseq.local_table.block(node.name.value.to_sym) - iseq.argument_size += 1 - end - - def visit_bodystmt(node) - visit(node.statements) - end - - def visit_call(node) - if node.is_a?(CallNode) - return( - visit_call( - CommandCall.new( - receiver: node.receiver, - operator: node.operator, - message: node.message, - arguments: node.arguments, - block: nil, - location: node.location - ) - ) - ) - end - - arg_parts = argument_parts(node.arguments) - argc = arg_parts.length - - # First we're going to check if we're calling a method on an array - # literal without any arguments. In that case there are some - # specializations we might be able to perform. - if argc == 0 && (node.message.is_a?(Ident) || node.message.is_a?(Op)) - case node.receiver - when ArrayLiteral - parts = node.receiver.contents&.parts || [] - - if parts.none? { |part| part.is_a?(ArgStar) } && - RubyVisitor.compile(node.receiver).nil? - case node.message.value - when "max" - visit(node.receiver.contents) - iseq.opt_newarray_max(parts.length) - return - when "min" - visit(node.receiver.contents) - iseq.opt_newarray_min(parts.length) - return - end - end - when StringLiteral - if RubyVisitor.compile(node.receiver).nil? - case node.message.value - when "-@" - iseq.opt_str_uminus(node.receiver.parts.first.value) - return - when "freeze" - iseq.opt_str_freeze(node.receiver.parts.first.value) - return - end - end - end - end - - if node.receiver - if node.receiver.is_a?(VarRef) - lookup = iseq.local_variable(node.receiver.value.value.to_sym) - - if lookup.local.is_a?(YARV::LocalTable::BlockLocal) - iseq.getblockparamproxy(lookup.index, lookup.level) - else - visit(node.receiver) - end - else - visit(node.receiver) - end - else - iseq.putself - end - - branchnil = - if node.operator&.value == "&." - iseq.dup - iseq.branchnil(-1) - end - - flag = 0 - - arg_parts.each do |arg_part| - case arg_part - when ArgBlock - argc -= 1 - flag |= YARV::VM_CALL_ARGS_BLOCKARG - visit(arg_part) - when ArgStar - flag |= YARV::VM_CALL_ARGS_SPLAT - visit(arg_part) - when ArgsForward - flag |= YARV::VM_CALL_ARGS_SPLAT | YARV::VM_CALL_ARGS_BLOCKARG - - lookup = iseq.local_table.find(:*, 0) - iseq.getlocal(lookup.index, lookup.level) - iseq.splatarray(arg_parts.length != 1) - - lookup = iseq.local_table.find(:&, 0) - iseq.getblockparamproxy(lookup.index, lookup.level) - when BareAssocHash - flag |= YARV::VM_CALL_KW_SPLAT - visit(arg_part) - else - visit(arg_part) - end - end - - block_iseq = visit(node.block) if node.block - flag |= YARV::VM_CALL_ARGS_SIMPLE if block_iseq.nil? && flag == 0 - flag |= YARV::VM_CALL_FCALL if node.receiver.nil? - - iseq.send(node.message.value.to_sym, argc, flag, block_iseq) - branchnil.patch!(iseq) if branchnil - end - - def visit_case(node) - visit(node.value) if node.value - - clauses = [] - else_clause = nil - current = node.consequent - - while current - clauses << current - - if (current = current.consequent).is_a?(Else) - else_clause = current - break - end - end - - branches = - clauses.map do |clause| - visit(clause.arguments) - iseq.topn(1) - iseq.send(:===, 1, YARV::VM_CALL_FCALL | YARV::VM_CALL_ARGS_SIMPLE) - [clause, iseq.branchif(:label_00)] - end - - iseq.pop - else_clause ? visit(else_clause) : iseq.putnil - iseq.leave - - branches.each_with_index do |(clause, branchif), index| - iseq.leave if index != 0 - branchif.patch!(iseq) - iseq.pop - visit(clause) - end - end - - def visit_class(node) - name = node.constant.constant.value.to_sym - class_iseq = - with_child_iseq(iseq.class_child_iseq(name, node.location)) do - iseq.event(:RUBY_EVENT_CLASS) - visit(node.bodystmt) - iseq.event(:RUBY_EVENT_END) - iseq.leave - end - - flags = YARV::DefineClass::TYPE_CLASS - - case node.constant - when ConstPathRef - flags |= YARV::DefineClass::FLAG_SCOPED - visit(node.constant.parent) - when ConstRef - iseq.putspecialobject(YARV::VM_SPECIAL_OBJECT_CONST_BASE) - when TopConstRef - flags |= YARV::DefineClass::FLAG_SCOPED - iseq.putobject(Object) - end - - if node.superclass - flags |= YARV::DefineClass::FLAG_HAS_SUPERCLASS - visit(node.superclass) - else - iseq.putnil - end - - iseq.defineclass(name, class_iseq, flags) - end - - def visit_command(node) - visit_call( - CommandCall.new( - receiver: nil, - operator: nil, - message: node.message, - arguments: node.arguments, - block: node.block, - location: node.location - ) - ) - end - - def visit_command_call(node) - visit_call( - CommandCall.new( - receiver: node.receiver, - operator: node.operator, - message: node.message, - arguments: node.arguments, - block: node.block, - location: node.location - ) - ) - end - - def visit_const_path_field(node) - visit(node.parent) - end - - def visit_const_path_ref(node) - names = constant_names(node) - iseq.opt_getconstant_path(names) - end - - def visit_def(node) - name = node.name.value.to_sym - method_iseq = iseq.method_child_iseq(name.to_s, node.location) - - with_child_iseq(method_iseq) do - visit(node.params) if node.params - iseq.event(:RUBY_EVENT_CALL) - visit(node.bodystmt) - iseq.event(:RUBY_EVENT_RETURN) - iseq.leave - end - - if node.target - visit(node.target) - iseq.definesmethod(name, method_iseq) - else - iseq.definemethod(name, method_iseq) - end - - iseq.putobject(name) - end - - def visit_defined(node) - case node.value - when Assign - # If we're assigning to a local variable, then we need to make sure - # that we put it into the local table. - if node.value.target.is_a?(VarField) && - node.value.target.value.is_a?(Ident) - iseq.local_table.plain(node.value.target.value.value.to_sym) - end - - iseq.putobject("assignment") - when VarRef - value = node.value.value - name = value.value.to_sym - - case value - when Const - iseq.putnil - iseq.defined(YARV::Defined::TYPE_CONST, name, "constant") - when CVar - iseq.putnil - iseq.defined(YARV::Defined::TYPE_CVAR, name, "class variable") - when GVar - iseq.putnil - iseq.defined(YARV::Defined::TYPE_GVAR, name, "global-variable") - when Ident - iseq.putobject("local-variable") - when IVar - iseq.putnil - iseq.defined(YARV::Defined::TYPE_IVAR, name, "instance-variable") - when Kw - case name - when :false - iseq.putobject("false") - when :nil - iseq.putobject("nil") - when :self - iseq.putobject("self") - when :true - iseq.putobject("true") - end - end - when VCall - iseq.putself - - name = node.value.value.value.to_sym - iseq.defined(YARV::Defined::TYPE_FUNC, name, "method") - when YieldNode - iseq.putnil - iseq.defined(YARV::Defined::TYPE_YIELD, false, "yield") - when ZSuper - iseq.putnil - iseq.defined(YARV::Defined::TYPE_ZSUPER, false, "super") - else - iseq.putobject("expression") - end - end - - def visit_dyna_symbol(node) - if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) - iseq.putobject(node.parts.first.value.to_sym) - end - end - - def visit_else(node) - visit(node.statements) - iseq.pop unless last_statement? - end - - def visit_elsif(node) - visit_if( - IfNode.new( - predicate: node.predicate, - statements: node.statements, - consequent: node.consequent, - location: node.location - ) - ) - end - - def visit_field(node) - visit(node.parent) - end - - def visit_float(node) - iseq.putobject(node.accept(RubyVisitor.new)) - end - - def visit_for(node) - visit(node.collection) - - name = node.index.value.value.to_sym - iseq.local_table.plain(name) - - block_iseq = - with_child_iseq(iseq.block_child_iseq(node.statements.location)) do - iseq.argument_options[:lead_num] ||= 0 - iseq.argument_options[:lead_num] += 1 - iseq.argument_options[:ambiguous_param0] = true - - iseq.argument_size += 1 - iseq.local_table.plain(2) - - iseq.getlocal(0, 0) - - local_variable = iseq.local_variable(name) - iseq.setlocal(local_variable.index, local_variable.level) - - iseq.event(:RUBY_EVENT_B_CALL) - iseq.nop - - visit(node.statements) - iseq.event(:RUBY_EVENT_B_RETURN) - iseq.leave - end - - iseq.send(:each, 0, 0, block_iseq) - end - - def visit_hash(node) - if (compiled = RubyVisitor.compile(node)) - iseq.duphash(compiled) - else - visit_all(node.assocs) - iseq.newhash(node.assocs.length * 2) - end - end - - def visit_heredoc(node) - if node.beginning.value.end_with?("`") - visit_xstring_literal(node) - elsif node.parts.length == 1 && node.parts.first.is_a?(TStringContent) - visit(node.parts.first) - else - length = visit_string_parts(node) - iseq.concatstrings(length) - end - end - - def visit_if(node) - if node.predicate.is_a?(RangeNode) - iseq.getspecial(YARV::VM_SVAR_FLIPFLOP_START, 0) - branchif = iseq.branchif(-1) - - visit(node.predicate.left) - branchunless_true = iseq.branchunless(-1) - - iseq.putobject(true) - iseq.setspecial(YARV::VM_SVAR_FLIPFLOP_START) - branchif.patch!(iseq) - - visit(node.predicate.right) - branchunless_false = iseq.branchunless(-1) - - iseq.putobject(false) - iseq.setspecial(YARV::VM_SVAR_FLIPFLOP_START) - branchunless_false.patch!(iseq) - - visit(node.statements) - iseq.leave - branchunless_true.patch!(iseq) - iseq.putnil - else - visit(node.predicate) - branchunless = iseq.branchunless(-1) - visit(node.statements) - - if last_statement? - iseq.leave - branchunless.patch!(iseq) - - node.consequent ? visit(node.consequent) : iseq.putnil - else - iseq.pop - - if node.consequent - jump = iseq.jump(-1) - branchunless.patch!(iseq) - visit(node.consequent) - jump[1] = iseq.label - else - branchunless.patch!(iseq) - end - end - end - end - - def visit_if_op(node) - visit_if( - IfNode.new( - predicate: node.predicate, - statements: node.truthy, - consequent: - Else.new( - keyword: Kw.new(value: "else", location: Location.default), - statements: node.falsy, - location: Location.default - ), - location: Location.default - ) - ) - end - - def visit_imaginary(node) - iseq.putobject(node.accept(RubyVisitor.new)) - end - - def visit_int(node) - iseq.putobject(node.accept(RubyVisitor.new)) - end - - def visit_kwrest_param(node) - iseq.argument_options[:kwrest] = iseq.argument_size - iseq.argument_size += 1 - iseq.local_table.plain(node.name.value.to_sym) - end - - def visit_label(node) - iseq.putobject(node.accept(RubyVisitor.new)) - end - - def visit_lambda(node) - lambda_iseq = - with_child_iseq(iseq.block_child_iseq(node.location)) do - iseq.event(:RUBY_EVENT_B_CALL) - visit(node.params) - visit(node.statements) - iseq.event(:RUBY_EVENT_B_RETURN) - iseq.leave - end - - iseq.putspecialobject(YARV::VM_SPECIAL_OBJECT_VMCORE) - iseq.send(:lambda, 0, YARV::VM_CALL_FCALL, lambda_iseq) - end - - def visit_lambda_var(node) - visit_block_var(node) - end - - def visit_massign(node) - visit(node.value) - iseq.dup - visit(node.target) - end - - def visit_method_add_block(node) - visit_call( - CommandCall.new( - receiver: node.call.receiver, - operator: node.call.operator, - message: node.call.message, - arguments: node.call.arguments, - block: node.block, - location: node.location - ) - ) - end - - def visit_mlhs(node) - lookups = [] - node.parts.each do |part| - case part - when VarField - lookups << visit(part) - end - end - - iseq.expandarray(lookups.length, 0) - lookups.each { |lookup| iseq.setlocal(lookup.index, lookup.level) } - end - - def visit_module(node) - name = node.constant.constant.value.to_sym - module_iseq = - with_child_iseq(iseq.module_child_iseq(name, node.location)) do - iseq.event(:RUBY_EVENT_CLASS) - visit(node.bodystmt) - iseq.event(:RUBY_EVENT_END) - iseq.leave - end - - flags = YARV::DefineClass::TYPE_MODULE - - case node.constant - when ConstPathRef - flags |= YARV::DefineClass::FLAG_SCOPED - visit(node.constant.parent) - when ConstRef - iseq.putspecialobject(YARV::VM_SPECIAL_OBJECT_CONST_BASE) - when TopConstRef - flags |= YARV::DefineClass::FLAG_SCOPED - iseq.putobject(Object) - end - - iseq.putnil - iseq.defineclass(name, module_iseq, flags) - end - - def visit_mrhs(node) - if (compiled = RubyVisitor.compile(node)) - iseq.duparray(compiled) - else - visit_all(node.parts) - iseq.newarray(node.parts.length) - end - end - - def visit_not(node) - visit(node.statement) - iseq.send(:!, 0) - end - - def visit_opassign(node) - flag = YARV::VM_CALL_ARGS_SIMPLE - if node.target.is_a?(ConstPathField) || node.target.is_a?(TopConstField) - flag |= YARV::VM_CALL_FCALL - end - - case (operator = node.operator.value.chomp("=").to_sym) - when :"&&" - branchunless = nil - - with_opassign(node) do - iseq.dup - branchunless = iseq.branchunless(-1) - iseq.pop - visit(node.value) - end - - case node.target - when ARefField - iseq.leave - branchunless.patch!(iseq) - iseq.setn(3) - iseq.adjuststack(3) - when ConstPathField, TopConstField - branchunless.patch!(iseq) - iseq.swap - iseq.pop - else - branchunless.patch!(iseq) - end - when :"||" - if node.target.is_a?(ConstPathField) || node.target.is_a?(TopConstField) - opassign_defined(node) - iseq.swap - iseq.pop - elsif node.target.is_a?(VarField) && - [Const, CVar, GVar].include?(node.target.value.class) - opassign_defined(node) - else - branchif = nil - - with_opassign(node) do - iseq.dup - branchif = iseq.branchif(-1) - iseq.pop - visit(node.value) - end - - if node.target.is_a?(ARefField) - iseq.leave - branchif.patch!(iseq) - iseq.setn(3) - iseq.adjuststack(3) - else - branchif.patch!(iseq) - end - end - else - with_opassign(node) do - visit(node.value) - iseq.send(operator, 1, flag) - end - end - end - - def visit_params(node) - argument_options = iseq.argument_options - - if node.requireds.any? - argument_options[:lead_num] = 0 - - node.requireds.each do |required| - iseq.local_table.plain(required.value.to_sym) - iseq.argument_size += 1 - argument_options[:lead_num] += 1 - end - end - - node.optionals.each do |(optional, value)| - index = iseq.local_table.size - name = optional.value.to_sym - - iseq.local_table.plain(name) - iseq.argument_size += 1 - - argument_options[:opt] = [iseq.label] unless argument_options.key?(:opt) - - visit(value) - iseq.setlocal(index, 0) - iseq.argument_options[:opt] << iseq.label - end - - visit(node.rest) if node.rest - - if node.posts.any? - argument_options[:post_start] = iseq.argument_size - argument_options[:post_num] = 0 - - node.posts.each do |post| - iseq.local_table.plain(post.value.to_sym) - iseq.argument_size += 1 - argument_options[:post_num] += 1 - end - end - - if node.keywords.any? - argument_options[:kwbits] = 0 - argument_options[:keyword] = [] - - keyword_bits_name = node.keyword_rest ? 3 : 2 - iseq.argument_size += 1 - keyword_bits_index = iseq.local_table.locals.size + node.keywords.size - - node.keywords.each_with_index do |(keyword, value), keyword_index| - name = keyword.value.chomp(":").to_sym - index = iseq.local_table.size - - iseq.local_table.plain(name) - iseq.argument_size += 1 - argument_options[:kwbits] += 1 - - if value.nil? - argument_options[:keyword] << name - elsif (compiled = RubyVisitor.compile(value)) - argument_options[:keyword] << [name, compiled] - else - argument_options[:keyword] << [name] - iseq.checkkeyword(keyword_bits_index, keyword_index) - branchif = iseq.branchif(-1) - visit(value) - iseq.setlocal(index, 0) - branchif.patch!(iseq) - end - end - - iseq.local_table.plain(keyword_bits_name) - end - - if node.keyword_rest.is_a?(ArgsForward) - iseq.local_table.plain(:*) - iseq.local_table.plain(:&) - - iseq.argument_options[:rest_start] = iseq.argument_size - iseq.argument_options[:block_start] = iseq.argument_size + 1 - - iseq.argument_size += 2 - elsif node.keyword_rest - visit(node.keyword_rest) - end - - visit(node.block) if node.block - end - - def visit_paren(node) - visit(node.contents) - end - - def visit_program(node) - node.statements.body.each do |statement| - break unless statement.is_a?(Comment) - - if statement.value == "# frozen_string_literal: true" - @frozen_string_literal = true - end - end - - preexes = [] - statements = [] - - node.statements.body.each do |statement| - case statement - when Comment, EmbDoc, EndContent, VoidStmt - # ignore - when BEGINBlock - preexes << statement - else - statements << statement - end - end - - top_iseq = - YARV::InstructionSequence.new( - :top, - "", - nil, - node.location, - frozen_string_literal: frozen_string_literal, - operands_unification: operands_unification, - specialized_instruction: specialized_instruction - ) - - with_child_iseq(top_iseq) do - visit_all(preexes) - - if statements.empty? - iseq.putnil - else - *statements, last_statement = statements - visit_all(statements) - with_last_statement { visit(last_statement) } - end - - iseq.leave - end - end - - def visit_qsymbols(node) - iseq.duparray(node.accept(RubyVisitor.new)) - end - - def visit_qwords(node) - if frozen_string_literal - iseq.duparray(node.accept(RubyVisitor.new)) - else - visit_all(node.elements) - iseq.newarray(node.elements.length) - end - end - - def visit_range(node) - if (compiled = RubyVisitor.compile(node)) - iseq.putobject(compiled) - else - visit(node.left) - visit(node.right) - iseq.newrange(node.operator.value == ".." ? 0 : 1) - end - end - - def visit_rassign(node) - iseq.putnil - - if node.operator.is_a?(Kw) - jumps = [] - - visit(node.value) - iseq.dup - - case node.pattern - when VarField - lookup = visit(node.pattern) - iseq.setlocal(lookup.index, lookup.level) - jumps << iseq.jump(-1) - else - jumps.concat(visit(node.pattern)) - end - - iseq.pop - iseq.pop - iseq.putobject(false) - iseq.leave - - jumps.each { |jump| jump[1] = iseq.label } - iseq.adjuststack(2) - iseq.putobject(true) - else - jumps_to_match = [] - - iseq.putnil - iseq.putobject(false) - iseq.putnil - iseq.putnil - visit(node.value) - iseq.dup - - # Visit the pattern. If it matches, - case node.pattern - when VarField - lookup = visit(node.pattern) - iseq.setlocal(lookup.index, lookup.level) - jumps_to_match << iseq.jump(-1) - else - jumps_to_match.concat(visit(node.pattern)) - end - - # First we're going to push the core onto the stack, then we'll check if - # the value to match is truthy. If it is, we'll jump down to raise - # NoMatchingPatternKeyError. Otherwise we'll raise - # NoMatchingPatternError. - iseq.putspecialobject(YARV::VM_SPECIAL_OBJECT_VMCORE) - iseq.topn(4) - branchif_no_key = iseq.branchif(-1) - - # Here we're going to raise NoMatchingPatternError. - iseq.putobject(NoMatchingPatternError) - iseq.putspecialobject(YARV::VM_SPECIAL_OBJECT_VMCORE) - iseq.putobject("%p: %s") - iseq.topn(4) - iseq.topn(7) - iseq.send(:"core#sprintf", 3) - iseq.send(:"core#raise", 2) - jump_to_exit = iseq.jump(-1) - - # Here we're going to raise NoMatchingPatternKeyError. - branchif_no_key.patch!(iseq) - iseq.putobject(NoMatchingPatternKeyError) - iseq.putspecialobject(YARV::VM_SPECIAL_OBJECT_VMCORE) - iseq.putobject("%p: %s") - iseq.topn(4) - iseq.topn(7) - iseq.send(:"core#sprintf", 3) - iseq.topn(7) - iseq.topn(9) - - # Super special behavior here because of the weird kw_arg handling. - iseq.stack.change_by(-(1 + 1) + 1) - call_data = { mid: :new, flag: YARV::VM_CALL_KWARG, orig_argc: 1, kw_arg: [:matchee, :key] } - - if specialized_instruction - iseq.push([:opt_send_without_block, call_data]) - else - iseq.push([:send, call_data, nil]) - end - - iseq.send(:"core#raise", 1) - - # This runs when the pattern fails to match. - jump_to_exit[1] = iseq.label - iseq.adjuststack(7) - iseq.putnil - iseq.leave - - # This runs when the pattern matches successfully. - jumps_to_match.each { |jump| jump[1] = iseq.label } - iseq.adjuststack(6) - iseq.putnil - end - end - - def visit_rational(node) - iseq.putobject(node.accept(RubyVisitor.new)) - end - - def visit_regexp_literal(node) - if (compiled = RubyVisitor.compile(node)) - iseq.putobject(compiled) - else - flags = RubyVisitor.new.visit_regexp_literal_flags(node) - length = visit_string_parts(node) - iseq.toregexp(flags, length) - end - end - - def visit_rest_param(node) - iseq.local_table.plain(node.name.value.to_sym) - iseq.argument_options[:rest_start] = iseq.argument_size - iseq.argument_size += 1 - end - - def visit_sclass(node) - visit(node.target) - iseq.putnil - - singleton_iseq = - with_child_iseq(iseq.singleton_class_child_iseq(node.location)) do - iseq.event(:RUBY_EVENT_CLASS) - visit(node.bodystmt) - iseq.event(:RUBY_EVENT_END) - iseq.leave - end - - iseq.defineclass( - :singletonclass, - singleton_iseq, - YARV::DefineClass::TYPE_SINGLETON_CLASS - ) - end - - def visit_statements(node) - statements = - node.body.select do |statement| - case statement - when Comment, EmbDoc, EndContent, VoidStmt - false - else - true - end - end - - statements.empty? ? iseq.putnil : visit_all(statements) - end - - def visit_string_concat(node) - value = node.left.parts.first.value + node.right.parts.first.value - - visit_string_literal( - StringLiteral.new( - parts: [TStringContent.new(value: value, location: node.location)], - quote: node.left.quote, - location: node.location - ) - ) - end - - def visit_string_embexpr(node) - visit(node.statements) - end - - def visit_string_literal(node) - if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) - visit(node.parts.first) - else - length = visit_string_parts(node) - iseq.concatstrings(length) - end - end - - def visit_super(node) - iseq.putself - visit(node.arguments) - iseq.invokesuper( - nil, - argument_parts(node.arguments).length, - YARV::VM_CALL_FCALL | YARV::VM_CALL_ARGS_SIMPLE | YARV::VM_CALL_SUPER, - nil - ) - end - - def visit_symbol_literal(node) - iseq.putobject(node.accept(RubyVisitor.new)) - end - - def visit_symbols(node) - if (compiled = RubyVisitor.compile(node)) - iseq.duparray(compiled) - else - node.elements.each do |element| - if element.parts.length == 1 && - element.parts.first.is_a?(TStringContent) - iseq.putobject(element.parts.first.value.to_sym) - else - length = visit_string_parts(element) - iseq.concatstrings(length) - iseq.intern - end - end - - iseq.newarray(node.elements.length) - end - end - - def visit_top_const_ref(node) - iseq.opt_getconstant_path(constant_names(node)) - end - - def visit_tstring_content(node) - if frozen_string_literal - iseq.putobject(node.accept(RubyVisitor.new)) - else - iseq.putstring(node.accept(RubyVisitor.new)) - end - end - - def visit_unary(node) - method_id = - case node.operator - when "+", "-" - "#{node.operator}@" - else - node.operator - end - - visit_call( - CommandCall.new( - receiver: node.statement, - operator: nil, - message: Ident.new(value: method_id, location: Location.default), - arguments: nil, - block: nil, - location: Location.default - ) - ) - end - - def visit_undef(node) - node.symbols.each_with_index do |symbol, index| - iseq.pop if index != 0 - iseq.putspecialobject(YARV::VM_SPECIAL_OBJECT_VMCORE) - iseq.putspecialobject(YARV::VM_SPECIAL_OBJECT_CBASE) - visit(symbol) - iseq.send(:"core#undef_method", 2) - end - end - - def visit_unless(node) - visit(node.predicate) - branchunless = iseq.branchunless(-1) - node.consequent ? visit(node.consequent) : iseq.putnil - - if last_statement? - iseq.leave - branchunless.patch!(iseq) - - visit(node.statements) - else - iseq.pop - - if node.consequent - jump = iseq.jump(-1) - branchunless.patch!(iseq) - visit(node.consequent) - jump[1] = iseq.label - else - branchunless.patch!(iseq) - end - end - end - - def visit_until(node) - jumps = [] - - jumps << iseq.jump(-1) - iseq.putnil - iseq.pop - jumps << iseq.jump(-1) - - label = iseq.label - visit(node.statements) - iseq.pop - jumps.each { |jump| jump[1] = iseq.label } - - visit(node.predicate) - iseq.branchunless(label) - iseq.putnil if last_statement? - end - - def visit_var_field(node) - case node.value - when CVar, IVar - name = node.value.value.to_sym - iseq.inline_storage_for(name) - when Ident - name = node.value.value.to_sym - - if (local_variable = iseq.local_variable(name)) - local_variable - else - iseq.local_table.plain(name) - iseq.local_variable(name) - end - end - end - - def visit_var_ref(node) - case node.value - when Const - iseq.opt_getconstant_path(constant_names(node)) - when CVar - name = node.value.value.to_sym - iseq.getclassvariable(name) - when GVar - iseq.getglobal(node.value.value.to_sym) - when Ident - lookup = iseq.local_variable(node.value.value.to_sym) - - case lookup.local - when YARV::LocalTable::BlockLocal - iseq.getblockparam(lookup.index, lookup.level) - when YARV::LocalTable::PlainLocal - iseq.getlocal(lookup.index, lookup.level) - end - when IVar - name = node.value.value.to_sym - iseq.getinstancevariable(name) - when Kw - case node.value.value - when "false" - iseq.putobject(false) - when "nil" - iseq.putnil - when "self" - iseq.putself - when "true" - iseq.putobject(true) - end - end - end - - def visit_vcall(node) - iseq.putself - - flag = - YARV::VM_CALL_FCALL | YARV::VM_CALL_VCALL | YARV::VM_CALL_ARGS_SIMPLE - iseq.send(node.value.value.to_sym, 0, flag) - end - - def visit_when(node) - visit(node.statements) - end - - def visit_while(node) - jumps = [] - - jumps << iseq.jump(-1) - iseq.putnil - iseq.pop - jumps << iseq.jump(-1) - - label = iseq.label - visit(node.statements) - iseq.pop - jumps.each { |jump| jump[1] = iseq.label } - - visit(node.predicate) - iseq.branchif(label) - iseq.putnil if last_statement? - end - - def visit_word(node) - if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) - visit(node.parts.first) - else - length = visit_string_parts(node) - iseq.concatstrings(length) - end - end - - def visit_words(node) - if frozen_string_literal && (compiled = RubyVisitor.compile(node)) - iseq.duparray(compiled) - else - visit_all(node.elements) - iseq.newarray(node.elements.length) - end - end - - def visit_xstring_literal(node) - iseq.putself - length = visit_string_parts(node) - iseq.concatstrings(node.parts.length) if length > 1 - iseq.send(:`, 1, YARV::VM_CALL_FCALL | YARV::VM_CALL_ARGS_SIMPLE) - end - - def visit_yield(node) - parts = argument_parts(node.arguments) - visit_all(parts) - iseq.invokeblock(nil, parts.length) - end - - def visit_zsuper(_node) - iseq.putself - iseq.invokesuper( - nil, - 0, - YARV::VM_CALL_FCALL | YARV::VM_CALL_ARGS_SIMPLE | YARV::VM_CALL_SUPER | - YARV::VM_CALL_ZSUPER, - nil - ) - end - - private - - # This is a helper that is used in places where arguments may be present - # or they may be wrapped in parentheses. It's meant to descend down the - # tree and return an array of argument nodes. - def argument_parts(node) - case node - when nil - [] - when Args - node.parts - when ArgParen - if node.arguments.is_a?(ArgsForward) - [node.arguments] - else - node.arguments.parts - end - when Paren - node.contents.parts - end - end - - # Constant names when they are being assigned or referenced come in as a - # tree, but it's more convenient to work with them as an array. This - # method converts them into that array. This is nice because it's the - # operand that goes to opt_getconstant_path in Ruby 3.2. - def constant_names(node) - current = node - names = [] - - while current.is_a?(ConstPathField) || current.is_a?(ConstPathRef) - names.unshift(current.constant.value.to_sym) - current = current.parent - end - - case current - when VarField, VarRef - names.unshift(current.value.value.to_sym) - when TopConstRef - names.unshift(current.constant.value.to_sym) - names.unshift(:"") - end - - names - end - - # For the most part when an OpAssign (operator assignment) node with a ||= - # operator is being compiled it's a matter of reading the target, checking - # if the value should be evaluated, evaluating it if so, and then writing - # the result back to the target. - # - # However, in certain kinds of assignments (X, ::X, X::Y, @@x, and $x) we - # first check if the value is defined using the defined instruction. I - # don't know why it is necessary, and suspect that it isn't. - def opassign_defined(node) - case node.target - when ConstPathField - visit(node.target.parent) - name = node.target.constant.value.to_sym - - iseq.dup - iseq.defined(YARV::Defined::TYPE_CONST_FROM, name, true) - when TopConstField - name = node.target.constant.value.to_sym - - iseq.putobject(Object) - iseq.dup - iseq.defined(YARV::Defined::TYPE_CONST_FROM, name, true) - when VarField - name = node.target.value.value.to_sym - iseq.putnil - - case node.target.value - when Const - iseq.defined(YARV::Defined::TYPE_CONST, name, true) - when CVar - iseq.defined(YARV::Defined::TYPE_CVAR, name, true) - when GVar - iseq.defined(YARV::Defined::TYPE_GVAR, name, true) - end - end - - branchunless = iseq.branchunless(-1) - - case node.target - when ConstPathField, TopConstField - iseq.dup - iseq.putobject(true) - iseq.getconstant(name) - when VarField - case node.target.value - when Const - iseq.opt_getconstant_path(constant_names(node.target)) - when CVar - iseq.getclassvariable(name) - when GVar - iseq.getglobal(name) - end - end - - iseq.dup - branchif = iseq.branchif(-1) - iseq.pop - - branchunless.patch!(iseq) - visit(node.value) - - case node.target - when ConstPathField, TopConstField - iseq.dupn(2) - iseq.swap - iseq.setconstant(name) - when VarField - iseq.dup - - case node.target.value - when Const - iseq.putspecialobject(YARV::VM_SPECIAL_OBJECT_CONST_BASE) - iseq.setconstant(name) - when CVar - iseq.setclassvariable(name) - when GVar - iseq.setglobal(name) - end - end - - branchif.patch!(iseq) - end - - # Whenever a value is interpolated into a string-like structure, these - # three instructions are pushed. - def push_interpolate - iseq.dup - iseq.objtostring( - :to_s, - 0, - YARV::VM_CALL_FCALL | YARV::VM_CALL_ARGS_SIMPLE - ) - iseq.anytostring - end - - # There are a lot of nodes in the AST that act as contains of parts of - # strings. This includes things like string literals, regular expressions, - # heredocs, etc. This method will visit all the parts of a string within - # those containers. - def visit_string_parts(node) - length = 0 - - unless node.parts.first.is_a?(TStringContent) - iseq.putobject("") - length += 1 - end - - node.parts.each do |part| - case part - when StringDVar - visit(part.variable) - push_interpolate - when StringEmbExpr - visit(part) - push_interpolate - when TStringContent - iseq.putobject(part.accept(RubyVisitor.new)) - end - - length += 1 - end - - length - end - - # The current instruction sequence that we're compiling is always stored - # on the compiler. When we descend into a node that has its own - # instruction sequence, this method can be called to temporarily set the - # new value of the instruction sequence, yield, and then set it back. - def with_child_iseq(child_iseq) - parent_iseq = iseq - - begin - @iseq = child_iseq - yield - child_iseq - ensure - @iseq = parent_iseq - end - end - - # When we're compiling the last statement of a set of statements within a - # scope, the instructions sometimes change from pops to leaves. These - # kinds of peephole optimizations can reduce the overall number of - # instructions. Therefore, we keep track of whether we're compiling the - # last statement of a scope and allow visit methods to query that - # information. - def with_last_statement - previous = @last_statement - @last_statement = true - - begin - yield - ensure - @last_statement = previous - end - end - - def last_statement? - @last_statement - end - - # OpAssign nodes can have a number of different kinds of nodes as their - # "target" (i.e., the left-hand side of the assignment). When compiling - # these nodes we typically need to first fetch the current value of the - # variable, then perform some kind of action, then store the result back - # into the variable. This method handles that by first fetching the value, - # then yielding to the block, then storing the result. - def with_opassign(node) - case node.target - when ARefField - iseq.putnil - visit(node.target.collection) - visit(node.target.index) - - iseq.dupn(2) - iseq.send(:[], 1) - - yield - - iseq.setn(3) - iseq.send(:[]=, 2) - iseq.pop - when ConstPathField - name = node.target.constant.value.to_sym - - visit(node.target.parent) - iseq.dup - iseq.putobject(true) - iseq.getconstant(name) - - yield - - if node.operator.value == "&&=" - iseq.dupn(2) - else - iseq.swap - iseq.topn(1) - end - - iseq.swap - iseq.setconstant(name) - when TopConstField - name = node.target.constant.value.to_sym - - iseq.putobject(Object) - iseq.dup - iseq.putobject(true) - iseq.getconstant(name) - - yield - - if node.operator.value == "&&=" - iseq.dupn(2) - else - iseq.swap - iseq.topn(1) - end - - iseq.swap - iseq.setconstant(name) - when VarField - case node.target.value - when Const - names = constant_names(node.target) - iseq.opt_getconstant_path(names) - - yield - - iseq.dup - iseq.putspecialobject(YARV::VM_SPECIAL_OBJECT_CONST_BASE) - iseq.setconstant(names.last) - when CVar - name = node.target.value.value.to_sym - iseq.getclassvariable(name) - - yield - - iseq.dup - iseq.setclassvariable(name) - when GVar - name = node.target.value.value.to_sym - iseq.getglobal(name) - - yield - - iseq.dup - iseq.setglobal(name) - when Ident - local_variable = visit(node.target) - iseq.getlocal(local_variable.index, local_variable.level) - - yield - - iseq.dup - iseq.setlocal(local_variable.index, local_variable.level) - when IVar - name = node.target.value.value.to_sym - iseq.getinstancevariable(name) - - yield - - iseq.dup - iseq.setinstancevariable(name) - end - end - end - end -end diff --git a/lib/syntax_tree/yarv.rb b/lib/syntax_tree/yarv.rb index 89920c6a..df8bc3ce 100644 --- a/lib/syntax_tree/yarv.rb +++ b/lib/syntax_tree/yarv.rb @@ -1,854 +1,11 @@ # frozen_string_literal: true module SyntaxTree + # This module provides an object representation of the YARV bytecode. module YARV - # This object is used to track the size of the stack at any given time. It - # is effectively a mini symbolic interpreter. It's necessary because when - # instruction sequences get serialized they include a :stack_max field on - # them. This field is used to determine how much stack space to allocate - # for the instruction sequence. - class Stack - attr_reader :current_size, :maximum_size - - def initialize - @current_size = 0 - @maximum_size = 0 - end - - def change_by(value) - @current_size += value - @maximum_size = @current_size if @current_size > @maximum_size - end + # Compile the given source into a YARV instruction sequence. + def self.compile(source, **options) + SyntaxTree.parse(source).accept(Compiler.new(**options)) end - - # This represents every local variable associated with an instruction - # sequence. There are two kinds of locals: plain locals that are what you - # expect, and block proxy locals, which represent local variables - # associated with blocks that were passed into the current instruction - # sequence. - class LocalTable - # A local representing a block passed into the current instruction - # sequence. - class BlockLocal - attr_reader :name - - def initialize(name) - @name = name - end - end - - # A regular local variable. - class PlainLocal - attr_reader :name - - def initialize(name) - @name = name - end - end - - # The result of looking up a local variable in the current local table. - class Lookup - attr_reader :local, :index, :level - - def initialize(local, index, level) - @local = local - @index = index - @level = level - end - end - - attr_reader :locals - - def initialize - @locals = [] - end - - def find(name, level) - index = locals.index { |local| local.name == name } - Lookup.new(locals[index], index, level) if index - end - - def has?(name) - locals.any? { |local| local.name == name } - end - - def names - locals.map(&:name) - end - - def size - locals.length - end - - # Add a BlockLocal to the local table. - def block(name) - locals << BlockLocal.new(name) unless has?(name) - end - - # Add a PlainLocal to the local table. - def plain(name) - locals << PlainLocal.new(name) unless has?(name) - end - - # This is the offset from the top of the stack where this local variable - # lives. - def offset(index) - size - (index - 3) - 1 - end - end - - # This class is meant to mirror RubyVM::InstructionSequence. It contains a - # list of instructions along with the metadata pertaining to them. It also - # functions as a builder for the instruction sequence. - class InstructionSequence - MAGIC = "YARVInstructionSequence/SimpleDataFormat" - - # This provides a handle to the rb_iseq_load function, which allows you to - # pass a serialized iseq to Ruby and have it return a - # RubyVM::InstructionSequence object. - ISEQ_LOAD = - Fiddle::Function.new( - Fiddle::Handle::DEFAULT["rb_iseq_load"], - [Fiddle::TYPE_VOIDP] * 3, - Fiddle::TYPE_VOIDP - ) - - # The type of the instruction sequence. - attr_reader :type - - # The name of the instruction sequence. - attr_reader :name - - # The parent instruction sequence, if there is one. - attr_reader :parent_iseq - - # The location of the root node of this instruction sequence. - attr_reader :location - - # This is the list of information about the arguments to this - # instruction sequence. - attr_accessor :argument_size - attr_reader :argument_options - - # The list of instructions for this instruction sequence. - attr_reader :insns - - # The table of local variables. - attr_reader :local_table - - # The hash of names of instance and class variables pointing to the - # index of their associated inline storage. - attr_reader :inline_storages - - # The index of the next inline storage that will be created. - attr_reader :storage_index - - # An object that will track the current size of the stack and the - # maximum size of the stack for this instruction sequence. - attr_reader :stack - - # These are various compilation options provided. - attr_reader :frozen_string_literal, - :operands_unification, - :specialized_instruction - - def initialize( - type, - name, - parent_iseq, - location, - frozen_string_literal: false, - operands_unification: true, - specialized_instruction: true - ) - @type = type - @name = name - @parent_iseq = parent_iseq - @location = location - - @argument_size = 0 - @argument_options = {} - - @local_table = LocalTable.new - @inline_storages = {} - @insns = [] - @storage_index = 0 - @stack = Stack.new - - @frozen_string_literal = frozen_string_literal - @operands_unification = operands_unification - @specialized_instruction = specialized_instruction - end - - ########################################################################## - # Query methods - ########################################################################## - - def local_variable(name, level = 0) - if (lookup = local_table.find(name, level)) - lookup - elsif parent_iseq - parent_iseq.local_variable(name, level + 1) - end - end - - def inline_storage - storage = storage_index - @storage_index += 1 - storage - end - - def inline_storage_for(name) - inline_storages[name] = inline_storage unless inline_storages.key?(name) - - inline_storages[name] - end - - def length - insns.inject(0) do |sum, insn| - case insn - when Integer, Symbol - sum - else - sum + insn.length - end - end - end - - def eval - compiled = to_a - - # Temporary hack until we get these working. - compiled[4][:node_id] = 11 - compiled[4][:node_ids] = [1, 0, 3, 2, 6, 7, 9, -1] - - Fiddle.dlunwrap(ISEQ_LOAD.call(Fiddle.dlwrap(compiled), 0, nil)).eval - end - - def to_a - versions = RUBY_VERSION.split(".").map(&:to_i) - - [ - MAGIC, - versions[0], - versions[1], - 1, - { - arg_size: argument_size, - local_size: local_table.size, - stack_max: stack.maximum_size - }, - name, - "", - "", - location.start_line, - type, - local_table.names, - argument_options, - [], - insns.map do |insn| - case insn - when Integer, Symbol - insn - when Array - case insn[0] - when :setlocal_WC_0, :setlocal_WC_1, :setlocal, :setblockparam - iseq = self - - case insn[0] - when :setlocal_WC_1 - iseq = iseq.parent_iseq - when :setlocal, :setblockparam - insn[2].times { iseq = iseq.parent_iseq } - end - - # Here we need to map the local variable index to the offset - # from the top of the stack where it will be stored. - [insn[0], iseq.local_table.offset(insn[1]), *insn[2..]] - when :send - # For any instructions that push instruction sequences onto the - # stack, we need to call #to_a on them as well. - [insn[0], insn[1], (insn[2].to_a if insn[2])] - when :once - [insn[0], insn[1].to_a, insn[2]] - else - insn - end - else - insn.to_a(self) - end - end - ] - end - - ########################################################################## - # Child instruction sequence methods - ########################################################################## - - def child_iseq(type, name, location) - InstructionSequence.new( - type, - name, - self, - location, - frozen_string_literal: frozen_string_literal, - operands_unification: operands_unification, - specialized_instruction: specialized_instruction - ) - end - - def block_child_iseq(location) - current = self - current = current.parent_iseq while current.type == :block - child_iseq(:block, "block in #{current.name}", location) - end - - def class_child_iseq(name, location) - child_iseq(:class, "", location) - end - - def method_child_iseq(name, location) - child_iseq(:method, name, location) - end - - def module_child_iseq(name, location) - child_iseq(:class, "", location) - end - - def singleton_class_child_iseq(location) - child_iseq(:class, "singleton class", location) - end - - ########################################################################## - # Instruction push methods - ########################################################################## - - def push(insn) - insns << insn - - case insn - when Integer, Symbol, Array - insn - else - stack.change_by(-insn.pops + insn.pushes) - insn - end - end - - # This creates a new label at the current length of the instruction - # sequence. It is used as the operand for jump instructions. - def label - name = :"label_#{length}" - insns.last == name ? name : event(name) - end - - def event(name) - push(name) - end - - def adjuststack(number) - push(AdjustStack.new(number)) - end - - def anytostring - push(AnyToString.new) - end - - def branchif(label) - push(BranchIf.new(label)) - end - - def branchnil(label) - push(BranchNil.new(label)) - end - - def branchunless(label) - push(BranchUnless.new(label)) - end - - def checkkeyword(keyword_bits_index, keyword_index) - push(CheckKeyword.new(keyword_bits_index, keyword_index)) - end - - def checkmatch(flag) - stack.change_by(-2 + 1) - push([:checkmatch, flag]) - end - - def checktype(type) - stack.change_by(-1 + 2) - push([:checktype, type]) - end - - def concatarray - push(ConcatArray.new) - end - - def concatstrings(number) - push(ConcatStrings.new(number)) - end - - def defined(type, name, message) - push(Defined.new(type, name, message)) - end - - def defineclass(name, class_iseq, flags) - push(DefineClass.new(name, class_iseq, flags)) - end - - def definemethod(name, method_iseq) - push(DefineMethod.new(name, method_iseq)) - end - - def definesmethod(name, method_iseq) - push(DefineSMethod.new(name, method_iseq)) - end - - def dup - push(Dup.new) - end - - def duparray(object) - push(DupArray.new(object)) - end - - def duphash(object) - push(DupHash.new(object)) - end - - def dupn(number) - push(DupN.new(number)) - end - - def expandarray(length, flags) - push(ExpandArray.new(length, flags)) - end - - def getblockparam(index, level) - push(GetBlockParam.new(index, level)) - end - - def getblockparamproxy(index, level) - push(GetBlockParamProxy.new(index, level)) - end - - def getclassvariable(name) - if RUBY_VERSION < "3.0" - push(Legacy::GetClassVariable.new(name)) - else - push(GetClassVariable.new(name, inline_storage_for(name))) - end - end - - def getconstant(name) - push(Legacy::GetConstant.new(name)) - end - - def getglobal(name) - push(GetGlobal.new(name)) - end - - def getinstancevariable(name) - if RUBY_VERSION < "3.2" - push(GetInstanceVariable.new(name, inline_storage_for(name))) - else - push(GetInstanceVariable.new(name, inline_storage)) - end - end - - def getlocal(index, level) - if operands_unification - # Specialize the getlocal instruction based on the level of the - # local variable. If it's 0 or 1, then there's a specialized - # instruction that will look at the current scope or the parent - # scope, respectively, and requires fewer operands. - case level - when 0 - push(GetLocalWC0.new(index)) - when 1 - push(GetLocalWC1.new(index)) - else - push(GetLocal.new(index, level)) - end - else - push(GetLocal.new(index, level)) - end - end - - def getspecial(key, type) - stack.change_by(-0 + 1) - push([:getspecial, key, type]) - end - - def intern - stack.change_by(-1 + 1) - push([:intern]) - end - - def invokeblock(method_id, argc, flag = VM_CALL_ARGS_SIMPLE) - stack.change_by(-argc + 1) - push([:invokeblock, call_data(method_id, argc, flag)]) - end - - def invokesuper(method_id, argc, flag, block_iseq) - stack.change_by(-(argc + 1) + 1) - - cdata = call_data(method_id, argc, flag) - push([:invokesuper, cdata, block_iseq]) - end - - def jump(index) - stack.change_by(0) - push([:jump, index]) - end - - def leave - stack.change_by(-1) - push([:leave]) - end - - def newarray(length) - stack.change_by(-length + 1) - push([:newarray, length]) - end - - def newarraykwsplat(length) - stack.change_by(-length + 1) - push([:newarraykwsplat, length]) - end - - def newhash(length) - stack.change_by(-length + 1) - push([:newhash, length]) - end - - def newrange(flag) - stack.change_by(-2 + 1) - push([:newrange, flag]) - end - - def nop - stack.change_by(0) - push([:nop]) - end - - def objtostring(method_id, argc, flag) - stack.change_by(-1 + 1) - push([:objtostring, call_data(method_id, argc, flag)]) - end - - def once(postexe_iseq, inline_storage) - stack.change_by(+1) - push([:once, postexe_iseq, inline_storage]) - end - - def opt_aref_with(object, method_id, argc, flag = VM_CALL_ARGS_SIMPLE) - stack.change_by(-1 + 1) - push([:opt_aref_with, object, call_data(method_id, argc, flag)]) - end - - def opt_aset_with(object, method_id, argc, flag = VM_CALL_ARGS_SIMPLE) - stack.change_by(-2 + 1) - push([:opt_aset_with, object, call_data(method_id, argc, flag)]) - end - - def opt_getconstant_path(names) - if RUBY_VERSION >= "3.2" - stack.change_by(+1) - push([:opt_getconstant_path, names]) - else - const_inline_storage = inline_storage - getinlinecache = opt_getinlinecache(-1, const_inline_storage) - - if names[0] == :"" - names.shift - pop - putobject(Object) - end - - names.each_with_index do |name, index| - putobject(index == 0) - getconstant(name) - end - - opt_setinlinecache(const_inline_storage) - getinlinecache[1] = label - end - end - - def opt_getinlinecache(offset, inline_storage) - stack.change_by(+1) - push([:opt_getinlinecache, offset, inline_storage]) - end - - def opt_newarray_max(length) - if specialized_instruction - stack.change_by(-length + 1) - push([:opt_newarray_max, length]) - else - newarray(length) - send(:max, 0) - end - end - - def opt_newarray_min(length) - if specialized_instruction - stack.change_by(-length + 1) - push([:opt_newarray_min, length]) - else - newarray(length) - send(:min, 0) - end - end - - def opt_setinlinecache(inline_storage) - stack.change_by(-1 + 1) - push([:opt_setinlinecache, inline_storage]) - end - - def opt_str_freeze(value) - if specialized_instruction - stack.change_by(+1) - push([:opt_str_freeze, value, call_data(:freeze, 0)]) - else - putstring(value) - send(:freeze, 0) - end - end - - def opt_str_uminus(value) - if specialized_instruction - stack.change_by(+1) - push([:opt_str_uminus, value, call_data(:-@, 0)]) - else - putstring(value) - send(:-@, 0) - end - end - - def pop - stack.change_by(-1) - push([:pop]) - end - - def putnil - stack.change_by(+1) - push([:putnil]) - end - - def putobject(object) - stack.change_by(+1) - - if operands_unification - # Specialize the putobject instruction based on the value of the - # object. If it's 0 or 1, then there's a specialized instruction - # that will push the object onto the stack and requires fewer - # operands. - if object.eql?(0) - push([:putobject_INT2FIX_0_]) - elsif object.eql?(1) - push([:putobject_INT2FIX_1_]) - else - push([:putobject, object]) - end - else - push([:putobject, object]) - end - end - - def putself - stack.change_by(+1) - push([:putself]) - end - - def putspecialobject(object) - stack.change_by(+1) - push([:putspecialobject, object]) - end - - def putstring(object) - stack.change_by(+1) - push([:putstring, object]) - end - - def send(method_id, argc, flag = VM_CALL_ARGS_SIMPLE, block_iseq = nil) - stack.change_by(-(argc + 1) + 1) - cdata = call_data(method_id, argc, flag) - - if specialized_instruction - # Specialize the send instruction. If it doesn't have a block - # attached, then we will replace it with an opt_send_without_block - # and do further specializations based on the called method and the - # number of arguments. - - # stree-ignore - if !block_iseq && (flag & VM_CALL_ARGS_BLOCKARG) == 0 - case [method_id, argc] - when [:length, 0] then push([:opt_length, cdata]) - when [:size, 0] then push([:opt_size, cdata]) - when [:empty?, 0] then push([:opt_empty_p, cdata]) - when [:nil?, 0] then push([:opt_nil_p, cdata]) - when [:succ, 0] then push([:opt_succ, cdata]) - when [:!, 0] then push([:opt_not, cdata]) - when [:+, 1] then push([:opt_plus, cdata]) - when [:-, 1] then push([:opt_minus, cdata]) - when [:*, 1] then push([:opt_mult, cdata]) - when [:/, 1] then push([:opt_div, cdata]) - when [:%, 1] then push([:opt_mod, cdata]) - when [:==, 1] then push([:opt_eq, cdata]) - when [:=~, 1] then push([:opt_regexpmatch2, cdata]) - when [:<, 1] then push([:opt_lt, cdata]) - when [:<=, 1] then push([:opt_le, cdata]) - when [:>, 1] then push([:opt_gt, cdata]) - when [:>=, 1] then push([:opt_ge, cdata]) - when [:<<, 1] then push([:opt_ltlt, cdata]) - when [:[], 1] then push([:opt_aref, cdata]) - when [:&, 1] then push([:opt_and, cdata]) - when [:|, 1] then push([:opt_or, cdata]) - when [:[]=, 2] then push([:opt_aset, cdata]) - when [:!=, 1] - push([:opt_neq, call_data(:==, 1), cdata]) - else - push([:opt_send_without_block, cdata]) - end - else - push([:send, cdata, block_iseq]) - end - else - push([:send, cdata, block_iseq]) - end - end - - def setblockparam(index, level) - stack.change_by(-1) - push([:setblockparam, index, level]) - end - - def setclassvariable(name) - stack.change_by(-1) - - if RUBY_VERSION >= "3.0" - push([:setclassvariable, name, inline_storage_for(name)]) - else - push([:setclassvariable, name]) - end - end - - def setconstant(name) - stack.change_by(-2) - push([:setconstant, name]) - end - - def setglobal(name) - stack.change_by(-1) - push([:setglobal, name]) - end - - def setinstancevariable(name) - stack.change_by(-1) - - if RUBY_VERSION >= "3.2" - push([:setinstancevariable, name, inline_storage]) - else - push([:setinstancevariable, name, inline_storage_for(name)]) - end - end - - def setlocal(index, level) - stack.change_by(-1) - - if operands_unification - # Specialize the setlocal instruction based on the level of the - # local variable. If it's 0 or 1, then there's a specialized - # instruction that will write to the current scope or the parent - # scope, respectively, and requires fewer operands. - case level - when 0 - push([:setlocal_WC_0, index]) - when 1 - push([:setlocal_WC_1, index]) - else - push([:setlocal, index, level]) - end - else - push([:setlocal, index, level]) - end - end - - def setn(number) - stack.change_by(-1 + 1) - push([:setn, number]) - end - - def setspecial(key) - stack.change_by(-1) - push([:setspecial, key]) - end - - def splatarray(flag) - stack.change_by(-1 + 1) - push([:splatarray, flag]) - end - - def swap - stack.change_by(-2 + 2) - push([:swap]) - end - - def topn(number) - stack.change_by(+1) - push([:topn, number]) - end - - def toregexp(options, length) - stack.change_by(-length + 1) - push([:toregexp, options, length]) - end - - private - - # This creates a call data object that is used as the operand for the - # send, invokesuper, and objtostring instructions. - def call_data(method_id, argc, flag = VM_CALL_ARGS_SIMPLE) - { mid: method_id, flag: flag, orig_argc: argc } - end - end - - # These constants correspond to the putspecialobject instruction. They are - # used to represent special objects that are pushed onto the stack. - VM_SPECIAL_OBJECT_VMCORE = 1 - VM_SPECIAL_OBJECT_CBASE = 2 - VM_SPECIAL_OBJECT_CONST_BASE = 3 - - # These constants correspond to the flag passed as part of the call data - # structure on the send instruction. They are used to represent various - # metadata about the callsite (e.g., were keyword arguments used?, was a - # block given?, etc.). - VM_CALL_ARGS_SPLAT = 1 << 0 - VM_CALL_ARGS_BLOCKARG = 1 << 1 - VM_CALL_FCALL = 1 << 2 - VM_CALL_VCALL = 1 << 3 - VM_CALL_ARGS_SIMPLE = 1 << 4 - VM_CALL_BLOCKISEQ = 1 << 5 - VM_CALL_KWARG = 1 << 6 - VM_CALL_KW_SPLAT = 1 << 7 - VM_CALL_TAILCALL = 1 << 8 - VM_CALL_SUPER = 1 << 9 - VM_CALL_ZSUPER = 1 << 10 - VM_CALL_OPT_SEND = 1 << 11 - VM_CALL_KW_SPLAT_MUT = 1 << 12 - - # These constants correspond to the setspecial instruction. - VM_SVAR_LASTLINE = 0 # $_ - VM_SVAR_BACKREF = 1 # $~ - VM_SVAR_FLIPFLOP_START = 2 # flipflop - - # These constants correspond to the checktype instruction. - VM_CHECKTYPE_ARRAY = 7 - - # These constants correspond to the checkmatch instruction. - VM_CHECKMATCH_TYPE_WHEN = 1 - VM_CHECKMATCH_TYPE_CASE = 2 - VM_CHECKMATCH_TYPE_RESCUE = 3 end end diff --git a/lib/syntax_tree/yarv/bf.rb b/lib/syntax_tree/yarv/bf.rb index 05c05705..0fb27f7e 100644 --- a/lib/syntax_tree/yarv/bf.rb +++ b/lib/syntax_tree/yarv/bf.rb @@ -20,7 +20,7 @@ def compile iseq.setglobal(:$tape) iseq.getglobal(:$tape) iseq.putobject(0) - iseq.send(:default=, 1) + iseq.send(YARV.calldata(:default=, 1)) # Set up the $cursor global variable that will hold the current position # in the tape. @@ -99,17 +99,17 @@ def change_by(iseq, value) iseq.getglobal(:$tape) iseq.getglobal(:$cursor) - iseq.send(:[], 1) + iseq.send(YARV.calldata(:[], 1)) if value < 0 iseq.putobject(-value) - iseq.send(:-, 1) + iseq.send(YARV.calldata(:-, 1)) else iseq.putobject(value) - iseq.send(:+, 1) + iseq.send(YARV.calldata(:+, 1)) end - iseq.send(:[]=, 2) + iseq.send(YARV.calldata(:[]=, 2)) end # $cursor += value @@ -118,10 +118,10 @@ def shift_by(iseq, value) if value < 0 iseq.putobject(-value) - iseq.send(:-, 1) + iseq.send(YARV.calldata(:-, 1)) else iseq.putobject(value) - iseq.send(:+, 1) + iseq.send(YARV.calldata(:+, 1)) end iseq.setglobal(:$cursor) @@ -133,10 +133,10 @@ def output_char(iseq) iseq.getglobal(:$tape) iseq.getglobal(:$cursor) - iseq.send(:[], 1) - iseq.send(:chr, 0) + iseq.send(YARV.calldata(:[], 1)) + iseq.send(YARV.calldata(:chr)) - iseq.send(:putc, 1) + iseq.send(YARV.calldata(:putc, 1)) end # $tape[$cursor] = $stdin.getc.ord @@ -145,10 +145,10 @@ def input_char(iseq) iseq.getglobal(:$cursor) iseq.getglobal(:$stdin) - iseq.send(:getc, 0) - iseq.send(:ord, 0) + iseq.send(YARV.calldata(:getc)) + iseq.send(YARV.calldata(:ord)) - iseq.send(:[]=, 2) + iseq.send(YARV.calldata(:[]=, 2)) end # unless $tape[$cursor] == 0 @@ -157,10 +157,10 @@ def loop_start(iseq) iseq.getglobal(:$tape) iseq.getglobal(:$cursor) - iseq.send(:[], 1) + iseq.send(YARV.calldata(:[], 1)) iseq.putobject(0) - iseq.send(:==, 1) + iseq.send(YARV.calldata(:==, 1)) branchunless = iseq.branchunless(-1) [start_label, branchunless] diff --git a/lib/syntax_tree/yarv/compiler.rb b/lib/syntax_tree/yarv/compiler.rb new file mode 100644 index 00000000..45f2bb59 --- /dev/null +++ b/lib/syntax_tree/yarv/compiler.rb @@ -0,0 +1,2164 @@ +# frozen_string_literal: true + +module SyntaxTree + module YARV + # This class is an experiment in transforming Syntax Tree nodes into their + # corresponding YARV instruction sequences. It attempts to mirror the + # behavior of RubyVM::InstructionSequence.compile. + # + # You use this as with any other visitor. First you parse code into a tree, + # then you visit it with this compiler. Visiting the root node of the tree + # will return a SyntaxTree::Visitor::Compiler::InstructionSequence object. + # With that object you can call #to_a on it, which will return a serialized + # form of the instruction sequence as an array. This array _should_ mirror + # the array given by RubyVM::InstructionSequence#to_a. + # + # As an example, here is how you would compile a single expression: + # + # program = SyntaxTree.parse("1 + 2") + # program.accept(SyntaxTree::YARV::Compiler.new).to_a + # + # [ + # "YARVInstructionSequence/SimpleDataFormat", + # 3, + # 1, + # 1, + # {:arg_size=>0, :local_size=>0, :stack_max=>2}, + # "", + # "", + # "", + # 1, + # :top, + # [], + # {}, + # [], + # [ + # [:putobject_INT2FIX_1_], + # [:putobject, 2], + # [:opt_plus, {:mid=>:+, :flag=>16, :orig_argc=>1}], + # [:leave] + # ] + # ] + # + # Note that this is the same output as calling: + # + # RubyVM::InstructionSequence.compile("1 + 2").to_a + # + class Compiler < BasicVisitor + # This visitor is responsible for converting Syntax Tree nodes into their + # corresponding Ruby structures. This is used to convert the operands of + # some instructions like putobject that push a Ruby object directly onto + # the stack. It is only used when the entire structure can be represented + # at compile-time, as opposed to constructed at run-time. + class RubyVisitor < BasicVisitor + # This error is raised whenever a node cannot be converted into a Ruby + # object at compile-time. + class CompilationError < StandardError + end + + # This will attempt to compile the given node. If it's possible, then + # it will return the compiled object. Otherwise it will return nil. + def self.compile(node) + node.accept(new) + rescue CompilationError + end + + def visit_array(node) + visit_all(node.contents.parts) + end + + def visit_bare_assoc_hash(node) + node.assocs.to_h do |assoc| + # We can only convert regular key-value pairs. A double splat ** + # operator means it has to be converted at run-time. + raise CompilationError unless assoc.is_a?(Assoc) + [visit(assoc.key), visit(assoc.value)] + end + end + + def visit_float(node) + node.value.to_f + end + + alias visit_hash visit_bare_assoc_hash + + def visit_imaginary(node) + node.value.to_c + end + + def visit_int(node) + node.value.to_i + end + + def visit_label(node) + node.value.chomp(":").to_sym + end + + def visit_mrhs(node) + visit_all(node.parts) + end + + def visit_qsymbols(node) + node.elements.map { |element| visit(element).to_sym } + end + + def visit_qwords(node) + visit_all(node.elements) + end + + def visit_range(node) + left, right = [visit(node.left), visit(node.right)] + node.operator.value === ".." ? left..right : left...right + end + + def visit_rational(node) + node.value.to_r + end + + def visit_regexp_literal(node) + if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) + Regexp.new(node.parts.first.value, visit_regexp_literal_flags(node)) + else + # Any interpolation of expressions or variables will result in the + # regular expression being constructed at run-time. + raise CompilationError + end + end + + # This isn't actually a visit method, though maybe it should be. It is + # responsible for converting the set of string options on a regular + # expression into its equivalent integer. + def visit_regexp_literal_flags(node) + node + .options + .chars + .inject(0) do |accum, option| + accum | + case option + when "i" + Regexp::IGNORECASE + when "x" + Regexp::EXTENDED + when "m" + Regexp::MULTILINE + else + raise "Unknown regexp option: #{option}" + end + end + end + + def visit_symbol_literal(node) + node.value.value.to_sym + end + + def visit_symbols(node) + node.elements.map { |element| visit(element).to_sym } + end + + def visit_tstring_content(node) + node.value + end + + def visit_var_ref(node) + raise CompilationError unless node.value.is_a?(Kw) + + case node.value.value + when "nil" + nil + when "true" + true + when "false" + false + else + raise CompilationError + end + end + + def visit_word(node) + if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) + node.parts.first.value + else + # Any interpolation of expressions or variables will result in the + # string being constructed at run-time. + raise CompilationError + end + end + + def visit_words(node) + visit_all(node.elements) + end + + def visit_unsupported(_node) + raise CompilationError + end + + # Please forgive the metaprogramming here. This is used to create visit + # methods for every node that we did not explicitly handle. By default + # each of these methods will raise a CompilationError. + handled = instance_methods(false) + (Visitor.instance_methods(false) - handled).each do |method| + alias_method method, :visit_unsupported + end + end + + # These options mirror the compilation options that we currently support + # that can be also passed to RubyVM::InstructionSequence.compile. + attr_reader :frozen_string_literal, + :operands_unification, + :specialized_instruction + + # The current instruction sequence that is being compiled. + attr_reader :iseq + + # A boolean to track if we're currently compiling the last statement + # within a set of statements. This information is necessary to determine + # if we need to return the value of the last statement. + attr_reader :last_statement + + def initialize( + frozen_string_literal: false, + operands_unification: true, + specialized_instruction: true + ) + @frozen_string_literal = frozen_string_literal + @operands_unification = operands_unification + @specialized_instruction = specialized_instruction + + @iseq = nil + @last_statement = false + end + + def visit_BEGIN(node) + visit(node.statements) + end + + def visit_CHAR(node) + if frozen_string_literal + iseq.putobject(node.value[1..]) + else + iseq.putstring(node.value[1..]) + end + end + + def visit_END(node) + once_iseq = + with_child_iseq(iseq.block_child_iseq(node.location)) do + postexe_iseq = + with_child_iseq(iseq.block_child_iseq(node.location)) do + iseq.event(:RUBY_EVENT_B_CALL) + + *statements, last_statement = node.statements.body + visit_all(statements) + with_last_statement { visit(last_statement) } + + iseq.event(:RUBY_EVENT_B_RETURN) + iseq.leave + end + + iseq.putspecialobject(PutSpecialObject::OBJECT_VMCORE) + iseq.send( + YARV.calldata(:"core#set_postexe", 0, CallData::CALL_FCALL), + postexe_iseq + ) + iseq.leave + end + + iseq.once(once_iseq, iseq.inline_storage) + iseq.pop + end + + def visit_alias(node) + iseq.putspecialobject(PutSpecialObject::OBJECT_VMCORE) + iseq.putspecialobject(PutSpecialObject::OBJECT_CBASE) + visit(node.left) + visit(node.right) + iseq.send(YARV.calldata(:"core#set_method_alias", 3)) + end + + def visit_aref(node) + calldata = YARV.calldata(:[], 1) + visit(node.collection) + + if !frozen_string_literal && specialized_instruction && + (node.index.parts.length == 1) + arg = node.index.parts.first + + if arg.is_a?(StringLiteral) && (arg.parts.length == 1) + string_part = arg.parts.first + + if string_part.is_a?(TStringContent) + iseq.opt_aref_with(string_part.value, calldata) + return + end + end + end + + visit(node.index) + iseq.send(calldata) + end + + def visit_arg_block(node) + visit(node.value) + end + + def visit_arg_paren(node) + visit(node.arguments) + end + + def visit_arg_star(node) + visit(node.value) + iseq.splatarray(false) + end + + def visit_args(node) + visit_all(node.parts) + end + + def visit_array(node) + if (compiled = RubyVisitor.compile(node)) + iseq.duparray(compiled) + elsif node.contents && node.contents.parts.length == 1 && + node.contents.parts.first.is_a?(BareAssocHash) && + node.contents.parts.first.assocs.length == 1 && + node.contents.parts.first.assocs.first.is_a?(AssocSplat) + iseq.putspecialobject(PutSpecialObject::OBJECT_VMCORE) + iseq.newhash(0) + visit(node.contents.parts.first) + iseq.send(YARV.calldata(:"core#hash_merge_kwd", 2)) + iseq.newarraykwsplat(1) + else + length = 0 + + node.contents.parts.each do |part| + if part.is_a?(ArgStar) + if length > 0 + iseq.newarray(length) + length = 0 + end + + visit(part.value) + iseq.concatarray + else + visit(part) + length += 1 + end + end + + iseq.newarray(length) if length > 0 + iseq.concatarray if length > 0 && length != node.contents.parts.length + end + end + + def visit_aryptn(node) + match_failures = [] + jumps_to_exit = [] + + # If there's a constant, then check if we match against that constant or + # not first. Branch to failure if we don't. + if node.constant + iseq.dup + visit(node.constant) + iseq.checkmatch(CheckMatch::TYPE_CASE) + match_failures << iseq.branchunless(-1) + end + + # First, check if the #deconstruct cache is nil. If it is, we're going + # to call #deconstruct on the object and cache the result. + iseq.topn(2) + branchnil = iseq.branchnil(-1) + + # Next, ensure that the cached value was cached correctly, otherwise + # fail the match. + iseq.topn(2) + match_failures << iseq.branchunless(-1) + + # Since we have a valid cached value, we can skip past the part where we + # call #deconstruct on the object. + iseq.pop + iseq.topn(1) + jump = iseq.jump(-1) + + # Check if the object responds to #deconstruct, fail the match + # otherwise. + branchnil.patch!(iseq) + iseq.dup + iseq.putobject(:deconstruct) + iseq.send(YARV.calldata(:respond_to?, 1)) + iseq.setn(3) + match_failures << iseq.branchunless(-1) + + # Call #deconstruct and ensure that it's an array, raise an error + # otherwise. + iseq.send(YARV.calldata(:deconstruct)) + iseq.setn(2) + iseq.dup + iseq.checktype(CheckType::TYPE_ARRAY) + match_error = iseq.branchunless(-1) + + # Ensure that the deconstructed array has the correct size, fail the + # match otherwise. + jump.patch!(iseq) + iseq.dup + iseq.send(YARV.calldata(:length)) + iseq.putobject(node.requireds.length) + iseq.send(YARV.calldata(:==, 1)) + match_failures << iseq.branchunless(-1) + + # For each required element, check if the deconstructed array contains + # the element, otherwise jump out to the top-level match failure. + iseq.dup + node.requireds.each_with_index do |required, index| + iseq.putobject(index) + iseq.send(YARV.calldata(:[], 1)) + + case required + when VarField + lookup = visit(required) + iseq.setlocal(lookup.index, lookup.level) + else + visit(required) + iseq.checkmatch(CheckMatch::TYPE_CASE) + match_failures << iseq.branchunless(-1) + end + + if index < node.requireds.length - 1 + iseq.dup + else + iseq.pop + jumps_to_exit << iseq.jump(-1) + end + end + + # Set up the routine here to raise an error to indicate that the type of + # the deconstructed array was incorrect. + match_error.patch!(iseq) + iseq.putspecialobject(PutSpecialObject::OBJECT_VMCORE) + iseq.putobject(TypeError) + iseq.putobject("deconstruct must return Array") + iseq.send(YARV.calldata(:"core#raise", 2)) + iseq.pop + + # Patch all of the match failures to jump here so that we pop a final + # value before returning to the parent node. + match_failures.each { |match_failure| match_failure.patch!(iseq) } + iseq.pop + jumps_to_exit + end + + def visit_assign(node) + case node.target + when ARefField + calldata = YARV.calldata(:[]=, 2) + + if !frozen_string_literal && specialized_instruction && + (node.target.index.parts.length == 1) + arg = node.target.index.parts.first + + if arg.is_a?(StringLiteral) && (arg.parts.length == 1) + string_part = arg.parts.first + + if string_part.is_a?(TStringContent) + visit(node.target.collection) + visit(node.value) + iseq.swap + iseq.topn(1) + iseq.opt_aset_with(string_part.value, calldata) + iseq.pop + return + end + end + end + + iseq.putnil + visit(node.target.collection) + visit(node.target.index) + visit(node.value) + iseq.setn(3) + iseq.send(calldata) + iseq.pop + when ConstPathField + names = constant_names(node.target) + name = names.pop + + if RUBY_VERSION >= "3.2" + iseq.opt_getconstant_path(names) + visit(node.value) + iseq.swap + iseq.topn(1) + iseq.swap + iseq.setconstant(name) + else + visit(node.value) + iseq.dup if last_statement? + iseq.opt_getconstant_path(names) + iseq.setconstant(name) + end + when Field + iseq.putnil + visit(node.target) + visit(node.value) + iseq.setn(2) + iseq.send(YARV.calldata(:"#{node.target.name.value}=", 1)) + iseq.pop + when TopConstField + name = node.target.constant.value.to_sym + + if RUBY_VERSION >= "3.2" + iseq.putobject(Object) + visit(node.value) + iseq.swap + iseq.topn(1) + iseq.swap + iseq.setconstant(name) + else + visit(node.value) + iseq.dup if last_statement? + iseq.putobject(Object) + iseq.setconstant(name) + end + when VarField + visit(node.value) + iseq.dup if last_statement? + + case node.target.value + when Const + iseq.putspecialobject(PutSpecialObject::OBJECT_CONST_BASE) + iseq.setconstant(node.target.value.value.to_sym) + when CVar + iseq.setclassvariable(node.target.value.value.to_sym) + when GVar + iseq.setglobal(node.target.value.value.to_sym) + when Ident + lookup = visit(node.target) + + if lookup.local.is_a?(LocalTable::BlockLocal) + iseq.setblockparam(lookup.index, lookup.level) + else + iseq.setlocal(lookup.index, lookup.level) + end + when IVar + iseq.setinstancevariable(node.target.value.value.to_sym) + end + end + end + + def visit_assoc(node) + visit(node.key) + visit(node.value) + end + + def visit_assoc_splat(node) + visit(node.value) + end + + def visit_backref(node) + iseq.getspecial(GetSpecial::SVAR_BACKREF, node.value[1..].to_i << 1) + end + + def visit_bare_assoc_hash(node) + if (compiled = RubyVisitor.compile(node)) + iseq.duphash(compiled) + else + visit_all(node.assocs) + end + end + + def visit_binary(node) + case node.operator + when :"&&" + visit(node.left) + iseq.dup + + branchunless = iseq.branchunless(-1) + iseq.pop + + visit(node.right) + branchunless.patch!(iseq) + when :"||" + visit(node.left) + iseq.dup + + branchif = iseq.branchif(-1) + iseq.pop + + visit(node.right) + branchif.patch!(iseq) + else + visit(node.left) + visit(node.right) + iseq.send(YARV.calldata(node.operator, 1)) + end + end + + def visit_block(node) + with_child_iseq(iseq.block_child_iseq(node.location)) do + iseq.event(:RUBY_EVENT_B_CALL) + visit(node.block_var) + visit(node.bodystmt) + iseq.event(:RUBY_EVENT_B_RETURN) + iseq.leave + end + end + + def visit_block_var(node) + params = node.params + + if params.requireds.length == 1 && params.optionals.empty? && + !params.rest && params.posts.empty? && params.keywords.empty? && + !params.keyword_rest && !params.block + iseq.argument_options[:ambiguous_param0] = true + end + + visit(node.params) + + node.locals.each { |local| iseq.local_table.plain(local.value.to_sym) } + end + + def visit_blockarg(node) + iseq.argument_options[:block_start] = iseq.argument_size + iseq.local_table.block(node.name.value.to_sym) + iseq.argument_size += 1 + end + + def visit_bodystmt(node) + visit(node.statements) + end + + def visit_call(node) + if node.is_a?(CallNode) + return( + visit_call( + CommandCall.new( + receiver: node.receiver, + operator: node.operator, + message: node.message, + arguments: node.arguments, + block: nil, + location: node.location + ) + ) + ) + end + + arg_parts = argument_parts(node.arguments) + argc = arg_parts.length + + # First we're going to check if we're calling a method on an array + # literal without any arguments. In that case there are some + # specializations we might be able to perform. + if argc == 0 && (node.message.is_a?(Ident) || node.message.is_a?(Op)) + case node.receiver + when ArrayLiteral + parts = node.receiver.contents&.parts || [] + + if parts.none? { |part| part.is_a?(ArgStar) } && + RubyVisitor.compile(node.receiver).nil? + case node.message.value + when "max" + visit(node.receiver.contents) + iseq.opt_newarray_max(parts.length) + return + when "min" + visit(node.receiver.contents) + iseq.opt_newarray_min(parts.length) + return + end + end + when StringLiteral + if RubyVisitor.compile(node.receiver).nil? + case node.message.value + when "-@" + iseq.opt_str_uminus(node.receiver.parts.first.value) + return + when "freeze" + iseq.opt_str_freeze(node.receiver.parts.first.value) + return + end + end + end + end + + if node.receiver + if node.receiver.is_a?(VarRef) + lookup = iseq.local_variable(node.receiver.value.value.to_sym) + + if lookup.local.is_a?(LocalTable::BlockLocal) + iseq.getblockparamproxy(lookup.index, lookup.level) + else + visit(node.receiver) + end + else + visit(node.receiver) + end + else + iseq.putself + end + + branchnil = + if node.operator&.value == "&." + iseq.dup + iseq.branchnil(-1) + end + + flag = 0 + + arg_parts.each do |arg_part| + case arg_part + when ArgBlock + argc -= 1 + flag |= CallData::CALL_ARGS_BLOCKARG + visit(arg_part) + when ArgStar + flag |= CallData::CALL_ARGS_SPLAT + visit(arg_part) + when ArgsForward + flag |= CallData::CALL_ARGS_SPLAT + flag |= CallData::CALL_ARGS_BLOCKARG + + lookup = iseq.local_table.find(:*) + iseq.getlocal(lookup.index, lookup.level) + iseq.splatarray(arg_parts.length != 1) + + lookup = iseq.local_table.find(:&) + iseq.getblockparamproxy(lookup.index, lookup.level) + when BareAssocHash + flag |= CallData::CALL_KW_SPLAT + visit(arg_part) + else + visit(arg_part) + end + end + + block_iseq = visit(node.block) if node.block + flag |= CallData::CALL_ARGS_SIMPLE if block_iseq.nil? && flag == 0 + flag |= CallData::CALL_FCALL if node.receiver.nil? + + iseq.send( + YARV.calldata(node.message.value.to_sym, argc, flag), + block_iseq + ) + branchnil.patch!(iseq) if branchnil + end + + def visit_case(node) + visit(node.value) if node.value + + clauses = [] + else_clause = nil + current = node.consequent + + while current + clauses << current + + if (current = current.consequent).is_a?(Else) + else_clause = current + break + end + end + + branches = + clauses.map do |clause| + visit(clause.arguments) + iseq.topn(1) + iseq.send( + YARV.calldata( + :===, + 1, + CallData::CALL_FCALL | CallData::CALL_ARGS_SIMPLE + ) + ) + [clause, iseq.branchif(:label_00)] + end + + iseq.pop + else_clause ? visit(else_clause) : iseq.putnil + iseq.leave + + branches.each_with_index do |(clause, branchif), index| + iseq.leave if index != 0 + branchif.patch!(iseq) + iseq.pop + visit(clause) + end + end + + def visit_class(node) + name = node.constant.constant.value.to_sym + class_iseq = + with_child_iseq(iseq.class_child_iseq(name, node.location)) do + iseq.event(:RUBY_EVENT_CLASS) + visit(node.bodystmt) + iseq.event(:RUBY_EVENT_END) + iseq.leave + end + + flags = DefineClass::TYPE_CLASS + + case node.constant + when ConstPathRef + flags |= DefineClass::FLAG_SCOPED + visit(node.constant.parent) + when ConstRef + iseq.putspecialobject(PutSpecialObject::OBJECT_CONST_BASE) + when TopConstRef + flags |= DefineClass::FLAG_SCOPED + iseq.putobject(Object) + end + + if node.superclass + flags |= DefineClass::FLAG_HAS_SUPERCLASS + visit(node.superclass) + else + iseq.putnil + end + + iseq.defineclass(name, class_iseq, flags) + end + + def visit_command(node) + visit_call( + CommandCall.new( + receiver: nil, + operator: nil, + message: node.message, + arguments: node.arguments, + block: node.block, + location: node.location + ) + ) + end + + def visit_command_call(node) + visit_call( + CommandCall.new( + receiver: node.receiver, + operator: node.operator, + message: node.message, + arguments: node.arguments, + block: node.block, + location: node.location + ) + ) + end + + def visit_const_path_field(node) + visit(node.parent) + end + + def visit_const_path_ref(node) + names = constant_names(node) + iseq.opt_getconstant_path(names) + end + + def visit_def(node) + name = node.name.value.to_sym + method_iseq = iseq.method_child_iseq(name.to_s, node.location) + + with_child_iseq(method_iseq) do + visit(node.params) if node.params + iseq.event(:RUBY_EVENT_CALL) + visit(node.bodystmt) + iseq.event(:RUBY_EVENT_RETURN) + iseq.leave + end + + if node.target + visit(node.target) + iseq.definesmethod(name, method_iseq) + else + iseq.definemethod(name, method_iseq) + end + + iseq.putobject(name) + end + + def visit_defined(node) + case node.value + when Assign + # If we're assigning to a local variable, then we need to make sure + # that we put it into the local table. + if node.value.target.is_a?(VarField) && + node.value.target.value.is_a?(Ident) + iseq.local_table.plain(node.value.target.value.value.to_sym) + end + + iseq.putobject("assignment") + when VarRef + value = node.value.value + name = value.value.to_sym + + case value + when Const + iseq.putnil + iseq.defined(Defined::TYPE_CONST, name, "constant") + when CVar + iseq.putnil + iseq.defined(Defined::TYPE_CVAR, name, "class variable") + when GVar + iseq.putnil + iseq.defined(Defined::TYPE_GVAR, name, "global-variable") + when Ident + iseq.putobject("local-variable") + when IVar + iseq.putnil + iseq.defined(Defined::TYPE_IVAR, name, "instance-variable") + when Kw + case name + when :false + iseq.putobject("false") + when :nil + iseq.putobject("nil") + when :self + iseq.putobject("self") + when :true + iseq.putobject("true") + end + end + when VCall + iseq.putself + + name = node.value.value.value.to_sym + iseq.defined(Defined::TYPE_FUNC, name, "method") + when YieldNode + iseq.putnil + iseq.defined(Defined::TYPE_YIELD, false, "yield") + when ZSuper + iseq.putnil + iseq.defined(Defined::TYPE_ZSUPER, false, "super") + else + iseq.putobject("expression") + end + end + + def visit_dyna_symbol(node) + if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) + iseq.putobject(node.parts.first.value.to_sym) + end + end + + def visit_else(node) + visit(node.statements) + iseq.pop unless last_statement? + end + + def visit_elsif(node) + visit_if( + IfNode.new( + predicate: node.predicate, + statements: node.statements, + consequent: node.consequent, + location: node.location + ) + ) + end + + def visit_field(node) + visit(node.parent) + end + + def visit_float(node) + iseq.putobject(node.accept(RubyVisitor.new)) + end + + def visit_for(node) + visit(node.collection) + + name = node.index.value.value.to_sym + iseq.local_table.plain(name) + + block_iseq = + with_child_iseq(iseq.block_child_iseq(node.statements.location)) do + iseq.argument_options[:lead_num] ||= 0 + iseq.argument_options[:lead_num] += 1 + iseq.argument_options[:ambiguous_param0] = true + + iseq.argument_size += 1 + iseq.local_table.plain(2) + + iseq.getlocal(0, 0) + + local_variable = iseq.local_variable(name) + iseq.setlocal(local_variable.index, local_variable.level) + + iseq.event(:RUBY_EVENT_B_CALL) + iseq.nop + + visit(node.statements) + iseq.event(:RUBY_EVENT_B_RETURN) + iseq.leave + end + + iseq.send(YARV.calldata(:each, 0, 0), block_iseq) + end + + def visit_hash(node) + if (compiled = RubyVisitor.compile(node)) + iseq.duphash(compiled) + else + visit_all(node.assocs) + iseq.newhash(node.assocs.length * 2) + end + end + + def visit_heredoc(node) + if node.beginning.value.end_with?("`") + visit_xstring_literal(node) + elsif node.parts.length == 1 && node.parts.first.is_a?(TStringContent) + visit(node.parts.first) + else + length = visit_string_parts(node) + iseq.concatstrings(length) + end + end + + def visit_if(node) + if node.predicate.is_a?(RangeNode) + iseq.getspecial(GetSpecial::SVAR_FLIPFLOP_START, 0) + branchif = iseq.branchif(-1) + + visit(node.predicate.left) + branchunless_true = iseq.branchunless(-1) + + iseq.putobject(true) + iseq.setspecial(GetSpecial::SVAR_FLIPFLOP_START) + branchif.patch!(iseq) + + visit(node.predicate.right) + branchunless_false = iseq.branchunless(-1) + + iseq.putobject(false) + iseq.setspecial(GetSpecial::SVAR_FLIPFLOP_START) + branchunless_false.patch!(iseq) + + visit(node.statements) + iseq.leave + branchunless_true.patch!(iseq) + iseq.putnil + else + visit(node.predicate) + branchunless = iseq.branchunless(-1) + visit(node.statements) + + if last_statement? + iseq.leave + branchunless.patch!(iseq) + + node.consequent ? visit(node.consequent) : iseq.putnil + else + iseq.pop + + if node.consequent + jump = iseq.jump(-1) + branchunless.patch!(iseq) + visit(node.consequent) + jump.patch!(iseq) + else + branchunless.patch!(iseq) + end + end + end + end + + def visit_if_op(node) + visit_if( + IfNode.new( + predicate: node.predicate, + statements: node.truthy, + consequent: + Else.new( + keyword: Kw.new(value: "else", location: Location.default), + statements: node.falsy, + location: Location.default + ), + location: Location.default + ) + ) + end + + def visit_imaginary(node) + iseq.putobject(node.accept(RubyVisitor.new)) + end + + def visit_int(node) + iseq.putobject(node.accept(RubyVisitor.new)) + end + + def visit_kwrest_param(node) + iseq.argument_options[:kwrest] = iseq.argument_size + iseq.argument_size += 1 + iseq.local_table.plain(node.name.value.to_sym) + end + + def visit_label(node) + iseq.putobject(node.accept(RubyVisitor.new)) + end + + def visit_lambda(node) + lambda_iseq = + with_child_iseq(iseq.block_child_iseq(node.location)) do + iseq.event(:RUBY_EVENT_B_CALL) + visit(node.params) + visit(node.statements) + iseq.event(:RUBY_EVENT_B_RETURN) + iseq.leave + end + + iseq.putspecialobject(PutSpecialObject::OBJECT_VMCORE) + iseq.send(YARV.calldata(:lambda, 0, CallData::CALL_FCALL), lambda_iseq) + end + + def visit_lambda_var(node) + visit_block_var(node) + end + + def visit_massign(node) + visit(node.value) + iseq.dup + visit(node.target) + end + + def visit_method_add_block(node) + visit_call( + CommandCall.new( + receiver: node.call.receiver, + operator: node.call.operator, + message: node.call.message, + arguments: node.call.arguments, + block: node.block, + location: node.location + ) + ) + end + + def visit_mlhs(node) + lookups = [] + node.parts.each do |part| + case part + when VarField + lookups << visit(part) + end + end + + iseq.expandarray(lookups.length, 0) + lookups.each { |lookup| iseq.setlocal(lookup.index, lookup.level) } + end + + def visit_module(node) + name = node.constant.constant.value.to_sym + module_iseq = + with_child_iseq(iseq.module_child_iseq(name, node.location)) do + iseq.event(:RUBY_EVENT_CLASS) + visit(node.bodystmt) + iseq.event(:RUBY_EVENT_END) + iseq.leave + end + + flags = DefineClass::TYPE_MODULE + + case node.constant + when ConstPathRef + flags |= DefineClass::FLAG_SCOPED + visit(node.constant.parent) + when ConstRef + iseq.putspecialobject(PutSpecialObject::OBJECT_CONST_BASE) + when TopConstRef + flags |= DefineClass::FLAG_SCOPED + iseq.putobject(Object) + end + + iseq.putnil + iseq.defineclass(name, module_iseq, flags) + end + + def visit_mrhs(node) + if (compiled = RubyVisitor.compile(node)) + iseq.duparray(compiled) + else + visit_all(node.parts) + iseq.newarray(node.parts.length) + end + end + + def visit_not(node) + visit(node.statement) + iseq.send(YARV.calldata(:!)) + end + + def visit_opassign(node) + flag = CallData::CALL_ARGS_SIMPLE + if node.target.is_a?(ConstPathField) || node.target.is_a?(TopConstField) + flag |= CallData::CALL_FCALL + end + + case (operator = node.operator.value.chomp("=").to_sym) + when :"&&" + branchunless = nil + + with_opassign(node) do + iseq.dup + branchunless = iseq.branchunless(-1) + iseq.pop + visit(node.value) + end + + case node.target + when ARefField + iseq.leave + branchunless.patch!(iseq) + iseq.setn(3) + iseq.adjuststack(3) + when ConstPathField, TopConstField + branchunless.patch!(iseq) + iseq.swap + iseq.pop + else + branchunless.patch!(iseq) + end + when :"||" + if node.target.is_a?(ConstPathField) || + node.target.is_a?(TopConstField) + opassign_defined(node) + iseq.swap + iseq.pop + elsif node.target.is_a?(VarField) && + [Const, CVar, GVar].include?(node.target.value.class) + opassign_defined(node) + else + branchif = nil + + with_opassign(node) do + iseq.dup + branchif = iseq.branchif(-1) + iseq.pop + visit(node.value) + end + + if node.target.is_a?(ARefField) + iseq.leave + branchif.patch!(iseq) + iseq.setn(3) + iseq.adjuststack(3) + else + branchif.patch!(iseq) + end + end + else + with_opassign(node) do + visit(node.value) + iseq.send(YARV.calldata(operator, 1, flag)) + end + end + end + + def visit_params(node) + argument_options = iseq.argument_options + + if node.requireds.any? + argument_options[:lead_num] = 0 + + node.requireds.each do |required| + iseq.local_table.plain(required.value.to_sym) + iseq.argument_size += 1 + argument_options[:lead_num] += 1 + end + end + + node.optionals.each do |(optional, value)| + index = iseq.local_table.size + name = optional.value.to_sym + + iseq.local_table.plain(name) + iseq.argument_size += 1 + + argument_options[:opt] = [iseq.label] unless argument_options.key?( + :opt + ) + + visit(value) + iseq.setlocal(index, 0) + iseq.argument_options[:opt] << iseq.label + end + + visit(node.rest) if node.rest + + if node.posts.any? + argument_options[:post_start] = iseq.argument_size + argument_options[:post_num] = 0 + + node.posts.each do |post| + iseq.local_table.plain(post.value.to_sym) + iseq.argument_size += 1 + argument_options[:post_num] += 1 + end + end + + if node.keywords.any? + argument_options[:kwbits] = 0 + argument_options[:keyword] = [] + + keyword_bits_name = node.keyword_rest ? 3 : 2 + iseq.argument_size += 1 + keyword_bits_index = iseq.local_table.locals.size + node.keywords.size + + node.keywords.each_with_index do |(keyword, value), keyword_index| + name = keyword.value.chomp(":").to_sym + index = iseq.local_table.size + + iseq.local_table.plain(name) + iseq.argument_size += 1 + argument_options[:kwbits] += 1 + + if value.nil? + argument_options[:keyword] << name + elsif (compiled = RubyVisitor.compile(value)) + argument_options[:keyword] << [name, compiled] + else + argument_options[:keyword] << [name] + iseq.checkkeyword(keyword_bits_index, keyword_index) + branchif = iseq.branchif(-1) + visit(value) + iseq.setlocal(index, 0) + branchif.patch!(iseq) + end + end + + iseq.local_table.plain(keyword_bits_name) + end + + if node.keyword_rest.is_a?(ArgsForward) + iseq.local_table.plain(:*) + iseq.local_table.plain(:&) + + iseq.argument_options[:rest_start] = iseq.argument_size + iseq.argument_options[:block_start] = iseq.argument_size + 1 + + iseq.argument_size += 2 + elsif node.keyword_rest + visit(node.keyword_rest) + end + + visit(node.block) if node.block + end + + def visit_paren(node) + visit(node.contents) + end + + def visit_program(node) + node.statements.body.each do |statement| + break unless statement.is_a?(Comment) + + if statement.value == "# frozen_string_literal: true" + @frozen_string_literal = true + end + end + + preexes = [] + statements = [] + + node.statements.body.each do |statement| + case statement + when Comment, EmbDoc, EndContent, VoidStmt + # ignore + when BEGINBlock + preexes << statement + else + statements << statement + end + end + + top_iseq = + InstructionSequence.new( + :top, + "", + nil, + node.location, + frozen_string_literal: frozen_string_literal, + operands_unification: operands_unification, + specialized_instruction: specialized_instruction + ) + + with_child_iseq(top_iseq) do + visit_all(preexes) + + if statements.empty? + iseq.putnil + else + *statements, last_statement = statements + visit_all(statements) + with_last_statement { visit(last_statement) } + end + + iseq.leave + end + end + + def visit_qsymbols(node) + iseq.duparray(node.accept(RubyVisitor.new)) + end + + def visit_qwords(node) + if frozen_string_literal + iseq.duparray(node.accept(RubyVisitor.new)) + else + visit_all(node.elements) + iseq.newarray(node.elements.length) + end + end + + def visit_range(node) + if (compiled = RubyVisitor.compile(node)) + iseq.putobject(compiled) + else + visit(node.left) + visit(node.right) + iseq.newrange(node.operator.value == ".." ? 0 : 1) + end + end + + def visit_rassign(node) + iseq.putnil + + if node.operator.is_a?(Kw) + jumps = [] + + visit(node.value) + iseq.dup + + case node.pattern + when VarField + lookup = visit(node.pattern) + iseq.setlocal(lookup.index, lookup.level) + jumps << iseq.jump(-1) + else + jumps.concat(visit(node.pattern)) + end + + iseq.pop + iseq.pop + iseq.putobject(false) + iseq.leave + + jumps.each { |jump| jump.patch!(iseq) } + iseq.adjuststack(2) + iseq.putobject(true) + else + jumps_to_match = [] + + iseq.putnil + iseq.putobject(false) + iseq.putnil + iseq.putnil + visit(node.value) + iseq.dup + + # Visit the pattern. If it matches, + case node.pattern + when VarField + lookup = visit(node.pattern) + iseq.setlocal(lookup.index, lookup.level) + jumps_to_match << iseq.jump(-1) + else + jumps_to_match.concat(visit(node.pattern)) + end + + # First we're going to push the core onto the stack, then we'll check + # if the value to match is truthy. If it is, we'll jump down to raise + # NoMatchingPatternKeyError. Otherwise we'll raise + # NoMatchingPatternError. + iseq.putspecialobject(PutSpecialObject::OBJECT_VMCORE) + iseq.topn(4) + branchif_no_key = iseq.branchif(-1) + + # Here we're going to raise NoMatchingPatternError. + iseq.putobject(NoMatchingPatternError) + iseq.putspecialobject(PutSpecialObject::OBJECT_VMCORE) + iseq.putobject("%p: %s") + iseq.topn(4) + iseq.topn(7) + iseq.send(YARV.calldata(:"core#sprintf", 3)) + iseq.send(YARV.calldata(:"core#raise", 2)) + jump_to_exit = iseq.jump(-1) + + # Here we're going to raise NoMatchingPatternKeyError. + branchif_no_key.patch!(iseq) + iseq.putobject(NoMatchingPatternKeyError) + iseq.putspecialobject(PutSpecialObject::OBJECT_VMCORE) + iseq.putobject("%p: %s") + iseq.topn(4) + iseq.topn(7) + iseq.send(YARV.calldata(:"core#sprintf", 3)) + iseq.topn(7) + iseq.topn(9) + iseq.send( + YARV.calldata(:new, 1, CallData::CALL_KWARG, %i[matchee key]) + ) + iseq.send(YARV.calldata(:"core#raise", 1)) + + # This runs when the pattern fails to match. + jump_to_exit.patch!(iseq) + iseq.adjuststack(7) + iseq.putnil + iseq.leave + + # This runs when the pattern matches successfully. + jumps_to_match.each { |jump| jump.patch!(iseq) } + iseq.adjuststack(6) + iseq.putnil + end + end + + def visit_rational(node) + iseq.putobject(node.accept(RubyVisitor.new)) + end + + def visit_regexp_literal(node) + if (compiled = RubyVisitor.compile(node)) + iseq.putobject(compiled) + else + flags = RubyVisitor.new.visit_regexp_literal_flags(node) + length = visit_string_parts(node) + iseq.toregexp(flags, length) + end + end + + def visit_rest_param(node) + iseq.local_table.plain(node.name.value.to_sym) + iseq.argument_options[:rest_start] = iseq.argument_size + iseq.argument_size += 1 + end + + def visit_sclass(node) + visit(node.target) + iseq.putnil + + singleton_iseq = + with_child_iseq(iseq.singleton_class_child_iseq(node.location)) do + iseq.event(:RUBY_EVENT_CLASS) + visit(node.bodystmt) + iseq.event(:RUBY_EVENT_END) + iseq.leave + end + + iseq.defineclass( + :singletonclass, + singleton_iseq, + DefineClass::TYPE_SINGLETON_CLASS + ) + end + + def visit_statements(node) + statements = + node.body.select do |statement| + case statement + when Comment, EmbDoc, EndContent, VoidStmt + false + else + true + end + end + + statements.empty? ? iseq.putnil : visit_all(statements) + end + + def visit_string_concat(node) + value = node.left.parts.first.value + node.right.parts.first.value + + visit_string_literal( + StringLiteral.new( + parts: [TStringContent.new(value: value, location: node.location)], + quote: node.left.quote, + location: node.location + ) + ) + end + + def visit_string_embexpr(node) + visit(node.statements) + end + + def visit_string_literal(node) + if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) + visit(node.parts.first) + else + length = visit_string_parts(node) + iseq.concatstrings(length) + end + end + + def visit_super(node) + iseq.putself + visit(node.arguments) + iseq.invokesuper( + YARV.calldata( + nil, + argument_parts(node.arguments).length, + CallData::CALL_FCALL | CallData::CALL_ARGS_SIMPLE | + CallData::CALL_SUPER + ), + nil + ) + end + + def visit_symbol_literal(node) + iseq.putobject(node.accept(RubyVisitor.new)) + end + + def visit_symbols(node) + if (compiled = RubyVisitor.compile(node)) + iseq.duparray(compiled) + else + node.elements.each do |element| + if element.parts.length == 1 && + element.parts.first.is_a?(TStringContent) + iseq.putobject(element.parts.first.value.to_sym) + else + length = visit_string_parts(element) + iseq.concatstrings(length) + iseq.intern + end + end + + iseq.newarray(node.elements.length) + end + end + + def visit_top_const_ref(node) + iseq.opt_getconstant_path(constant_names(node)) + end + + def visit_tstring_content(node) + if frozen_string_literal + iseq.putobject(node.accept(RubyVisitor.new)) + else + iseq.putstring(node.accept(RubyVisitor.new)) + end + end + + def visit_unary(node) + method_id = + case node.operator + when "+", "-" + "#{node.operator}@" + else + node.operator + end + + visit_call( + CommandCall.new( + receiver: node.statement, + operator: nil, + message: Ident.new(value: method_id, location: Location.default), + arguments: nil, + block: nil, + location: Location.default + ) + ) + end + + def visit_undef(node) + node.symbols.each_with_index do |symbol, index| + iseq.pop if index != 0 + iseq.putspecialobject(PutSpecialObject::OBJECT_VMCORE) + iseq.putspecialobject(PutSpecialObject::OBJECT_CBASE) + visit(symbol) + iseq.send(YARV.calldata(:"core#undef_method", 2)) + end + end + + def visit_unless(node) + visit(node.predicate) + branchunless = iseq.branchunless(-1) + node.consequent ? visit(node.consequent) : iseq.putnil + + if last_statement? + iseq.leave + branchunless.patch!(iseq) + + visit(node.statements) + else + iseq.pop + + if node.consequent + jump = iseq.jump(-1) + branchunless.patch!(iseq) + visit(node.consequent) + jump.patch!(iseq.label) + else + branchunless.patch!(iseq) + end + end + end + + def visit_until(node) + jumps = [] + + jumps << iseq.jump(-1) + iseq.putnil + iseq.pop + jumps << iseq.jump(-1) + + label = iseq.label + visit(node.statements) + iseq.pop + jumps.each { |jump| jump.patch!(iseq) } + + visit(node.predicate) + iseq.branchunless(label) + iseq.putnil if last_statement? + end + + def visit_var_field(node) + case node.value + when CVar, IVar + name = node.value.value.to_sym + iseq.inline_storage_for(name) + when Ident + name = node.value.value.to_sym + + if (local_variable = iseq.local_variable(name)) + local_variable + else + iseq.local_table.plain(name) + iseq.local_variable(name) + end + end + end + + def visit_var_ref(node) + case node.value + when Const + iseq.opt_getconstant_path(constant_names(node)) + when CVar + name = node.value.value.to_sym + iseq.getclassvariable(name) + when GVar + iseq.getglobal(node.value.value.to_sym) + when Ident + lookup = iseq.local_variable(node.value.value.to_sym) + + case lookup.local + when LocalTable::BlockLocal + iseq.getblockparam(lookup.index, lookup.level) + when LocalTable::PlainLocal + iseq.getlocal(lookup.index, lookup.level) + end + when IVar + name = node.value.value.to_sym + iseq.getinstancevariable(name) + when Kw + case node.value.value + when "false" + iseq.putobject(false) + when "nil" + iseq.putnil + when "self" + iseq.putself + when "true" + iseq.putobject(true) + end + end + end + + def visit_vcall(node) + iseq.putself + iseq.send( + YARV.calldata( + node.value.value.to_sym, + 0, + CallData::CALL_FCALL | CallData::CALL_VCALL | + CallData::CALL_ARGS_SIMPLE + ) + ) + end + + def visit_when(node) + visit(node.statements) + end + + def visit_while(node) + jumps = [] + + jumps << iseq.jump(-1) + iseq.putnil + iseq.pop + jumps << iseq.jump(-1) + + label = iseq.label + visit(node.statements) + iseq.pop + jumps.each { |jump| jump.patch!(iseq) } + + visit(node.predicate) + iseq.branchif(label) + iseq.putnil if last_statement? + end + + def visit_word(node) + if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) + visit(node.parts.first) + else + length = visit_string_parts(node) + iseq.concatstrings(length) + end + end + + def visit_words(node) + if frozen_string_literal && (compiled = RubyVisitor.compile(node)) + iseq.duparray(compiled) + else + visit_all(node.elements) + iseq.newarray(node.elements.length) + end + end + + def visit_xstring_literal(node) + iseq.putself + length = visit_string_parts(node) + iseq.concatstrings(node.parts.length) if length > 1 + iseq.send( + YARV.calldata( + :`, + 1, + CallData::CALL_FCALL | CallData::CALL_ARGS_SIMPLE + ) + ) + end + + def visit_yield(node) + parts = argument_parts(node.arguments) + visit_all(parts) + iseq.invokeblock(YARV.calldata(nil, parts.length)) + end + + def visit_zsuper(_node) + iseq.putself + iseq.invokesuper( + YARV.calldata( + nil, + 0, + CallData::CALL_FCALL | CallData::CALL_ARGS_SIMPLE | + CallData::CALL_SUPER | CallData::CALL_ZSUPER + ), + nil + ) + end + + private + + # This is a helper that is used in places where arguments may be present + # or they may be wrapped in parentheses. It's meant to descend down the + # tree and return an array of argument nodes. + def argument_parts(node) + case node + when nil + [] + when Args + node.parts + when ArgParen + if node.arguments.is_a?(ArgsForward) + [node.arguments] + else + node.arguments.parts + end + when Paren + node.contents.parts + end + end + + # Constant names when they are being assigned or referenced come in as a + # tree, but it's more convenient to work with them as an array. This + # method converts them into that array. This is nice because it's the + # operand that goes to opt_getconstant_path in Ruby 3.2. + def constant_names(node) + current = node + names = [] + + while current.is_a?(ConstPathField) || current.is_a?(ConstPathRef) + names.unshift(current.constant.value.to_sym) + current = current.parent + end + + case current + when VarField, VarRef + names.unshift(current.value.value.to_sym) + when TopConstRef + names.unshift(current.constant.value.to_sym) + names.unshift(:"") + end + + names + end + + # For the most part when an OpAssign (operator assignment) node with a ||= + # operator is being compiled it's a matter of reading the target, checking + # if the value should be evaluated, evaluating it if so, and then writing + # the result back to the target. + # + # However, in certain kinds of assignments (X, ::X, X::Y, @@x, and $x) we + # first check if the value is defined using the defined instruction. I + # don't know why it is necessary, and suspect that it isn't. + def opassign_defined(node) + case node.target + when ConstPathField + visit(node.target.parent) + name = node.target.constant.value.to_sym + + iseq.dup + iseq.defined(Defined::TYPE_CONST_FROM, name, true) + when TopConstField + name = node.target.constant.value.to_sym + + iseq.putobject(Object) + iseq.dup + iseq.defined(Defined::TYPE_CONST_FROM, name, true) + when VarField + name = node.target.value.value.to_sym + iseq.putnil + + case node.target.value + when Const + iseq.defined(Defined::TYPE_CONST, name, true) + when CVar + iseq.defined(Defined::TYPE_CVAR, name, true) + when GVar + iseq.defined(Defined::TYPE_GVAR, name, true) + end + end + + branchunless = iseq.branchunless(-1) + + case node.target + when ConstPathField, TopConstField + iseq.dup + iseq.putobject(true) + iseq.getconstant(name) + when VarField + case node.target.value + when Const + iseq.opt_getconstant_path(constant_names(node.target)) + when CVar + iseq.getclassvariable(name) + when GVar + iseq.getglobal(name) + end + end + + iseq.dup + branchif = iseq.branchif(-1) + iseq.pop + + branchunless.patch!(iseq) + visit(node.value) + + case node.target + when ConstPathField, TopConstField + iseq.dupn(2) + iseq.swap + iseq.setconstant(name) + when VarField + iseq.dup + + case node.target.value + when Const + iseq.putspecialobject(PutSpecialObject::OBJECT_CONST_BASE) + iseq.setconstant(name) + when CVar + iseq.setclassvariable(name) + when GVar + iseq.setglobal(name) + end + end + + branchif.patch!(iseq) + end + + # Whenever a value is interpolated into a string-like structure, these + # three instructions are pushed. + def push_interpolate + iseq.dup + iseq.objtostring( + YARV.calldata( + :to_s, + 0, + CallData::CALL_FCALL | CallData::CALL_ARGS_SIMPLE + ) + ) + iseq.anytostring + end + + # There are a lot of nodes in the AST that act as contains of parts of + # strings. This includes things like string literals, regular expressions, + # heredocs, etc. This method will visit all the parts of a string within + # those containers. + def visit_string_parts(node) + length = 0 + + unless node.parts.first.is_a?(TStringContent) + iseq.putobject("") + length += 1 + end + + node.parts.each do |part| + case part + when StringDVar + visit(part.variable) + push_interpolate + when StringEmbExpr + visit(part) + push_interpolate + when TStringContent + iseq.putobject(part.accept(RubyVisitor.new)) + end + + length += 1 + end + + length + end + + # The current instruction sequence that we're compiling is always stored + # on the compiler. When we descend into a node that has its own + # instruction sequence, this method can be called to temporarily set the + # new value of the instruction sequence, yield, and then set it back. + def with_child_iseq(child_iseq) + parent_iseq = iseq + + begin + @iseq = child_iseq + yield + child_iseq + ensure + @iseq = parent_iseq + end + end + + # When we're compiling the last statement of a set of statements within a + # scope, the instructions sometimes change from pops to leaves. These + # kinds of peephole optimizations can reduce the overall number of + # instructions. Therefore, we keep track of whether we're compiling the + # last statement of a scope and allow visit methods to query that + # information. + def with_last_statement + previous = @last_statement + @last_statement = true + + begin + yield + ensure + @last_statement = previous + end + end + + def last_statement? + @last_statement + end + + # OpAssign nodes can have a number of different kinds of nodes as their + # "target" (i.e., the left-hand side of the assignment). When compiling + # these nodes we typically need to first fetch the current value of the + # variable, then perform some kind of action, then store the result back + # into the variable. This method handles that by first fetching the value, + # then yielding to the block, then storing the result. + def with_opassign(node) + case node.target + when ARefField + iseq.putnil + visit(node.target.collection) + visit(node.target.index) + + iseq.dupn(2) + iseq.send(YARV.calldata(:[], 1)) + + yield + + iseq.setn(3) + iseq.send(YARV.calldata(:[]=, 2)) + iseq.pop + when ConstPathField + name = node.target.constant.value.to_sym + + visit(node.target.parent) + iseq.dup + iseq.putobject(true) + iseq.getconstant(name) + + yield + + if node.operator.value == "&&=" + iseq.dupn(2) + else + iseq.swap + iseq.topn(1) + end + + iseq.swap + iseq.setconstant(name) + when TopConstField + name = node.target.constant.value.to_sym + + iseq.putobject(Object) + iseq.dup + iseq.putobject(true) + iseq.getconstant(name) + + yield + + if node.operator.value == "&&=" + iseq.dupn(2) + else + iseq.swap + iseq.topn(1) + end + + iseq.swap + iseq.setconstant(name) + when VarField + case node.target.value + when Const + names = constant_names(node.target) + iseq.opt_getconstant_path(names) + + yield + + iseq.dup + iseq.putspecialobject(PutSpecialObject::OBJECT_CONST_BASE) + iseq.setconstant(names.last) + when CVar + name = node.target.value.value.to_sym + iseq.getclassvariable(name) + + yield + + iseq.dup + iseq.setclassvariable(name) + when GVar + name = node.target.value.value.to_sym + iseq.getglobal(name) + + yield + + iseq.dup + iseq.setglobal(name) + when Ident + local_variable = visit(node.target) + iseq.getlocal(local_variable.index, local_variable.level) + + yield + + iseq.dup + iseq.setlocal(local_variable.index, local_variable.level) + when IVar + name = node.target.value.value.to_sym + iseq.getinstancevariable(name) + + yield + + iseq.dup + iseq.setinstancevariable(name) + end + end + end + end + end +end diff --git a/lib/syntax_tree/yarv/disassembler.rb b/lib/syntax_tree/yarv/disassembler.rb index 7a6e8893..d606e3cc 100644 --- a/lib/syntax_tree/yarv/disassembler.rb +++ b/lib/syntax_tree/yarv/disassembler.rb @@ -87,166 +87,113 @@ def disassemble(iseq) when GetLocalWC0 local = iseq.local_table.locals[insn.index] clause << VarRef(Ident(local.name.to_s)) - when Array - case insn[0] - when :jump - clause << Assign(disasm_label.field, node_for(insn[1])) - clause << Next(Args([])) - when :leave - value = Args([clause.pop]) - clause << (iseq.type == :top ? Break(value) : ReturnNode(value)) - when :opt_and - left, right = clause.pop(2) - clause << Binary(left, :&, right) - when :opt_aref - collection, arg = clause.pop(2) - clause << ARef(collection, Args([arg])) - when :opt_aset - collection, arg, value = clause.pop(3) + when Jump + clause << Assign(disasm_label.field, node_for(insn.label)) + clause << Next(Args([])) + when Leave + value = Args([clause.pop]) + clause << (iseq.type == :top ? Break(value) : ReturnNode(value)) + when OptAnd, OptDiv, OptEq, OptGE, OptGT, OptLE, OptLT, OptLTLT, + OptMinus, OptMod, OptMult, OptOr, OptPlus + left, right = clause.pop(2) + clause << Binary(left, insn.calldata.method, right) + when OptAref + collection, arg = clause.pop(2) + clause << ARef(collection, Args([arg])) + when OptAset + collection, arg, value = clause.pop(3) - clause << if value.is_a?(Binary) && value.left.is_a?(ARef) && - collection === value.left.collection && - arg === value.left.index.parts[0] - OpAssign( - ARefField(collection, Args([arg])), - Op("#{value.operator}="), - value.right - ) - else - Assign(ARefField(collection, Args([arg])), value) - end - when :opt_div - left, right = clause.pop(2) - clause << Binary(left, :/, right) - when :opt_eq - left, right = clause.pop(2) - clause << Binary(left, :==, right) - when :opt_ge - left, right = clause.pop(2) - clause << Binary(left, :>=, right) - when :opt_gt - left, right = clause.pop(2) - clause << Binary(left, :>, right) - when :opt_le - left, right = clause.pop(2) - clause << Binary(left, :<=, right) - when :opt_lt - left, right = clause.pop(2) - clause << Binary(left, :<, right) - when :opt_ltlt - left, right = clause.pop(2) - clause << Binary(left, :<<, right) - when :opt_minus - left, right = clause.pop(2) - clause << Binary(left, :-, right) - when :opt_mod - left, right = clause.pop(2) - clause << Binary(left, :%, right) - when :opt_mult - left, right = clause.pop(2) - clause << Binary(left, :*, right) - when :opt_neq - left, right = clause.pop(2) - clause << Binary(left, :"!=", right) - when :opt_or - left, right = clause.pop(2) - clause << Binary(left, :|, right) - when :opt_plus - left, right = clause.pop(2) - clause << Binary(left, :+, right) - when :opt_send_without_block - if insn[1][:flag] & VM_CALL_FCALL > 0 - if insn[1][:orig_argc] == 0 - clause.pop - clause << CallNode(nil, nil, Ident(insn[1][:mid]), Args([])) - elsif insn[1][:orig_argc] == 1 && insn[1][:mid].end_with?("=") - _receiver, argument = clause.pop(2) - clause << Assign( - CallNode(nil, nil, Ident(insn[1][:mid][0..-2]), nil), - argument - ) - else - _receiver, *arguments = clause.pop(insn[1][:orig_argc] + 1) - clause << CallNode( - nil, - nil, - Ident(insn[1][:mid]), - ArgParen(Args(arguments)) - ) - end - else - if insn[1][:orig_argc] == 0 - clause << CallNode( - clause.pop, - Period("."), - Ident(insn[1][:mid]), - nil - ) - elsif insn[1][:orig_argc] == 1 && insn[1][:mid].end_with?("=") - receiver, argument = clause.pop(2) - clause << Assign( - CallNode( - receiver, - Period("."), - Ident(insn[1][:mid][0..-2]), - nil - ), - argument - ) - else - receiver, *arguments = clause.pop(insn[1][:orig_argc] + 1) - clause << CallNode( - receiver, - Period("."), - Ident(insn[1][:mid]), - ArgParen(Args(arguments)) - ) - end - end - when :putobject - case insn[1] - when Float - clause << FloatLiteral(insn[1].inspect) - when Integer - clause << Int(insn[1].inspect) - else - raise "Unknown object type: #{insn[1].class.name}" - end - when :putobject_INT2FIX_0_ - clause << Int("0") - when :putobject_INT2FIX_1_ - clause << Int("1") - when :putself - clause << VarRef(Kw("self")) - when :setglobal - target = GVar(insn[1].to_s) - value = clause.pop + clause << if value.is_a?(Binary) && value.left.is_a?(ARef) && + collection === value.left.collection && + arg === value.left.index.parts[0] + OpAssign( + ARefField(collection, Args([arg])), + Op("#{value.operator}="), + value.right + ) + else + Assign(ARefField(collection, Args([arg])), value) + end + when OptNEq + left, right = clause.pop(2) + clause << Binary(left, :"!=", right) + when OptSendWithoutBlock + method = insn.calldata.method.to_s + argc = insn.calldata.argc - clause << if value.is_a?(Binary) && VarRef(target) === value.left - OpAssign( - VarField(target), - Op("#{value.operator}="), - value.right + if insn.calldata.flag?(CallData::CALL_FCALL) + if argc == 0 + clause.pop + clause << CallNode(nil, nil, Ident(method), Args([])) + elsif argc == 1 && method.end_with?("=") + _receiver, argument = clause.pop(2) + clause << Assign( + CallNode(nil, nil, Ident(method[0..-2]), nil), + argument ) else - Assign(VarField(target), value) + _receiver, *arguments = clause.pop(argc + 1) + clause << CallNode( + nil, + nil, + Ident(method), + ArgParen(Args(arguments)) + ) end - when :setlocal_WC_0 - target = Ident(local_name(insn[1], 0)) - value = clause.pop - - clause << if value.is_a?(Binary) && VarRef(target) === value.left - OpAssign( - VarField(target), - Op("#{value.operator}="), - value.right + else + if argc == 0 + clause << CallNode(clause.pop, Period("."), Ident(method), nil) + elsif argc == 1 && method.end_with?("=") + receiver, argument = clause.pop(2) + clause << Assign( + CallNode(receiver, Period("."), Ident(method[0..-2]), nil), + argument ) else - Assign(VarField(target), value) + receiver, *arguments = clause.pop(argc + 1) + clause << CallNode( + receiver, + Period("."), + Ident(method), + ArgParen(Args(arguments)) + ) end + end + when PutObject + case insn.object + when Float + clause << FloatLiteral(insn.object.inspect) + when Integer + clause << Int(insn.object.inspect) + else + raise "Unknown object type: #{insn.object.class.name}" + end + when PutObjectInt2Fix0 + clause << Int("0") + when PutObjectInt2Fix1 + clause << Int("1") + when PutSelf + clause << VarRef(Kw("self")) + when SetGlobal + target = GVar(insn.name.to_s) + value = clause.pop + + clause << if value.is_a?(Binary) && VarRef(target) === value.left + OpAssign(VarField(target), Op("#{value.operator}="), value.right) + else + Assign(VarField(target), value) + end + when SetLocalWC0 + target = Ident(local_name(insn.index, 0)) + value = clause.pop + + clause << if value.is_a?(Binary) && VarRef(target) === value.left + OpAssign(VarField(target), Op("#{value.operator}="), value.right) else - raise "Unknown instruction #{insn[0]}" + Assign(VarField(target), value) end + else + raise "Unknown instruction #{insn[0]}" end end diff --git a/lib/syntax_tree/yarv/instruction_sequence.rb b/lib/syntax_tree/yarv/instruction_sequence.rb new file mode 100644 index 00000000..c59d02c7 --- /dev/null +++ b/lib/syntax_tree/yarv/instruction_sequence.rb @@ -0,0 +1,671 @@ +# frozen_string_literal: true + +module SyntaxTree + # This module provides an object representation of the YARV bytecode. + module YARV + # This class is meant to mirror RubyVM::InstructionSequence. It contains a + # list of instructions along with the metadata pertaining to them. It also + # functions as a builder for the instruction sequence. + class InstructionSequence + MAGIC = "YARVInstructionSequence/SimpleDataFormat" + + # This provides a handle to the rb_iseq_load function, which allows you to + # pass a serialized iseq to Ruby and have it return a + # RubyVM::InstructionSequence object. + ISEQ_LOAD = + Fiddle::Function.new( + Fiddle::Handle::DEFAULT["rb_iseq_load"], + [Fiddle::TYPE_VOIDP] * 3, + Fiddle::TYPE_VOIDP + ) + + # This object is used to track the size of the stack at any given time. It + # is effectively a mini symbolic interpreter. It's necessary because when + # instruction sequences get serialized they include a :stack_max field on + # them. This field is used to determine how much stack space to allocate + # for the instruction sequence. + class Stack + attr_reader :current_size, :maximum_size + + def initialize + @current_size = 0 + @maximum_size = 0 + end + + def change_by(value) + @current_size += value + @maximum_size = @current_size if @current_size > @maximum_size + end + end + + # The type of the instruction sequence. + attr_reader :type + + # The name of the instruction sequence. + attr_reader :name + + # The parent instruction sequence, if there is one. + attr_reader :parent_iseq + + # The location of the root node of this instruction sequence. + attr_reader :location + + # This is the list of information about the arguments to this + # instruction sequence. + attr_accessor :argument_size + attr_reader :argument_options + + # The list of instructions for this instruction sequence. + attr_reader :insns + + # The table of local variables. + attr_reader :local_table + + # The hash of names of instance and class variables pointing to the + # index of their associated inline storage. + attr_reader :inline_storages + + # The index of the next inline storage that will be created. + attr_reader :storage_index + + # An object that will track the current size of the stack and the + # maximum size of the stack for this instruction sequence. + attr_reader :stack + + # These are various compilation options provided. + attr_reader :frozen_string_literal, + :operands_unification, + :specialized_instruction + + def initialize( + type, + name, + parent_iseq, + location, + frozen_string_literal: false, + operands_unification: true, + specialized_instruction: true + ) + @type = type + @name = name + @parent_iseq = parent_iseq + @location = location + + @argument_size = 0 + @argument_options = {} + + @local_table = LocalTable.new + @inline_storages = {} + @insns = [] + @storage_index = 0 + @stack = Stack.new + + @frozen_string_literal = frozen_string_literal + @operands_unification = operands_unification + @specialized_instruction = specialized_instruction + end + + ########################################################################## + # Query methods + ########################################################################## + + def local_variable(name, level = 0) + if (lookup = local_table.find(name, level)) + lookup + elsif parent_iseq + parent_iseq.local_variable(name, level + 1) + end + end + + def inline_storage + storage = storage_index + @storage_index += 1 + storage + end + + def inline_storage_for(name) + inline_storages[name] = inline_storage unless inline_storages.key?(name) + + inline_storages[name] + end + + def length + insns.inject(0) do |sum, insn| + case insn + when Integer, Symbol + sum + else + sum + insn.length + end + end + end + + def eval + compiled = to_a + + # Temporary hack until we get these working. + compiled[4][:node_id] = 11 + compiled[4][:node_ids] = [1, 0, 3, 2, 6, 7, 9, -1] + + Fiddle.dlunwrap(ISEQ_LOAD.call(Fiddle.dlwrap(compiled), 0, nil)).eval + end + + def to_a + versions = RUBY_VERSION.split(".").map(&:to_i) + + [ + MAGIC, + versions[0], + versions[1], + 1, + { + arg_size: argument_size, + local_size: local_table.size, + stack_max: stack.maximum_size + }, + name, + "", + "", + location.start_line, + type, + local_table.names, + argument_options, + [], + insns.map do |insn| + insn.is_a?(Integer) || insn.is_a?(Symbol) ? insn : insn.to_a(self) + end + ] + end + + ########################################################################## + # Child instruction sequence methods + ########################################################################## + + def child_iseq(type, name, location) + InstructionSequence.new( + type, + name, + self, + location, + frozen_string_literal: frozen_string_literal, + operands_unification: operands_unification, + specialized_instruction: specialized_instruction + ) + end + + def block_child_iseq(location) + current = self + current = current.parent_iseq while current.type == :block + child_iseq(:block, "block in #{current.name}", location) + end + + def class_child_iseq(name, location) + child_iseq(:class, "", location) + end + + def method_child_iseq(name, location) + child_iseq(:method, name, location) + end + + def module_child_iseq(name, location) + child_iseq(:class, "", location) + end + + def singleton_class_child_iseq(location) + child_iseq(:class, "singleton class", location) + end + + ########################################################################## + # Instruction push methods + ########################################################################## + + def push(insn) + insns << insn + + case insn + when Integer, Symbol, Array + insn + else + stack.change_by(-insn.pops + insn.pushes) + insn + end + end + + # This creates a new label at the current length of the instruction + # sequence. It is used as the operand for jump instructions. + def label + name = :"label_#{length}" + insns.last == name ? name : event(name) + end + + def event(name) + push(name) + end + + def adjuststack(number) + push(AdjustStack.new(number)) + end + + def anytostring + push(AnyToString.new) + end + + def branchif(label) + push(BranchIf.new(label)) + end + + def branchnil(label) + push(BranchNil.new(label)) + end + + def branchunless(label) + push(BranchUnless.new(label)) + end + + def checkkeyword(keyword_bits_index, keyword_index) + push(CheckKeyword.new(keyword_bits_index, keyword_index)) + end + + def checkmatch(type) + push(CheckMatch.new(type)) + end + + def checktype(type) + push(CheckType.new(type)) + end + + def concatarray + push(ConcatArray.new) + end + + def concatstrings(number) + push(ConcatStrings.new(number)) + end + + def defined(type, name, message) + push(Defined.new(type, name, message)) + end + + def defineclass(name, class_iseq, flags) + push(DefineClass.new(name, class_iseq, flags)) + end + + def definemethod(name, method_iseq) + push(DefineMethod.new(name, method_iseq)) + end + + def definesmethod(name, method_iseq) + push(DefineSMethod.new(name, method_iseq)) + end + + def dup + push(Dup.new) + end + + def duparray(object) + push(DupArray.new(object)) + end + + def duphash(object) + push(DupHash.new(object)) + end + + def dupn(number) + push(DupN.new(number)) + end + + def expandarray(length, flags) + push(ExpandArray.new(length, flags)) + end + + def getblockparam(index, level) + push(GetBlockParam.new(index, level)) + end + + def getblockparamproxy(index, level) + push(GetBlockParamProxy.new(index, level)) + end + + def getclassvariable(name) + if RUBY_VERSION < "3.0" + push(Legacy::GetClassVariable.new(name)) + else + push(GetClassVariable.new(name, inline_storage_for(name))) + end + end + + def getconstant(name) + push(GetConstant.new(name)) + end + + def getglobal(name) + push(GetGlobal.new(name)) + end + + def getinstancevariable(name) + if RUBY_VERSION < "3.2" + push(GetInstanceVariable.new(name, inline_storage_for(name))) + else + push(GetInstanceVariable.new(name, inline_storage)) + end + end + + def getlocal(index, level) + if operands_unification + # Specialize the getlocal instruction based on the level of the + # local variable. If it's 0 or 1, then there's a specialized + # instruction that will look at the current scope or the parent + # scope, respectively, and requires fewer operands. + case level + when 0 + push(GetLocalWC0.new(index)) + when 1 + push(GetLocalWC1.new(index)) + else + push(GetLocal.new(index, level)) + end + else + push(GetLocal.new(index, level)) + end + end + + def getspecial(key, type) + push(GetSpecial.new(key, type)) + end + + def intern + push(Intern.new) + end + + def invokeblock(calldata) + push(InvokeBlock.new(calldata)) + end + + def invokesuper(calldata, block_iseq) + push(InvokeSuper.new(calldata, block_iseq)) + end + + def jump(label) + push(Jump.new(label)) + end + + def leave + push(Leave.new) + end + + def newarray(number) + push(NewArray.new(number)) + end + + def newarraykwsplat(number) + push(NewArrayKwSplat.new(number)) + end + + def newhash(number) + push(NewHash.new(number)) + end + + def newrange(exclude_end) + push(NewRange.new(exclude_end)) + end + + def nop + push(Nop.new) + end + + def objtostring(calldata) + push(ObjToString.new(calldata)) + end + + def once(iseq, cache) + push(Once.new(iseq, cache)) + end + + def opt_aref_with(object, calldata) + push(OptArefWith.new(object, calldata)) + end + + def opt_aset_with(object, calldata) + push(OptAsetWith.new(object, calldata)) + end + + def opt_getconstant_path(names) + if RUBY_VERSION < "3.2" + cache = inline_storage + getinlinecache = opt_getinlinecache(-1, cache) + + if names[0] == :"" + names.shift + pop + putobject(Object) + end + + names.each_with_index do |name, index| + putobject(index == 0) + getconstant(name) + end + + opt_setinlinecache(cache) + getinlinecache.patch!(self) + else + push(OptGetConstantPath.new(names)) + end + end + + def opt_getinlinecache(label, cache) + push(Legacy::OptGetInlineCache.new(label, cache)) + end + + def opt_newarray_max(length) + if specialized_instruction + push(OptNewArrayMax.new(length)) + else + newarray(length) + send(YARV.calldata(:max)) + end + end + + def opt_newarray_min(length) + if specialized_instruction + push(OptNewArrayMin.new(length)) + else + newarray(length) + send(YARV.calldata(:min)) + end + end + + def opt_setinlinecache(cache) + push(Legacy::OptSetInlineCache.new(cache)) + end + + def opt_str_freeze(object) + if specialized_instruction + push(OptStrFreeze.new(object, YARV.calldata(:freeze))) + else + putstring(object) + send(YARV.calldata(:freeze)) + end + end + + def opt_str_uminus(object) + if specialized_instruction + push(OptStrUMinus.new(object, YARV.calldata(:-@))) + else + putstring(object) + send(YARV.calldata(:-@)) + end + end + + def pop + push(Pop.new) + end + + def putnil + push(PutNil.new) + end + + def putobject(object) + if operands_unification + # Specialize the putobject instruction based on the value of the + # object. If it's 0 or 1, then there's a specialized instruction + # that will push the object onto the stack and requires fewer + # operands. + if object.eql?(0) + push(PutObjectInt2Fix0.new) + elsif object.eql?(1) + push(PutObjectInt2Fix1.new) + else + push(PutObject.new(object)) + end + else + push(PutObject.new(object)) + end + end + + def putself + push(PutSelf.new) + end + + def putspecialobject(object) + push(PutSpecialObject.new(object)) + end + + def putstring(object) + push(PutString.new(object)) + end + + def send(calldata, block_iseq = nil) + if specialized_instruction && !block_iseq && + !calldata.flag?(CallData::CALL_ARGS_BLOCKARG) + # Specialize the send instruction. If it doesn't have a block + # attached, then we will replace it with an opt_send_without_block + # and do further specializations based on the called method and the + # number of arguments. + case [calldata.method, calldata.argc] + when [:length, 0] + push(OptLength.new(calldata)) + when [:size, 0] + push(OptSize.new(calldata)) + when [:empty?, 0] + push(OptEmptyP.new(calldata)) + when [:nil?, 0] + push(OptNilP.new(calldata)) + when [:succ, 0] + push(OptSucc.new(calldata)) + when [:!, 0] + push(OptNot.new(calldata)) + when [:+, 1] + push(OptPlus.new(calldata)) + when [:-, 1] + push(OptMinus.new(calldata)) + when [:*, 1] + push(OptMult.new(calldata)) + when [:/, 1] + push(OptDiv.new(calldata)) + when [:%, 1] + push(OptMod.new(calldata)) + when [:==, 1] + push(OptEq.new(calldata)) + when [:!=, 1] + push(OptNEq.new(YARV.calldata(:==, 1), calldata)) + when [:=~, 1] + push(OptRegExpMatch2.new(calldata)) + when [:<, 1] + push(OptLT.new(calldata)) + when [:<=, 1] + push(OptLE.new(calldata)) + when [:>, 1] + push(OptGT.new(calldata)) + when [:>=, 1] + push(OptGE.new(calldata)) + when [:<<, 1] + push(OptLTLT.new(calldata)) + when [:[], 1] + push(OptAref.new(calldata)) + when [:&, 1] + push(OptAnd.new(calldata)) + when [:|, 1] + push(OptOr.new(calldata)) + when [:[]=, 2] + push(OptAset.new(calldata)) + else + push(OptSendWithoutBlock.new(calldata)) + end + else + push(Send.new(calldata, block_iseq)) + end + end + + def setblockparam(index, level) + push(SetBlockParam.new(index, level)) + end + + def setclassvariable(name) + if RUBY_VERSION < "3.0" + push(Legacy::SetClassVariable.new(name)) + else + push(SetClassVariable.new(name, inline_storage_for(name))) + end + end + + def setconstant(name) + push(SetConstant.new(name)) + end + + def setglobal(name) + push(SetGlobal.new(name)) + end + + def setinstancevariable(name) + if RUBY_VERSION < "3.2" + push(SetInstanceVariable.new(name, inline_storage_for(name))) + else + push(SetInstanceVariable.new(name, inline_storage)) + end + end + + def setlocal(index, level) + if operands_unification + # Specialize the setlocal instruction based on the level of the + # local variable. If it's 0 or 1, then there's a specialized + # instruction that will write to the current scope or the parent + # scope, respectively, and requires fewer operands. + case level + when 0 + push(SetLocalWC0.new(index)) + when 1 + push(SetLocalWC1.new(index)) + else + push(SetLocal.new(index, level)) + end + else + push(SetLocal.new(index, level)) + end + end + + def setn(number) + push(SetN.new(number)) + end + + def setspecial(key) + push(SetSpecial.new(key)) + end + + def splatarray(flag) + push(SplatArray.new(flag)) + end + + def swap + push(Swap.new) + end + + def topn(number) + push(TopN.new(number)) + end + + def toregexp(options, length) + push(ToRegExp.new(options, length)) + end + end + end +end diff --git a/lib/syntax_tree/yarv/instructions.rb b/lib/syntax_tree/yarv/instructions.rb index e6853a87..5a23bbf0 100644 --- a/lib/syntax_tree/yarv/instructions.rb +++ b/lib/syntax_tree/yarv/instructions.rb @@ -2,6 +2,58 @@ module SyntaxTree module YARV + # This is an operand to various YARV instructions that represents the + # information about a specific call site. + class CallData + CALL_ARGS_SPLAT = 1 << 0 + CALL_ARGS_BLOCKARG = 1 << 1 + CALL_FCALL = 1 << 2 + CALL_VCALL = 1 << 3 + CALL_ARGS_SIMPLE = 1 << 4 + CALL_BLOCKISEQ = 1 << 5 + CALL_KWARG = 1 << 6 + CALL_KW_SPLAT = 1 << 7 + CALL_TAILCALL = 1 << 8 + CALL_SUPER = 1 << 9 + CALL_ZSUPER = 1 << 10 + CALL_OPT_SEND = 1 << 11 + CALL_KW_SPLAT_MUT = 1 << 12 + + attr_reader :method, :argc, :flags, :kw_arg + + def initialize( + method, + argc = 0, + flags = CallData::CALL_ARGS_SIMPLE, + kw_arg = nil + ) + @method = method + @argc = argc + @flags = flags + @kw_arg = kw_arg + end + + def flag?(mask) + (flags & mask) > 0 + end + + def to_h + result = { mid: method, flag: flags, orig_argc: argc } + result[:kw_arg] = kw_arg if kw_arg + result + end + end + + # A convenience method for creating a CallData object. + def self.calldata( + method, + argc = 0, + flags = CallData::CALL_ARGS_SIMPLE, + kw_arg = nil + ) + CallData.new(method, argc, flags, kw_arg) + end + # ### Summary # # `adjuststack` accepts a single integer argument and removes that many @@ -260,6 +312,109 @@ def pushes end end + # ### Summary + # + # `checkmatch` checks if the current pattern matches the current value. It + # pops the target and the pattern off the stack and pushes a boolean onto + # the stack if it matches or not. + # + # ### Usage + # + # ~~~ruby + # foo in Foo + # ~~~ + # + class CheckMatch + TYPE_WHEN = 1 + TYPE_CASE = 2 + TYPE_RESCUE = 3 + + attr_reader :type + + def initialize(type) + @type = type + end + + def to_a(_iseq) + [:checkmatch, type] + end + + def length + 2 + end + + def pops + 2 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `checktype` checks if the value on top of the stack is of a certain type. + # The type is the only argument. It pops the value off the stack and pushes + # a boolean onto the stack indicating whether or not the value is of the + # given type. + # + # ### Usage + # + # ~~~ruby + # foo in [bar] + # ~~~ + # + class CheckType + TYPE_OBJECT = 0x01 + TYPE_CLASS = 0x02 + TYPE_MODULE = 0x03 + TYPE_FLOAT = 0x04 + TYPE_STRING = 0x05 + TYPE_REGEXP = 0x06 + TYPE_ARRAY = 0x07 + TYPE_HASH = 0x08 + TYPE_STRUCT = 0x09 + TYPE_BIGNUM = 0x0a + TYPE_FILE = 0x0b + TYPE_DATA = 0x0c + TYPE_MATCH = 0x0d + TYPE_COMPLEX = 0x0e + TYPE_RATIONAL = 0x0f + TYPE_NIL = 0x11 + TYPE_TRUE = 0x12 + TYPE_FALSE = 0x13 + TYPE_SYMBOL = 0x14 + TYPE_FIXNUM = 0x15 + TYPE_UNDEF = 0x16 + + attr_reader :type + + def initialize(type) + @type = type + end + + def to_a(_iseq) + [:checktype, type] + end + + def length + 2 + end + + def pops + 1 + end + + def pushes + # TODO: This is incorrect. The instruction only pushes a single value + # onto the stack. However, if this is set to 1, we no longer match the + # output of RubyVM::InstructionSequence. So leaving this here until we + # can investigate further. + 2 + end + end + # ### Summary # # `concatarray` concatenates the two Arrays on top of the stack. @@ -800,6 +955,42 @@ def pushes end end + # ### Summary + # + # `getconstant` performs a constant lookup and pushes the value of the + # constant onto the stack. It pops both the class it should look in and + # whether or not it should look globally as well. + # + # ### Usage + # + # ~~~ruby + # Constant + # ~~~ + # + class GetConstant + attr_reader :name + + def initialize(name) + @name = name + end + + def to_a(_iseq) + [:getconstant, name] + end + + def length + 2 + end + + def pops + 2 + end + + def pushes + 1 + end + end + # ### Summary # # `getglobal` pushes the value of a global variables onto the stack. @@ -991,84 +1182,2425 @@ def pushes end end - # This module contains the instructions that used to be a part of YARV but - # have been replaced or removed in more recent versions. - module Legacy - # ### Summary - # - # `getclassvariable` looks for a class variable in the current class and - # pushes its value onto the stack. - # - # This version of the `getclassvariable` instruction is no longer used - # since in Ruby 3.0 it gained an inline cache.` - # - # ### Usage - # - # ~~~ruby - # @@class_variable - # ~~~ - # - class GetClassVariable - attr_reader :name - - def initialize(name) - @name = name - end - - def to_a(_iseq) - [:getclassvariable, name] - end - - def length - 2 - end - - def pops - 0 - end - - def pushes - 1 - end - end - - # ### Summary - # - # `getconstant` performs a constant lookup and pushes the value of the - # constant onto the stack. It pops both the class it should look in and - # whether or not it should look globally as well. - # - # This instruction is no longer used since in Ruby 3.2 it was replaced by - # the consolidated `opt_getconstant_path` instruction. - # - # ### Usage - # - # ~~~ruby - # Constant - # ~~~ - # - class GetConstant - attr_reader :name - - def initialize(name) - @name = name - end - - def to_a(_iseq) - [:getconstant, name] - end - - def length - 2 - end - - def pops - 2 - end - - def pushes - 1 - end + # ### Summary + # + # `getspecial` pushes the value of a special local variable onto the stack. + # + # ### Usage + # + # ~~~ruby + # [true] + # ~~~ + # + class GetSpecial + SVAR_LASTLINE = 0 # $_ + SVAR_BACKREF = 1 # $~ + SVAR_FLIPFLOP_START = 2 # flipflop + + attr_reader :key, :type + + def initialize(key, type) + @key = key + @type = type + end + + def to_a(_iseq) + [:getspecial, key, type] + end + + def length + 3 + end + + def pops + 0 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `intern` converts the top element of the stack to a symbol and pushes the + # symbol onto the stack. + # + # ### Usage + # + # ~~~ruby + # :"#{"foo"}" + # ~~~ + # + class Intern + def to_a(_iseq) + [:intern] + end + + def length + 1 + end + + def pops + 1 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `invokeblock` invokes the block given to the current method. It pops the + # arguments for the block off the stack and pushes the result of running the + # block onto the stack. + # + # ### Usage + # + # ~~~ruby + # def foo + # yield + # end + # ~~~ + # + class InvokeBlock + attr_reader :calldata + + def initialize(calldata) + @calldata = calldata + end + + def to_a(_iseq) + [:invokeblock, calldata.to_h] + end + + def length + 2 + end + + def pops + calldata.argc + end + + def pushes + 1 + end + end + + # ### Summary + # + # `invokesuper` is similar to the `send` instruction, except that it calls + # the super method. It pops the receiver and arguments off the stack and + # pushes the return value onto the stack. + # + # ### Usage + # + # ~~~ruby + # def foo + # super + # end + # ~~~ + # + class InvokeSuper + attr_reader :calldata, :block_iseq + + def initialize(calldata, block_iseq) + @calldata = calldata + @block_iseq = block_iseq + end + + def to_a(_iseq) + [:invokesuper, calldata.to_h, block_iseq&.to_a] + end + + def length + 1 + end + + def pops + argb = (calldata.flag?(CallData::CALL_ARGS_BLOCKARG) ? 1 : 0) + argb + calldata.argc + 1 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `jump` unconditionally jumps to the label given as its only argument. + # + # ### Usage + # + # ~~~ruby + # x = 0 + # if x == 0 + # puts "0" + # else + # puts "2" + # end + # ~~~ + # + class Jump + attr_reader :label + + def initialize(label) + @label = label + end + + def patch!(iseq) + @label = iseq.label + end + + def to_a(_iseq) + [:jump, label] + end + + def length + 2 + end + + def pops + 0 + end + + def pushes + 0 + end + end + + # ### Summary + # + # `leave` exits the current frame. + # + # ### Usage + # + # ~~~ruby + # ;; + # ~~~ + # + class Leave + def to_a(_iseq) + [:leave] + end + + def length + 1 + end + + def pops + 1 + end + + def pushes + # TODO: This is wrong. It should be 1. But it's 0 for now because + # otherwise the stack size is incorrectly calculated. + 0 + end + end + + # ### Summary + # + # `newarray` puts a new array initialized with `number` values from the + # stack. It pops `number` values off the stack and pushes the array onto the + # stack. + # + # ### Usage + # + # ~~~ruby + # ["string"] + # ~~~ + # + class NewArray + attr_reader :number + + def initialize(number) + @number = number + end + + def to_a(_iseq) + [:newarray, number] + end + + def length + 2 + end + + def pops + number + end + + def pushes + 1 + end + end + + # ### Summary + # + # `newarraykwsplat` is a specialized version of `newarray` that takes a ** + # splat argument. It pops `number` values off the stack and pushes the array + # onto the stack. + # + # ### Usage + # + # ~~~ruby + # ["string", **{ foo: "bar" }] + # ~~~ + # + class NewArrayKwSplat + attr_reader :number + + def initialize(number) + @number = number + end + + def to_a(_iseq) + [:newarraykwsplat, number] + end + + def length + 2 + end + + def pops + number + end + + def pushes + 1 + end + end + + # ### Summary + # + # `newhash` puts a new hash onto the stack, using `number` elements from the + # stack. `number` needs to be even. It pops `number` elements off the stack + # and pushes a hash onto the stack. + # + # ### Usage + # + # ~~~ruby + # def foo(key, value) + # { key => value } + # end + # ~~~ + # + class NewHash + attr_reader :number + + def initialize(number) + @number = number + end + + def to_a(_iseq) + [:newhash, number] + end + + def length + 2 + end + + def pops + number + end + + def pushes + 1 + end + end + + # ### Summary + # + # `newrange` creates a new range object from the top two values on the + # stack. It pops both of them off, and then pushes on the new range. It + # takes one argument which is 0 if the end is included or 1 if the end value + # is excluded. + # + # ### Usage + # + # ~~~ruby + # x = 0 + # y = 1 + # p (x..y), (x...y) + # ~~~ + # + class NewRange + attr_reader :exclude_end + + def initialize(exclude_end) + @exclude_end = exclude_end + end + + def to_a(_iseq) + [:newrange, exclude_end] + end + + def length + 2 + end + + def pops + 2 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `nop` is a no-operation instruction. It is used to pad the instruction + # sequence so there is a place for other instructions to jump to. + # + # ### Usage + # + # ~~~ruby + # raise rescue true + # ~~~ + # + class Nop + def to_a(_iseq) + [:nop] + end + + def length + 1 + end + + def pops + 0 + end + + def pushes + 0 + end + end + + # ### Summary + # + # `objtostring` pops a value from the stack, calls `to_s` on that value and + # then pushes the result back to the stack. + # + # It has various fast paths for classes like String, Symbol, Module, Class, + # etc. For everything else it calls `to_s`. + # + # ### Usage + # + # ~~~ruby + # "#{5}" + # ~~~ + # + class ObjToString + attr_reader :calldata + + def initialize(calldata) + @calldata = calldata + end + + def to_a(_iseq) + [:objtostring, calldata.to_h] + end + + def length + 2 + end + + def pops + 1 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `once` is an instruction that wraps an instruction sequence and ensures + # that is it only ever executed once for the lifetime of the program. It + # uses a cache to ensure that it is only executed once. It pushes the result + # of running the instruction sequence onto the stack. + # + # ### Usage + # + # ~~~ruby + # END { puts "END" } + # ~~~ + # + class Once + attr_reader :iseq, :cache + + def initialize(iseq, cache) + @iseq = iseq + @cache = cache + end + + def to_a(_iseq) + [:once, iseq.to_a, cache] + end + + def length + 3 + end + + def pops + 0 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `opt_and` is a specialization of the `opt_send_without_block` instruction + # that occurs when the `&` operator is used. There is a fast path for if + # both operands are integers. It pops both the receiver and the argument off + # the stack and pushes on the result. + # + # ### Usage + # + # ~~~ruby + # 2 & 3 + # ~~~ + # + class OptAnd + attr_reader :calldata + + def initialize(calldata) + @calldata = calldata + end + + def to_a(_iseq) + [:opt_and, calldata.to_h] + end + + def length + 2 + end + + def pops + 2 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `opt_aref` is a specialization of the `opt_send_without_block` instruction + # that occurs when the `[]` operator is used. There are fast paths if the + # receiver is an integer, array, or hash. + # + # ### Usage + # + # ~~~ruby + # 7[2] + # ~~~ + # + class OptAref + attr_reader :calldata + + def initialize(calldata) + @calldata = calldata + end + + def to_a(_iseq) + [:opt_aref, calldata.to_h] + end + + def length + 2 + end + + def pops + 2 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `opt_aref_with` is a specialization of the `opt_aref` instruction that + # occurs when the `[]` operator is used with a string argument known at + # compile time. There are fast paths if the receiver is a hash. It pops the + # receiver off the stack and pushes on the result. + # + # ### Usage + # + # ~~~ruby + # { 'test' => true }['test'] + # ~~~ + # + class OptArefWith + attr_reader :object, :calldata + + def initialize(object, calldata) + @object = object + @calldata = calldata + end + + def to_a(_iseq) + [:opt_aref_with, object, calldata.to_h] + end + + def length + 3 + end + + def pops + 1 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `opt_aset` is an instruction for setting the hash value by the key in + # the `recv[obj] = set` format. It is a specialization of the + # `opt_send_without_block` instruction. It pops the receiver, the key, and + # the value off the stack and pushes on the result. + # + # ### Usage + # + # ~~~ruby + # {}[:key] = value + # ~~~ + # + class OptAset + attr_reader :calldata + + def initialize(calldata) + @calldata = calldata + end + + def to_a(_iseq) + [:opt_aset, calldata.to_h] + end + + def length + 2 + end + + def pops + 3 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `opt_aset_with` is an instruction for setting the hash value by the known + # string key in the `recv[obj] = set` format. It pops the receiver and the + # value off the stack and pushes on the result. + # + # ### Usage + # + # ~~~ruby + # {}["key"] = value + # ~~~ + # + class OptAsetWith + attr_reader :object, :calldata + + def initialize(object, calldata) + @object = object + @calldata = calldata + end + + def to_a(_iseq) + [:opt_aset_with, object, calldata.to_h] + end + + def length + 3 + end + + def pops + 2 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `opt_div` is a specialization of the `opt_send_without_block` instruction + # that occurs when the `/` operator is used. There are fast paths for if + # both operands are integers, or if both operands are floats. It pops both + # the receiver and the argument off the stack and pushes on the result. + # + # ### Usage + # + # ~~~ruby + # 2 / 3 + # ~~~ + # + class OptDiv + attr_reader :calldata + + def initialize(calldata) + @calldata = calldata + end + + def to_a(_iseq) + [:opt_div, calldata.to_h] + end + + def length + 2 + end + + def pops + 2 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `opt_empty_p` is an optimization applied when the method `empty?` is + # called. It pops the receiver off the stack and pushes on the result of the + # method call. + # + # ### Usage + # + # ~~~ruby + # "".empty? + # ~~~ + # + class OptEmptyP + attr_reader :calldata + + def initialize(calldata) + @calldata = calldata + end + + def to_a(_iseq) + [:opt_empty_p, calldata.to_h] + end + + def length + 2 + end + + def pops + 1 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `opt_eq` is a specialization of the `opt_send_without_block` instruction + # that occurs when the == operator is used. Fast paths exist when both + # operands are integers, floats, symbols or strings. It pops both the + # receiver and the argument off the stack and pushes on the result. + # + # ### Usage + # + # ~~~ruby + # 2 == 2 + # ~~~ + # + class OptEq + attr_reader :calldata + + def initialize(calldata) + @calldata = calldata + end + + def to_a(_iseq) + [:opt_eq, calldata.to_h] + end + + def length + 2 + end + + def pops + 2 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `opt_ge` is a specialization of the `opt_send_without_block` instruction + # that occurs when the >= operator is used. Fast paths exist when both + # operands are integers or floats. It pops both the receiver and the + # argument off the stack and pushes on the result. + # + # ### Usage + # + # ~~~ruby + # 4 >= 3 + # ~~~ + # + class OptGE + attr_reader :calldata + + def initialize(calldata) + @calldata = calldata + end + + def to_a(_iseq) + [:opt_ge, calldata.to_h] + end + + def length + 2 + end + + def pops + 2 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `opt_getconstant_path` performs a constant lookup on a chain of constant + # names. It accepts as its argument an array of constant names, and pushes + # the value of the constant onto the stack. + # + # ### Usage + # + # ~~~ruby + # ::Object + # ~~~ + # + class OptGetConstantPath + attr_reader :names + + def initialize(names) + @names = names + end + + def to_a(_iseq) + [:opt_getconstant_path, names] + end + + def length + 2 + end + + def pops + 0 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `opt_gt` is a specialization of the `opt_send_without_block` instruction + # that occurs when the > operator is used. Fast paths exist when both + # operands are integers or floats. It pops both the receiver and the + # argument off the stack and pushes on the result. + # + # ### Usage + # + # ~~~ruby + # 4 > 3 + # ~~~ + # + class OptGT + attr_reader :calldata + + def initialize(calldata) + @calldata = calldata + end + + def to_a(_iseq) + [:opt_gt, calldata.to_h] + end + + def length + 2 + end + + def pops + 2 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `opt_le` is a specialization of the `opt_send_without_block` instruction + # that occurs when the <= operator is used. Fast paths exist when both + # operands are integers or floats. It pops both the receiver and the + # argument off the stack and pushes on the result. + # + # ### Usage + # + # ~~~ruby + # 3 <= 4 + # ~~~ + # + class OptLE + attr_reader :calldata + + def initialize(calldata) + @calldata = calldata + end + + def to_a(_iseq) + [:opt_le, calldata.to_h] + end + + def length + 2 + end + + def pops + 2 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `opt_length` is a specialization of `opt_send_without_block`, when the + # `length` method is called. There are fast paths when the receiver is + # either a string, hash, or array. It pops the receiver off the stack and + # pushes on the result of the method call. + # + # ### Usage + # + # ~~~ruby + # "".length + # ~~~ + # + class OptLength + attr_reader :calldata + + def initialize(calldata) + @calldata = calldata + end + + def to_a(_iseq) + [:opt_length, calldata.to_h] + end + + def length + 2 + end + + def pops + 1 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `opt_lt` is a specialization of the `opt_send_without_block` instruction + # that occurs when the < operator is used. Fast paths exist when both + # operands are integers or floats. It pops both the receiver and the + # argument off the stack and pushes on the result. + # + # ### Usage + # + # ~~~ruby + # 3 < 4 + # ~~~ + # + class OptLT + attr_reader :calldata + + def initialize(calldata) + @calldata = calldata + end + + def to_a(_iseq) + [:opt_lt, calldata.to_h] + end + + def length + 2 + end + + def pops + 2 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `opt_ltlt` is a specialization of the `opt_send_without_block` instruction + # that occurs when the `<<` operator is used. Fast paths exists when the + # receiver is either a String or an Array. It pops both the receiver and the + # argument off the stack and pushes on the result. + # + # ### Usage + # + # ~~~ruby + # "" << 2 + # ~~~ + # + class OptLTLT + attr_reader :calldata + + def initialize(calldata) + @calldata = calldata + end + + def to_a(_iseq) + [:opt_ltlt, calldata.to_h] + end + + def length + 2 + end + + def pops + 2 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `opt_minus` is a specialization of the `opt_send_without_block` + # instruction that occurs when the `-` operator is used. There are fast + # paths for if both operands are integers or if both operands are floats. It + # pops both the receiver and the argument off the stack and pushes on the + # result. + # + # ### Usage + # + # ~~~ruby + # 3 - 2 + # ~~~ + # + class OptMinus + attr_reader :calldata + + def initialize(calldata) + @calldata = calldata + end + + def to_a(_iseq) + [:opt_minus, calldata.to_h] + end + + def length + 2 + end + + def pops + 2 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `opt_mod` is a specialization of the `opt_send_without_block` instruction + # that occurs when the `%` operator is used. There are fast paths for if + # both operands are integers or if both operands are floats. It pops both + # the receiver and the argument off the stack and pushes on the result. + # + # ### Usage + # + # ~~~ruby + # 4 % 2 + # ~~~ + # + class OptMod + attr_reader :calldata + + def initialize(calldata) + @calldata = calldata + end + + def to_a(_iseq) + [:opt_mod, calldata.to_h] + end + + def length + 2 + end + + def pops + 2 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `opt_mult` is a specialization of the `opt_send_without_block` instruction + # that occurs when the `*` operator is used. There are fast paths for if + # both operands are integers or floats. It pops both the receiver and the + # argument off the stack and pushes on the result. + # + # ### Usage + # + # ~~~ruby + # 3 * 2 + # ~~~ + # + class OptMult + attr_reader :calldata + + def initialize(calldata) + @calldata = calldata + end + + def to_a(_iseq) + [:opt_mult, calldata.to_h] + end + + def length + 2 + end + + def pops + 2 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `opt_neq` is an optimization that tests whether two values at the top of + # the stack are not equal by testing their equality and calling the `!` on + # the result. This allows `opt_neq` to use the fast paths optimized in + # `opt_eq` when both operands are Integers, Floats, Symbols, or Strings. It + # pops both the receiver and the argument off the stack and pushes on the + # result. + # + # ### Usage + # + # ~~~ruby + # 2 != 2 + # ~~~ + # + class OptNEq + attr_reader :eq_calldata, :neq_calldata + + def initialize(eq_calldata, neq_calldata) + @eq_calldata = eq_calldata + @neq_calldata = neq_calldata + end + + def to_a(_iseq) + [:opt_neq, eq_calldata.to_h, neq_calldata.to_h] + end + + def length + 3 + end + + def pops + 2 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `opt_newarray_max` is a specialization that occurs when the `max` method + # is called on an array literal. It pops the values of the array off the + # stack and pushes on the result. + # + # ### Usage + # + # ~~~ruby + # [1, 2, 3].max + # ~~~ + # + class OptNewArrayMax + attr_reader :number + + def initialize(number) + @number = number + end + + def to_a(_iseq) + [:opt_newarray_max, number] + end + + def length + 2 + end + + def pops + number + end + + def pushes + 1 + end + end + + # ### Summary + # + # `opt_newarray_min` is a specialization that occurs when the `min` method + # is called on an array literal. It pops the values of the array off the + # stack and pushes on the result. + # + # ### Usage + # + # ~~~ruby + # [1, 2, 3].min + # ~~~ + # + class OptNewArrayMin + attr_reader :number + + def initialize(number) + @number = number + end + + def to_a(_iseq) + [:opt_newarray_min, number] + end + + def length + 2 + end + + def pops + number + end + + def pushes + 1 + end + end + + # ### Summary + # + # `opt_nil_p` is an optimization applied when the method `nil?` is called. + # It returns true immediately when the receiver is `nil` and defers to the + # `nil?` method in other cases. It pops the receiver off the stack and + # pushes on the result. + # + # ### Usage + # + # ~~~ruby + # "".nil? + # ~~~ + # + class OptNilP + attr_reader :calldata + + def initialize(calldata) + @calldata = calldata + end + + def to_a(_iseq) + [:opt_nil_p, calldata.to_h] + end + + def length + 2 + end + + def pops + 1 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `opt_not` negates the value on top of the stack by calling the `!` method + # on it. It pops the receiver off the stack and pushes on the result. + # + # ### Usage + # + # ~~~ruby + # !true + # ~~~ + # + class OptNot + attr_reader :calldata + + def initialize(calldata) + @calldata = calldata + end + + def to_a(_iseq) + [:opt_not, calldata.to_h] + end + + def length + 2 + end + + def pops + 1 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `opt_or` is a specialization of the `opt_send_without_block` instruction + # that occurs when the `|` operator is used. There is a fast path for if + # both operands are integers. It pops both the receiver and the argument off + # the stack and pushes on the result. + # + # ### Usage + # + # ~~~ruby + # 2 | 3 + # ~~~ + # + class OptOr + attr_reader :calldata + + def initialize(calldata) + @calldata = calldata + end + + def to_a(_iseq) + [:opt_or, calldata.to_h] + end + + def length + 2 + end + + def pops + 2 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `opt_plus` is a specialization of the `opt_send_without_block` instruction + # that occurs when the `+` operator is used. There are fast paths for if + # both operands are integers, floats, strings, or arrays. It pops both the + # receiver and the argument off the stack and pushes on the result. + # + # ### Usage + # + # ~~~ruby + # 2 + 3 + # ~~~ + # + class OptPlus + attr_reader :calldata + + def initialize(calldata) + @calldata = calldata + end + + def to_a(_iseq) + [:opt_plus, calldata.to_h] + end + + def length + 2 + end + + def pops + 2 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `opt_regexpmatch2` is a specialization of the `opt_send_without_block` + # instruction that occurs when the `=~` operator is used. It pops both the + # receiver and the argument off the stack and pushes on the result. + # + # ### Usage + # + # ~~~ruby + # /a/ =~ "a" + # ~~~ + # + class OptRegExpMatch2 + attr_reader :calldata + + def initialize(calldata) + @calldata = calldata + end + + def to_a(_iseq) + [:opt_regexpmatch2, calldata.to_h] + end + + def length + 2 + end + + def pops + 2 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `opt_send_without_block` is a specialization of the send instruction that + # occurs when a method is being called without a block. It pops the receiver + # and the arguments off the stack and pushes on the result. + # + # ### Usage + # + # ~~~ruby + # puts "Hello, world!" + # ~~~ + # + class OptSendWithoutBlock + attr_reader :calldata + + def initialize(calldata) + @calldata = calldata + end + + def to_a(_iseq) + [:opt_send_without_block, calldata.to_h] + end + + def length + 2 + end + + def pops + 1 + calldata.argc + end + + def pushes + 1 + end + end + + # ### Summary + # + # `opt_size` is a specialization of `opt_send_without_block`, when the + # `size` method is called. There are fast paths when the receiver is either + # a string, hash, or array. It pops the receiver off the stack and pushes on + # the result. + # + # ### Usage + # + # ~~~ruby + # "".size + # ~~~ + # + class OptSize + attr_reader :calldata + + def initialize(calldata) + @calldata = calldata + end + + def to_a(_iseq) + [:opt_size, calldata.to_h] + end + + def length + 2 + end + + def pops + 1 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `opt_str_freeze` pushes a frozen known string value with no interpolation + # onto the stack using the #freeze method. If the method gets overridden, + # this will fall back to a send. + # + # ### Usage + # + # ~~~ruby + # "hello".freeze + # ~~~ + # + class OptStrFreeze + attr_reader :object, :calldata + + def initialize(object, calldata) + @object = object + @calldata = calldata + end + + def to_a(_iseq) + [:opt_str_freeze, object, calldata.to_h] + end + + def length + 3 + end + + def pops + 0 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `opt_str_uminus` pushes a frozen known string value with no interpolation + # onto the stack. If the method gets overridden, this will fall back to a + # send. + # + # ### Usage + # + # ~~~ruby + # -"string" + # ~~~ + # + class OptStrUMinus + attr_reader :object, :calldata + + def initialize(object, calldata) + @object = object + @calldata = calldata + end + + def to_a(_iseq) + [:opt_str_uminus, object, calldata.to_h] + end + + def length + 3 + end + + def pops + 0 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `opt_succ` is a specialization of the `opt_send_without_block` instruction + # when the method being called is `succ`. Fast paths exist when the receiver + # is either a String or a Fixnum. It pops the receiver off the stack and + # pushes on the result. + # + # ### Usage + # + # ~~~ruby + # "".succ + # ~~~ + # + class OptSucc + attr_reader :calldata + + def initialize(calldata) + @calldata = calldata + end + + def to_a(_iseq) + [:opt_succ, calldata.to_h] + end + + def length + 2 + end + + def pops + 1 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `pop` pops the top value off the stack. + # + # ### Usage + # + # ~~~ruby + # a ||= 2 + # ~~~ + # + class Pop + def to_a(_iseq) + [:pop] + end + + def length + 1 + end + + def pops + 1 + end + + def pushes + 0 + end + end + + # ### Summary + # + # `putnil` pushes a global nil object onto the stack. + # + # ### Usage + # + # ~~~ruby + # nil + # ~~~ + # + class PutNil + def to_a(_iseq) + [:putnil] + end + + def length + 1 + end + + def pops + 0 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `putobject` pushes a known value onto the stack. + # + # ### Usage + # + # ~~~ruby + # 5 + # ~~~ + # + class PutObject + attr_reader :object + + def initialize(object) + @object = object + end + + def to_a(_iseq) + [:putobject, object] + end + + def length + 2 + end + + def pops + 0 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `putobject_INT2FIX_0_` pushes 0 on the stack. It is a specialized + # instruction resulting from the operand unification optimization. It is + # equivalent to `putobject 0`. + # + # ### Usage + # + # ~~~ruby + # 0 + # ~~~ + # + class PutObjectInt2Fix0 + def to_a(_iseq) + [:putobject_INT2FIX_0_] + end + + def length + 1 + end + + def pops + 0 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `putobject_INT2FIX_1_` pushes 1 on the stack. It is a specialized + # instruction resulting from the operand unification optimization. It is + # equivalent to `putobject 1`. + # + # ### Usage + # + # ~~~ruby + # 1 + # ~~~ + # + class PutObjectInt2Fix1 + def to_a(_iseq) + [:putobject_INT2FIX_1_] + end + + def length + 1 + end + + def pops + 0 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `putself` pushes the current value of self onto the stack. + # + # ### Usage + # + # ~~~ruby + # puts "Hello, world!" + # ~~~ + # + class PutSelf + def to_a(_iseq) + [:putself] + end + + def length + 1 + end + + def pops + 0 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `putspecialobject` pushes one of three special objects onto the stack. + # These are either the VM core special object, the class base special + # object, or the constant base special object. + # + # ### Usage + # + # ~~~ruby + # alias foo bar + # ~~~ + # + class PutSpecialObject + OBJECT_VMCORE = 1 + OBJECT_CBASE = 2 + OBJECT_CONST_BASE = 3 + + attr_reader :object + + def initialize(object) + @object = object + end + + def to_a(_iseq) + [:putspecialobject, object] + end + + def length + 2 + end + + def pops + 0 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `putstring` pushes an unfrozen string literal onto the stack. + # + # ### Usage + # + # ~~~ruby + # "foo" + # ~~~ + # + class PutString + attr_reader :object + + def initialize(object) + @object = object + end + + def to_a(_iseq) + [:putstring, object] + end + + def length + 2 + end + + def pops + 0 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `send` invokes a method with an optional block. It pops its receiver and + # the arguments for the method off the stack and pushes the return value + # onto the stack. It has two arguments: the calldata for the call site and + # the optional block instruction sequence. + # + # ### Usage + # + # ~~~ruby + # "hello".tap { |i| p i } + # ~~~ + # + class Send + attr_reader :calldata, :block_iseq + + def initialize(calldata, block_iseq) + @calldata = calldata + @block_iseq = block_iseq + end + + def to_a(_iseq) + [:send, calldata.to_h, block_iseq&.to_a] + end + + def length + 3 + end + + def pops + argb = (calldata.flag?(CallData::CALL_ARGS_BLOCKARG) ? 1 : 0) + argb + calldata.argc + 1 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `setblockparam` sets the value of a block local variable on a frame + # determined by the level and index arguments. The level is the number of + # frames back to look and the index is the index in the local table. It pops + # the value it is setting off the stack. + # + # ### Usage + # + # ~~~ruby + # def foo(&bar) + # bar = baz + # end + # ~~~ + # + class SetBlockParam + attr_reader :index, :level + + def initialize(index, level) + @index = index + @level = level + end + + def to_a(iseq) + current = iseq + level.times { current = current.parent_iseq } + [:setblockparam, current.local_table.offset(index), level] + end + + def length + 3 + end + + def pops + 1 + end + + def pushes + 0 + end + end + + # ### Summary + # + # `setclassvariable` looks for a class variable in the current class and + # sets its value to the value it pops off the top of the stack. It uses an + # inline cache to reduce the need to lookup the class variable in the class + # hierarchy every time. + # + # ### Usage + # + # ~~~ruby + # @@class_variable = 1 + # ~~~ + # + class SetClassVariable + attr_reader :name, :cache + + def initialize(name, cache) + @name = name + @cache = cache + end + + def to_a(_iseq) + [:setclassvariable, name, cache] + end + + def length + 3 + end + + def pops + 1 + end + + def pushes + 0 + end + end + + # ### Summary + # + # `setconstant` pops two values off the stack: the value to set the + # constant to and the constant base to set it in. + # + # ### Usage + # + # ~~~ruby + # Constant = 1 + # ~~~ + # + class SetConstant + attr_reader :name + + def initialize(name) + @name = name + end + + def to_a(_iseq) + [:setconstant, name] + end + + def length + 2 + end + + def pops + 2 + end + + def pushes + 0 + end + end + + # ### Summary + # + # `setglobal` sets the value of a global variable to a value popped off the + # top of the stack. + # + # ### Usage + # + # ~~~ruby + # $global = 5 + # ~~~ + # + class SetGlobal + attr_reader :name + + def initialize(name) + @name = name + end + + def to_a(_iseq) + [:setglobal, name] + end + + def length + 2 + end + + def pops + 1 + end + + def pushes + 0 + end + end + + # ### Summary + # + # `setinstancevariable` pops a value off the top of the stack and then sets + # the instance variable associated with the instruction to that value. + # + # This instruction has two forms, but both have the same structure. Before + # Ruby 3.2, the inline cache corresponded to both the get and set + # instructions and could be shared. Since Ruby 3.2, it uses object shapes + # instead so the caches are unique per instruction. + # + # ### Usage + # + # ~~~ruby + # @instance_variable = 1 + # ~~~ + # + class SetInstanceVariable + attr_reader :name, :cache + + def initialize(name, cache) + @name = name + @cache = cache + end + + def to_a(_iseq) + [:setinstancevariable, name, cache] + end + + def length + 3 + end + + def pops + 1 + end + + def pushes + 0 + end + end + + # ### Summary + # + # `setlocal` sets the value of a local variable on a frame determined by the + # level and index arguments. The level is the number of frames back to + # look and the index is the index in the local table. It pops the value it + # is setting off the stack. + # + # ### Usage + # + # ~~~ruby + # value = 5 + # tap { tap { value = 10 } } + # ~~~ + # + class SetLocal + attr_reader :index, :level + + def initialize(index, level) + @index = index + @level = level + end + + def to_a(iseq) + current = iseq + level.times { current = current.parent_iseq } + [:setlocal, current.local_table.offset(index), level] + end + + def length + 3 + end + + def pops + 1 + end + + def pushes + 0 + end + end + + # ### Summary + # + # `setlocal_WC_0` is a specialized version of the `setlocal` instruction. It + # sets the value of a local variable on the current frame to the value at + # the top of the stack as determined by the index given as its only + # argument. + # + # ### Usage + # + # ~~~ruby + # value = 5 + # ~~~ + # + class SetLocalWC0 + attr_reader :index + + def initialize(index) + @index = index + end + + def to_a(iseq) + [:setlocal_WC_0, iseq.local_table.offset(index)] + end + + def length + 2 + end + + def pops + 1 + end + + def pushes + 0 + end + end + + # ### Summary + # + # `setlocal_WC_1` is a specialized version of the `setlocal` instruction. It + # sets the value of a local variable on the parent frame to the value at the + # top of the stack as determined by the index given as its only argument. + # + # ### Usage + # + # ~~~ruby + # value = 5 + # self.then { value = 10 } + # ~~~ + # + class SetLocalWC1 + attr_reader :index + + def initialize(index) + @index = index + end + + def to_a(iseq) + [:setlocal_WC_1, iseq.parent_iseq.local_table.offset(index)] + end + + def length + 2 + end + + def pops + 1 + end + + def pushes + 0 + end + end + + # ### Summary + # + # `setn` sets a value in the stack to a value popped off the top of the + # stack. It then pushes that value onto the top of the stack as well. + # + # ### Usage + # + # ~~~ruby + # {}[:key] = 'val' + # ~~~ + # + class SetN + attr_reader :number + + def initialize(number) + @number = number + end + + def to_a(_iseq) + [:setn, number] + end + + def length + 2 + end + + def pops + 1 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `setspecial` pops a value off the top of the stack and sets a special + # local variable to that value. The special local variable is determined by + # the key given as its only argument. + # + # ### Usage + # + # ~~~ruby + # baz if (foo == 1) .. (bar == 1) + # ~~~ + # + class SetSpecial + attr_reader :key + + def initialize(key) + @key = key + end + + def to_a(_iseq) + [:setspecial, key] + end + + def length + 2 + end + + def pops + 1 + end + + def pushes + 0 + end + end + + # ### Summary + # + # `splatarray` coerces the array object at the top of the stack into Array + # by calling `to_a`. It pushes a duplicate of the array if there is a flag, + # and the original array if there isn't one. + # + # ### Usage + # + # ~~~ruby + # x = *(5) + # ~~~ + # + class SplatArray + attr_reader :flag + + def initialize(flag) + @flag = flag + end + + def to_a(_iseq) + [:splatarray, flag] + end + + def length + 2 + end + + def pops + 1 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `swap` swaps the top two elements in the stack. + # + # ### TracePoint + # + # `swap` does not dispatch any events. + # + # ### Usage + # + # ~~~ruby + # !!defined?([[]]) + # ~~~ + # + class Swap + def to_a(_iseq) + [:swap] + end + + def length + 1 + end + + def pops + 2 + end + + def pushes + 2 + end + end + + # ### Summary + # + # `topn` pushes a single value onto the stack that is a copy of the value + # within the stack that is `number` of slots down from the top. + # + # ### Usage + # + # ~~~ruby + # case 3 + # when 1..5 + # puts "foo" + # end + # ~~~ + # + class TopN + attr_reader :number + + def initialize(number) + @number = number + end + + def to_a(_iseq) + [:topn, number] + end + + def length + 2 + end + + def pops + 0 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `toregexp` pops a number of values off the stack, combines them into a new + # regular expression, and pushes the new regular expression onto the stack. + # + # ### Usage + # + # ~~~ruby + # /foo #{bar}/ + # ~~~ + # + class ToRegExp + attr_reader :options, :length + + def initialize(options, length) + @options = options + @length = length + end + + def to_a(_iseq) + [:toregexp, options, length] + end + + def pops + length + end + + def pushes + 1 end end end diff --git a/lib/syntax_tree/yarv/legacy.rb b/lib/syntax_tree/yarv/legacy.rb new file mode 100644 index 00000000..45dfe768 --- /dev/null +++ b/lib/syntax_tree/yarv/legacy.rb @@ -0,0 +1,169 @@ +# frozen_string_literal: true + +module SyntaxTree + module YARV + # This module contains the instructions that used to be a part of YARV but + # have been replaced or removed in more recent versions. + module Legacy + # ### Summary + # + # `getclassvariable` looks for a class variable in the current class and + # pushes its value onto the stack. + # + # This version of the `getclassvariable` instruction is no longer used + # since in Ruby 3.0 it gained an inline cache.` + # + # ### Usage + # + # ~~~ruby + # @@class_variable + # ~~~ + # + class GetClassVariable + attr_reader :name + + def initialize(name) + @name = name + end + + def to_a(_iseq) + [:getclassvariable, name] + end + + def length + 2 + end + + def pops + 0 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `opt_getinlinecache` is a wrapper around a series of `putobject` and + # `getconstant` instructions that allows skipping past them if the inline + # cache is currently set. It pushes the value of the cache onto the stack + # if it is set, otherwise it pushes `nil`. + # + # This instruction is no longer used since in Ruby 3.2 it was replaced by + # the consolidated `opt_getconstant_path` instruction. + # + # ### Usage + # + # ~~~ruby + # Constant + # ~~~ + # + class OptGetInlineCache + attr_reader :label, :cache + + def initialize(label, cache) + @label = label + @cache = cache + end + + def patch!(iseq) + @label = iseq.label + end + + def to_a(_iseq) + [:opt_getinlinecache, label, cache] + end + + def length + 3 + end + + def pops + 0 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `opt_setinlinecache` sets an inline cache for a constant lookup. It pops + # the value it should set off the top of the stack. It then pushes that + # value back onto the top of the stack. + # + # This instruction is no longer used since in Ruby 3.2 it was replaced by + # the consolidated `opt_getconstant_path` instruction. + # + # ### Usage + # + # ~~~ruby + # Constant + # ~~~ + # + class OptSetInlineCache + attr_reader :cache + + def initialize(cache) + @cache = cache + end + + def to_a(_iseq) + [:opt_setinlinecache, cache] + end + + def length + 2 + end + + def pops + 1 + end + + def pushes + 1 + end + end + + # ### Summary + # + # `setclassvariable` looks for a class variable in the current class and + # sets its value to the value it pops off the top of the stack. + # + # This version of the `setclassvariable` instruction is no longer used + # since in Ruby 3.0 it gained an inline cache. + # + # ### Usage + # + # ~~~ruby + # @@class_variable = 1 + # ~~~ + # + class SetClassVariable + attr_reader :name + + def initialize(name) + @name = name + end + + def to_a(_iseq) + [:setclassvariable, name] + end + + def length + 2 + end + + def pops + 1 + end + + def pushes + 0 + end + end + end + end +end diff --git a/lib/syntax_tree/yarv/local_table.rb b/lib/syntax_tree/yarv/local_table.rb new file mode 100644 index 00000000..5eac346c --- /dev/null +++ b/lib/syntax_tree/yarv/local_table.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +module SyntaxTree + module YARV + # This represents every local variable associated with an instruction + # sequence. There are two kinds of locals: plain locals that are what you + # expect, and block proxy locals, which represent local variables + # associated with blocks that were passed into the current instruction + # sequence. + class LocalTable + # A local representing a block passed into the current instruction + # sequence. + class BlockLocal + attr_reader :name + + def initialize(name) + @name = name + end + end + + # A regular local variable. + class PlainLocal + attr_reader :name + + def initialize(name) + @name = name + end + end + + # The result of looking up a local variable in the current local table. + class Lookup + attr_reader :local, :index, :level + + def initialize(local, index, level) + @local = local + @index = index + @level = level + end + end + + attr_reader :locals + + def initialize + @locals = [] + end + + def find(name, level = 0) + index = locals.index { |local| local.name == name } + Lookup.new(locals[index], index, level) if index + end + + def has?(name) + locals.any? { |local| local.name == name } + end + + def names + locals.map(&:name) + end + + def size + locals.length + end + + # Add a BlockLocal to the local table. + def block(name) + locals << BlockLocal.new(name) unless has?(name) + end + + # Add a PlainLocal to the local table. + def plain(name) + locals << PlainLocal.new(name) unless has?(name) + end + + # This is the offset from the top of the stack where this local variable + # lives. + def offset(index) + size - (index - 3) - 1 + end + end + end +end diff --git a/test/compiler_test.rb b/test/compiler_test.rb index c2472432..6b185dea 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -485,13 +485,12 @@ def assert_compiles(source, **options) assert_equal( serialize_iseq(RubyVM::InstructionSequence.compile(source, **options)), - serialize_iseq(program.accept(Compiler.new(**options))) + serialize_iseq(program.accept(YARV::Compiler.new(**options))) ) end def assert_evaluates(expected, source, **options) - program = SyntaxTree.parse(source) - assert_equal expected, program.accept(Compiler.new(**options)).eval + assert_equal expected, YARV.compile(source, **options).eval end end end diff --git a/test/yarv_test.rb b/test/yarv_test.rb index 55cdb657..02514a93 100644 --- a/test/yarv_test.rb +++ b/test/yarv_test.rb @@ -47,8 +47,8 @@ def test_bf private def assert_disassembles(expected, source) - iseq = SyntaxTree.parse(source).accept(Compiler.new) - actual = Formatter.format(source, YARV::Disassembler.new(iseq).to_ruby) + ruby = YARV::Disassembler.new(YARV.compile(source)).to_ruby + actual = Formatter.format(source, ruby) assert_equal expected, actual end end From b6fb92ee9fe39bec7e547a307742c915e78bf5d4 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 22 Nov 2022 16:24:04 -0500 Subject: [PATCH 271/536] Get it working on TruffleRuby --- lib/syntax_tree/yarv/instruction_sequence.rb | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/lib/syntax_tree/yarv/instruction_sequence.rb b/lib/syntax_tree/yarv/instruction_sequence.rb index c59d02c7..411f4692 100644 --- a/lib/syntax_tree/yarv/instruction_sequence.rb +++ b/lib/syntax_tree/yarv/instruction_sequence.rb @@ -13,11 +13,14 @@ class InstructionSequence # pass a serialized iseq to Ruby and have it return a # RubyVM::InstructionSequence object. ISEQ_LOAD = - Fiddle::Function.new( - Fiddle::Handle::DEFAULT["rb_iseq_load"], - [Fiddle::TYPE_VOIDP] * 3, - Fiddle::TYPE_VOIDP - ) + begin + Fiddle::Function.new( + Fiddle::Handle::DEFAULT["rb_iseq_load"], + [Fiddle::TYPE_VOIDP] * 3, + Fiddle::TYPE_VOIDP + ) + rescue NameError + end # This object is used to track the size of the stack at any given time. It # is effectively a mini symbolic interpreter. It's necessary because when @@ -141,6 +144,7 @@ def length end def eval + raise "Unsupported platform" if ISEQ_LOAD.nil? compiled = to_a # Temporary hack until we get these working. From be9465d49edf5fe71b470aefeff1893289d68070 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 22 Nov 2022 16:45:20 -0500 Subject: [PATCH 272/536] Handle inline_const_cache=false --- lib/syntax_tree/yarv/compiler.rb | 4 +++ lib/syntax_tree/yarv/instruction_sequence.rb | 32 +++++++++++++++----- test/compiler_test.rb | 1 + 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/lib/syntax_tree/yarv/compiler.rb b/lib/syntax_tree/yarv/compiler.rb index 45f2bb59..21d335ce 100644 --- a/lib/syntax_tree/yarv/compiler.rb +++ b/lib/syntax_tree/yarv/compiler.rb @@ -204,6 +204,7 @@ def visit_unsupported(_node) # These options mirror the compilation options that we currently support # that can be also passed to RubyVM::InstructionSequence.compile. attr_reader :frozen_string_literal, + :inline_const_cache, :operands_unification, :specialized_instruction @@ -217,10 +218,12 @@ def visit_unsupported(_node) def initialize( frozen_string_literal: false, + inline_const_cache: true, operands_unification: true, specialized_instruction: true ) @frozen_string_literal = frozen_string_literal + @inline_const_cache = inline_const_cache @operands_unification = operands_unification @specialized_instruction = specialized_instruction @@ -1374,6 +1377,7 @@ def visit_program(node) nil, node.location, frozen_string_literal: frozen_string_literal, + inline_const_cache: inline_const_cache, operands_unification: operands_unification, specialized_instruction: specialized_instruction ) diff --git a/lib/syntax_tree/yarv/instruction_sequence.rb b/lib/syntax_tree/yarv/instruction_sequence.rb index 411f4692..4754618e 100644 --- a/lib/syntax_tree/yarv/instruction_sequence.rb +++ b/lib/syntax_tree/yarv/instruction_sequence.rb @@ -77,6 +77,7 @@ def change_by(value) # These are various compilation options provided. attr_reader :frozen_string_literal, + :inline_const_cache, :operands_unification, :specialized_instruction @@ -86,6 +87,7 @@ def initialize( parent_iseq, location, frozen_string_literal: false, + inline_const_cache: true, operands_unification: true, specialized_instruction: true ) @@ -104,6 +106,7 @@ def initialize( @stack = Stack.new @frozen_string_literal = frozen_string_literal + @inline_const_cache = inline_const_cache @operands_unification = operands_unification @specialized_instruction = specialized_instruction end @@ -192,6 +195,7 @@ def child_iseq(type, name, location) self, location, frozen_string_literal: frozen_string_literal, + inline_const_cache: inline_const_cache, operands_unification: operands_unification, specialized_instruction: specialized_instruction ) @@ -434,14 +438,24 @@ def opt_aset_with(object, calldata) end def opt_getconstant_path(names) - if RUBY_VERSION < "3.2" - cache = inline_storage - getinlinecache = opt_getinlinecache(-1, cache) - - if names[0] == :"" + if RUBY_VERSION < "3.2" || !inline_const_cache + cache = nil + getinlinecache = nil + + if inline_const_cache + cache = inline_storage + getinlinecache = opt_getinlinecache(-1, cache) + + if names[0] == :"" + names.shift + pop + putobject(Object) + end + elsif names[0] == :"" names.shift - pop putobject(Object) + else + putnil end names.each_with_index do |name, index| @@ -449,8 +463,10 @@ def opt_getconstant_path(names) getconstant(name) end - opt_setinlinecache(cache) - getinlinecache.patch!(self) + if inline_const_cache + opt_setinlinecache(cache) + getinlinecache.patch!(self) + end else push(OptGetConstantPath.new(names)) end diff --git a/test/compiler_test.rb b/test/compiler_test.rb index 6b185dea..387a726d 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -432,6 +432,7 @@ class CompilerTest < Minitest::Test { frozen_string_literal: true }, { operands_unification: false }, { specialized_instruction: false }, + { inline_const_cache: false }, { operands_unification: false, specialized_instruction: false } ] From 4631b5c1708ac71fc53614924ccf1b6155203b94 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 23 Nov 2022 09:31:49 -0500 Subject: [PATCH 273/536] Convert options into an object --- lib/syntax_tree/yarv.rb | 4 +- lib/syntax_tree/yarv/compiler.rb | 92 ++++++++++++-------- lib/syntax_tree/yarv/instruction_sequence.rb | 56 ++++-------- test/compiler_test.rb | 22 ++--- 4 files changed, 87 insertions(+), 87 deletions(-) diff --git a/lib/syntax_tree/yarv.rb b/lib/syntax_tree/yarv.rb index df8bc3ce..1e759ad1 100644 --- a/lib/syntax_tree/yarv.rb +++ b/lib/syntax_tree/yarv.rb @@ -4,8 +4,8 @@ module SyntaxTree # This module provides an object representation of the YARV bytecode. module YARV # Compile the given source into a YARV instruction sequence. - def self.compile(source, **options) - SyntaxTree.parse(source).accept(Compiler.new(**options)) + def self.compile(source, options = Compiler::Options.new) + SyntaxTree.parse(source).accept(Compiler.new(options)) end end end diff --git a/lib/syntax_tree/yarv/compiler.rb b/lib/syntax_tree/yarv/compiler.rb index 21d335ce..5d717bd1 100644 --- a/lib/syntax_tree/yarv/compiler.rb +++ b/lib/syntax_tree/yarv/compiler.rb @@ -45,6 +45,53 @@ module YARV # RubyVM::InstructionSequence.compile("1 + 2").to_a # class Compiler < BasicVisitor + # This represents a set of options that can be passed to the compiler to + # control how it compiles the code. It mirrors the options that can be + # passed to RubyVM::InstructionSequence.compile, except it only includes + # options that actually change the behavior. + class Options + def initialize( + frozen_string_literal: false, + inline_const_cache: true, + operands_unification: true, + specialized_instruction: true + ) + @frozen_string_literal = frozen_string_literal + @inline_const_cache = inline_const_cache + @operands_unification = operands_unification + @specialized_instruction = specialized_instruction + end + + def to_hash + { + frozen_string_literal: @frozen_string_literal, + inline_const_cache: @inline_const_cache, + operands_unification: @operands_unification, + specialized_instruction: @specialized_instruction + } + end + + def frozen_string_literal! + @frozen_string_literal = true + end + + def frozen_string_literal? + @frozen_string_literal + end + + def inline_const_cache? + @inline_const_cache + end + + def operands_unification? + @operands_unification + end + + def specialized_instruction? + @specialized_instruction + end + end + # This visitor is responsible for converting Syntax Tree nodes into their # corresponding Ruby structures. This is used to convert the operands of # some instructions like putobject that push a Ruby object directly onto @@ -203,10 +250,7 @@ def visit_unsupported(_node) # These options mirror the compilation options that we currently support # that can be also passed to RubyVM::InstructionSequence.compile. - attr_reader :frozen_string_literal, - :inline_const_cache, - :operands_unification, - :specialized_instruction + attr_reader :options # The current instruction sequence that is being compiled. attr_reader :iseq @@ -216,17 +260,8 @@ def visit_unsupported(_node) # if we need to return the value of the last statement. attr_reader :last_statement - def initialize( - frozen_string_literal: false, - inline_const_cache: true, - operands_unification: true, - specialized_instruction: true - ) - @frozen_string_literal = frozen_string_literal - @inline_const_cache = inline_const_cache - @operands_unification = operands_unification - @specialized_instruction = specialized_instruction - + def initialize(options) + @options = options @iseq = nil @last_statement = false end @@ -236,7 +271,7 @@ def visit_BEGIN(node) end def visit_CHAR(node) - if frozen_string_literal + if options.frozen_string_literal? iseq.putobject(node.value[1..]) else iseq.putstring(node.value[1..]) @@ -282,7 +317,7 @@ def visit_aref(node) calldata = YARV.calldata(:[], 1) visit(node.collection) - if !frozen_string_literal && specialized_instruction && + if !options.frozen_string_literal? && options.specialized_instruction? && (node.index.parts.length == 1) arg = node.index.parts.first @@ -453,7 +488,7 @@ def visit_assign(node) when ARefField calldata = YARV.calldata(:[]=, 2) - if !frozen_string_literal && specialized_instruction && + if !options.frozen_string_literal? && options.specialized_instruction? && (node.target.index.parts.length == 1) arg = node.target.index.parts.first @@ -1352,7 +1387,7 @@ def visit_program(node) break unless statement.is_a?(Comment) if statement.value == "# frozen_string_literal: true" - @frozen_string_literal = true + options.frozen_string_literal! end end @@ -1370,18 +1405,7 @@ def visit_program(node) end end - top_iseq = - InstructionSequence.new( - :top, - "", - nil, - node.location, - frozen_string_literal: frozen_string_literal, - inline_const_cache: inline_const_cache, - operands_unification: operands_unification, - specialized_instruction: specialized_instruction - ) - + top_iseq = InstructionSequence.new(:top, "", nil, node.location, options) with_child_iseq(top_iseq) do visit_all(preexes) @@ -1402,7 +1426,7 @@ def visit_qsymbols(node) end def visit_qwords(node) - if frozen_string_literal + if options.frozen_string_literal? iseq.duparray(node.accept(RubyVisitor.new)) else visit_all(node.elements) @@ -1632,7 +1656,7 @@ def visit_top_const_ref(node) end def visit_tstring_content(node) - if frozen_string_literal + if options.frozen_string_literal? iseq.putobject(node.accept(RubyVisitor.new)) else iseq.putstring(node.accept(RubyVisitor.new)) @@ -1808,7 +1832,7 @@ def visit_word(node) end def visit_words(node) - if frozen_string_literal && (compiled = RubyVisitor.compile(node)) + if options.frozen_string_literal? && (compiled = RubyVisitor.compile(node)) iseq.duparray(compiled) else visit_all(node.elements) diff --git a/lib/syntax_tree/yarv/instruction_sequence.rb b/lib/syntax_tree/yarv/instruction_sequence.rb index 4754618e..156070da 100644 --- a/lib/syntax_tree/yarv/instruction_sequence.rb +++ b/lib/syntax_tree/yarv/instruction_sequence.rb @@ -76,21 +76,9 @@ def change_by(value) attr_reader :stack # These are various compilation options provided. - attr_reader :frozen_string_literal, - :inline_const_cache, - :operands_unification, - :specialized_instruction - - def initialize( - type, - name, - parent_iseq, - location, - frozen_string_literal: false, - inline_const_cache: true, - operands_unification: true, - specialized_instruction: true - ) + attr_reader :options + + def initialize(type, name, parent_iseq, location, options = Compiler::Options.new) @type = type @name = name @parent_iseq = parent_iseq @@ -105,10 +93,7 @@ def initialize( @storage_index = 0 @stack = Stack.new - @frozen_string_literal = frozen_string_literal - @inline_const_cache = inline_const_cache - @operands_unification = operands_unification - @specialized_instruction = specialized_instruction + @options = options end ########################################################################## @@ -189,16 +174,7 @@ def to_a ########################################################################## def child_iseq(type, name, location) - InstructionSequence.new( - type, - name, - self, - location, - frozen_string_literal: frozen_string_literal, - inline_const_cache: inline_const_cache, - operands_unification: operands_unification, - specialized_instruction: specialized_instruction - ) + InstructionSequence.new(type, name, self, location, options) end def block_child_iseq(location) @@ -359,7 +335,7 @@ def getinstancevariable(name) end def getlocal(index, level) - if operands_unification + if options.operands_unification? # Specialize the getlocal instruction based on the level of the # local variable. If it's 0 or 1, then there's a specialized # instruction that will look at the current scope or the parent @@ -438,11 +414,11 @@ def opt_aset_with(object, calldata) end def opt_getconstant_path(names) - if RUBY_VERSION < "3.2" || !inline_const_cache + if RUBY_VERSION < "3.2" || !options.inline_const_cache? cache = nil getinlinecache = nil - if inline_const_cache + if options.inline_const_cache? cache = inline_storage getinlinecache = opt_getinlinecache(-1, cache) @@ -463,7 +439,7 @@ def opt_getconstant_path(names) getconstant(name) end - if inline_const_cache + if options.inline_const_cache? opt_setinlinecache(cache) getinlinecache.patch!(self) end @@ -477,7 +453,7 @@ def opt_getinlinecache(label, cache) end def opt_newarray_max(length) - if specialized_instruction + if options.specialized_instruction? push(OptNewArrayMax.new(length)) else newarray(length) @@ -486,7 +462,7 @@ def opt_newarray_max(length) end def opt_newarray_min(length) - if specialized_instruction + if options.specialized_instruction? push(OptNewArrayMin.new(length)) else newarray(length) @@ -499,7 +475,7 @@ def opt_setinlinecache(cache) end def opt_str_freeze(object) - if specialized_instruction + if options.specialized_instruction? push(OptStrFreeze.new(object, YARV.calldata(:freeze))) else putstring(object) @@ -508,7 +484,7 @@ def opt_str_freeze(object) end def opt_str_uminus(object) - if specialized_instruction + if options.specialized_instruction? push(OptStrUMinus.new(object, YARV.calldata(:-@))) else putstring(object) @@ -525,7 +501,7 @@ def putnil end def putobject(object) - if operands_unification + if options.operands_unification? # Specialize the putobject instruction based on the value of the # object. If it's 0 or 1, then there's a specialized instruction # that will push the object onto the stack and requires fewer @@ -555,7 +531,7 @@ def putstring(object) end def send(calldata, block_iseq = nil) - if specialized_instruction && !block_iseq && + if options.specialized_instruction? && !block_iseq && !calldata.flag?(CallData::CALL_ARGS_BLOCKARG) # Specialize the send instruction. If it doesn't have a block # attached, then we will replace it with an opt_send_without_block @@ -645,7 +621,7 @@ def setinstancevariable(name) end def setlocal(index, level) - if operands_unification + if options.operands_unification? # Specialize the setlocal instruction based on the level of the # local variable. If it's 0 or 1, then there's a specialized # instruction that will write to the current scope or the parent diff --git a/test/compiler_test.rb b/test/compiler_test.rb index 387a726d..5a602417 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -428,12 +428,12 @@ class CompilerTest < Minitest::Test # These are the combinations of instructions that we're going to test. OPTIONS = [ - {}, - { frozen_string_literal: true }, - { operands_unification: false }, - { specialized_instruction: false }, - { inline_const_cache: false }, - { operands_unification: false, specialized_instruction: false } + YARV::Compiler::Options.new, + YARV::Compiler::Options.new(frozen_string_literal: true), + YARV::Compiler::Options.new(operands_unification: false), + YARV::Compiler::Options.new(specialized_instruction: false), + YARV::Compiler::Options.new(inline_const_cache: false), + YARV::Compiler::Options.new(operands_unification: false, specialized_instruction: false) ] OPTIONS.each do |options| @@ -441,7 +441,7 @@ class CompilerTest < Minitest::Test CASES.each do |source| define_method(:"test_#{source}_#{suffix}") do - assert_compiles(source, **options) + assert_compiles(source, options) end end end @@ -481,17 +481,17 @@ def serialize_iseq(iseq) serialized end - def assert_compiles(source, **options) + def assert_compiles(source, options) program = SyntaxTree.parse(source) assert_equal( serialize_iseq(RubyVM::InstructionSequence.compile(source, **options)), - serialize_iseq(program.accept(YARV::Compiler.new(**options))) + serialize_iseq(program.accept(YARV::Compiler.new(options))) ) end - def assert_evaluates(expected, source, **options) - assert_equal expected, YARV.compile(source, **options).eval + def assert_evaluates(expected, source) + assert_equal expected, YARV.compile(source).eval end end end From da1e46604d56941de004ce561da5b56e7eae1bde Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 23 Nov 2022 09:38:33 -0500 Subject: [PATCH 274/536] Support the tailcall_optimization flag --- lib/syntax_tree/yarv/compiler.rb | 30 ++++++++++++++++++++++++++++-- test/compiler_test.rb | 6 +++--- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/lib/syntax_tree/yarv/compiler.rb b/lib/syntax_tree/yarv/compiler.rb index 5d717bd1..4b0587fc 100644 --- a/lib/syntax_tree/yarv/compiler.rb +++ b/lib/syntax_tree/yarv/compiler.rb @@ -54,12 +54,14 @@ def initialize( frozen_string_literal: false, inline_const_cache: true, operands_unification: true, - specialized_instruction: true + specialized_instruction: true, + tailcall_optimization: false ) @frozen_string_literal = frozen_string_literal @inline_const_cache = inline_const_cache @operands_unification = operands_unification @specialized_instruction = specialized_instruction + @tailcall_optimization = tailcall_optimization end def to_hash @@ -67,7 +69,8 @@ def to_hash frozen_string_literal: @frozen_string_literal, inline_const_cache: @inline_const_cache, operands_unification: @operands_unification, - specialized_instruction: @specialized_instruction + specialized_instruction: @specialized_instruction, + tailcall_optimization: @tailcall_optimization } end @@ -90,6 +93,10 @@ def operands_unification? def specialized_instruction? @specialized_instruction end + + def tailcall_optimization? + @tailcall_optimization + end end # This visitor is responsible for converting Syntax Tree nodes into their @@ -716,12 +723,17 @@ def visit_call(node) end end + # Track whether or not this is a method call on a block proxy receiver. + # If it is, we can potentially do tailcall optimizations on it. + block_receiver = false + if node.receiver if node.receiver.is_a?(VarRef) lookup = iseq.local_variable(node.receiver.value.value.to_sym) if lookup.local.is_a?(LocalTable::BlockLocal) iseq.getblockparamproxy(lookup.index, lookup.level) + block_receiver = true else visit(node.receiver) end @@ -752,6 +764,7 @@ def visit_call(node) when ArgsForward flag |= CallData::CALL_ARGS_SPLAT flag |= CallData::CALL_ARGS_BLOCKARG + flag |= CallData::CALL_TAILCALL if options.tailcall_optimization? lookup = iseq.local_table.find(:*) iseq.getlocal(lookup.index, lookup.level) @@ -768,9 +781,22 @@ def visit_call(node) end block_iseq = visit(node.block) if node.block + + # If there's no block and we don't already have any special flags set, + # then we can safely call this simple arguments. Note that has to be the + # first flag we set after looking at the arguments to get the flags + # correct. flag |= CallData::CALL_ARGS_SIMPLE if block_iseq.nil? && flag == 0 + + # If there's no receiver, then this is an "fcall". flag |= CallData::CALL_FCALL if node.receiver.nil? + # If we're calling a method on the passed block object and we have + # tailcall optimizations turned on, then we can set the tailcall flag. + if block_receiver && options.tailcall_optimization? + flag |= CallData::CALL_TAILCALL + end + iseq.send( YARV.calldata(node.message.value.to_sym, argc, flag), block_iseq diff --git a/test/compiler_test.rb b/test/compiler_test.rb index 5a602417..02343ca2 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -433,14 +433,14 @@ class CompilerTest < Minitest::Test YARV::Compiler::Options.new(operands_unification: false), YARV::Compiler::Options.new(specialized_instruction: false), YARV::Compiler::Options.new(inline_const_cache: false), - YARV::Compiler::Options.new(operands_unification: false, specialized_instruction: false) + YARV::Compiler::Options.new(tailcall_optimization: true) ] OPTIONS.each do |options| - suffix = options.inspect + suffix = options.to_hash.map { |k, v| "#{k}=#{v}" }.join("&") CASES.each do |source| - define_method(:"test_#{source}_#{suffix}") do + define_method(:"test_#{source}_(#{suffix})") do assert_compiles(source, options) end end From 85df98f85dc297e16bc27003f2202728c871687e Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 23 Nov 2022 09:59:58 -0500 Subject: [PATCH 275/536] Provide shims for methods that should compile --- lib/syntax_tree/yarv/compiler.rb | 52 ++++++++++++++++++++++++++++++++ test/compiler_test.rb | 2 ++ 2 files changed, 54 insertions(+) diff --git a/lib/syntax_tree/yarv/compiler.rb b/lib/syntax_tree/yarv/compiler.rb index 4b0587fc..bdc31ab3 100644 --- a/lib/syntax_tree/yarv/compiler.rb +++ b/lib/syntax_tree/yarv/compiler.rb @@ -54,12 +54,14 @@ def initialize( frozen_string_literal: false, inline_const_cache: true, operands_unification: true, + peephole_optimization: true, specialized_instruction: true, tailcall_optimization: false ) @frozen_string_literal = frozen_string_literal @inline_const_cache = inline_const_cache @operands_unification = operands_unification + @peephole_optimization = peephole_optimization @specialized_instruction = specialized_instruction @tailcall_optimization = tailcall_optimization end @@ -69,6 +71,7 @@ def to_hash frozen_string_literal: @frozen_string_literal, inline_const_cache: @inline_const_cache, operands_unification: @operands_unification, + peephole_optimization: @peephole_optimization, specialized_instruction: @specialized_instruction, tailcall_optimization: @tailcall_optimization } @@ -90,6 +93,10 @@ def operands_unification? @operands_unification end + def peephole_optimization? + @peephole_optimization + end + def specialized_instruction? @specialized_instruction end @@ -608,6 +615,9 @@ def visit_bare_assoc_hash(node) end end + def visit_begin(node) + end + def visit_binary(node) case node.operator when :"&&" @@ -669,6 +679,9 @@ def visit_bodystmt(node) visit(node.statements) end + def visit_break(node) + end + def visit_call(node) if node.is_a?(CallNode) return( @@ -1016,6 +1029,9 @@ def visit_elsif(node) ) end + def visit_ensure(node) + end + def visit_field(node) visit(node.parent) end @@ -1024,6 +1040,9 @@ def visit_float(node) iseq.putobject(node.accept(RubyVisitor.new)) end + def visit_fndptn(node) + end + def visit_for(node) visit(node.collection) @@ -1064,6 +1083,9 @@ def visit_hash(node) end end + def visit_hshptn(node) + end + def visit_heredoc(node) if node.beginning.value.end_with?("`") visit_xstring_literal(node) @@ -1143,6 +1165,9 @@ def visit_imaginary(node) iseq.putobject(node.accept(RubyVisitor.new)) end + def visit_in(node) + end + def visit_int(node) iseq.putobject(node.accept(RubyVisitor.new)) end @@ -1243,6 +1268,9 @@ def visit_mrhs(node) end end + def visit_next(node) + end + def visit_not(node) visit(node.statement) iseq.send(YARV.calldata(:!)) @@ -1408,6 +1436,12 @@ def visit_paren(node) visit(node.contents) end + def visit_pinned_begin(node) + end + + def visit_pinned_var_ref(node) + end + def visit_program(node) node.statements.body.each do |statement| break unless statement.is_a?(Comment) @@ -1566,6 +1600,9 @@ def visit_rational(node) iseq.putobject(node.accept(RubyVisitor.new)) end + def visit_redo(node) + end + def visit_regexp_literal(node) if (compiled = RubyVisitor.compile(node)) iseq.putobject(compiled) @@ -1576,12 +1613,27 @@ def visit_regexp_literal(node) end end + def visit_rescue(node) + end + + def visit_rescue_ex(node) + end + + def visit_rescue_mod(node) + end + def visit_rest_param(node) iseq.local_table.plain(node.name.value.to_sym) iseq.argument_options[:rest_start] = iseq.argument_size iseq.argument_size += 1 end + def visit_retry(node) + end + + def visit_return(node) + end + def visit_sclass(node) visit(node.target) iseq.putnil diff --git a/test/compiler_test.rb b/test/compiler_test.rb index 02343ca2..9ea7f21b 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -431,6 +431,8 @@ class CompilerTest < Minitest::Test YARV::Compiler::Options.new, YARV::Compiler::Options.new(frozen_string_literal: true), YARV::Compiler::Options.new(operands_unification: false), + # TODO: have this work when peephole optimizations are turned off. + # YARV::Compiler::Options.new(peephole_optimization: false), YARV::Compiler::Options.new(specialized_instruction: false), YARV::Compiler::Options.new(inline_const_cache: false), YARV::Compiler::Options.new(tailcall_optimization: true) From 83cdfbbc60adb200aa2d9fa7477c81ee7ab2e6c7 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 23 Nov 2022 10:05:31 -0500 Subject: [PATCH 276/536] Provide missing instructions --- lib/syntax_tree/yarv/instructions.rb | 84 ++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/lib/syntax_tree/yarv/instructions.rb b/lib/syntax_tree/yarv/instructions.rb index 5a23bbf0..3fcdadb3 100644 --- a/lib/syntax_tree/yarv/instructions.rb +++ b/lib/syntax_tree/yarv/instructions.rb @@ -1840,6 +1840,54 @@ def pushes end end + # ### Summary + # + # `opt_case_dispatch` is a branch instruction that moves the control flow + # for case statements that have clauses where they can all be used as hash + # keys for an internal hash. + # + # It has two arguments: the `case_dispatch_hash` and an `else_label`. It + # pops one value off the stack: a hash key. `opt_case_dispatch` looks up the + # key in the `case_dispatch_hash` and jumps to the corresponding label if + # there is one. If there is no value in the `case_dispatch_hash`, + # `opt_case_dispatch` jumps to the `else_label` index. + # + # ### Usage + # + # ~~~ruby + # case 1 + # when 1 + # puts "foo" + # else + # puts "bar" + # end + # ~~~ + # + class OptCaseDispatch + attr_reader :case_dispatch_hash, :else_label + + def initialize(case_dispatch_hash, else_label) + @case_dispatch_hash = case_dispatch_hash + @else_label = else_label + end + + def to_a(_iseq) + [:opt_case_dispatch, case_dispatch_hash, else_label] + end + + def length + 3 + end + + def pops + 1 + end + + def pushes + 0 + end + end + # ### Summary # # `opt_div` is a specialization of the `opt_send_without_block` instruction @@ -3534,6 +3582,42 @@ def pushes end end + # ### Summary + # + # `throw` pops a value off the top of the stack and throws it. It is caught + # using the instruction sequence's (or an ancestor's) catch table. It pushes + # on the result of throwing the value. + # + # ### Usage + # + # ~~~ruby + # [1, 2, 3].map { break 2 } + # ~~~ + # + class Throw + attr_reader :type + + def initialize(type) + @type = type + end + + def to_a(_iseq) + [:throw, type] + end + + def length + 2 + end + + def pops + 1 + end + + def pushes + 1 + end + end + # ### Summary # # `topn` pushes a single value onto the stack that is a copy of the value From a43005d8a04e277f57b9cbf88d925197de13a367 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 23 Nov 2022 11:06:26 -0500 Subject: [PATCH 277/536] Allow converting from compiled iseq to YARV iseq --- lib/syntax_tree/yarv/compiler.rb | 21 +- lib/syntax_tree/yarv/instruction_sequence.rb | 224 ++++++++++++++++++- lib/syntax_tree/yarv/instructions.rb | 9 + test/compiler_test.rb | 25 ++- 4 files changed, 267 insertions(+), 12 deletions(-) diff --git a/lib/syntax_tree/yarv/compiler.rb b/lib/syntax_tree/yarv/compiler.rb index bdc31ab3..f876cb3b 100644 --- a/lib/syntax_tree/yarv/compiler.rb +++ b/lib/syntax_tree/yarv/compiler.rb @@ -331,8 +331,8 @@ def visit_aref(node) calldata = YARV.calldata(:[], 1) visit(node.collection) - if !options.frozen_string_literal? && options.specialized_instruction? && - (node.index.parts.length == 1) + if !options.frozen_string_literal? && + options.specialized_instruction? && (node.index.parts.length == 1) arg = node.index.parts.first if arg.is_a?(StringLiteral) && (arg.parts.length == 1) @@ -502,7 +502,8 @@ def visit_assign(node) when ARefField calldata = YARV.calldata(:[]=, 2) - if !options.frozen_string_literal? && options.specialized_instruction? && + if !options.frozen_string_literal? && + options.specialized_instruction? && (node.target.index.parts.length == 1) arg = node.target.index.parts.first @@ -1085,7 +1086,7 @@ def visit_hash(node) def visit_hshptn(node) end - + def visit_heredoc(node) if node.beginning.value.end_with?("`") visit_xstring_literal(node) @@ -1465,7 +1466,14 @@ def visit_program(node) end end - top_iseq = InstructionSequence.new(:top, "", nil, node.location, options) + top_iseq = + InstructionSequence.new( + :top, + "", + nil, + node.location, + options + ) with_child_iseq(top_iseq) do visit_all(preexes) @@ -1910,7 +1918,8 @@ def visit_word(node) end def visit_words(node) - if options.frozen_string_literal? && (compiled = RubyVisitor.compile(node)) + if options.frozen_string_literal? && + (compiled = RubyVisitor.compile(node)) iseq.duparray(compiled) else visit_all(node.elements) diff --git a/lib/syntax_tree/yarv/instruction_sequence.rb b/lib/syntax_tree/yarv/instruction_sequence.rb index 156070da..c6395f65 100644 --- a/lib/syntax_tree/yarv/instruction_sequence.rb +++ b/lib/syntax_tree/yarv/instruction_sequence.rb @@ -78,7 +78,13 @@ def change_by(value) # These are various compilation options provided. attr_reader :options - def initialize(type, name, parent_iseq, location, options = Compiler::Options.new) + def initialize( + type, + name, + parent_iseq, + location, + options = Compiler::Options.new + ) @type = type @name = name @parent_iseq = parent_iseq @@ -413,6 +419,10 @@ def opt_aset_with(object, calldata) push(OptAsetWith.new(object, calldata)) end + def opt_case_dispatch(case_dispatch_hash, else_label) + push(OptCaseDispatch.new(case_dispatch_hash, else_label)) + end + def opt_getconstant_path(names) if RUBY_VERSION < "3.2" || !options.inline_const_cache? cache = nil @@ -655,6 +665,10 @@ def swap push(Swap.new) end + def throw(type) + push(Throw.new(type)) + end + def topn(number) push(TopN.new(number)) end @@ -662,6 +676,214 @@ def topn(number) def toregexp(options, length) push(ToRegExp.new(options, length)) end + + # This method will create a new instruction sequence from a serialized + # RubyVM::InstructionSequence object. + def self.from(source, options = Compiler::Options.new, parent_iseq = nil) + iseq = new(source[9], source[5], parent_iseq, Location.default, options) + + # set up the correct argument size + iseq.argument_size = source[4][:arg_size] + + # set up all of the locals + source[10].each { |local| iseq.local_table.plain(local) } + + # set up the argument options + iseq.argument_options.merge!(source[11]) + + # set up all of the instructions + source[13].each do |insn| + # skip line numbers + next if insn.is_a?(Integer) + + # put events into the array and then continue + if insn.is_a?(Symbol) + iseq.event(insn) + next + end + + type, *opnds = insn + case type + when :adjuststack + iseq.adjuststack(opnds[0]) + when :anytostring + iseq.anytostring + when :branchif + iseq.branchif(opnds[0]) + when :branchnil + iseq.branchnil(opnds[0]) + when :branchunless + iseq.branchunless(opnds[0]) + when :checkkeyword + iseq.checkkeyword(iseq.local_table.size - opnds[0] + 2, opnds[1]) + when :checkmatch + iseq.checkmatch(opnds[0]) + when :checktype + iseq.checktype(opnds[0]) + when :concatarray + iseq.concatarray + when :concatstrings + iseq.concatstrings(opnds[0]) + when :defineclass + iseq.defineclass(opnds[0], from(opnds[1], options, iseq), opnds[2]) + when :defined + iseq.defined(opnds[0], opnds[1], opnds[2]) + when :definemethod + iseq.definemethod(opnds[0], from(opnds[1], options, iseq)) + when :definesmethod + iseq.definesmethod(opnds[0], from(opnds[1], options, iseq)) + when :dup + iseq.dup + when :duparray + iseq.duparray(opnds[0]) + when :duphash + iseq.duphash(opnds[0]) + when :dupn + iseq.dupn(opnds[0]) + when :expandarray + iseq.expandarray(opnds[0], opnds[1]) + when :getblockparam, :getblockparamproxy, :getlocal, :getlocal_WC_0, + :getlocal_WC_1, :setblockparam, :setlocal, :setlocal_WC_0, + :setlocal_WC_1 + current = iseq + level = 0 + + case type + when :getlocal_WC_1, :setlocal_WC_1 + level = 1 + when :getblockparam, :getblockparamproxy, :getlocal, :setblockparam, + :setlocal + level = opnds[1] + end + + level.times { current = current.parent_iseq } + index = current.local_table.size - opnds[0] + 2 + + case type + when :getblockparam + iseq.getblockparam(index, level) + when :getblockparamproxy + iseq.getblockparamproxy(index, level) + when :getlocal, :getlocal_WC_0, :getlocal_WC_1 + iseq.getlocal(index, level) + when :setblockparam + iseq.setblockparam(index, level) + when :setlocal, :setlocal_WC_0, :setlocal_WC_1 + iseq.setlocal(index, level) + end + when :getclassvariable + iseq.push(GetClassVariable.new(opnds[0], opnds[1])) + when :getconstant + iseq.getconstant(opnds[0]) + when :getglobal + iseq.getglobal(opnds[0]) + when :getinstancevariable + iseq.push(GetInstanceVariable.new(opnds[0], opnds[1])) + when :getspecial + iseq.getspecial(opnds[0], opnds[1]) + when :intern + iseq.intern + when :invokeblock + iseq.invokeblock(CallData.from(opnds[0])) + when :invokesuper + block_iseq = opnds[1] ? from(opnds[1], options, iseq) : nil + iseq.invokesuper(CallData.from(opnds[0]), block_iseq) + when :jump + iseq.jump(opnds[0]) + when :leave + iseq.leave + when :newarray + iseq.newarray(opnds[0]) + when :newarraykwsplat + iseq.newarraykwsplat(opnds[0]) + when :newhash + iseq.newhash(opnds[0]) + when :newrange + iseq.newrange(opnds[0]) + when :nop + iseq.nop + when :objtostring + iseq.objtostring(CallData.from(opnds[0])) + when :once + iseq.once(from(opnds[0], options, iseq), opnds[1]) + when :opt_and, :opt_aref, :opt_aset, :opt_div, :opt_empty_p, :opt_eq, + :opt_ge, :opt_gt, :opt_le, :opt_length, :opt_lt, :opt_ltlt, + :opt_minus, :opt_mod, :opt_mult, :opt_nil_p, :opt_not, :opt_or, + :opt_plus, :opt_regexpmatch2, :opt_send_without_block, :opt_size, + :opt_succ + iseq.send(CallData.from(opnds[0]), nil) + when :opt_aref_with + iseq.opt_aref_with(opnds[0], CallData.from(opnds[1])) + when :opt_aset_with + iseq.opt_aset_with(opnds[0], CallData.from(opnds[1])) + when :opt_case_dispatch + iseq.opt_case_dispatch(opnds[0], opnds[1]) + when :opt_getconstant_path + iseq.opt_getconstant_path(opnds[0]) + when :opt_getinlinecache + iseq.opt_getinlinecache(opnds[0], opnds[1]) + when :opt_newarray_max + iseq.opt_newarray_max(opnds[0]) + when :opt_newarray_min + iseq.opt_newarray_min(opnds[0]) + when :opt_neq + iseq.push( + OptNEq.new(CallData.from(opnds[0]), CallData.from(opnds[1])) + ) + when :opt_setinlinecache + iseq.opt_setinlinecache(opnds[0]) + when :opt_str_freeze + iseq.opt_str_freeze(opnds[0]) + when :opt_str_uminus + iseq.opt_str_uminus(opnds[0]) + when :pop + iseq.pop + when :putnil + iseq.putnil + when :putobject + iseq.putobject(opnds[0]) + when :putobject_INT2FIX_0_ + iseq.putobject(0) + when :putobject_INT2FIX_1_ + iseq.putobject(1) + when :putself + iseq.putself + when :putstring + iseq.putstring(opnds[0]) + when :putspecialobject + iseq.putspecialobject(opnds[0]) + when :send + block_iseq = opnds[1] ? from(opnds[1], options, iseq) : nil + iseq.send(CallData.from(opnds[0]), block_iseq) + when :setclassvariable + iseq.push(SetClassVariable.new(opnds[0], opnds[1])) + when :setconstant + iseq.setconstant(opnds[0]) + when :setglobal + iseq.setglobal(opnds[0]) + when :setinstancevariable + iseq.push(SetInstanceVariable.new(opnds[0], opnds[1])) + when :setn + iseq.setn(opnds[0]) + when :setspecial + iseq.setspecial(opnds[0]) + when :splatarray + iseq.splatarray(opnds[0]) + when :swap + iseq.swap + when :throw + iseq.throw(opnds[0]) + when :topn + iseq.topn(opnds[0]) + when :toregexp + iseq.toregexp(opnds[0], opnds[1]) + else + raise "Unknown instruction type: #{type}" + end + end + + iseq + end end end end diff --git a/lib/syntax_tree/yarv/instructions.rb b/lib/syntax_tree/yarv/instructions.rb index 3fcdadb3..9c816072 100644 --- a/lib/syntax_tree/yarv/instructions.rb +++ b/lib/syntax_tree/yarv/instructions.rb @@ -42,6 +42,15 @@ def to_h result[:kw_arg] = kw_arg if kw_arg result end + + def self.from(serialized) + new( + serialized[:mid], + serialized[:orig_argc], + serialized[:flag], + serialized[:kw_arg] + ) + end end # A convenience method for creating a CallData object. diff --git a/test/compiler_test.rb b/test/compiler_test.rb index 9ea7f21b..1f4a5299 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -439,12 +439,16 @@ class CompilerTest < Minitest::Test ] OPTIONS.each do |options| - suffix = options.to_hash.map { |k, v| "#{k}=#{v}" }.join("&") + suffix = options.to_hash.map { |key, value| "#{key}=#{value}" }.join("&") CASES.each do |source| - define_method(:"test_#{source}_(#{suffix})") do + define_method(:"test_compiles_#{source}_(#{suffix})") do assert_compiles(source, options) end + + define_method(:"test_loads_#{source}_(#{suffix})") do + assert_loads(source, options) + end end end @@ -483,12 +487,23 @@ def serialize_iseq(iseq) serialized end + # Check that the compiled instruction sequence matches the expected + # instruction sequence. def assert_compiles(source, options) - program = SyntaxTree.parse(source) - assert_equal( serialize_iseq(RubyVM::InstructionSequence.compile(source, **options)), - serialize_iseq(program.accept(YARV::Compiler.new(options))) + serialize_iseq(YARV.compile(source, options)) + ) + end + + # Check that the compiled instruction sequence matches the instruction + # sequence created directly from the compiled instruction sequence. + def assert_loads(source, options) + compiled = RubyVM::InstructionSequence.compile(source, **options) + + assert_equal( + serialize_iseq(compiled), + serialize_iseq(YARV::InstructionSequence.from(compiled.to_a, options)) ) end From 5dcd6722b6ccec6e95ade74d08d3260fdd292a54 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 23 Nov 2022 12:37:45 -0500 Subject: [PATCH 278/536] Use label objects instead of symbols --- lib/syntax_tree/yarv/bf.rb | 2 +- lib/syntax_tree/yarv/compiler.rb | 86 +++++++++++--------- lib/syntax_tree/yarv/instruction_sequence.rb | 69 +++++++++++++--- lib/syntax_tree/yarv/instructions.rb | 18 ++-- lib/syntax_tree/yarv/legacy.rb | 2 +- 5 files changed, 115 insertions(+), 62 deletions(-) diff --git a/lib/syntax_tree/yarv/bf.rb b/lib/syntax_tree/yarv/bf.rb index 0fb27f7e..9b037305 100644 --- a/lib/syntax_tree/yarv/bf.rb +++ b/lib/syntax_tree/yarv/bf.rb @@ -153,7 +153,7 @@ def input_char(iseq) # unless $tape[$cursor] == 0 def loop_start(iseq) - start_label = iseq.label + start_label = iseq.label_at_index iseq.getglobal(:$tape) iseq.getglobal(:$cursor) diff --git a/lib/syntax_tree/yarv/compiler.rb b/lib/syntax_tree/yarv/compiler.rb index f876cb3b..5f4f6ac0 100644 --- a/lib/syntax_tree/yarv/compiler.rb +++ b/lib/syntax_tree/yarv/compiler.rb @@ -417,7 +417,8 @@ def visit_aryptn(node) # First, check if the #deconstruct cache is nil. If it is, we're going # to call #deconstruct on the object and cache the result. iseq.topn(2) - branchnil = iseq.branchnil(-1) + deconstruct_label = iseq.label + iseq.branchnil(deconstruct_label) # Next, ensure that the cached value was cached correctly, otherwise # fail the match. @@ -432,7 +433,7 @@ def visit_aryptn(node) # Check if the object responds to #deconstruct, fail the match # otherwise. - branchnil.patch!(iseq) + iseq.event(deconstruct_label) iseq.dup iseq.putobject(:deconstruct) iseq.send(YARV.calldata(:respond_to?, 1)) @@ -634,11 +635,12 @@ def visit_binary(node) visit(node.left) iseq.dup - branchif = iseq.branchif(-1) + skip_right_label = iseq.label + iseq.branchif(skip_right_label) iseq.pop visit(node.right) - branchif.patch!(iseq) + iseq.push(skip_right_label) else visit(node.left) visit(node.right) @@ -758,11 +760,12 @@ def visit_call(node) iseq.putself end - branchnil = - if node.operator&.value == "&." - iseq.dup - iseq.branchnil(-1) - end + after_call_label = nil + if node.operator&.value == "&." + iseq.dup + after_call_label = iseq.label + iseq.branchnil(after_call_label) + end flag = 0 @@ -815,7 +818,7 @@ def visit_call(node) YARV.calldata(node.message.value.to_sym, argc, flag), block_iseq ) - branchnil.patch!(iseq) if branchnil + iseq.event(after_call_label) if after_call_label end def visit_case(node) @@ -845,16 +848,19 @@ def visit_case(node) CallData::CALL_FCALL | CallData::CALL_ARGS_SIMPLE ) ) - [clause, iseq.branchif(:label_00)] + + label = iseq.label + iseq.branchif(label) + [clause, label] end iseq.pop else_clause ? visit(else_clause) : iseq.putnil iseq.leave - branches.each_with_index do |(clause, branchif), index| + branches.each_with_index do |(clause, label), index| iseq.leave if index != 0 - branchif.patch!(iseq) + iseq.push(label) iseq.pop visit(clause) end @@ -1100,26 +1106,28 @@ def visit_heredoc(node) def visit_if(node) if node.predicate.is_a?(RangeNode) + true_label = iseq.label + iseq.getspecial(GetSpecial::SVAR_FLIPFLOP_START, 0) - branchif = iseq.branchif(-1) + iseq.branchif(true_label) visit(node.predicate.left) - branchunless_true = iseq.branchunless(-1) + end_branch = iseq.branchunless(-1) iseq.putobject(true) iseq.setspecial(GetSpecial::SVAR_FLIPFLOP_START) - branchif.patch!(iseq) + iseq.push(true_label) visit(node.predicate.right) - branchunless_false = iseq.branchunless(-1) + false_branch = iseq.branchunless(-1) iseq.putobject(false) iseq.setspecial(GetSpecial::SVAR_FLIPFLOP_START) - branchunless_false.patch!(iseq) + false_branch.patch!(iseq) visit(node.statements) iseq.leave - branchunless_true.patch!(iseq) + end_branch.patch!(iseq) iseq.putnil else visit(node.predicate) @@ -1317,22 +1325,22 @@ def visit_opassign(node) [Const, CVar, GVar].include?(node.target.value.class) opassign_defined(node) else - branchif = nil + skip_value_label = iseq.label with_opassign(node) do iseq.dup - branchif = iseq.branchif(-1) + iseq.branchif(skip_value_label) iseq.pop visit(node.value) end if node.target.is_a?(ARefField) iseq.leave - branchif.patch!(iseq) + iseq.push(skip_value_label) iseq.setn(3) iseq.adjuststack(3) else - branchif.patch!(iseq) + iseq.push(skip_value_label) end end else @@ -1363,13 +1371,11 @@ def visit_params(node) iseq.local_table.plain(name) iseq.argument_size += 1 - argument_options[:opt] = [iseq.label] unless argument_options.key?( - :opt - ) + argument_options[:opt] = [iseq.label_at_index] unless argument_options.key?(:opt) visit(value) iseq.setlocal(index, 0) - iseq.argument_options[:opt] << iseq.label + iseq.argument_options[:opt] << iseq.label_at_index end visit(node.rest) if node.rest @@ -1406,12 +1412,14 @@ def visit_params(node) elsif (compiled = RubyVisitor.compile(value)) argument_options[:keyword] << [name, compiled] else + skip_value_label = iseq.label + argument_options[:keyword] << [name] iseq.checkkeyword(keyword_bits_index, keyword_index) - branchif = iseq.branchif(-1) + iseq.branchif(skip_value_label) visit(value) iseq.setlocal(index, 0) - branchif.patch!(iseq) + iseq.push(skip_value_label) end end @@ -1558,13 +1566,15 @@ def visit_rassign(node) jumps_to_match.concat(visit(node.pattern)) end + no_key_label = iseq.label + # First we're going to push the core onto the stack, then we'll check # if the value to match is truthy. If it is, we'll jump down to raise # NoMatchingPatternKeyError. Otherwise we'll raise # NoMatchingPatternError. iseq.putspecialobject(PutSpecialObject::OBJECT_VMCORE) iseq.topn(4) - branchif_no_key = iseq.branchif(-1) + iseq.branchif(no_key_label) # Here we're going to raise NoMatchingPatternError. iseq.putobject(NoMatchingPatternError) @@ -1577,7 +1587,7 @@ def visit_rassign(node) jump_to_exit = iseq.jump(-1) # Here we're going to raise NoMatchingPatternKeyError. - branchif_no_key.patch!(iseq) + iseq.push(no_key_label) iseq.putobject(NoMatchingPatternKeyError) iseq.putspecialobject(PutSpecialObject::OBJECT_VMCORE) iseq.putobject("%p: %s") @@ -1797,7 +1807,7 @@ def visit_unless(node) jump = iseq.jump(-1) branchunless.patch!(iseq) visit(node.consequent) - jump.patch!(iseq.label) + jump.patch!(iseq.label_at_index) else branchunless.patch!(iseq) end @@ -1812,7 +1822,7 @@ def visit_until(node) iseq.pop jumps << iseq.jump(-1) - label = iseq.label + label = iseq.label_at_index visit(node.statements) iseq.pop jumps.each { |jump| jump.patch!(iseq) } @@ -1891,6 +1901,7 @@ def visit_when(node) end def visit_while(node) + repeat_label = iseq.label jumps = [] jumps << iseq.jump(-1) @@ -1898,13 +1909,13 @@ def visit_while(node) iseq.pop jumps << iseq.jump(-1) - label = iseq.label + iseq.push(repeat_label) visit(node.statements) iseq.pop jumps.each { |jump| jump.patch!(iseq) } visit(node.predicate) - iseq.branchif(label) + iseq.branchif(repeat_label) iseq.putnil if last_statement? end @@ -2060,7 +2071,8 @@ def opassign_defined(node) end iseq.dup - branchif = iseq.branchif(-1) + skip_value_label = iseq.label + iseq.branchif(skip_value_label) iseq.pop branchunless.patch!(iseq) @@ -2085,7 +2097,7 @@ def opassign_defined(node) end end - branchif.patch!(iseq) + iseq.push(skip_value_label) end # Whenever a value is interpolated into a string-like structure, these diff --git a/lib/syntax_tree/yarv/instruction_sequence.rb b/lib/syntax_tree/yarv/instruction_sequence.rb index c6395f65..e47a18ea 100644 --- a/lib/syntax_tree/yarv/instruction_sequence.rb +++ b/lib/syntax_tree/yarv/instruction_sequence.rb @@ -41,6 +41,21 @@ def change_by(value) end end + # This represents the destination of instructions that jump. Initially it + # does not track its position so that when we perform optimizations the + # indices don't get messed up. + class Label + attr_reader :name + + def initialize(name = nil) + @name = name + end + + def patch!(name) + @name = name + end + end + # The type of the instruction sequence. attr_reader :type @@ -129,7 +144,7 @@ def inline_storage_for(name) def length insns.inject(0) do |sum, insn| case insn - when Integer, Symbol + when Integer, Label, Symbol sum else sum + insn.length @@ -151,6 +166,20 @@ def eval def to_a versions = RUBY_VERSION.split(".").map(&:to_i) + # First, set it up so that all of the labels get their correct name. + insns.inject(0) do |length, insn| + case insn + when Integer, Symbol + length + when Label + insn.patch!(:"label_#{length}") + length + else + length + insn.length + end + end + + # Next, return the instruction sequence as an array. [ MAGIC, versions[0], @@ -170,7 +199,14 @@ def to_a argument_options, [], insns.map do |insn| - insn.is_a?(Integer) || insn.is_a?(Symbol) ? insn : insn.to_a(self) + case insn + when Integer, Symbol + insn + when Label + insn.name + else + insn.to_a(self) + end end ] end @@ -209,11 +245,15 @@ def singleton_class_child_iseq(location) # Instruction push methods ########################################################################## + def label + Label.new + end + def push(insn) insns << insn case insn - when Integer, Symbol, Array + when Array, Integer, Label, Symbol insn else stack.change_by(-insn.pops + insn.pushes) @@ -221,9 +261,7 @@ def push(insn) end end - # This creates a new label at the current length of the instruction - # sequence. It is used as the operand for jump instructions. - def label + def label_at_index name = :"label_#{length}" insns.last == name ? name : event(name) end @@ -691,27 +729,38 @@ def self.from(source, options = Compiler::Options.new, parent_iseq = nil) # set up the argument options iseq.argument_options.merge!(source[11]) + # set up the labels object so that the labels are shared between the + # location in the instruction sequence and the instructions that + # reference them + labels = Hash.new { |hash, name| hash[name] = Label.new(name) } + # set up all of the instructions source[13].each do |insn| # skip line numbers next if insn.is_a?(Integer) - # put events into the array and then continue + # add events and labels if insn.is_a?(Symbol) - iseq.event(insn) + if insn.start_with?("label_") + iseq.push(labels[insn]) + else + iseq.push(insn) + end next end + # add instructions, mapped to our own instruction classes type, *opnds = insn + case type when :adjuststack iseq.adjuststack(opnds[0]) when :anytostring iseq.anytostring when :branchif - iseq.branchif(opnds[0]) + iseq.branchif(labels[opnds[0]]) when :branchnil - iseq.branchnil(opnds[0]) + iseq.branchnil(labels[opnds[0]]) when :branchunless iseq.branchunless(opnds[0]) when :checkkeyword diff --git a/lib/syntax_tree/yarv/instructions.rb b/lib/syntax_tree/yarv/instructions.rb index 9c816072..c340cd4e 100644 --- a/lib/syntax_tree/yarv/instructions.rb +++ b/lib/syntax_tree/yarv/instructions.rb @@ -159,12 +159,8 @@ def initialize(label) @label = label end - def patch!(iseq) - @label = iseq.label - end - def to_a(_iseq) - [:branchif, label] + [:branchif, label.name] end def length @@ -204,12 +200,8 @@ def initialize(label) @label = label end - def patch!(iseq) - @label = iseq.label - end - def to_a(_iseq) - [:branchnil, label] + [:branchnil, label.name] end def length @@ -249,7 +241,7 @@ def initialize(label) end def patch!(iseq) - @label = iseq.label + @label = iseq.label_at_index end def to_a(_iseq) @@ -297,7 +289,7 @@ def initialize(keyword_bits_index, keyword_index) end def patch!(iseq) - @label = iseq.label + @label = iseq.label_at_index end def to_a(iseq) @@ -1360,7 +1352,7 @@ def initialize(label) end def patch!(iseq) - @label = iseq.label + @label = iseq.label_at_index end def to_a(_iseq) diff --git a/lib/syntax_tree/yarv/legacy.rb b/lib/syntax_tree/yarv/legacy.rb index 45dfe768..20588974 100644 --- a/lib/syntax_tree/yarv/legacy.rb +++ b/lib/syntax_tree/yarv/legacy.rb @@ -68,7 +68,7 @@ def initialize(label, cache) end def patch!(iseq) - @label = iseq.label + @label = iseq.label_at_index end def to_a(_iseq) From 633ab9bea7f542b098c975296e7e6044faefdb51 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 23 Nov 2022 14:26:28 -0500 Subject: [PATCH 279/536] Start using labels for jumps --- lib/syntax_tree/yarv/bf.rb | 12 +- lib/syntax_tree/yarv/compiler.rb | 335 ++++++++++--------- lib/syntax_tree/yarv/disassembler.rb | 21 +- lib/syntax_tree/yarv/instruction_sequence.rb | 15 +- lib/syntax_tree/yarv/instructions.rb | 16 +- lib/syntax_tree/yarv/legacy.rb | 6 +- 6 files changed, 196 insertions(+), 209 deletions(-) diff --git a/lib/syntax_tree/yarv/bf.rb b/lib/syntax_tree/yarv/bf.rb index 9b037305..78c01af5 100644 --- a/lib/syntax_tree/yarv/bf.rb +++ b/lib/syntax_tree/yarv/bf.rb @@ -153,23 +153,25 @@ def input_char(iseq) # unless $tape[$cursor] == 0 def loop_start(iseq) - start_label = iseq.label_at_index + start_label = iseq.label + end_label = iseq.label + iseq.push(start_label) iseq.getglobal(:$tape) iseq.getglobal(:$cursor) iseq.send(YARV.calldata(:[], 1)) iseq.putobject(0) iseq.send(YARV.calldata(:==, 1)) + iseq.branchunless(end_label) - branchunless = iseq.branchunless(-1) - [start_label, branchunless] + [start_label, end_label] end # Jump back to the start of the loop. - def loop_end(iseq, start_label, branchunless) + def loop_end(iseq, start_label, end_label) iseq.jump(start_label) - branchunless.patch!(iseq) + iseq.push(end_label) end end end diff --git a/lib/syntax_tree/yarv/compiler.rb b/lib/syntax_tree/yarv/compiler.rb index 5f4f6ac0..3bcfc598 100644 --- a/lib/syntax_tree/yarv/compiler.rb +++ b/lib/syntax_tree/yarv/compiler.rb @@ -402,100 +402,6 @@ def visit_array(node) end def visit_aryptn(node) - match_failures = [] - jumps_to_exit = [] - - # If there's a constant, then check if we match against that constant or - # not first. Branch to failure if we don't. - if node.constant - iseq.dup - visit(node.constant) - iseq.checkmatch(CheckMatch::TYPE_CASE) - match_failures << iseq.branchunless(-1) - end - - # First, check if the #deconstruct cache is nil. If it is, we're going - # to call #deconstruct on the object and cache the result. - iseq.topn(2) - deconstruct_label = iseq.label - iseq.branchnil(deconstruct_label) - - # Next, ensure that the cached value was cached correctly, otherwise - # fail the match. - iseq.topn(2) - match_failures << iseq.branchunless(-1) - - # Since we have a valid cached value, we can skip past the part where we - # call #deconstruct on the object. - iseq.pop - iseq.topn(1) - jump = iseq.jump(-1) - - # Check if the object responds to #deconstruct, fail the match - # otherwise. - iseq.event(deconstruct_label) - iseq.dup - iseq.putobject(:deconstruct) - iseq.send(YARV.calldata(:respond_to?, 1)) - iseq.setn(3) - match_failures << iseq.branchunless(-1) - - # Call #deconstruct and ensure that it's an array, raise an error - # otherwise. - iseq.send(YARV.calldata(:deconstruct)) - iseq.setn(2) - iseq.dup - iseq.checktype(CheckType::TYPE_ARRAY) - match_error = iseq.branchunless(-1) - - # Ensure that the deconstructed array has the correct size, fail the - # match otherwise. - jump.patch!(iseq) - iseq.dup - iseq.send(YARV.calldata(:length)) - iseq.putobject(node.requireds.length) - iseq.send(YARV.calldata(:==, 1)) - match_failures << iseq.branchunless(-1) - - # For each required element, check if the deconstructed array contains - # the element, otherwise jump out to the top-level match failure. - iseq.dup - node.requireds.each_with_index do |required, index| - iseq.putobject(index) - iseq.send(YARV.calldata(:[], 1)) - - case required - when VarField - lookup = visit(required) - iseq.setlocal(lookup.index, lookup.level) - else - visit(required) - iseq.checkmatch(CheckMatch::TYPE_CASE) - match_failures << iseq.branchunless(-1) - end - - if index < node.requireds.length - 1 - iseq.dup - else - iseq.pop - jumps_to_exit << iseq.jump(-1) - end - end - - # Set up the routine here to raise an error to indicate that the type of - # the deconstructed array was incorrect. - match_error.patch!(iseq) - iseq.putspecialobject(PutSpecialObject::OBJECT_VMCORE) - iseq.putobject(TypeError) - iseq.putobject("deconstruct must return Array") - iseq.send(YARV.calldata(:"core#raise", 2)) - iseq.pop - - # Patch all of the match failures to jump here so that we pop a final - # value before returning to the parent node. - match_failures.each { |match_failure| match_failure.patch!(iseq) } - iseq.pop - jumps_to_exit end def visit_assign(node) @@ -623,14 +529,15 @@ def visit_begin(node) def visit_binary(node) case node.operator when :"&&" + done_label = iseq.label + visit(node.left) iseq.dup + iseq.branchunless(done_label) - branchunless = iseq.branchunless(-1) iseq.pop - visit(node.right) - branchunless.patch!(iseq) + iseq.push(done_label) when :"||" visit(node.left) iseq.dup @@ -1107,48 +1014,52 @@ def visit_heredoc(node) def visit_if(node) if node.predicate.is_a?(RangeNode) true_label = iseq.label + false_label = iseq.label + end_label = iseq.label iseq.getspecial(GetSpecial::SVAR_FLIPFLOP_START, 0) iseq.branchif(true_label) visit(node.predicate.left) - end_branch = iseq.branchunless(-1) + iseq.branchunless(end_label) iseq.putobject(true) iseq.setspecial(GetSpecial::SVAR_FLIPFLOP_START) iseq.push(true_label) visit(node.predicate.right) - false_branch = iseq.branchunless(-1) + iseq.branchunless(false_label) iseq.putobject(false) iseq.setspecial(GetSpecial::SVAR_FLIPFLOP_START) - false_branch.patch!(iseq) + iseq.push(false_label) visit(node.statements) iseq.leave - end_branch.patch!(iseq) + iseq.push(end_label) iseq.putnil else + consequent_label = iseq.label + visit(node.predicate) - branchunless = iseq.branchunless(-1) + iseq.branchunless(consequent_label) visit(node.statements) if last_statement? iseq.leave - branchunless.patch!(iseq) - + iseq.push(consequent_label) node.consequent ? visit(node.consequent) : iseq.putnil else iseq.pop if node.consequent - jump = iseq.jump(-1) - branchunless.patch!(iseq) + done_label = iseq.label + iseq.jump(done_label) + iseq.push(consequent_label) visit(node.consequent) - jump.patch!(iseq) + iseq.push(done_label) else - branchunless.patch!(iseq) + iseq.push(consequent_label) end end end @@ -1174,9 +1085,6 @@ def visit_imaginary(node) iseq.putobject(node.accept(RubyVisitor.new)) end - def visit_in(node) - end - def visit_int(node) iseq.putobject(node.accept(RubyVisitor.new)) end @@ -1293,11 +1201,11 @@ def visit_opassign(node) case (operator = node.operator.value.chomp("=").to_sym) when :"&&" - branchunless = nil + done_label = iseq.label with_opassign(node) do iseq.dup - branchunless = iseq.branchunless(-1) + iseq.branchunless(done_label) iseq.pop visit(node.value) end @@ -1305,15 +1213,15 @@ def visit_opassign(node) case node.target when ARefField iseq.leave - branchunless.patch!(iseq) + iseq.push(done_label) iseq.setn(3) iseq.adjuststack(3) when ConstPathField, TopConstField - branchunless.patch!(iseq) + iseq.push(done_label) iseq.swap iseq.pop else - branchunless.patch!(iseq) + iseq.push(done_label) end when :"||" if node.target.is_a?(ConstPathField) || @@ -1524,30 +1432,25 @@ def visit_rassign(node) iseq.putnil if node.operator.is_a?(Kw) - jumps = [] + match_label = iseq.label visit(node.value) iseq.dup - case node.pattern - when VarField - lookup = visit(node.pattern) - iseq.setlocal(lookup.index, lookup.level) - jumps << iseq.jump(-1) - else - jumps.concat(visit(node.pattern)) - end + visit_pattern(node.pattern, match_label) iseq.pop iseq.pop iseq.putobject(false) iseq.leave - jumps.each { |jump| jump.patch!(iseq) } + iseq.push(match_label) iseq.adjuststack(2) iseq.putobject(true) else - jumps_to_match = [] + no_key_label = iseq.label + end_leave_label = iseq.label + end_label = iseq.label iseq.putnil iseq.putobject(false) @@ -1556,17 +1459,7 @@ def visit_rassign(node) visit(node.value) iseq.dup - # Visit the pattern. If it matches, - case node.pattern - when VarField - lookup = visit(node.pattern) - iseq.setlocal(lookup.index, lookup.level) - jumps_to_match << iseq.jump(-1) - else - jumps_to_match.concat(visit(node.pattern)) - end - - no_key_label = iseq.label + visit_pattern(node.pattern, end_label) # First we're going to push the core onto the stack, then we'll check # if the value to match is truthy. If it is, we'll jump down to raise @@ -1584,7 +1477,7 @@ def visit_rassign(node) iseq.topn(7) iseq.send(YARV.calldata(:"core#sprintf", 3)) iseq.send(YARV.calldata(:"core#raise", 2)) - jump_to_exit = iseq.jump(-1) + iseq.jump(end_leave_label) # Here we're going to raise NoMatchingPatternKeyError. iseq.push(no_key_label) @@ -1601,14 +1494,12 @@ def visit_rassign(node) ) iseq.send(YARV.calldata(:"core#raise", 1)) - # This runs when the pattern fails to match. - jump_to_exit.patch!(iseq) + iseq.push(end_leave_label) iseq.adjuststack(7) iseq.putnil iseq.leave - # This runs when the pattern matches successfully. - jumps_to_match.each { |jump| jump.patch!(iseq) } + iseq.push(end_label) iseq.adjuststack(6) iseq.putnil end @@ -1791,44 +1682,47 @@ def visit_undef(node) end def visit_unless(node) + statements_label = iseq.label + visit(node.predicate) - branchunless = iseq.branchunless(-1) + iseq.branchunless(statements_label) node.consequent ? visit(node.consequent) : iseq.putnil if last_statement? iseq.leave - branchunless.patch!(iseq) - + iseq.push(statements_label) visit(node.statements) else iseq.pop if node.consequent - jump = iseq.jump(-1) - branchunless.patch!(iseq) + done_label = iseq.label + iseq.jump(done_label) + iseq.push(statements_label) visit(node.consequent) - jump.patch!(iseq.label_at_index) + iseq.push(done_label) else - branchunless.patch!(iseq) + iseq.push(statements_label) end end end def visit_until(node) - jumps = [] + predicate_label = iseq.label + statements_label = iseq.label - jumps << iseq.jump(-1) + iseq.jump(predicate_label) iseq.putnil iseq.pop - jumps << iseq.jump(-1) + iseq.jump(predicate_label) - label = iseq.label_at_index + iseq.push(statements_label) visit(node.statements) iseq.pop - jumps.each { |jump| jump.patch!(iseq) } + iseq.push(predicate_label) visit(node.predicate) - iseq.branchunless(label) + iseq.branchunless(statements_label) iseq.putnil if last_statement? end @@ -1901,21 +1795,21 @@ def visit_when(node) end def visit_while(node) - repeat_label = iseq.label - jumps = [] + predicate_label = iseq.label + statements_label = iseq.label - jumps << iseq.jump(-1) + iseq.jump(predicate_label) iseq.putnil iseq.pop - jumps << iseq.jump(-1) + iseq.jump(predicate_label) - iseq.push(repeat_label) + iseq.push(statements_label) visit(node.statements) iseq.pop - jumps.each { |jump| jump.patch!(iseq) } + iseq.push(predicate_label) visit(node.predicate) - iseq.branchif(repeat_label) + iseq.branchif(statements_label) iseq.putnil if last_statement? end @@ -2025,6 +1919,9 @@ def constant_names(node) # first check if the value is defined using the defined instruction. I # don't know why it is necessary, and suspect that it isn't. def opassign_defined(node) + value_label = iseq.label + skip_value_label = iseq.label + case node.target when ConstPathField visit(node.target.parent) @@ -2052,7 +1949,7 @@ def opassign_defined(node) end end - branchunless = iseq.branchunless(-1) + iseq.branchunless(value_label) case node.target when ConstPathField, TopConstField @@ -2071,11 +1968,10 @@ def opassign_defined(node) end iseq.dup - skip_value_label = iseq.label iseq.branchif(skip_value_label) - iseq.pop - branchunless.patch!(iseq) + iseq.pop + iseq.push(value_label) visit(node.value) case node.target @@ -2114,6 +2010,111 @@ def push_interpolate iseq.anytostring end + # Visit a type of pattern in a pattern match. + def visit_pattern(node, end_label) + case node + when AryPtn + length_label = iseq.label + match_failure_label = iseq.label + match_error_label = iseq.label + + # If there's a constant, then check if we match against that constant or + # not first. Branch to failure if we don't. + if node.constant + iseq.dup + visit(node.constant) + iseq.checkmatch(CheckMatch::TYPE_CASE) + iseq.branchunless(match_failure_label) + end + + # First, check if the #deconstruct cache is nil. If it is, we're going + # to call #deconstruct on the object and cache the result. + iseq.topn(2) + deconstruct_label = iseq.label + iseq.branchnil(deconstruct_label) + + # Next, ensure that the cached value was cached correctly, otherwise + # fail the match. + iseq.topn(2) + iseq.branchunless(match_failure_label) + + # Since we have a valid cached value, we can skip past the part where we + # call #deconstruct on the object. + iseq.pop + iseq.topn(1) + iseq.jump(length_label) + + # Check if the object responds to #deconstruct, fail the match + # otherwise. + iseq.event(deconstruct_label) + iseq.dup + iseq.putobject(:deconstruct) + iseq.send(YARV.calldata(:respond_to?, 1)) + iseq.setn(3) + iseq.branchunless(match_failure_label) + + # Call #deconstruct and ensure that it's an array, raise an error + # otherwise. + iseq.send(YARV.calldata(:deconstruct)) + iseq.setn(2) + iseq.dup + iseq.checktype(CheckType::TYPE_ARRAY) + iseq.branchunless(match_error_label) + + # Ensure that the deconstructed array has the correct size, fail the + # match otherwise. + iseq.push(length_label) + iseq.dup + iseq.send(YARV.calldata(:length)) + iseq.putobject(node.requireds.length) + iseq.send(YARV.calldata(:==, 1)) + iseq.branchunless(match_failure_label) + + # For each required element, check if the deconstructed array contains + # the element, otherwise jump out to the top-level match failure. + iseq.dup + node.requireds.each_with_index do |required, index| + iseq.putobject(index) + iseq.send(YARV.calldata(:[], 1)) + + case required + when VarField + lookup = visit(required) + iseq.setlocal(lookup.index, lookup.level) + else + visit(required) + iseq.checkmatch(CheckMatch::TYPE_CASE) + iseq.branchunless(match_failure_label) + end + + if index < node.requireds.length - 1 + iseq.dup + else + iseq.pop + iseq.jump(end_label) + end + end + + # Set up the routine here to raise an error to indicate that the type of + # the deconstructed array was incorrect. + iseq.push(match_error_label) + iseq.putspecialobject(PutSpecialObject::OBJECT_VMCORE) + iseq.putobject(TypeError) + iseq.putobject("deconstruct must return Array") + iseq.send(YARV.calldata(:"core#raise", 2)) + iseq.pop + + # Patch all of the match failures to jump here so that we pop a final + # value before returning to the parent node. + iseq.push(match_failure_label) + iseq.pop + when VarField + lookup = visit(node) + iseq.setlocal(lookup.index, lookup.level) + iseq.jump(end_label) + end + end + # There are a lot of nodes in the AST that act as contains of parts of # strings. This includes things like string literals, regular expressions, # heredocs, etc. This method will visit all the parts of a string within diff --git a/lib/syntax_tree/yarv/disassembler.rb b/lib/syntax_tree/yarv/disassembler.rb index d606e3cc..757b8b40 100644 --- a/lib/syntax_tree/yarv/disassembler.rb +++ b/lib/syntax_tree/yarv/disassembler.rb @@ -54,21 +54,20 @@ def disassemble(iseq) clauses = {} clause = [] + iseq.to_a iseq.insns.each do |insn| case insn - when Symbol - if insn.start_with?("label_") - unless clause.last.is_a?(Next) - clause << Assign(disasm_label.field, node_for(insn)) - end - - clauses[label] = clause - clause = [] - label = insn + when InstructionSequence::Label + unless clause.last.is_a?(Next) + clause << Assign(disasm_label.field, node_for(insn.name)) end + + clauses[label] = clause + clause = [] + label = insn.name when BranchUnless body = [ - Assign(disasm_label.field, node_for(insn.label)), + Assign(disasm_label.field, node_for(insn.label.name)), Next(Args([])) ] @@ -88,7 +87,7 @@ def disassemble(iseq) local = iseq.local_table.locals[insn.index] clause << VarRef(Ident(local.name.to_s)) when Jump - clause << Assign(disasm_label.field, node_for(insn.label)) + clause << Assign(disasm_label.field, node_for(insn.label.name)) clause << Next(Args([])) when Leave value = Args([clause.pop]) diff --git a/lib/syntax_tree/yarv/instruction_sequence.rb b/lib/syntax_tree/yarv/instruction_sequence.rb index e47a18ea..097fda38 100644 --- a/lib/syntax_tree/yarv/instruction_sequence.rb +++ b/lib/syntax_tree/yarv/instruction_sequence.rb @@ -464,11 +464,12 @@ def opt_case_dispatch(case_dispatch_hash, else_label) def opt_getconstant_path(names) if RUBY_VERSION < "3.2" || !options.inline_const_cache? cache = nil - getinlinecache = nil + cache_filled_label = nil if options.inline_const_cache? cache = inline_storage - getinlinecache = opt_getinlinecache(-1, cache) + cache_filled_label = label + opt_getinlinecache(cache_filled_label, cache) if names[0] == :"" names.shift @@ -489,7 +490,7 @@ def opt_getconstant_path(names) if options.inline_const_cache? opt_setinlinecache(cache) - getinlinecache.patch!(self) + push(cache_filled_label) end else push(OptGetConstantPath.new(names)) @@ -762,7 +763,7 @@ def self.from(source, options = Compiler::Options.new, parent_iseq = nil) when :branchnil iseq.branchnil(labels[opnds[0]]) when :branchunless - iseq.branchunless(opnds[0]) + iseq.branchunless(labels[opnds[0]]) when :checkkeyword iseq.checkkeyword(iseq.local_table.size - opnds[0] + 2, opnds[1]) when :checkmatch @@ -838,7 +839,7 @@ def self.from(source, options = Compiler::Options.new, parent_iseq = nil) block_iseq = opnds[1] ? from(opnds[1], options, iseq) : nil iseq.invokesuper(CallData.from(opnds[0]), block_iseq) when :jump - iseq.jump(opnds[0]) + iseq.jump(labels[opnds[0]]) when :leave iseq.leave when :newarray @@ -866,11 +867,11 @@ def self.from(source, options = Compiler::Options.new, parent_iseq = nil) when :opt_aset_with iseq.opt_aset_with(opnds[0], CallData.from(opnds[1])) when :opt_case_dispatch - iseq.opt_case_dispatch(opnds[0], opnds[1]) + iseq.opt_case_dispatch(opnds[0], labels[opnds[1]]) when :opt_getconstant_path iseq.opt_getconstant_path(opnds[0]) when :opt_getinlinecache - iseq.opt_getinlinecache(opnds[0], opnds[1]) + iseq.opt_getinlinecache(labels[opnds[0]], opnds[1]) when :opt_newarray_max iseq.opt_newarray_max(opnds[0]) when :opt_newarray_min diff --git a/lib/syntax_tree/yarv/instructions.rb b/lib/syntax_tree/yarv/instructions.rb index c340cd4e..8ec1f068 100644 --- a/lib/syntax_tree/yarv/instructions.rb +++ b/lib/syntax_tree/yarv/instructions.rb @@ -240,12 +240,8 @@ def initialize(label) @label = label end - def patch!(iseq) - @label = iseq.label_at_index - end - def to_a(_iseq) - [:branchunless, label] + [:branchunless, label.name] end def length @@ -288,10 +284,6 @@ def initialize(keyword_bits_index, keyword_index) @keyword_index = keyword_index end - def patch!(iseq) - @label = iseq.label_at_index - end - def to_a(iseq) [ :checkkeyword, @@ -1351,12 +1343,8 @@ def initialize(label) @label = label end - def patch!(iseq) - @label = iseq.label_at_index - end - def to_a(_iseq) - [:jump, label] + [:jump, label.name] end def length diff --git a/lib/syntax_tree/yarv/legacy.rb b/lib/syntax_tree/yarv/legacy.rb index 20588974..82f7560d 100644 --- a/lib/syntax_tree/yarv/legacy.rb +++ b/lib/syntax_tree/yarv/legacy.rb @@ -67,12 +67,8 @@ def initialize(label, cache) @cache = cache end - def patch!(iseq) - @label = iseq.label_at_index - end - def to_a(_iseq) - [:opt_getinlinecache, label, cache] + [:opt_getinlinecache, label.name, cache] end def length From f87fc563b0127bbe661bb43b424ca379e3a20aa4 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Sat, 26 Nov 2022 13:20:21 -0500 Subject: [PATCH 280/536] Create a linked list for nodes --- lib/syntax_tree/yarv/compiler.rb | 28 +++---- lib/syntax_tree/yarv/instruction_sequence.rb | 78 ++++++++++++++++---- 2 files changed, 76 insertions(+), 30 deletions(-) diff --git a/lib/syntax_tree/yarv/compiler.rb b/lib/syntax_tree/yarv/compiler.rb index 3bcfc598..f6d40f30 100644 --- a/lib/syntax_tree/yarv/compiler.rb +++ b/lib/syntax_tree/yarv/compiler.rb @@ -1260,15 +1260,13 @@ def visit_opassign(node) end def visit_params(node) - argument_options = iseq.argument_options - if node.requireds.any? - argument_options[:lead_num] = 0 + iseq.argument_options[:lead_num] = 0 node.requireds.each do |required| iseq.local_table.plain(required.value.to_sym) iseq.argument_size += 1 - argument_options[:lead_num] += 1 + iseq.argument_options[:lead_num] += 1 end end @@ -1279,7 +1277,9 @@ def visit_params(node) iseq.local_table.plain(name) iseq.argument_size += 1 - argument_options[:opt] = [iseq.label_at_index] unless argument_options.key?(:opt) + unless iseq.argument_options.key?(:opt) + iseq.argument_options[:opt] = [iseq.label_at_index] + end visit(value) iseq.setlocal(index, 0) @@ -1289,19 +1289,19 @@ def visit_params(node) visit(node.rest) if node.rest if node.posts.any? - argument_options[:post_start] = iseq.argument_size - argument_options[:post_num] = 0 + iseq.argument_options[:post_start] = iseq.argument_size + iseq.argument_options[:post_num] = 0 node.posts.each do |post| iseq.local_table.plain(post.value.to_sym) iseq.argument_size += 1 - argument_options[:post_num] += 1 + iseq.argument_options[:post_num] += 1 end end if node.keywords.any? - argument_options[:kwbits] = 0 - argument_options[:keyword] = [] + iseq.argument_options[:kwbits] = 0 + iseq.argument_options[:keyword] = [] keyword_bits_name = node.keyword_rest ? 3 : 2 iseq.argument_size += 1 @@ -1313,16 +1313,16 @@ def visit_params(node) iseq.local_table.plain(name) iseq.argument_size += 1 - argument_options[:kwbits] += 1 + iseq.argument_options[:kwbits] += 1 if value.nil? - argument_options[:keyword] << name + iseq.argument_options[:keyword] << name elsif (compiled = RubyVisitor.compile(value)) - argument_options[:keyword] << [name, compiled] + iseq.argument_options[:keyword] << [name, compiled] else skip_value_label = iseq.label - argument_options[:keyword] << [name] + iseq.argument_options[:keyword] << [name] iseq.checkkeyword(keyword_bits_index, keyword_index) iseq.branchif(skip_value_label) visit(value) diff --git a/lib/syntax_tree/yarv/instruction_sequence.rb b/lib/syntax_tree/yarv/instruction_sequence.rb index 097fda38..42910266 100644 --- a/lib/syntax_tree/yarv/instruction_sequence.rb +++ b/lib/syntax_tree/yarv/instruction_sequence.rb @@ -7,6 +7,50 @@ module YARV # list of instructions along with the metadata pertaining to them. It also # functions as a builder for the instruction sequence. class InstructionSequence + # When the list of instructions is first being created, it's stored as a + # linked list. This is to make it easier to perform peephole optimizations + # and other transformations like instruction specialization. + class InstructionList + class Node + attr_reader :instruction + attr_accessor :next_node + + def initialize(instruction, next_node = nil) + @instruction = instruction + @next_node = next_node + end + end + + attr_reader :head_node, :tail_node + + def initialize + @head_node = nil + @tail_node = nil + end + + def each + return to_enum(__method__) unless block_given? + node = head_node + + while node + yield node.instruction + node = node.next_node + end + end + + def push(instruction) + node = Node.new(instruction) + + if head_node.nil? + @head_node = node + @tail_node = node + else + @tail_node.next_node = node + @tail_node = node + end + end + end + MAGIC = "YARVInstructionSequence/SimpleDataFormat" # This provides a handle to the rb_iseq_load function, which allows you to @@ -110,7 +154,7 @@ def initialize( @local_table = LocalTable.new @inline_storages = {} - @insns = [] + @insns = InstructionList.new @storage_index = 0 @stack = Stack.new @@ -142,7 +186,7 @@ def inline_storage_for(name) end def length - insns.inject(0) do |sum, insn| + insns.each.inject(0) do |sum, insn| case insn when Integer, Label, Symbol sum @@ -167,7 +211,7 @@ def to_a versions = RUBY_VERSION.split(".").map(&:to_i) # First, set it up so that all of the labels get their correct name. - insns.inject(0) do |length, insn| + insns.each.inject(0) do |length, insn| case insn when Integer, Symbol length @@ -179,6 +223,18 @@ def to_a end end + # Next, dump all of the instructions into a flat list. + dumped = insns.each.map do |insn| + case insn + when Integer, Symbol + insn + when Label + insn.name + else + insn.to_a(self) + end + end + # Next, return the instruction sequence as an array. [ MAGIC, @@ -198,16 +254,7 @@ def to_a local_table.names, argument_options, [], - insns.map do |insn| - case insn - when Integer, Symbol - insn - when Label - insn.name - else - insn.to_a(self) - end - end + dumped ] end @@ -250,7 +297,7 @@ def label end def push(insn) - insns << insn + insns.push(insn) case insn when Array, Integer, Label, Symbol @@ -262,8 +309,7 @@ def push(insn) end def label_at_index - name = :"label_#{length}" - insns.last == name ? name : event(name) + push(:"label_#{length}") end def event(name) From 2115177c7f74faafdf6760e9d926417c7c648bde Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Sat, 26 Nov 2022 13:27:18 -0500 Subject: [PATCH 281/536] Fix opt table to use labels --- lib/syntax_tree/yarv/compiler.rb | 9 ++++++-- lib/syntax_tree/yarv/instruction_sequence.rb | 22 +++++++++++--------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/lib/syntax_tree/yarv/compiler.rb b/lib/syntax_tree/yarv/compiler.rb index f6d40f30..c0d89239 100644 --- a/lib/syntax_tree/yarv/compiler.rb +++ b/lib/syntax_tree/yarv/compiler.rb @@ -1278,12 +1278,17 @@ def visit_params(node) iseq.argument_size += 1 unless iseq.argument_options.key?(:opt) - iseq.argument_options[:opt] = [iseq.label_at_index] + start_label = iseq.label + iseq.push(start_label) + iseq.argument_options[:opt] = [start_label] end visit(value) iseq.setlocal(index, 0) - iseq.argument_options[:opt] << iseq.label_at_index + + arg_given_label = iseq.label + iseq.push(arg_given_label) + iseq.argument_options[:opt] << arg_given_label end visit(node.rest) if node.rest diff --git a/lib/syntax_tree/yarv/instruction_sequence.rb b/lib/syntax_tree/yarv/instruction_sequence.rb index 42910266..63904923 100644 --- a/lib/syntax_tree/yarv/instruction_sequence.rb +++ b/lib/syntax_tree/yarv/instruction_sequence.rb @@ -235,6 +235,9 @@ def to_a end end + dumped_options = argument_options.dup + dumped_options[:opt].map!(&:name) if dumped_options[:opt] + # Next, return the instruction sequence as an array. [ MAGIC, @@ -252,7 +255,7 @@ def to_a location.start_line, type, local_table.names, - argument_options, + dumped_options, [], dumped ] @@ -308,10 +311,6 @@ def push(insn) end end - def label_at_index - push(:"label_#{length}") - end - def event(name) push(name) end @@ -767,6 +766,11 @@ def toregexp(options, length) def self.from(source, options = Compiler::Options.new, parent_iseq = nil) iseq = new(source[9], source[5], parent_iseq, Location.default, options) + # set up the labels object so that the labels are shared between the + # location in the instruction sequence and the instructions that + # reference them + labels = Hash.new { |hash, name| hash[name] = Label.new(name) } + # set up the correct argument size iseq.argument_size = source[4][:arg_size] @@ -775,11 +779,9 @@ def self.from(source, options = Compiler::Options.new, parent_iseq = nil) # set up the argument options iseq.argument_options.merge!(source[11]) - - # set up the labels object so that the labels are shared between the - # location in the instruction sequence and the instructions that - # reference them - labels = Hash.new { |hash, name| hash[name] = Label.new(name) } + if iseq.argument_options[:opt] + iseq.argument_options[:opt].map! { |opt| labels[opt] } + end # set up all of the instructions source[13].each do |insn| From 69d2dfa143361357c2684da17d3c2df3b5ed85c2 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Sat, 26 Nov 2022 13:52:27 -0500 Subject: [PATCH 282/536] Specialize in a separate pass --- lib/syntax_tree/yarv/compiler.rb | 11 -- lib/syntax_tree/yarv/instruction_sequence.rb | 185 ++++++++++--------- 2 files changed, 102 insertions(+), 94 deletions(-) diff --git a/lib/syntax_tree/yarv/compiler.rb b/lib/syntax_tree/yarv/compiler.rb index c0d89239..362ce32f 100644 --- a/lib/syntax_tree/yarv/compiler.rb +++ b/lib/syntax_tree/yarv/compiler.rb @@ -632,17 +632,6 @@ def visit_call(node) return end end - when StringLiteral - if RubyVisitor.compile(node.receiver).nil? - case node.message.value - when "-@" - iseq.opt_str_uminus(node.receiver.parts.first.value) - return - when "freeze" - iseq.opt_str_freeze(node.receiver.parts.first.value) - return - end - end end end diff --git a/lib/syntax_tree/yarv/instruction_sequence.rb b/lib/syntax_tree/yarv/instruction_sequence.rb index 63904923..dc2f7da8 100644 --- a/lib/syntax_tree/yarv/instruction_sequence.rb +++ b/lib/syntax_tree/yarv/instruction_sequence.rb @@ -12,8 +12,7 @@ class InstructionSequence # and other transformations like instruction specialization. class InstructionList class Node - attr_reader :instruction - attr_accessor :next_node + attr_accessor :instruction, :next_node def initialize(instruction, next_node = nil) @instruction = instruction @@ -29,11 +28,16 @@ def initialize end def each + return to_enum(__method__) unless block_given? + each_node { |node| yield node.instruction } + end + + def each_node return to_enum(__method__) unless block_given? node = head_node while node - yield node.instruction + yield node node = node.next_node end end @@ -210,7 +214,10 @@ def eval def to_a versions = RUBY_VERSION.split(".").map(&:to_i) - # First, set it up so that all of the labels get their correct name. + # First, specialize any instructions that need to be specialized. + specialize_instructions! if options.specialized_instruction? + + # Next, set it up so that all of the labels get their correct name. insns.each.inject(0) do |length, insn| case insn when Integer, Symbol @@ -261,6 +268,92 @@ def to_a ] end + def specialize_instructions! + insns.each_node do |node| + case node.instruction + when PutObject, PutString + next unless node.next_node + next if node.instruction.is_a?(PutObject) && !node.instruction.object.is_a?(String) + + next_node = node.next_node + next unless next_node.instruction.is_a?(Send) + next if next_node.instruction.block_iseq + + calldata = next_node.instruction.calldata + next unless calldata.flags == CallData::CALL_ARGS_SIMPLE + + case calldata.method + when :freeze + node.instruction = OptStrFreeze.new(node.instruction.object, calldata) + node.next_node = next_node.next_node + when :-@ + node.instruction = OptStrUMinus.new(node.instruction.object, calldata) + node.next_node = next_node.next_node + end + when Send + calldata = node.instruction.calldata + + if !node.instruction.block_iseq && !calldata.flag?(CallData::CALL_ARGS_BLOCKARG) + # Specialize the send instruction. If it doesn't have a block + # attached, then we will replace it with an opt_send_without_block + # and do further specializations based on the called method and + # the number of arguments. + node.instruction = + case [calldata.method, calldata.argc] + when [:length, 0] + OptLength.new(calldata) + when [:size, 0] + OptSize.new(calldata) + when [:empty?, 0] + OptEmptyP.new(calldata) + when [:nil?, 0] + OptNilP.new(calldata) + when [:succ, 0] + OptSucc.new(calldata) + when [:!, 0] + OptNot.new(calldata) + when [:+, 1] + OptPlus.new(calldata) + when [:-, 1] + OptMinus.new(calldata) + when [:*, 1] + OptMult.new(calldata) + when [:/, 1] + OptDiv.new(calldata) + when [:%, 1] + OptMod.new(calldata) + when [:==, 1] + OptEq.new(calldata) + when [:!=, 1] + OptNEq.new(YARV.calldata(:==, 1), calldata) + when [:=~, 1] + OptRegExpMatch2.new(calldata) + when [:<, 1] + OptLT.new(calldata) + when [:<=, 1] + OptLE.new(calldata) + when [:>, 1] + OptGT.new(calldata) + when [:>=, 1] + OptGE.new(calldata) + when [:<<, 1] + OptLTLT.new(calldata) + when [:[], 1] + OptAref.new(calldata) + when [:&, 1] + OptAnd.new(calldata) + when [:|, 1] + OptOr.new(calldata) + when [:[]=, 2] + OptAset.new(calldata) + else + OptSendWithoutBlock.new(calldata) + end + end + end + end + end + ########################################################################## # Child instruction sequence methods ########################################################################## @@ -568,24 +661,6 @@ def opt_setinlinecache(cache) push(Legacy::OptSetInlineCache.new(cache)) end - def opt_str_freeze(object) - if options.specialized_instruction? - push(OptStrFreeze.new(object, YARV.calldata(:freeze))) - else - putstring(object) - send(YARV.calldata(:freeze)) - end - end - - def opt_str_uminus(object) - if options.specialized_instruction? - push(OptStrUMinus.new(object, YARV.calldata(:-@))) - else - putstring(object) - send(YARV.calldata(:-@)) - end - end - def pop push(Pop.new) end @@ -625,65 +700,7 @@ def putstring(object) end def send(calldata, block_iseq = nil) - if options.specialized_instruction? && !block_iseq && - !calldata.flag?(CallData::CALL_ARGS_BLOCKARG) - # Specialize the send instruction. If it doesn't have a block - # attached, then we will replace it with an opt_send_without_block - # and do further specializations based on the called method and the - # number of arguments. - case [calldata.method, calldata.argc] - when [:length, 0] - push(OptLength.new(calldata)) - when [:size, 0] - push(OptSize.new(calldata)) - when [:empty?, 0] - push(OptEmptyP.new(calldata)) - when [:nil?, 0] - push(OptNilP.new(calldata)) - when [:succ, 0] - push(OptSucc.new(calldata)) - when [:!, 0] - push(OptNot.new(calldata)) - when [:+, 1] - push(OptPlus.new(calldata)) - when [:-, 1] - push(OptMinus.new(calldata)) - when [:*, 1] - push(OptMult.new(calldata)) - when [:/, 1] - push(OptDiv.new(calldata)) - when [:%, 1] - push(OptMod.new(calldata)) - when [:==, 1] - push(OptEq.new(calldata)) - when [:!=, 1] - push(OptNEq.new(YARV.calldata(:==, 1), calldata)) - when [:=~, 1] - push(OptRegExpMatch2.new(calldata)) - when [:<, 1] - push(OptLT.new(calldata)) - when [:<=, 1] - push(OptLE.new(calldata)) - when [:>, 1] - push(OptGT.new(calldata)) - when [:>=, 1] - push(OptGE.new(calldata)) - when [:<<, 1] - push(OptLTLT.new(calldata)) - when [:[], 1] - push(OptAref.new(calldata)) - when [:&, 1] - push(OptAnd.new(calldata)) - when [:|, 1] - push(OptOr.new(calldata)) - when [:[]=, 2] - push(OptAset.new(calldata)) - else - push(OptSendWithoutBlock.new(calldata)) - end - else - push(Send.new(calldata, block_iseq)) - end + push(Send.new(calldata, block_iseq)) end def setblockparam(index, level) @@ -931,9 +948,11 @@ def self.from(source, options = Compiler::Options.new, parent_iseq = nil) when :opt_setinlinecache iseq.opt_setinlinecache(opnds[0]) when :opt_str_freeze - iseq.opt_str_freeze(opnds[0]) + iseq.putstring(opnds[0]) + iseq.send(YARV.calldata(:freeze)) when :opt_str_uminus - iseq.opt_str_uminus(opnds[0]) + iseq.putstring(opnds[0]) + iseq.send(YARV.calldata(:-@)) when :pop iseq.pop when :putnil From 80de9c9d4e1ddfc73fab479df69d77ce7367de69 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Sat, 26 Nov 2022 14:03:46 -0500 Subject: [PATCH 283/536] Specialize using the linked list --- lib/syntax_tree/yarv/compiler.rb | 29 +---- lib/syntax_tree/yarv/instruction_sequence.rb | 115 +++++++++++-------- 2 files changed, 67 insertions(+), 77 deletions(-) diff --git a/lib/syntax_tree/yarv/compiler.rb b/lib/syntax_tree/yarv/compiler.rb index 362ce32f..9016c136 100644 --- a/lib/syntax_tree/yarv/compiler.rb +++ b/lib/syntax_tree/yarv/compiler.rb @@ -608,33 +608,6 @@ def visit_call(node) ) end - arg_parts = argument_parts(node.arguments) - argc = arg_parts.length - - # First we're going to check if we're calling a method on an array - # literal without any arguments. In that case there are some - # specializations we might be able to perform. - if argc == 0 && (node.message.is_a?(Ident) || node.message.is_a?(Op)) - case node.receiver - when ArrayLiteral - parts = node.receiver.contents&.parts || [] - - if parts.none? { |part| part.is_a?(ArgStar) } && - RubyVisitor.compile(node.receiver).nil? - case node.message.value - when "max" - visit(node.receiver.contents) - iseq.opt_newarray_max(parts.length) - return - when "min" - visit(node.receiver.contents) - iseq.opt_newarray_min(parts.length) - return - end - end - end - end - # Track whether or not this is a method call on a block proxy receiver. # If it is, we can potentially do tailcall optimizations on it. block_receiver = false @@ -663,6 +636,8 @@ def visit_call(node) iseq.branchnil(after_call_label) end + arg_parts = argument_parts(node.arguments) + argc = arg_parts.length flag = 0 arg_parts.each do |arg_part| diff --git a/lib/syntax_tree/yarv/instruction_sequence.rb b/lib/syntax_tree/yarv/instruction_sequence.rb index dc2f7da8..ff324d92 100644 --- a/lib/syntax_tree/yarv/instruction_sequence.rb +++ b/lib/syntax_tree/yarv/instruction_sequence.rb @@ -190,14 +190,16 @@ def inline_storage_for(name) end def length - insns.each.inject(0) do |sum, insn| - case insn - when Integer, Label, Symbol - sum - else - sum + insn.length + insns + .each + .inject(0) do |sum, insn| + case insn + when Integer, Label, Symbol + sum + else + sum + insn.length + end end - end end def eval @@ -218,29 +220,32 @@ def to_a specialize_instructions! if options.specialized_instruction? # Next, set it up so that all of the labels get their correct name. - insns.each.inject(0) do |length, insn| - case insn - when Integer, Symbol - length - when Label - insn.patch!(:"label_#{length}") - length - else - length + insn.length + insns + .each + .inject(0) do |length, insn| + case insn + when Integer, Symbol + length + when Label + insn.patch!(:"label_#{length}") + length + else + length + insn.length + end end - end # Next, dump all of the instructions into a flat list. - dumped = insns.each.map do |insn| - case insn - when Integer, Symbol - insn - when Label - insn.name - else - insn.to_a(self) + dumped = + insns.each.map do |insn| + case insn + when Integer, Symbol + insn + when Label + insn.name + else + insn.to_a(self) + end end - end dumped_options = argument_options.dup dumped_options[:opt].map!(&:name) if dumped_options[:opt] @@ -271,9 +276,31 @@ def to_a def specialize_instructions! insns.each_node do |node| case node.instruction + when NewArray + next unless node.next_node + + next_node = node.next_node + next unless next_node.instruction.is_a?(Send) + next if next_node.instruction.block_iseq + + calldata = next_node.instruction.calldata + next unless calldata.flags == CallData::CALL_ARGS_SIMPLE + next unless calldata.argc == 0 + + case calldata.method + when :max + node.instruction = OptNewArrayMax.new(node.instruction.number) + node.next_node = next_node.next_node + when :min + node.instruction = OptNewArrayMin.new(node.instruction.number) + node.next_node = next_node.next_node + end when PutObject, PutString next unless node.next_node - next if node.instruction.is_a?(PutObject) && !node.instruction.object.is_a?(String) + if node.instruction.is_a?(PutObject) && + !node.instruction.object.is_a?(String) + next + end next_node = node.next_node next unless next_node.instruction.is_a?(Send) @@ -281,19 +308,23 @@ def specialize_instructions! calldata = next_node.instruction.calldata next unless calldata.flags == CallData::CALL_ARGS_SIMPLE + next unless calldata.argc == 0 case calldata.method when :freeze - node.instruction = OptStrFreeze.new(node.instruction.object, calldata) + node.instruction = + OptStrFreeze.new(node.instruction.object, calldata) node.next_node = next_node.next_node when :-@ - node.instruction = OptStrUMinus.new(node.instruction.object, calldata) + node.instruction = + OptStrUMinus.new(node.instruction.object, calldata) node.next_node = next_node.next_node end when Send calldata = node.instruction.calldata - if !node.instruction.block_iseq && !calldata.flag?(CallData::CALL_ARGS_BLOCKARG) + if !node.instruction.block_iseq && + !calldata.flag?(CallData::CALL_ARGS_BLOCKARG) # Specialize the send instruction. If it doesn't have a block # attached, then we will replace it with an opt_send_without_block # and do further specializations based on the called method and @@ -639,24 +670,6 @@ def opt_getinlinecache(label, cache) push(Legacy::OptGetInlineCache.new(label, cache)) end - def opt_newarray_max(length) - if options.specialized_instruction? - push(OptNewArrayMax.new(length)) - else - newarray(length) - send(YARV.calldata(:max)) - end - end - - def opt_newarray_min(length) - if options.specialized_instruction? - push(OptNewArrayMin.new(length)) - else - newarray(length) - send(YARV.calldata(:min)) - end - end - def opt_setinlinecache(cache) push(Legacy::OptSetInlineCache.new(cache)) end @@ -938,9 +951,11 @@ def self.from(source, options = Compiler::Options.new, parent_iseq = nil) when :opt_getinlinecache iseq.opt_getinlinecache(labels[opnds[0]], opnds[1]) when :opt_newarray_max - iseq.opt_newarray_max(opnds[0]) + iseq.newarray(opnds[0]) + iseq.send(YARV.calldata(:max)) when :opt_newarray_min - iseq.opt_newarray_min(opnds[0]) + iseq.newarray(opnds[0]) + iseq.send(YARV.calldata(:min)) when :opt_neq iseq.push( OptNEq.new(CallData.from(opnds[0]), CallData.from(opnds[1])) From f3ed30d2157dd6351d0cf2fce1d91148f1432318 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Sat, 26 Nov 2022 14:21:01 -0500 Subject: [PATCH 284/536] Have the instruction list point to values not necessarily instructions --- lib/syntax_tree/yarv/instruction_sequence.rb | 48 +++++++++----------- 1 file changed, 21 insertions(+), 27 deletions(-) diff --git a/lib/syntax_tree/yarv/instruction_sequence.rb b/lib/syntax_tree/yarv/instruction_sequence.rb index ff324d92..a994c6d2 100644 --- a/lib/syntax_tree/yarv/instruction_sequence.rb +++ b/lib/syntax_tree/yarv/instruction_sequence.rb @@ -12,10 +12,10 @@ class InstructionSequence # and other transformations like instruction specialization. class InstructionList class Node - attr_accessor :instruction, :next_node + attr_accessor :value, :next_node - def initialize(instruction, next_node = nil) - @instruction = instruction + def initialize(value, next_node = nil) + @value = value @next_node = next_node end end @@ -29,7 +29,7 @@ def initialize def each return to_enum(__method__) unless block_given? - each_node { |node| yield node.instruction } + each_node { |node| yield node.value } end def each_node @@ -37,7 +37,7 @@ def each_node node = head_node while node - yield node + yield node, node.value node = node.next_node end end @@ -274,62 +274,56 @@ def to_a end def specialize_instructions! - insns.each_node do |node| - case node.instruction + insns.each_node do |node, value| + case value when NewArray next unless node.next_node next_node = node.next_node - next unless next_node.instruction.is_a?(Send) - next if next_node.instruction.block_iseq + next unless next_node.value.is_a?(Send) + next if next_node.value.block_iseq - calldata = next_node.instruction.calldata + calldata = next_node.value.calldata next unless calldata.flags == CallData::CALL_ARGS_SIMPLE next unless calldata.argc == 0 case calldata.method when :max - node.instruction = OptNewArrayMax.new(node.instruction.number) + node.value = OptNewArrayMax.new(value.number) node.next_node = next_node.next_node when :min - node.instruction = OptNewArrayMin.new(node.instruction.number) + node.value = OptNewArrayMin.new(value.number) node.next_node = next_node.next_node end when PutObject, PutString next unless node.next_node - if node.instruction.is_a?(PutObject) && - !node.instruction.object.is_a?(String) - next - end + next if value.is_a?(PutObject) && !value.object.is_a?(String) next_node = node.next_node - next unless next_node.instruction.is_a?(Send) - next if next_node.instruction.block_iseq + next unless next_node.value.is_a?(Send) + next if next_node.value.block_iseq - calldata = next_node.instruction.calldata + calldata = next_node.value.calldata next unless calldata.flags == CallData::CALL_ARGS_SIMPLE next unless calldata.argc == 0 case calldata.method when :freeze - node.instruction = - OptStrFreeze.new(node.instruction.object, calldata) + node.value = OptStrFreeze.new(value.object, calldata) node.next_node = next_node.next_node when :-@ - node.instruction = - OptStrUMinus.new(node.instruction.object, calldata) + node.value = OptStrUMinus.new(value.object, calldata) node.next_node = next_node.next_node end when Send - calldata = node.instruction.calldata + calldata = value.calldata - if !node.instruction.block_iseq && - !calldata.flag?(CallData::CALL_ARGS_BLOCKARG) + if !value.block_iseq && !calldata.flag?(CallData::CALL_ARGS_BLOCKARG) # Specialize the send instruction. If it doesn't have a block # attached, then we will replace it with an opt_send_without_block # and do further specializations based on the called method and # the number of arguments. - node.instruction = + node.value = case [calldata.method, calldata.argc] when [:length, 0] OptLength.new(calldata) From b422b428f8089e723732b5a586d5d97bdc18ead6 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Sat, 26 Nov 2022 14:29:54 -0500 Subject: [PATCH 285/536] Give a reference on the labels to their container nodes --- lib/syntax_tree/yarv/instruction_sequence.rb | 30 +++++++++++--------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/lib/syntax_tree/yarv/instruction_sequence.rb b/lib/syntax_tree/yarv/instruction_sequence.rb index a994c6d2..5469f6f7 100644 --- a/lib/syntax_tree/yarv/instruction_sequence.rb +++ b/lib/syntax_tree/yarv/instruction_sequence.rb @@ -95,12 +95,18 @@ def change_by(value) class Label attr_reader :name + # When we're serializing the instruction sequence, we need to be able to + # look up the label from the branch instructions and then access the + # subsequent node. So we'll store the reference here. + attr_reader :node + def initialize(name = nil) @name = name end - def patch!(name) + def patch!(name, node) @name = name + @node = node end end @@ -220,19 +226,17 @@ def to_a specialize_instructions! if options.specialized_instruction? # Next, set it up so that all of the labels get their correct name. - insns - .each - .inject(0) do |length, insn| - case insn - when Integer, Symbol - length - when Label - insn.patch!(:"label_#{length}") - length - else - length + insn.length - end + length = 0 + insns.each_node do |node, value| + case value + when Integer, Symbol + # skip + when Label + value.patch!(:"label_#{length}", node) + else + length += value.length end + end # Next, dump all of the instructions into a flat list. dumped = From 14df44ed9b4c01845e0402a9514c0d40e05bddd7 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Sat, 26 Nov 2022 16:03:06 -0500 Subject: [PATCH 286/536] Begin peephole optimizations --- lib/syntax_tree/yarv/instruction_sequence.rb | 50 +++++++++++++++----- 1 file changed, 38 insertions(+), 12 deletions(-) diff --git a/lib/syntax_tree/yarv/instruction_sequence.rb b/lib/syntax_tree/yarv/instruction_sequence.rb index 5469f6f7..e8e30b3b 100644 --- a/lib/syntax_tree/yarv/instruction_sequence.rb +++ b/lib/syntax_tree/yarv/instruction_sequence.rb @@ -52,6 +52,8 @@ def push(instruction) @tail_node.next_node = node @tail_node = node end + + node end end @@ -98,15 +100,14 @@ class Label # When we're serializing the instruction sequence, we need to be able to # look up the label from the branch instructions and then access the # subsequent node. So we'll store the reference here. - attr_reader :node + attr_accessor :node def initialize(name = nil) @name = name end - def patch!(name, node) + def patch!(name) @name = name - @node = node end end @@ -222,8 +223,9 @@ def eval def to_a versions = RUBY_VERSION.split(".").map(&:to_i) - # First, specialize any instructions that need to be specialized. + # First, handle any compilation options that we need to. specialize_instructions! if options.specialized_instruction? + peephole_optimize! if options.peephole_optimization? # Next, set it up so that all of the labels get their correct name. length = 0 @@ -232,7 +234,7 @@ def to_a when Integer, Symbol # skip when Label - value.patch!(:"label_#{length}", node) + value.patch!(:"label_#{length}") else length += value.length end @@ -383,6 +385,27 @@ def specialize_instructions! end end + def peephole_optimize! + insns.each_node do |node, value| + case value + when Jump + # jump LABEL + # ... + # LABEL: + # leave + # => + # leave + # ... + # LABEL: + # leave + # case value.label.node.next_node&.value + # when Leave + # node.value = Leave.new + # end + end + end + end + ########################################################################## # Child instruction sequence methods ########################################################################## @@ -421,15 +444,18 @@ def label Label.new end - def push(insn) - insns.push(insn) + def push(value) + node = insns.push(value) - case insn - when Array, Integer, Label, Symbol - insn + case value + when Array, Integer, Symbol + value + when Label + value.node = node + value else - stack.change_by(-insn.pops + insn.pushes) - insn + stack.change_by(-value.pops + value.pushes) + value end end From b998a6ea9a5a9564dafc0cd422a77f03e3937c26 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 28 Nov 2022 11:15:38 -0500 Subject: [PATCH 287/536] Add a bit of execution --- .rubocop.yml | 18 + lib/syntax_tree/yarv.rb | 277 +++++ lib/syntax_tree/yarv/compiler.rb | 12 +- lib/syntax_tree/yarv/instruction_sequence.rb | 178 ++- lib/syntax_tree/yarv/instructions.rb | 1062 +++++++++++++++++- lib/syntax_tree/yarv/legacy.rb | 8 + 6 files changed, 1487 insertions(+), 68 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index b7ba43e8..c81fdb59 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -31,12 +31,18 @@ Lint/InterpolationCheck: Lint/MissingSuper: Enabled: false +Lint/NonLocalExitFromIterator: + Enabled: false + Lint/RedundantRequireStatement: Enabled: false Lint/SuppressedException: Enabled: false +Lint/UnderscorePrefixedVariableName: + Enabled: false + Lint/UnusedMethodArgument: AllowUnusedKeywordArguments: true @@ -55,6 +61,9 @@ Naming/RescuedExceptionsVariableName: Naming/VariableNumber: Enabled: false +Security/Eval: + Enabled: false + Style/AccessorGrouping: Enabled: false @@ -64,9 +73,18 @@ Style/CaseEquality: Style/CaseLikeIf: Enabled: false +Style/ClassVars: + Enabled: false + +Style/DocumentDynamicEvalDefinition: + Enabled: false + Style/Documentation: Enabled: false +Style/EndBlock: + Enabled: false + Style/ExplicitBlockArgument: Enabled: false diff --git a/lib/syntax_tree/yarv.rb b/lib/syntax_tree/yarv.rb index 1e759ad1..74f2598e 100644 --- a/lib/syntax_tree/yarv.rb +++ b/lib/syntax_tree/yarv.rb @@ -1,11 +1,288 @@ # frozen_string_literal: true +require "forwardable" + module SyntaxTree # This module provides an object representation of the YARV bytecode. module YARV + class VM + class Jump + attr_reader :name + + def initialize(name) + @name = name + end + end + + class Leave + attr_reader :value + + def initialize(value) + @value = value + end + end + + class Frame + attr_reader :iseq, :parent, :stack_index, :_self, :nesting, :svars + + def initialize(iseq, parent, stack_index, _self, nesting) + @iseq = iseq + @parent = parent + @stack_index = stack_index + @_self = _self + @nesting = nesting + @svars = {} + end + end + + class TopFrame < Frame + def initialize(iseq) + super(iseq, nil, 0, TOPLEVEL_BINDING.eval("self"), [Object]) + end + end + + class BlockFrame < Frame + def initialize(iseq, parent, stack_index) + super(iseq, parent, stack_index, parent._self, parent.nesting) + end + end + + class MethodFrame < Frame + attr_reader :name, :block + + def initialize(iseq, parent, stack_index, _self, name, block) + super(iseq, parent, stack_index, _self, parent.nesting) + @name = name + @block = block + end + end + + class ClassFrame < Frame + def initialize(iseq, parent, stack_index, _self) + super(iseq, parent, stack_index, _self, parent.nesting + [_self]) + end + end + + class FrozenCore + define_method("core#hash_merge_kwd") { |left, right| left.merge(right) } + + define_method("core#hash_merge_ptr") do |hash, *values| + hash.merge(values.each_slice(2).to_h) + end + + define_method("core#set_method_alias") do |clazz, new_name, old_name| + clazz.alias_method(new_name, old_name) + end + + define_method("core#set_variable_alias") do |new_name, old_name| + # Using eval here since there isn't a reflection API to be able to + # alias global variables. + eval("alias #{new_name} #{old_name}", binding, __FILE__, __LINE__) + end + + define_method("core#set_postexe") { |&block| END { block.call } } + + define_method("core#undef_method") do |clazz, name| + clazz.undef_method(name) + end + end + + FROZEN_CORE = FrozenCore.new.freeze + + extend Forwardable + + attr_reader :stack + def_delegators :stack, :push, :pop + + attr_reader :frame + def_delegators :frame, :_self + + def initialize + @stack = [] + @frame = nil + end + + ########################################################################## + # Helper methods for frames + ########################################################################## + + def run_frame(frame) + # First, set the current frame to the given value. + @frame = frame + + # Next, set up the local table for the frame. This is actually incorrect + # as it could use the values already on the stack, but for now we're + # just doing this for simplicity. + frame.iseq.local_table.size.times { push(nil) } + + # Yield so that some frame-specific setup can be done. + yield if block_given? + + # This hash is going to hold a mapping of label names to their + # respective indices in our instruction list. + labels = {} + + # This array is going to hold our instructions. + insns = [] + + # Here we're going to preprocess the instruction list from the + # instruction sequence to set up the labels hash and the insns array. + frame.iseq.insns.each do |insn| + case insn + when Integer, Symbol + # skip + when InstructionSequence::Label + labels[insn.name] = insns.length + else + insns << insn + end + end + + # Finally we can execute the instructions one at a time. If they return + # jumps or leaves we will handle those appropriately. + pc = 0 + while pc < insns.length + insn = insns[pc] + pc += 1 + + case (result = insn.call(self)) + when Jump + pc = labels[result.name] + when Leave + return result.value + end + end + ensure + @stack = stack[0...frame.stack_index] + @frame = frame.parent + end + + def run_top_frame(iseq) + run_frame(TopFrame.new(iseq)) + end + + def run_block_frame(iseq, *args, &block) + run_frame(BlockFrame.new(iseq, frame, stack.length)) do + locals = [*args, block] + iseq.local_table.size.times do |index| + local_set(index, 0, locals.shift) + end + end + end + + def run_class_frame(iseq, clazz) + run_frame(ClassFrame.new(iseq, frame, stack.length, clazz)) + end + + def run_method_frame(name, iseq, _self, *args, **kwargs, &block) + run_frame( + MethodFrame.new(iseq, frame, stack.length, _self, name, block) + ) do + locals = [*args, block] + + if iseq.argument_options[:keyword] + # First, set up the keyword bits array. + keyword_bits = + iseq.argument_options[:keyword].map do |config| + kwargs.key?(config.is_a?(Array) ? config[0] : config) + end + + iseq.local_table.locals.each_with_index do |local, index| + # If this is the keyword bits local, then set it appropriately. + if local.name == 2 + locals.insert(index, keyword_bits) + next + end + + # First, find the configuration for this local in the keywords + # list if it exists. + name = local.name + config = + iseq.argument_options[:keyword].find do |keyword| + keyword.is_a?(Array) ? keyword[0] == name : keyword == name + end + + # If the configuration doesn't exist, then the local is not a + # keyword local. + next unless config + + if !config.is_a?(Array) + # required keyword + locals.insert(index, kwargs.fetch(name)) + elsif !config[1].nil? + # optional keyword with embedded default value + locals.insert(index, kwargs.fetch(name, config[1])) + else + # optional keyword with expression default value + locals.insert(index, nil) + end + end + end + + iseq.local_table.size.times do |index| + local_set(index, 0, locals.shift) + end + end + end + + ########################################################################## + # Helper methods for instructions + ########################################################################## + + def const_base + frame.nesting.last + end + + def frame_at(level) + current = frame + level.times { current = current.parent } + current + end + + def frame_svar + current = frame + current = current.parent while current.is_a?(BlockFrame) + current + end + + def frame_yield + current = frame + current = current.parent until current.is_a?(MethodFrame) + current + end + + def frozen_core + FROZEN_CORE + end + + def jump(label) + Jump.new(label.name) + end + + def leave + Leave.new(pop) + end + + def local_get(index, level) + stack[frame_at(level).stack_index + index] + end + + def local_set(index, level, value) + stack[frame_at(level).stack_index + index] = value + end + end + # Compile the given source into a YARV instruction sequence. def self.compile(source, options = Compiler::Options.new) SyntaxTree.parse(source).accept(Compiler.new(options)) end + + # Compile and interpret the given source. + def self.interpret(source, options = Compiler::Options.new) + iseq = RubyVM::InstructionSequence.compile(source, **options) + iseq = InstructionSequence.from(iseq.to_a) + iseq.specialize_instructions! + VM.new.run_top_frame(iseq) + end end end diff --git a/lib/syntax_tree/yarv/compiler.rb b/lib/syntax_tree/yarv/compiler.rb index 9016c136..194b758b 100644 --- a/lib/syntax_tree/yarv/compiler.rb +++ b/lib/syntax_tree/yarv/compiler.rb @@ -1987,8 +1987,8 @@ def visit_pattern(node, end_label) match_failure_label = iseq.label match_error_label = iseq.label - # If there's a constant, then check if we match against that constant or - # not first. Branch to failure if we don't. + # If there's a constant, then check if we match against that constant + # or not first. Branch to failure if we don't. if node.constant iseq.dup visit(node.constant) @@ -2007,8 +2007,8 @@ def visit_pattern(node, end_label) iseq.topn(2) iseq.branchunless(match_failure_label) - # Since we have a valid cached value, we can skip past the part where we - # call #deconstruct on the object. + # Since we have a valid cached value, we can skip past the part where + # we call #deconstruct on the object. iseq.pop iseq.topn(1) iseq.jump(length_label) @@ -2064,8 +2064,8 @@ def visit_pattern(node, end_label) end end - # Set up the routine here to raise an error to indicate that the type of - # the deconstructed array was incorrect. + # Set up the routine here to raise an error to indicate that the type + # of the deconstructed array was incorrect. iseq.push(match_error_label) iseq.putspecialobject(PutSpecialObject::OBJECT_VMCORE) iseq.putobject(TypeError) diff --git a/lib/syntax_tree/yarv/instruction_sequence.rb b/lib/syntax_tree/yarv/instruction_sequence.rb index e8e30b3b..f20981df 100644 --- a/lib/syntax_tree/yarv/instruction_sequence.rb +++ b/lib/syntax_tree/yarv/instruction_sequence.rb @@ -20,6 +20,7 @@ def initialize(value, next_node = nil) end end + include Enumerable attr_reader :head_node, :tail_node def initialize @@ -109,6 +110,10 @@ def initialize(name = nil) def patch!(name) @name = name end + + def inspect + name.inspect + end end # The type of the instruction sequence. @@ -128,6 +133,9 @@ def patch!(name) attr_accessor :argument_size attr_reader :argument_options + # The catch table for this instruction sequence. + attr_reader :catch_table + # The list of instructions for this instruction sequence. attr_reader :insns @@ -162,6 +170,7 @@ def initialize( @argument_size = 0 @argument_options = {} + @catch_table = [] @local_table = LocalTable.new @inline_storages = {} @@ -229,20 +238,20 @@ def to_a # Next, set it up so that all of the labels get their correct name. length = 0 - insns.each_node do |node, value| - case value + insns.each do |insn| + case insn when Integer, Symbol # skip when Label - value.patch!(:"label_#{length}") + insn.patch!(:"label_#{length}") else - length += value.length + length += insn.length end end # Next, dump all of the instructions into a flat list. dumped = - insns.each.map do |insn| + insns.map do |insn| case insn when Integer, Symbol insn @@ -274,7 +283,7 @@ def to_a type, local_table.names, dumped_options, - [], + catch_table.map(&:to_a), dumped ] end @@ -324,7 +333,8 @@ def specialize_instructions! when Send calldata = value.calldata - if !value.block_iseq && !calldata.flag?(CallData::CALL_ARGS_BLOCKARG) + if !value.block_iseq && + !calldata.flag?(CallData::CALL_ARGS_BLOCKARG) # Specialize the send instruction. If it doesn't have a block # attached, then we will replace it with an opt_send_without_block # and do further specializations based on the called method and @@ -386,24 +396,24 @@ def specialize_instructions! end def peephole_optimize! - insns.each_node do |node, value| - case value - when Jump - # jump LABEL - # ... - # LABEL: - # leave - # => - # leave - # ... - # LABEL: - # leave - # case value.label.node.next_node&.value - # when Leave - # node.value = Leave.new - # end - end - end + # insns.each_node do |node, value| + # case value + # when Jump + # # jump LABEL + # # ... + # # LABEL: + # # leave + # # => + # # leave + # # ... + # # LABEL: + # # leave + # # case value.label.node.next_node&.value + # # when Leave + # # node.value = Leave.new + # # end + # end + # end end ########################################################################## @@ -436,6 +446,77 @@ def singleton_class_child_iseq(location) child_iseq(:class, "singleton class", location) end + ########################################################################## + # Catch table methods + ########################################################################## + + class CatchEntry + attr_reader :iseq, :begin_label, :end_label, :exit_label + + def initialize(iseq, begin_label, end_label, exit_label) + @iseq = iseq + @begin_label = begin_label + @end_label = end_label + @exit_label = exit_label + end + end + + class CatchBreak < CatchEntry + def to_a + [:break, iseq.to_a, begin_label.name, end_label.name, exit_label.name] + end + end + + class CatchNext < CatchEntry + def to_a + [:next, nil, begin_label.name, end_label.name, exit_label.name] + end + end + + class CatchRedo < CatchEntry + def to_a + [:redo, nil, begin_label.name, end_label.name, exit_label.name] + end + end + + class CatchRescue < CatchEntry + def to_a + [ + :rescue, + iseq.to_a, + begin_label.name, + end_label.name, + exit_label.name + ] + end + end + + class CatchRetry < CatchEntry + def to_a + [:retry, nil, begin_label.name, end_label.name, exit_label.name] + end + end + + def catch_break(iseq, begin_label, end_label, exit_label) + catch_table << CatchBreak.new(iseq, begin_label, end_label, exit_label) + end + + def catch_next(begin_label, end_label, exit_label) + catch_table << CatchNext.new(nil, begin_label, end_label, exit_label) + end + + def catch_redo(begin_label, end_label, exit_label) + catch_table << CatchRedo.new(nil, begin_label, end_label, exit_label) + end + + def catch_rescue(iseq, begin_label, end_label, exit_label) + catch_table << CatchRescue.new(iseq, begin_label, end_label, exit_label) + end + + def catch_retry(begin_label, end_label, exit_label) + catch_table << CatchRetry.new(nil, begin_label, end_label, exit_label) + end + ########################################################################## # Instruction push methods ########################################################################## @@ -837,6 +918,46 @@ def self.from(source, options = Compiler::Options.new, parent_iseq = nil) iseq.argument_options[:opt].map! { |opt| labels[opt] } end + # set up the catch table + source[12].each do |entry| + case entry[0] + when :break + iseq.catch_break( + from(entry[1]), + labels[entry[2]], + labels[entry[3]], + labels[entry[4]] + ) + when :next + iseq.catch_next( + labels[entry[2]], + labels[entry[3]], + labels[entry[4]] + ) + when :rescue + iseq.catch_rescue( + from(entry[1]), + labels[entry[2]], + labels[entry[3]], + labels[entry[4]] + ) + when :redo + iseq.catch_redo( + labels[entry[2]], + labels[entry[3]], + labels[entry[4]] + ) + when :retry + iseq.catch_retry( + labels[entry[2]], + labels[entry[3]], + labels[entry[4]] + ) + else + raise "unknown catch type: #{entry[0]}" + end + end + # set up all of the instructions source[13].each do |insn| # skip line numbers @@ -969,7 +1090,12 @@ def self.from(source, options = Compiler::Options.new, parent_iseq = nil) when :opt_aset_with iseq.opt_aset_with(opnds[0], CallData.from(opnds[1])) when :opt_case_dispatch - iseq.opt_case_dispatch(opnds[0], labels[opnds[1]]) + hash = + opnds[0] + .each_slice(2) + .to_h + .transform_values { |value| labels[value] } + iseq.opt_case_dispatch(hash, labels[opnds[1]]) when :opt_getconstant_path iseq.opt_getconstant_path(opnds[0]) when :opt_getinlinecache diff --git a/lib/syntax_tree/yarv/instructions.rb b/lib/syntax_tree/yarv/instructions.rb index 8ec1f068..0b60bd13 100644 --- a/lib/syntax_tree/yarv/instructions.rb +++ b/lib/syntax_tree/yarv/instructions.rb @@ -98,6 +98,14 @@ def pops def pushes 0 end + + def canonical + self + end + + def call(vm) + vm.pop(number) + end end # ### Summary @@ -134,6 +142,20 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + original, value = vm.pop(2) + + if value.is_a?(String) + vm.push(value) + else + vm.push("#<#{original.class.name}:0000>") + end + end end # ### Summary @@ -174,6 +196,14 @@ def pops def pushes 0 end + + def canonical + self + end + + def call(vm) + vm.jump(label) if vm.pop + end end # ### Summary @@ -215,6 +245,14 @@ def pops def pushes 0 end + + def canonical + self + end + + def call(vm) + vm.jump(label) if vm.pop.nil? + end end # ### Summary @@ -255,6 +293,14 @@ def pops def pushes 0 end + + def canonical + self + end + + def call(vm) + vm.jump(label) unless vm.pop + end end # ### Summary @@ -303,6 +349,14 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + vm.push(vm.local_get(keyword_bits_index, 0)[keyword_index]) + end end # ### Summary @@ -343,6 +397,14 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + raise NotImplementedError, "checkmatch" + end end # ### Summary @@ -406,6 +468,61 @@ def pushes # can investigate further. 2 end + + def canonical + self + end + + def call(vm) + object = vm.pop + result = + case type + when TYPE_OBJECT + raise NotImplementedError, "checktype TYPE_OBJECT" + when TYPE_CLASS + object.is_a?(Class) + when TYPE_MODULE + object.is_a?(Module) + when TYPE_FLOAT + object.is_a?(Float) + when TYPE_STRING + object.is_a?(String) + when TYPE_REGEXP + object.is_a?(Regexp) + when TYPE_ARRAY + object.is_a?(Array) + when TYPE_HASH + object.is_a?(Hash) + when TYPE_STRUCT + object.is_a?(Struct) + when TYPE_BIGNUM + raise NotImplementedError, "checktype TYPE_BIGNUM" + when TYPE_FILE + object.is_a?(File) + when TYPE_DATA + raise NotImplementedError, "checktype TYPE_DATA" + when TYPE_MATCH + raise NotImplementedError, "checktype TYPE_MATCH" + when TYPE_COMPLEX + object.is_a?(Complex) + when TYPE_RATIONAL + object.is_a?(Rational) + when TYPE_NIL + object.nil? + when TYPE_TRUE + object == true + when TYPE_FALSE + object == false + when TYPE_SYMBOL + object.is_a?(Symbol) + when TYPE_FIXNUM + object.is_a?(Integer) + when TYPE_UNDEF + raise NotImplementedError, "checktype TYPE_UNDEF" + end + + vm.push(result) + end end # ### Summary @@ -438,6 +555,15 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + left, right = vm.pop(2) + vm.push([*left, *right]) + end end # ### Summary @@ -477,6 +603,14 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + vm.push(vm.pop(number).join) + end end # ### Summary @@ -524,6 +658,20 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + object, superclass = vm.pop(2) + iseq = class_iseq + + clazz = Class.new(superclass || Object) + vm.push(vm.run_class_frame(iseq, clazz)) + + object.const_set(name, clazz) + end end # ### Summary @@ -579,6 +727,46 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + object = vm.pop + + result = + case type + when TYPE_NIL, TYPE_SELF, TYPE_TRUE, TYPE_FALSE, TYPE_ASGN, TYPE_EXPR + message + when TYPE_IVAR + message if vm._self.instance_variable_defined?(name) + when TYPE_LVAR + raise NotImplementedError, "defined TYPE_LVAR" + when TYPE_GVAR + message if global_variables.include?(name) + when TYPE_CVAR + clazz = vm._self + clazz = clazz.singleton_class unless clazz.is_a?(Module) + message if clazz.class_variable_defined?(name) + when TYPE_CONST + raise NotImplementedError, "defined TYPE_CONST" + when TYPE_METHOD + raise NotImplementedError, "defined TYPE_METHOD" + when TYPE_YIELD + raise NotImplementedError, "defined TYPE_YIELD" + when TYPE_ZSUPER + raise NotImplementedError, "defined TYPE_ZSUPER" + when TYPE_REF + raise NotImplementedError, "defined TYPE_REF" + when TYPE_FUNC + message if object.respond_to?(name, true) + when TYPE_CONST_FROM + raise NotImplementedError, "defined TYPE_CONST_FROM" + end + + vm.push(result) + end end # ### Summary @@ -595,15 +783,15 @@ def pushes # ~~~ # class DefineMethod - attr_reader :name, :method_iseq + attr_reader :method_name, :method_iseq - def initialize(name, method_iseq) - @name = name + def initialize(method_name, method_iseq) + @method_name = method_name @method_iseq = method_iseq end def to_a(_iseq) - [:definemethod, name, method_iseq.to_a] + [:definemethod, method_name, method_iseq.to_a] end def length @@ -617,6 +805,21 @@ def pops def pushes 0 end + + def canonical + self + end + + def call(vm) + name = method_name + iseq = method_iseq + + vm + ._self + .__send__(:define_method, name) do |*args, **kwargs, &block| + vm.run_method_frame(name, iseq, self, *args, **kwargs, &block) + end + end end # ### Summary @@ -634,15 +837,15 @@ def pushes # ~~~ # class DefineSMethod - attr_reader :name, :method_iseq + attr_reader :method_name, :method_iseq - def initialize(name, method_iseq) - @name = name + def initialize(method_name, method_iseq) + @method_name = method_name @method_iseq = method_iseq end def to_a(_iseq) - [:definesmethod, name, method_iseq.to_a] + [:definesmethod, method_name, method_iseq.to_a] end def length @@ -656,6 +859,21 @@ def pops def pushes 0 end + + def canonical + self + end + + def call(vm) + name = method_name + iseq = method_iseq + + vm + ._self + .__send__(:define_singleton_method, name) do |*args, **kwargs, &block| + vm.run_method_frame(name, iseq, self, *args, **kwargs, &block) + end + end end # ### Summary @@ -684,6 +902,14 @@ def pops def pushes 2 end + + def canonical + self + end + + def call(vm) + vm.push(vm.stack.last.dup) + end end # ### Summary @@ -718,6 +944,14 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + vm.push(object.dup) + end end # ### Summary @@ -752,6 +986,14 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + vm.push(object.dup) + end end # ### Summary @@ -786,6 +1028,16 @@ def pops def pushes number end + + def canonical + self + end + + def call(vm) + values = vm.pop(number) + vm.push(*values) + vm.push(*values) + end end # ### Summary @@ -823,6 +1075,14 @@ def pops def pushes number end + + def canonical + self + end + + def call(vm) + raise NotImplementedError, "expandarray" + end end # ### Summary @@ -867,6 +1127,14 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + vm.push(vm.local_get(index, level)) + end end # ### Summary @@ -909,6 +1177,14 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + vm.push(vm.local_get(index, level)) + end end # ### Summary @@ -946,6 +1222,16 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + clazz = vm._self + clazz = clazz.class unless clazz.is_a?(Class) + vm.push(clazz.class_variable_get(name)) + end end # ### Summary @@ -982,6 +1268,24 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + # const_base, allow_nil = + vm.pop(2) + + vm.frame.nesting.reverse_each do |clazz| + if clazz.const_defined?(name) + vm.push(clazz.const_get(name)) + return + end + end + + raise NameError, "uninitialized constant #{name}" + end end # ### Summary @@ -1016,6 +1320,16 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + # Evaluating the name of the global variable because there isn't a + # reflection API for global variables. + vm.push(eval(name.to_s, binding, __FILE__, __LINE__)) + end end # ### Summary @@ -1058,34 +1372,47 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + method = Object.instance_method(:instance_variable_get) + vm.push(method.bind(vm._self).call(name)) + end end # ### Summary # - # `getlocal_WC_0` is a specialized version of the `getlocal` instruction. It - # fetches the value of a local variable from the current frame determined by - # the index given as its only argument. + # `getlocal` fetches the value of a local variable from a frame determined + # by the level and index arguments. The level is the number of frames back + # to look and the index is the index in the local table. It pushes the value + # it finds onto the stack. # # ### Usage # # ~~~ruby # value = 5 - # value + # tap { tap { value } } # ~~~ # - class GetLocalWC0 - attr_reader :index + class GetLocal + attr_reader :index, :level - def initialize(index) + def initialize(index, level) @index = index + @level = level end def to_a(iseq) - [:getlocal_WC_0, iseq.local_table.offset(index)] + current = iseq + level.times { current = current.parent_iseq } + [:getlocal, current.local_table.offset(index), level] end def length - 2 + 3 end def pops @@ -1095,22 +1422,30 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + vm.push(vm.local_get(index, level)) + end end # ### Summary # - # `getlocal_WC_1` is a specialized version of the `getlocal` instruction. It - # fetches the value of a local variable from the parent frame determined by + # `getlocal_WC_0` is a specialized version of the `getlocal` instruction. It + # fetches the value of a local variable from the current frame determined by # the index given as its only argument. # # ### Usage # # ~~~ruby # value = 5 - # self.then { value } + # value # ~~~ # - class GetLocalWC1 + class GetLocalWC0 attr_reader :index def initialize(index) @@ -1118,7 +1453,7 @@ def initialize(index) end def to_a(iseq) - [:getlocal_WC_1, iseq.parent_iseq.local_table.offset(index)] + [:getlocal_WC_0, iseq.local_table.offset(index)] end def length @@ -1132,38 +1467,42 @@ def pops def pushes 1 end + + def canonical + GetLocal.new(index, 0) + end + + def call(vm) + canonical.call(vm) + end end # ### Summary # - # `getlocal` fetches the value of a local variable from a frame determined - # by the level and index arguments. The level is the number of frames back - # to look and the index is the index in the local table. It pushes the value - # it finds onto the stack. + # `getlocal_WC_1` is a specialized version of the `getlocal` instruction. It + # fetches the value of a local variable from the parent frame determined by + # the index given as its only argument. # # ### Usage # # ~~~ruby # value = 5 - # tap { tap { value } } + # self.then { value } # ~~~ # - class GetLocal - attr_reader :index, :level + class GetLocalWC1 + attr_reader :index - def initialize(index, level) + def initialize(index) @index = index - @level = level end def to_a(iseq) - current = iseq - level.times { current = current.parent_iseq } - [:getlocal, current.local_table.offset(index), level] + [:getlocal_WC_1, iseq.parent_iseq.local_table.offset(index)] end def length - 3 + 2 end def pops @@ -1173,6 +1512,14 @@ def pops def pushes 1 end + + def canonical + GetLocal.new(index, 1) + end + + def call(vm) + canonical.call(vm) + end end # ### Summary @@ -1212,6 +1559,21 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + case key + when SVAR_LASTLINE + raise NotImplementedError, "getspecial SVAR_LASTLINE" + when SVAR_BACKREF + raise NotImplementedError, "getspecial SVAR_BACKREF" + when SVAR_FLIPFLOP_START + vm.frame_svar.svars[SVAR_FLIPFLOP_START] + end + end end # ### Summary @@ -1241,6 +1603,14 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + vm.push(vm.pop.to_sym) + end end # ### Summary @@ -1279,6 +1649,14 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + vm.push(vm.frame_yield.block.call(*vm.pop(calldata.argc))) + end end # ### Summary @@ -1319,6 +1697,32 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + block = + if (iseq = block_iseq) + ->(*args, **kwargs, &blk) do + vm.run_block_frame(iseq, *args, **kwargs, &blk) + end + end + + keywords = + if calldata.kw_arg + calldata.kw_arg.zip(vm.pop(calldata.kw_arg.length)).to_h + else + {} + end + + arguments = vm.pop(calldata.argc) + receiver = vm.pop + + method = receiver.method(vm.frame.name).super_method + vm.push(method.call(*arguments, **keywords, &block)) + end end # ### Summary @@ -1358,6 +1762,14 @@ def pops def pushes 0 end + + def canonical + self + end + + def call(vm) + vm.jump(label) + end end # ### Summary @@ -1388,6 +1800,14 @@ def pushes # otherwise the stack size is incorrectly calculated. 0 end + + def canonical + self + end + + def call(vm) + vm.leave + end end # ### Summary @@ -1424,6 +1844,14 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + vm.push(vm.pop(number)) + end end # ### Summary @@ -1460,6 +1888,14 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + vm.push(vm.pop(number)) + end end # ### Summary @@ -1498,6 +1934,14 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + vm.push(vm.pop(number).each_slice(2).to_h) + end end # ### Summary @@ -1537,6 +1981,14 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + vm.push(Range.new(*vm.pop(2), exclude_end == 1)) + end end # ### Summary @@ -1566,6 +2018,13 @@ def pops def pushes 0 end + + def canonical + self + end + + def call(vm) + end end # ### Summary @@ -1604,6 +2063,14 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + vm.push(vm.pop.to_s) + end end # ### Summary @@ -1642,6 +2109,16 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + return if @executed + vm.push(vm.run_block_frame(iseq)) + @executed = true + end end # ### Summary @@ -1679,6 +2156,14 @@ def pops def pushes 1 end + + def canonical + Send.new(calldata, nil) + end + + def call(vm) + canonical.call(vm) + end end # ### Summary @@ -1715,6 +2200,14 @@ def pops def pushes 1 end + + def canonical + Send.new(calldata, nil) + end + + def call(vm) + canonical.call(vm) + end end # ### Summary @@ -1753,6 +2246,14 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + vm.push(vm.pop[object]) + end end # ### Summary @@ -1790,6 +2291,14 @@ def pops def pushes 1 end + + def canonical + Send.new(calldata, nil) + end + + def call(vm) + canonical.call(vm) + end end # ### Summary @@ -1827,6 +2336,15 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + hash, value = vm.pop(2) + vm.push(hash[object] = value) + end end # ### Summary @@ -1861,7 +2379,11 @@ def initialize(case_dispatch_hash, else_label) end def to_a(_iseq) - [:opt_case_dispatch, case_dispatch_hash, else_label] + [ + :opt_case_dispatch, + case_dispatch_hash.flat_map { |key, value| [key, value.name] }, + else_label + ] end def length @@ -1875,6 +2397,14 @@ def pops def pushes 0 end + + def canonical + self + end + + def call(vm) + vm.jump(case_dispatch_hash.fetch(vm.pop, else_label)) + end end # ### Summary @@ -1912,6 +2442,14 @@ def pops def pushes 1 end + + def canonical + Send.new(calldata, nil) + end + + def call(vm) + canonical.call(vm) + end end # ### Summary @@ -1948,6 +2486,14 @@ def pops def pushes 1 end + + def canonical + Send.new(calldata, nil) + end + + def call(vm) + canonical.call(vm) + end end # ### Summary @@ -1985,6 +2531,14 @@ def pops def pushes 1 end + + def canonical + Send.new(calldata, nil) + end + + def call(vm) + canonical.call(vm) + end end # ### Summary @@ -2022,6 +2576,14 @@ def pops def pushes 1 end + + def canonical + Send.new(calldata, nil) + end + + def call(vm) + canonical.call(vm) + end end # ### Summary @@ -2058,6 +2620,21 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + current = vm._self + current = current.class unless current.is_a?(Class) + + names.each do |name| + current = name == :"" ? Object : current.const_get(name) + end + + vm.push(current) + end end # ### Summary @@ -2095,6 +2672,14 @@ def pops def pushes 1 end + + def canonical + Send.new(calldata, nil) + end + + def call(vm) + canonical.call(vm) + end end # ### Summary @@ -2132,6 +2717,14 @@ def pops def pushes 1 end + + def canonical + Send.new(calldata, nil) + end + + def call(vm) + canonical.call(vm) + end end # ### Summary @@ -2169,6 +2762,14 @@ def pops def pushes 1 end + + def canonical + Send.new(calldata, nil) + end + + def call(vm) + canonical.call(vm) + end end # ### Summary @@ -2206,6 +2807,14 @@ def pops def pushes 1 end + + def canonical + Send.new(calldata, nil) + end + + def call(vm) + canonical.call(vm) + end end # ### Summary @@ -2243,6 +2852,14 @@ def pops def pushes 1 end + + def canonical + Send.new(calldata, nil) + end + + def call(vm) + canonical.call(vm) + end end # ### Summary @@ -2281,6 +2898,14 @@ def pops def pushes 1 end + + def canonical + Send.new(calldata, nil) + end + + def call(vm) + canonical.call(vm) + end end # ### Summary @@ -2318,6 +2943,14 @@ def pops def pushes 1 end + + def canonical + Send.new(calldata, nil) + end + + def call(vm) + canonical.call(vm) + end end # ### Summary @@ -2355,6 +2988,14 @@ def pops def pushes 1 end + + def canonical + Send.new(calldata, nil) + end + + def call(vm) + canonical.call(vm) + end end # ### Summary @@ -2395,6 +3036,15 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + receiver, argument = vm.pop(2) + vm.push(receiver != argument) + end end # ### Summary @@ -2431,6 +3081,14 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + vm.push(vm.pop(number).max) + end end # ### Summary @@ -2467,6 +3125,14 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + vm.push(vm.pop(number).min) + end end # ### Summary @@ -2504,6 +3170,14 @@ def pops def pushes 1 end + + def canonical + Send.new(calldata, nil) + end + + def call(vm) + canonical.call(vm) + end end # ### Summary @@ -2539,6 +3213,14 @@ def pops def pushes 1 end + + def canonical + Send.new(calldata, nil) + end + + def call(vm) + canonical.call(vm) + end end # ### Summary @@ -2576,6 +3258,14 @@ def pops def pushes 1 end + + def canonical + Send.new(calldata, nil) + end + + def call(vm) + canonical.call(vm) + end end # ### Summary @@ -2613,6 +3303,14 @@ def pops def pushes 1 end + + def canonical + Send.new(calldata, nil) + end + + def call(vm) + canonical.call(vm) + end end # ### Summary @@ -2649,6 +3347,14 @@ def pops def pushes 1 end + + def canonical + Send.new(calldata, nil) + end + + def call(vm) + canonical.call(vm) + end end # ### Summary @@ -2685,6 +3391,14 @@ def pops def pushes 1 end + + def canonical + Send.new(calldata, nil) + end + + def call(vm) + canonical.call(vm) + end end # ### Summary @@ -2722,6 +3436,14 @@ def pops def pushes 1 end + + def canonical + Send.new(calldata, nil) + end + + def call(vm) + canonical.call(vm) + end end # ### Summary @@ -2759,6 +3481,14 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + vm.push(object.freeze) + end end # ### Summary @@ -2796,6 +3526,14 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + vm.push(-object) + end end # ### Summary @@ -2833,6 +3571,14 @@ def pops def pushes 1 end + + def canonical + Send.new(calldata, nil) + end + + def call(vm) + canonical.call(vm) + end end # ### Summary @@ -2861,6 +3607,14 @@ def pops def pushes 0 end + + def canonical + self + end + + def call(vm) + vm.pop + end end # ### Summary @@ -2889,6 +3643,14 @@ def pops def pushes 1 end + + def canonical + PutObject.new(nil) + end + + def call(vm) + canonical.call(vm) + end end # ### Summary @@ -2923,6 +3685,14 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + vm.push(object) + end end # ### Summary @@ -2953,6 +3723,14 @@ def pops def pushes 1 end + + def canonical + PutObject.new(0) + end + + def call(vm) + canonical.call(vm) + end end # ### Summary @@ -2983,6 +3761,14 @@ def pops def pushes 1 end + + def canonical + PutObject.new(1) + end + + def call(vm) + canonical.call(vm) + end end # ### Summary @@ -3011,6 +3797,14 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + vm.push(vm._self) + end end # ### Summary @@ -3051,6 +3845,23 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + case object + when OBJECT_VMCORE + vm.push(vm.frozen_core) + when OBJECT_CBASE + value = vm._self + value = value.singleton_class unless value.is_a?(Class) + vm.push(value) + when OBJECT_CONST_BASE + vm.push(vm.const_base) + end + end end # ### Summary @@ -3085,6 +3896,14 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + vm.push(object.dup) + end end # ### Summary @@ -3124,6 +3943,33 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + block = + if (iseq = block_iseq) + ->(*args, **kwargs, &blk) do + vm.run_block_frame(iseq, *args, **kwargs, &blk) + end + end + + keywords = + if calldata.kw_arg + calldata.kw_arg.zip(vm.pop(calldata.kw_arg.length)).to_h + else + {} + end + + arguments = vm.pop(calldata.argc) + receiver = vm.pop + + vm.push( + receiver.__send__(calldata.method, *arguments, **keywords, &block) + ) + end end # ### Summary @@ -3166,6 +4012,14 @@ def pops def pushes 0 end + + def canonical + self + end + + def call(vm) + vm.local_set(index, level, vm.pop) + end end # ### Summary @@ -3204,6 +4058,16 @@ def pops def pushes 0 end + + def canonical + self + end + + def call(vm) + clazz = vm._self + clazz = clazz.class unless clazz.is_a?(Class) + clazz.class_variable_set(name, vm.pop) + end end # ### Summary @@ -3239,6 +4103,15 @@ def pops def pushes 0 end + + def canonical + self + end + + def call(vm) + value, parent = vm.pop(2) + parent.const_set(name, value) + end end # ### Summary @@ -3274,6 +4147,16 @@ def pops def pushes 0 end + + def canonical + self + end + + def call(vm) + # Evaluating the name of the global variable because there isn't a + # reflection API for global variables. + eval("#{name} = vm.pop", binding, __FILE__, __LINE__) + end end # ### Summary @@ -3315,6 +4198,15 @@ def pops def pushes 0 end + + def canonical + self + end + + def call(vm) + method = Object.instance_method(:instance_variable_set) + method.bind(vm._self).call(name, vm.pop) + end end # ### Summary @@ -3356,6 +4248,14 @@ def pops def pushes 0 end + + def canonical + self + end + + def call(vm) + vm.local_set(index, level, vm.pop) + end end # ### Summary @@ -3393,6 +4293,14 @@ def pops def pushes 0 end + + def canonical + SetLocal.new(index, 0) + end + + def call(vm) + canonical.call(vm) + end end # ### Summary @@ -3430,6 +4338,14 @@ def pops def pushes 0 end + + def canonical + SetLocal.new(index, 1) + end + + def call(vm) + canonical.call(vm) + end end # ### Summary @@ -3465,6 +4381,14 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + vm.stack[-number - 1] = vm.stack.last + end end # ### Summary @@ -3501,6 +4425,21 @@ def pops def pushes 0 end + + def canonical + self + end + + def call(vm) + case key + when GetSpecial::SVAR_LASTLINE + raise NotImplementedError, "svar SVAR_LASTLINE" + when GetSpecial::SVAR_BACKREF + raise NotImplementedError, "setspecial SVAR_BACKREF" + when GetSpecial::SVAR_FLIPFLOP_START + vm.frame_svar.svars[GetSpecial::SVAR_FLIPFLOP_START] + end + end end # ### Summary @@ -3537,6 +4476,14 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + vm.push(*vm.pop) + end end # ### Summary @@ -3569,6 +4516,15 @@ def pops def pushes 2 end + + def canonical + self + end + + def call(vm) + left, right = vm.pop(2) + vm.push(right, left) + end end # ### Summary @@ -3584,6 +4540,16 @@ def pushes # ~~~ # class Throw + TAG_NONE = 0x0 + TAG_RETURN = 0x1 + TAG_BREAK = 0x2 + TAG_NEXT = 0x3 + TAG_RETRY = 0x4 + TAG_REDO = 0x5 + TAG_RAISE = 0x6 + TAG_THROW = 0x7 + TAG_FATAL = 0x8 + attr_reader :type def initialize(type) @@ -3605,6 +4571,14 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + raise NotImplementedError, "throw" + end end # ### Summary @@ -3643,6 +4617,14 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + vm.push(vm.stack[-number - 1]) + end end # ### Summary @@ -3675,6 +4657,14 @@ def pops def pushes 1 end + + def canonical + self + end + + def call(vm) + vm.push(Regexp.new(vm.pop(length).join, options)) + end end end end diff --git a/lib/syntax_tree/yarv/legacy.rb b/lib/syntax_tree/yarv/legacy.rb index 82f7560d..93c4e4c3 100644 --- a/lib/syntax_tree/yarv/legacy.rb +++ b/lib/syntax_tree/yarv/legacy.rb @@ -82,6 +82,10 @@ def pops def pushes 1 end + + def call(vm) + vm.push(nil) + end end # ### Summary @@ -121,6 +125,10 @@ def pops def pushes 1 end + + def call(vm) + vm.push(vm.pop) + end end # ### Summary From 70064564221d38748366abc264368cbb5f8042b3 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 29 Nov 2022 11:02:04 -0500 Subject: [PATCH 288/536] Add an entire compile! step --- lib/syntax_tree/yarv.rb | 1 - lib/syntax_tree/yarv/bf.rb | 1 + lib/syntax_tree/yarv/compiler.rb | 4 + lib/syntax_tree/yarv/disassembler.rb | 3 +- lib/syntax_tree/yarv/instruction_sequence.rb | 103 +++++++++++-------- lib/syntax_tree/yarv/instructions.rb | 43 ++++++++ 6 files changed, 111 insertions(+), 44 deletions(-) diff --git a/lib/syntax_tree/yarv.rb b/lib/syntax_tree/yarv.rb index 74f2598e..97592d4d 100644 --- a/lib/syntax_tree/yarv.rb +++ b/lib/syntax_tree/yarv.rb @@ -281,7 +281,6 @@ def self.compile(source, options = Compiler::Options.new) def self.interpret(source, options = Compiler::Options.new) iseq = RubyVM::InstructionSequence.compile(source, **options) iseq = InstructionSequence.from(iseq.to_a) - iseq.specialize_instructions! VM.new.run_top_frame(iseq) end end diff --git a/lib/syntax_tree/yarv/bf.rb b/lib/syntax_tree/yarv/bf.rb index 78c01af5..f642fb2f 100644 --- a/lib/syntax_tree/yarv/bf.rb +++ b/lib/syntax_tree/yarv/bf.rb @@ -74,6 +74,7 @@ def compile end iseq.leave + iseq.compile! iseq end diff --git a/lib/syntax_tree/yarv/compiler.rb b/lib/syntax_tree/yarv/compiler.rb index 194b758b..3ea6d22a 100644 --- a/lib/syntax_tree/yarv/compiler.rb +++ b/lib/syntax_tree/yarv/compiler.rb @@ -1359,6 +1359,7 @@ def visit_program(node) node.location, options ) + with_child_iseq(top_iseq) do visit_all(preexes) @@ -1372,6 +1373,9 @@ def visit_program(node) iseq.leave end + + top_iseq.compile! + top_iseq end def visit_qsymbols(node) diff --git a/lib/syntax_tree/yarv/disassembler.rb b/lib/syntax_tree/yarv/disassembler.rb index 757b8b40..af325c31 100644 --- a/lib/syntax_tree/yarv/disassembler.rb +++ b/lib/syntax_tree/yarv/disassembler.rb @@ -54,7 +54,6 @@ def disassemble(iseq) clauses = {} clause = [] - iseq.to_a iseq.insns.each do |insn| case insn when InstructionSequence::Label @@ -192,7 +191,7 @@ def disassemble(iseq) Assign(VarField(target), value) end else - raise "Unknown instruction #{insn[0]}" + raise "Unknown instruction #{insn}" end end diff --git a/lib/syntax_tree/yarv/instruction_sequence.rb b/lib/syntax_tree/yarv/instruction_sequence.rb index f20981df..e3d0c2fc 100644 --- a/lib/syntax_tree/yarv/instruction_sequence.rb +++ b/lib/syntax_tree/yarv/instruction_sequence.rb @@ -232,24 +232,7 @@ def eval def to_a versions = RUBY_VERSION.split(".").map(&:to_i) - # First, handle any compilation options that we need to. - specialize_instructions! if options.specialized_instruction? - peephole_optimize! if options.peephole_optimization? - - # Next, set it up so that all of the labels get their correct name. - length = 0 - insns.each do |insn| - case insn - when Integer, Symbol - # skip - when Label - insn.patch!(:"label_#{length}") - else - length += insn.length - end - end - - # Next, dump all of the instructions into a flat list. + # Dump all of the instructions into a flat list. dumped = insns.map do |insn| case insn @@ -288,6 +271,65 @@ def to_a ] end + def disasm + output = StringIO.new + output << "== disasm: #:1 (#{location.start_line},#{location.start_column})-(#{location.end_line},#{location.end_column})> (catch: FALSE)\n" + + length = 0 + events = [] + + insns.each do |insn| + case insn + when Integer + # skip + when Symbol + events << insn + when Label + # skip + else + output << "%04d " % length + output << insn.disasm(self) + output << "\n" + end + + length += insn.length + end + + output.string + end + + # This method converts our linked list of instructions into a final array + # and performs any other compilation steps necessary. + def compile! + specialize_instructions! if options.specialized_instruction? + + length = 0 + insns.each do |insn| + case insn + when Integer, Symbol + # skip + when Label + insn.patch!(:"label_#{length}") + when DefineClass + insn.class_iseq.compile! + length += insn.length + when DefineMethod, DefineSMethod + insn.method_iseq.compile! + length += insn.length + when InvokeSuper, Send + insn.block_iseq.compile! if insn.block_iseq + length += insn.length + when Once + insn.iseq.compile! + length += insn.length + else + length += insn.length + end + end + + @insns = insns.to_a + end + def specialize_instructions! insns.each_node do |node, value| case value @@ -333,8 +375,7 @@ def specialize_instructions! when Send calldata = value.calldata - if !value.block_iseq && - !calldata.flag?(CallData::CALL_ARGS_BLOCKARG) + if !value.block_iseq && !calldata.flag?(CallData::CALL_ARGS_BLOCKARG) # Specialize the send instruction. If it doesn't have a block # attached, then we will replace it with an opt_send_without_block # and do further specializations based on the called method and @@ -395,27 +436,6 @@ def specialize_instructions! end end - def peephole_optimize! - # insns.each_node do |node, value| - # case value - # when Jump - # # jump LABEL - # # ... - # # LABEL: - # # leave - # # => - # # leave - # # ... - # # LABEL: - # # leave - # # case value.label.node.next_node&.value - # # when Leave - # # node.value = Leave.new - # # end - # end - # end - end - ########################################################################## # Child instruction sequence methods ########################################################################## @@ -1164,6 +1184,7 @@ def self.from(source, options = Compiler::Options.new, parent_iseq = nil) end end + iseq.compile! if iseq.type == :top iseq end end diff --git a/lib/syntax_tree/yarv/instructions.rb b/lib/syntax_tree/yarv/instructions.rb index 0b60bd13..c146bdbf 100644 --- a/lib/syntax_tree/yarv/instructions.rb +++ b/lib/syntax_tree/yarv/instructions.rb @@ -33,6 +33,25 @@ def initialize( @kw_arg = kw_arg end + def disasm + flag_names = [] + flag_names << :ARGS_SPLAT if flag?(CALL_ARGS_SPLAT) + flag_names << :ARGS_BLOCKARG if flag?(CALL_ARGS_BLOCKARG) + flag_names << :FCALL if flag?(CALL_FCALL) + flag_names << :VCALL if flag?(CALL_VCALL) + flag_names << :ARGS_SIMPLE if flag?(CALL_ARGS_SIMPLE) + flag_names << :BLOCKISEQ if flag?(CALL_BLOCKISEQ) + flag_names << :KWARG if flag?(CALL_KWARG) + flag_names << :KW_SPLAT if flag?(CALL_KW_SPLAT) + flag_names << :TAILCALL if flag?(CALL_TAILCALL) + flag_names << :SUPER if flag?(CALL_SUPER) + flag_names << :ZSUPER if flag?(CALL_ZSUPER) + flag_names << :OPT_SEND if flag?(CALL_OPT_SEND) + flag_names << :KW_SPLAT_MUT if flag?(CALL_KW_SPLAT_MUT) + + "" + end + def flag?(mask) (flags & mask) > 0 end @@ -1783,6 +1802,10 @@ def call(vm) # ~~~ # class Leave + def disasm(_iseq) + "leave" + end + def to_a(_iseq) [:leave] end @@ -2973,6 +2996,10 @@ def initialize(calldata) @calldata = calldata end + def disasm(_iseq) + "%-38s %s" % ["opt_mult", calldata.disasm] + end + def to_a(_iseq) [:opt_mult, calldata.to_h] end @@ -3288,6 +3315,10 @@ def initialize(calldata) @calldata = calldata end + def disasm(iseq) + "%-38s %s" % ["opt_plus", calldata.disasm] + end + def to_a(_iseq) [:opt_plus, calldata.to_h] end @@ -3670,6 +3701,10 @@ def initialize(object) @object = object end + def disasm(_iseq) + "%-38s %s" % ["putobject", object.inspect] + end + def to_a(_iseq) [:putobject, object] end @@ -3708,6 +3743,10 @@ def call(vm) # ~~~ # class PutObjectInt2Fix0 + def disasm(_iseq) + "putobject_INT2FIX_0_" + end + def to_a(_iseq) [:putobject_INT2FIX_0_] end @@ -3746,6 +3785,10 @@ def call(vm) # ~~~ # class PutObjectInt2Fix1 + def disasm(_iseq) + "putobject_INT2FIX_1_" + end + def to_a(_iseq) [:putobject_INT2FIX_1_] end From 46ab8292ef0f88f5969e4dece3c45a2c8c968d74 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 29 Nov 2022 12:58:33 -0500 Subject: [PATCH 289/536] Allow calling disasm on instructions --- .rubocop.yml | 9 + lib/syntax_tree.rb | 1 + lib/syntax_tree/yarv/disasm_formatter.rb | 211 +++++++ lib/syntax_tree/yarv/instruction_sequence.rb | 37 +- lib/syntax_tree/yarv/instructions.rb | 558 +++++++++++++++++-- lib/syntax_tree/yarv/legacy.rb | 19 + lib/syntax_tree/yarv/local_table.rb | 8 + test/compiler_test.rb | 11 + 8 files changed, 793 insertions(+), 61 deletions(-) create mode 100644 lib/syntax_tree/yarv/disasm_formatter.rb diff --git a/.rubocop.yml b/.rubocop.yml index c81fdb59..daf5a824 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -16,6 +16,12 @@ Layout/LineLength: Lint/AmbiguousBlockAssociation: Enabled: false +Lint/AmbiguousOperatorPrecedence: + Enabled: false + +Lint/AmbiguousRange: + Enabled: false + Lint/BooleanSymbol: Enabled: false @@ -91,6 +97,9 @@ Style/ExplicitBlockArgument: Style/FormatString: Enabled: false +Style/FormatStringToken: + Enabled: false + Style/GuardClause: Enabled: false diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index b2ff8414..eadb485d 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -30,6 +30,7 @@ require_relative "syntax_tree/yarv" require_relative "syntax_tree/yarv/bf" require_relative "syntax_tree/yarv/compiler" +require_relative "syntax_tree/yarv/disasm_formatter" require_relative "syntax_tree/yarv/disassembler" require_relative "syntax_tree/yarv/instruction_sequence" require_relative "syntax_tree/yarv/instructions" diff --git a/lib/syntax_tree/yarv/disasm_formatter.rb b/lib/syntax_tree/yarv/disasm_formatter.rb new file mode 100644 index 00000000..566bc8fd --- /dev/null +++ b/lib/syntax_tree/yarv/disasm_formatter.rb @@ -0,0 +1,211 @@ +# frozen_string_literal: true + +module SyntaxTree + module YARV + class DisasmFormatter + attr_reader :output, :queue + attr_reader :current_prefix, :current_iseq + + def initialize + @output = StringIO.new + @queue = [] + + @current_prefix = "" + @current_iseq = nil + end + + ######################################################################## + # Helpers for various instructions + ######################################################################## + + def calldata(value) + flag_names = [] + flag_names << :ARGS_SPLAT if value.flag?(CallData::CALL_ARGS_SPLAT) + if value.flag?(CallData::CALL_ARGS_BLOCKARG) + flag_names << :ARGS_BLOCKARG + end + flag_names << :FCALL if value.flag?(CallData::CALL_FCALL) + flag_names << :VCALL if value.flag?(CallData::CALL_VCALL) + flag_names << :ARGS_SIMPLE if value.flag?(CallData::CALL_ARGS_SIMPLE) + flag_names << :BLOCKISEQ if value.flag?(CallData::CALL_BLOCKISEQ) + flag_names << :KWARG if value.flag?(CallData::CALL_KWARG) + flag_names << :KW_SPLAT if value.flag?(CallData::CALL_KW_SPLAT) + flag_names << :TAILCALL if value.flag?(CallData::CALL_TAILCALL) + flag_names << :SUPER if value.flag?(CallData::CALL_SUPER) + flag_names << :ZSUPER if value.flag?(CallData::CALL_ZSUPER) + flag_names << :OPT_SEND if value.flag?(CallData::CALL_OPT_SEND) + flag_names << :KW_SPLAT_MUT if value.flag?(CallData::CALL_KW_SPLAT_MUT) + + parts = [] + parts << "mid:#{value.method}" if value.method + parts << "argc:#{value.argc}" + parts << "kw:[#{value.kw_arg.join(", ")}]" if value.kw_arg + parts << flag_names.join("|") if flag_names.any? + + "" + end + + def enqueue(iseq) + queue << iseq + end + + def event(name) + case name + when :RUBY_EVENT_B_CALL + "Bc" + when :RUBY_EVENT_B_RETURN + "Br" + when :RUBY_EVENT_CALL + "Ca" + when :RUBY_EVENT_CLASS + "Cl" + when :RUBY_EVENT_END + "En" + when :RUBY_EVENT_LINE + "Li" + when :RUBY_EVENT_RETURN + "Re" + else + raise "Unknown event: #{name}" + end + end + + def inline_storage(cache) + "" + end + + def instruction(name, operands = []) + operands.empty? ? name : "%-38s %s" % [name, operands.join(", ")] + end + + def label(value) + value.name["label_".length..] + end + + def local(index, explicit: nil, implicit: nil) + current = current_iseq + (explicit || implicit).times { current = current.parent_iseq } + + value = "#{current.local_table.name_at(index)}@#{index}" + value << ", #{explicit}" if explicit + value + end + + def object(value) + value.inspect + end + + ######################################################################## + # Main entrypoint + ######################################################################## + + def format! + while (@current_iseq = queue.shift) + output << "\n" if output.pos > 0 + format_iseq(@current_iseq) + end + + output.string + end + + private + + def format_iseq(iseq) + output << "#{current_prefix}== disasm: " + output << "#:1 " + + location = iseq.location + output << "(#{location.start_line},#{location.start_column})-" + output << "(#{location.end_line},#{location.end_column})" + output << "> " + + if iseq.catch_table.any? + output << "(catch: TRUE)\n" + output << "#{current_prefix}== catch table\n" + + with_prefix("#{current_prefix}| ") do + iseq.catch_table.each do |entry| + case entry + when InstructionSequence::CatchBreak + output << "#{current_prefix}catch type: break\n" + format_iseq(entry.iseq) + when InstructionSequence::CatchNext + output << "#{current_prefix}catch type: next\n" + when InstructionSequence::CatchRedo + output << "#{current_prefix}catch type: redo\n" + when InstructionSequence::CatchRescue + output << "#{current_prefix}catch type: rescue\n" + format_iseq(entry.iseq) + end + end + end + + output << "#{current_prefix}|#{"-" * 72}\n" + else + output << "(catch: FALSE)\n" + end + + if (local_table = iseq.local_table) && !local_table.empty? + output << "#{current_prefix}local table (size: #{local_table.size})\n" + + locals = + local_table.locals.each_with_index.map do |local, index| + "[%2d] %s@%d" % [local_table.offset(index), local.name, index] + end + + output << "#{current_prefix}#{locals.join(" ")}\n" + end + + length = 0 + events = [] + lines = [] + + iseq.insns.each do |insn| + case insn + when Integer + lines << insn + when Symbol + events << event(insn) + when InstructionSequence::Label + # skip + else + output << "#{current_prefix}%04d " % length + + disasm = insn.disasm(self) + output << disasm + + if lines.any? + output << " " * (65 - disasm.length) if disasm.length < 65 + elsif events.any? + output << " " * (39 - disasm.length) if disasm.length < 39 + end + + if lines.any? + output << "(%4d)" % lines.last + lines.clear + end + + if events.any? + output << "[#{events.join}]" + events.clear + end + + output << "\n" + length += insn.length + end + end + end + + def with_prefix(value) + previous = @current_prefix + + begin + @current_prefix = value + yield + ensure + @current_prefix = previous + end + end + end + end +end diff --git a/lib/syntax_tree/yarv/instruction_sequence.rb b/lib/syntax_tree/yarv/instruction_sequence.rb index e3d0c2fc..ee5390a1 100644 --- a/lib/syntax_tree/yarv/instruction_sequence.rb +++ b/lib/syntax_tree/yarv/instruction_sequence.rb @@ -272,30 +272,9 @@ def to_a end def disasm - output = StringIO.new - output << "== disasm: #:1 (#{location.start_line},#{location.start_column})-(#{location.end_line},#{location.end_column})> (catch: FALSE)\n" - - length = 0 - events = [] - - insns.each do |insn| - case insn - when Integer - # skip - when Symbol - events << insn - when Label - # skip - else - output << "%04d " % length - output << insn.disasm(self) - output << "\n" - end - - length += insn.length - end - - output.string + formatter = DisasmFormatter.new + formatter.enqueue(self) + formatter.format! end # This method converts our linked list of instructions into a final array @@ -375,7 +354,8 @@ def specialize_instructions! when Send calldata = value.calldata - if !value.block_iseq && !calldata.flag?(CallData::CALL_ARGS_BLOCKARG) + if !value.block_iseq && + !calldata.flag?(CallData::CALL_ARGS_BLOCKARG) # Specialize the send instruction. If it doesn't have a block # attached, then we will replace it with an opt_send_without_block # and do further specializations based on the called method and @@ -980,8 +960,11 @@ def self.from(source, options = Compiler::Options.new, parent_iseq = nil) # set up all of the instructions source[13].each do |insn| - # skip line numbers - next if insn.is_a?(Integer) + # add line numbers + if insn.is_a?(Integer) + iseq.push(insn) + next + end # add events and labels if insn.is_a?(Symbol) diff --git a/lib/syntax_tree/yarv/instructions.rb b/lib/syntax_tree/yarv/instructions.rb index c146bdbf..772f1bb3 100644 --- a/lib/syntax_tree/yarv/instructions.rb +++ b/lib/syntax_tree/yarv/instructions.rb @@ -33,25 +33,6 @@ def initialize( @kw_arg = kw_arg end - def disasm - flag_names = [] - flag_names << :ARGS_SPLAT if flag?(CALL_ARGS_SPLAT) - flag_names << :ARGS_BLOCKARG if flag?(CALL_ARGS_BLOCKARG) - flag_names << :FCALL if flag?(CALL_FCALL) - flag_names << :VCALL if flag?(CALL_VCALL) - flag_names << :ARGS_SIMPLE if flag?(CALL_ARGS_SIMPLE) - flag_names << :BLOCKISEQ if flag?(CALL_BLOCKISEQ) - flag_names << :KWARG if flag?(CALL_KWARG) - flag_names << :KW_SPLAT if flag?(CALL_KW_SPLAT) - flag_names << :TAILCALL if flag?(CALL_TAILCALL) - flag_names << :SUPER if flag?(CALL_SUPER) - flag_names << :ZSUPER if flag?(CALL_ZSUPER) - flag_names << :OPT_SEND if flag?(CALL_OPT_SEND) - flag_names << :KW_SPLAT_MUT if flag?(CALL_KW_SPLAT_MUT) - - "" - end - def flag?(mask) (flags & mask) > 0 end @@ -102,6 +83,10 @@ def initialize(number) @number = number end + def disasm(fmt) + fmt.instruction("adjuststack", [fmt.object(number)]) + end + def to_a(_iseq) [:adjuststack, number] end @@ -146,6 +131,10 @@ def call(vm) # ~~~ # class AnyToString + def disasm(fmt) + fmt.instruction("anytostring") + end + def to_a(_iseq) [:anytostring] end @@ -200,6 +189,10 @@ def initialize(label) @label = label end + def disasm(fmt) + fmt.instruction("branchif", [fmt.label(label)]) + end + def to_a(_iseq) [:branchif, label.name] end @@ -249,6 +242,10 @@ def initialize(label) @label = label end + def disasm(fmt) + fmt.instruction("branchnil", [fmt.label(label)]) + end + def to_a(_iseq) [:branchnil, label.name] end @@ -297,6 +294,10 @@ def initialize(label) @label = label end + def disasm(fmt) + fmt.instruction("branchunless", [fmt.label(label)]) + end + def to_a(_iseq) [:branchunless, label.name] end @@ -349,6 +350,13 @@ def initialize(keyword_bits_index, keyword_index) @keyword_index = keyword_index end + def disasm(fmt) + fmt.instruction( + "checkkeyword", + [fmt.object(keyword_bits_index), fmt.object(keyword_index)] + ) + end + def to_a(iseq) [ :checkkeyword, @@ -401,6 +409,10 @@ def initialize(type) @type = type end + def disasm(fmt) + fmt.instruction("checkmatch", [fmt.object(type)]) + end + def to_a(_iseq) [:checkmatch, type] end @@ -468,6 +480,56 @@ def initialize(type) @type = type end + def disasm(fmt) + name = + case type + when TYPE_OBJECT + "T_OBJECT" + when TYPE_CLASS + "T_CLASS" + when TYPE_MODULE + "T_MODULE" + when TYPE_FLOAT + "T_FLOAT" + when TYPE_STRING + "T_STRING" + when TYPE_REGEXP + "T_REGEXP" + when TYPE_ARRAY + "T_ARRAY" + when TYPE_HASH + "T_HASH" + when TYPE_STRUCT + "T_STRUCT" + when TYPE_BIGNUM + "T_BIGNUM" + when TYPE_FILE + "T_FILE" + when TYPE_DATA + "T_DATA" + when TYPE_MATCH + "T_MATCH" + when TYPE_COMPLEX + "T_COMPLEX" + when TYPE_RATIONAL + "T_RATIONAL" + when TYPE_NIL + "T_NIL" + when TYPE_TRUE + "T_TRUE" + when TYPE_FALSE + "T_FALSE" + when TYPE_SYMBOL + "T_SYMBOL" + when TYPE_FIXNUM + "T_FIXNUM" + when TYPE_UNDEF + "T_UNDEF" + end + + fmt.instruction("checktype", [name]) + end + def to_a(_iseq) [:checktype, type] end @@ -559,6 +621,10 @@ def call(vm) # ~~~ # class ConcatArray + def disasm(fmt) + fmt.instruction("concatarray") + end + def to_a(_iseq) [:concatarray] end @@ -607,6 +673,10 @@ def initialize(number) @number = number end + def disasm(fmt) + fmt.instruction("concatstrings", [fmt.object(number)]) + end + def to_a(_iseq) [:concatstrings, number] end @@ -662,6 +732,14 @@ def initialize(name, class_iseq, flags) @flags = flags end + def disasm(fmt) + fmt.enqueue(class_iseq) + fmt.instruction( + "defineclass", + [fmt.object(name), class_iseq.name, fmt.object(flags)] + ) + end + def to_a(_iseq) [:defineclass, name, class_iseq.to_a, flags] end @@ -731,6 +809,51 @@ def initialize(type, name, message) @message = message end + def disasm(fmt) + type_name = + case type + when TYPE_NIL + "nil" + when TYPE_IVAR + "ivar" + when TYPE_LVAR + "lvar" + when TYPE_GVAR + "gvar" + when TYPE_CVAR + "cvar" + when TYPE_CONST + "const" + when TYPE_METHOD + "method" + when TYPE_YIELD + "yield" + when TYPE_ZSUPER + "zsuper" + when TYPE_SELF + "self" + when TYPE_TRUE + "true" + when TYPE_FALSE + "false" + when TYPE_ASGN + "asgn" + when TYPE_EXPR + "expr" + when TYPE_REF + "ref" + when TYPE_FUNC + "func" + when TYPE_CONST_FROM + "constant-from" + end + + fmt.instruction( + "defined", + [type_name, fmt.object(name), fmt.object(message)] + ) + end + def to_a(_iseq) [:defined, type, name, message] end @@ -809,6 +932,14 @@ def initialize(method_name, method_iseq) @method_iseq = method_iseq end + def disasm(fmt) + fmt.enqueue(method_iseq) + fmt.instruction( + "definemethod", + [fmt.object(method_name), method_iseq.name] + ) + end + def to_a(_iseq) [:definemethod, method_name, method_iseq.to_a] end @@ -863,6 +994,14 @@ def initialize(method_name, method_iseq) @method_iseq = method_iseq end + def disasm(fmt) + fmt.enqueue(method_iseq) + fmt.instruction( + "definesmethod", + [fmt.object(method_name), method_iseq.name] + ) + end + def to_a(_iseq) [:definesmethod, method_name, method_iseq.to_a] end @@ -906,6 +1045,10 @@ def call(vm) # ~~~ # class Dup + def disasm(fmt) + fmt.instruction("dup") + end + def to_a(_iseq) [:dup] end @@ -948,6 +1091,10 @@ def initialize(object) @object = object end + def disasm(fmt) + fmt.instruction("duparray", [fmt.object(object)]) + end + def to_a(_iseq) [:duparray, object] end @@ -990,6 +1137,10 @@ def initialize(object) @object = object end + def disasm(fmt) + fmt.instruction("duphash", [fmt.object(object)]) + end + def to_a(_iseq) [:duphash, object] end @@ -1032,6 +1183,10 @@ def initialize(number) @number = number end + def disasm(fmt) + fmt.instruction("dupn", [fmt.object(number)]) + end + def to_a(_iseq) [:dupn, number] end @@ -1079,6 +1234,10 @@ def initialize(number, flags) @flags = flags end + def disasm(fmt) + fmt.instruction("expandarray", [fmt.object(number), fmt.object(flags)]) + end + def to_a(_iseq) [:expandarray, number, flags] end @@ -1129,6 +1288,10 @@ def initialize(index, level) @level = level end + def disasm(fmt) + fmt.instruction("getblockparam", [fmt.local(index, explicit: level)]) + end + def to_a(iseq) current = iseq level.times { current = iseq.parent_iseq } @@ -1179,6 +1342,13 @@ def initialize(index, level) @level = level end + def disasm(fmt) + fmt.instruction( + "getblockparamproxy", + [fmt.local(index, explicit: level)] + ) + end + def to_a(iseq) current = iseq level.times { current = iseq.parent_iseq } @@ -1226,6 +1396,13 @@ def initialize(name, cache) @cache = cache end + def disasm(fmt) + fmt.instruction( + "getclassvariable", + [fmt.object(name), fmt.inline_storage(cache)] + ) + end + def to_a(_iseq) [:getclassvariable, name, cache] end @@ -1272,6 +1449,10 @@ def initialize(name) @name = name end + def disasm(fmt) + fmt.instruction("getconstant", [fmt.object(name)]) + end + def to_a(_iseq) [:getconstant, name] end @@ -1324,6 +1505,10 @@ def initialize(name) @name = name end + def disasm(fmt) + fmt.instruction("getglobal", [fmt.object(name)]) + end + def to_a(_iseq) [:getglobal, name] end @@ -1376,6 +1561,13 @@ def initialize(name, cache) @cache = cache end + def disasm(fmt) + fmt.instruction( + "getinstancevariable", + [fmt.object(name), fmt.inline_storage(cache)] + ) + end + def to_a(_iseq) [:getinstancevariable, name, cache] end @@ -1424,6 +1616,10 @@ def initialize(index, level) @level = level end + def disasm(fmt) + fmt.instruction("getlocal", [fmt.local(index, explicit: level)]) + end + def to_a(iseq) current = iseq level.times { current = current.parent_iseq } @@ -1471,6 +1667,10 @@ def initialize(index) @index = index end + def disasm(fmt) + fmt.instruction("getlocal_WC_0", [fmt.local(index, implicit: 0)]) + end + def to_a(iseq) [:getlocal_WC_0, iseq.local_table.offset(index)] end @@ -1516,6 +1716,10 @@ def initialize(index) @index = index end + def disasm(fmt) + fmt.instruction("getlocal_WC_1", [fmt.local(index, implicit: 1)]) + end + def to_a(iseq) [:getlocal_WC_1, iseq.parent_iseq.local_table.offset(index)] end @@ -1548,7 +1752,7 @@ def call(vm) # ### Usage # # ~~~ruby - # [true] + # 1 if (a == 1) .. (b == 2) # ~~~ # class GetSpecial @@ -1563,6 +1767,10 @@ def initialize(key, type) @type = type end + def disasm(fmt) + fmt.instruction("getspecial", [fmt.object(key), fmt.object(type)]) + end + def to_a(_iseq) [:getspecial, key, type] end @@ -1607,6 +1815,10 @@ def call(vm) # ~~~ # class Intern + def disasm(fmt) + fmt.instruction("intern") + end + def to_a(_iseq) [:intern] end @@ -1653,6 +1865,10 @@ def initialize(calldata) @calldata = calldata end + def disasm(fmt) + fmt.instruction("invokeblock", [fmt.calldata(calldata)]) + end + def to_a(_iseq) [:invokeblock, calldata.to_h] end @@ -1700,6 +1916,14 @@ def initialize(calldata, block_iseq) @block_iseq = block_iseq end + def disasm(fmt) + fmt.enqueue(block_iseq) if block_iseq + fmt.instruction( + "invokesuper", + [fmt.calldata(calldata), block_iseq&.name || "nil"] + ) + end + def to_a(_iseq) [:invokesuper, calldata.to_h, block_iseq&.to_a] end @@ -1766,6 +1990,10 @@ def initialize(label) @label = label end + def disasm(fmt) + fmt.instruction("jump", [fmt.label(label)]) + end + def to_a(_iseq) [:jump, label.name] end @@ -1802,8 +2030,8 @@ def call(vm) # ~~~ # class Leave - def disasm(_iseq) - "leave" + def disasm(fmt) + fmt.instruction("leave") end def to_a(_iseq) @@ -1852,6 +2080,10 @@ def initialize(number) @number = number end + def disasm(fmt) + fmt.instruction("newarray", [fmt.object(number)]) + end + def to_a(_iseq) [:newarray, number] end @@ -1896,6 +2128,10 @@ def initialize(number) @number = number end + def disasm(fmt) + fmt.instruction("newarraykwsplat", [fmt.object(number)]) + end + def to_a(_iseq) [:newarraykwsplat, number] end @@ -1942,6 +2178,10 @@ def initialize(number) @number = number end + def disasm(fmt) + fmt.instruction("newhash", [fmt.object(number)]) + end + def to_a(_iseq) [:newhash, number] end @@ -1989,6 +2229,10 @@ def initialize(exclude_end) @exclude_end = exclude_end end + def disasm(fmt) + fmt.instruction("newrange", [fmt.object(exclude_end)]) + end + def to_a(_iseq) [:newrange, exclude_end] end @@ -2026,6 +2270,10 @@ def call(vm) # ~~~ # class Nop + def disasm(fmt) + fmt.instruction("nop") + end + def to_a(_iseq) [:nop] end @@ -2071,6 +2319,10 @@ def initialize(calldata) @calldata = calldata end + def disasm(fmt) + fmt.instruction("objtostring", [fmt.calldata(calldata)]) + end + def to_a(_iseq) [:objtostring, calldata.to_h] end @@ -2117,6 +2369,11 @@ def initialize(iseq, cache) @cache = cache end + def disasm(fmt) + fmt.enqueue(iseq) + fmt.instruction("once", [iseq.name, fmt.inline_storage(cache)]) + end + def to_a(_iseq) [:once, iseq.to_a, cache] end @@ -2164,6 +2421,10 @@ def initialize(calldata) @calldata = calldata end + def disasm(fmt) + fmt.instruction("opt_and", [fmt.calldata(calldata)]) + end + def to_a(_iseq) [:opt_and, calldata.to_h] end @@ -2208,6 +2469,10 @@ def initialize(calldata) @calldata = calldata end + def disasm(fmt) + fmt.instruction("opt_aref", [fmt.calldata(calldata)]) + end + def to_a(_iseq) [:opt_aref, calldata.to_h] end @@ -2254,6 +2519,13 @@ def initialize(object, calldata) @calldata = calldata end + def disasm(fmt) + fmt.instruction( + "opt_aref_with", + [fmt.object(object), fmt.calldata(calldata)] + ) + end + def to_a(_iseq) [:opt_aref_with, object, calldata.to_h] end @@ -2299,6 +2571,10 @@ def initialize(calldata) @calldata = calldata end + def disasm(fmt) + fmt.instruction("opt_aset", [fmt.calldata(calldata)]) + end + def to_a(_iseq) [:opt_aset, calldata.to_h] end @@ -2344,6 +2620,13 @@ def initialize(object, calldata) @calldata = calldata end + def disasm(fmt) + fmt.instruction( + "opt_aset_with", + [fmt.object(object), fmt.calldata(calldata)] + ) + end + def to_a(_iseq) [:opt_aset_with, object, calldata.to_h] end @@ -2401,6 +2684,13 @@ def initialize(case_dispatch_hash, else_label) @else_label = else_label end + def disasm(fmt) + fmt.instruction( + "opt_case_dispatch", + ["", fmt.label(else_label)] + ) + end + def to_a(_iseq) [ :opt_case_dispatch, @@ -2450,6 +2740,10 @@ def initialize(calldata) @calldata = calldata end + def disasm(fmt) + fmt.instruction("opt_div", [fmt.calldata(calldata)]) + end + def to_a(_iseq) [:opt_div, calldata.to_h] end @@ -2494,6 +2788,10 @@ def initialize(calldata) @calldata = calldata end + def disasm(fmt) + fmt.instruction("opt_empty_p", [fmt.calldata(calldata)]) + end + def to_a(_iseq) [:opt_empty_p, calldata.to_h] end @@ -2539,6 +2837,10 @@ def initialize(calldata) @calldata = calldata end + def disasm(fmt) + fmt.instruction("opt_eq", [fmt.calldata(calldata)]) + end + def to_a(_iseq) [:opt_eq, calldata.to_h] end @@ -2584,6 +2886,10 @@ def initialize(calldata) @calldata = calldata end + def disasm(fmt) + fmt.instruction("opt_ge", [fmt.calldata(calldata)]) + end + def to_a(_iseq) [:opt_ge, calldata.to_h] end @@ -2628,6 +2934,11 @@ def initialize(names) @names = names end + def disasm(fmt) + cache = "" + fmt.instruction("opt_getconstant_path", [cache]) + end + def to_a(_iseq) [:opt_getconstant_path, names] end @@ -2680,6 +2991,10 @@ def initialize(calldata) @calldata = calldata end + def disasm(fmt) + fmt.instruction("opt_gt", [fmt.calldata(calldata)]) + end + def to_a(_iseq) [:opt_gt, calldata.to_h] end @@ -2725,6 +3040,10 @@ def initialize(calldata) @calldata = calldata end + def disasm(fmt) + fmt.instruction("opt_le", [fmt.calldata(calldata)]) + end + def to_a(_iseq) [:opt_le, calldata.to_h] end @@ -2770,6 +3089,10 @@ def initialize(calldata) @calldata = calldata end + def disasm(fmt) + fmt.instruction("opt_length", [fmt.calldata(calldata)]) + end + def to_a(_iseq) [:opt_length, calldata.to_h] end @@ -2815,6 +3138,10 @@ def initialize(calldata) @calldata = calldata end + def disasm(fmt) + fmt.instruction("opt_lt", [fmt.calldata(calldata)]) + end + def to_a(_iseq) [:opt_lt, calldata.to_h] end @@ -2860,6 +3187,10 @@ def initialize(calldata) @calldata = calldata end + def disasm(fmt) + fmt.instruction("opt_ltlt", [fmt.calldata(calldata)]) + end + def to_a(_iseq) [:opt_ltlt, calldata.to_h] end @@ -2906,6 +3237,10 @@ def initialize(calldata) @calldata = calldata end + def disasm(fmt) + fmt.instruction("opt_minus", [fmt.calldata(calldata)]) + end + def to_a(_iseq) [:opt_minus, calldata.to_h] end @@ -2951,6 +3286,10 @@ def initialize(calldata) @calldata = calldata end + def disasm(fmt) + fmt.instruction("opt_mod", [fmt.calldata(calldata)]) + end + def to_a(_iseq) [:opt_mod, calldata.to_h] end @@ -2996,8 +3335,8 @@ def initialize(calldata) @calldata = calldata end - def disasm(_iseq) - "%-38s %s" % ["opt_mult", calldata.disasm] + def disasm(fmt) + fmt.instruction("opt_mult", [fmt.calldata(calldata)]) end def to_a(_iseq) @@ -3048,6 +3387,13 @@ def initialize(eq_calldata, neq_calldata) @neq_calldata = neq_calldata end + def disasm(fmt) + fmt.instruction( + "opt_neq", + [fmt.calldata(eq_calldata), fmt.calldata(neq_calldata)] + ) + end + def to_a(_iseq) [:opt_neq, eq_calldata.to_h, neq_calldata.to_h] end @@ -3083,7 +3429,7 @@ def call(vm) # ### Usage # # ~~~ruby - # [1, 2, 3].max + # [a, b, c].max # ~~~ # class OptNewArrayMax @@ -3093,6 +3439,10 @@ def initialize(number) @number = number end + def disasm(fmt) + fmt.instruction("opt_newarray_max", [fmt.object(number)]) + end + def to_a(_iseq) [:opt_newarray_max, number] end @@ -3127,7 +3477,7 @@ def call(vm) # ### Usage # # ~~~ruby - # [1, 2, 3].min + # [a, b, c].min # ~~~ # class OptNewArrayMin @@ -3137,6 +3487,10 @@ def initialize(number) @number = number end + def disasm(fmt) + fmt.instruction("opt_newarray_min", [fmt.object(number)]) + end + def to_a(_iseq) [:opt_newarray_min, number] end @@ -3182,6 +3536,10 @@ def initialize(calldata) @calldata = calldata end + def disasm(fmt) + fmt.instruction("opt_nil_p", [fmt.calldata(calldata)]) + end + def to_a(_iseq) [:opt_nil_p, calldata.to_h] end @@ -3225,6 +3583,10 @@ def initialize(calldata) @calldata = calldata end + def disasm(fmt) + fmt.instruction("opt_not", [fmt.calldata(calldata)]) + end + def to_a(_iseq) [:opt_not, calldata.to_h] end @@ -3270,6 +3632,10 @@ def initialize(calldata) @calldata = calldata end + def disasm(fmt) + fmt.instruction("opt_or", [fmt.calldata(calldata)]) + end + def to_a(_iseq) [:opt_or, calldata.to_h] end @@ -3315,8 +3681,8 @@ def initialize(calldata) @calldata = calldata end - def disasm(iseq) - "%-38s %s" % ["opt_plus", calldata.disasm] + def disasm(fmt) + fmt.instruction("opt_plus", [fmt.calldata(calldata)]) end def to_a(_iseq) @@ -3363,6 +3729,10 @@ def initialize(calldata) @calldata = calldata end + def disasm(fmt) + fmt.instruction("opt_regexpmatch2", [fmt.calldata(calldata)]) + end + def to_a(_iseq) [:opt_regexpmatch2, calldata.to_h] end @@ -3407,6 +3777,10 @@ def initialize(calldata) @calldata = calldata end + def disasm(fmt) + fmt.instruction("opt_send_without_block", [fmt.calldata(calldata)]) + end + def to_a(_iseq) [:opt_send_without_block, calldata.to_h] end @@ -3452,6 +3826,10 @@ def initialize(calldata) @calldata = calldata end + def disasm(fmt) + fmt.instruction("opt_size", [fmt.calldata(calldata)]) + end + def to_a(_iseq) [:opt_size, calldata.to_h] end @@ -3497,6 +3875,13 @@ def initialize(object, calldata) @calldata = calldata end + def disasm(fmt) + fmt.instruction( + "opt_str_freeze", + [fmt.object(object), fmt.calldata(calldata)] + ) + end + def to_a(_iseq) [:opt_str_freeze, object, calldata.to_h] end @@ -3542,6 +3927,13 @@ def initialize(object, calldata) @calldata = calldata end + def disasm(fmt) + fmt.instruction( + "opt_str_uminus", + [fmt.object(object), fmt.calldata(calldata)] + ) + end + def to_a(_iseq) [:opt_str_uminus, object, calldata.to_h] end @@ -3587,6 +3979,10 @@ def initialize(calldata) @calldata = calldata end + def disasm(fmt) + fmt.instruction("opt_succ", [fmt.calldata(calldata)]) + end + def to_a(_iseq) [:opt_succ, calldata.to_h] end @@ -3623,6 +4019,10 @@ def call(vm) # ~~~ # class Pop + def disasm(fmt) + fmt.instruction("pop") + end + def to_a(_iseq) [:pop] end @@ -3659,6 +4059,10 @@ def call(vm) # ~~~ # class PutNil + def disasm(fmt) + fmt.instruction("putnil") + end + def to_a(_iseq) [:putnil] end @@ -3701,8 +4105,8 @@ def initialize(object) @object = object end - def disasm(_iseq) - "%-38s %s" % ["putobject", object.inspect] + def disasm(fmt) + fmt.instruction("putobject", [fmt.object(object)]) end def to_a(_iseq) @@ -3743,8 +4147,8 @@ def call(vm) # ~~~ # class PutObjectInt2Fix0 - def disasm(_iseq) - "putobject_INT2FIX_0_" + def disasm(fmt) + fmt.instruction("putobject_INT2FIX_0_") end def to_a(_iseq) @@ -3785,8 +4189,8 @@ def call(vm) # ~~~ # class PutObjectInt2Fix1 - def disasm(_iseq) - "putobject_INT2FIX_1_" + def disasm(fmt) + fmt.instruction("putobject_INT2FIX_1_") end def to_a(_iseq) @@ -3825,6 +4229,10 @@ def call(vm) # ~~~ # class PutSelf + def disasm(fmt) + fmt.instruction("putself") + end + def to_a(_iseq) [:putself] end @@ -3873,6 +4281,10 @@ def initialize(object) @object = object end + def disasm(fmt) + fmt.instruction("putspecialobject", [fmt.object(object)]) + end + def to_a(_iseq) [:putspecialobject, object] end @@ -3924,6 +4336,10 @@ def initialize(object) @object = object end + def disasm(fmt) + fmt.instruction("putstring", [fmt.object(object)]) + end + def to_a(_iseq) [:putstring, object] end @@ -3970,6 +4386,14 @@ def initialize(calldata, block_iseq) @block_iseq = block_iseq end + def disasm(fmt) + fmt.enqueue(block_iseq) if block_iseq + fmt.instruction( + "send", + [fmt.calldata(calldata), block_iseq&.name || "nil"] + ) + end + def to_a(_iseq) [:send, calldata.to_h, block_iseq&.to_a] end @@ -4038,6 +4462,10 @@ def initialize(index, level) @level = level end + def disasm(fmt) + fmt.instruction("setblockparam", [fmt.local(index, explicit: level)]) + end + def to_a(iseq) current = iseq level.times { current = current.parent_iseq } @@ -4086,6 +4514,13 @@ def initialize(name, cache) @cache = cache end + def disasm(fmt) + fmt.instruction( + "setclassvariable", + [fmt.object(name), fmt.inline_storage(cache)] + ) + end + def to_a(_iseq) [:setclassvariable, name, cache] end @@ -4131,6 +4566,10 @@ def initialize(name) @name = name end + def disasm(fmt) + fmt.instruction("setconstant", [fmt.object(name)]) + end + def to_a(_iseq) [:setconstant, name] end @@ -4175,6 +4614,10 @@ def initialize(name) @name = name end + def disasm(fmt) + fmt.instruction("setglobal", [fmt.object(name)]) + end + def to_a(_iseq) [:setglobal, name] end @@ -4226,6 +4669,13 @@ def initialize(name, cache) @cache = cache end + def disasm(fmt) + fmt.instruction( + "setinstancevariable", + [fmt.object(name), fmt.inline_storage(cache)] + ) + end + def to_a(_iseq) [:setinstancevariable, name, cache] end @@ -4274,6 +4724,10 @@ def initialize(index, level) @level = level end + def disasm(fmt) + fmt.instruction("setlocal", [fmt.local(index, explicit: level)]) + end + def to_a(iseq) current = iseq level.times { current = current.parent_iseq } @@ -4321,6 +4775,10 @@ def initialize(index) @index = index end + def disasm(fmt) + fmt.instruction("setlocal_WC_0", [fmt.local(index, implicit: 0)]) + end + def to_a(iseq) [:setlocal_WC_0, iseq.local_table.offset(index)] end @@ -4366,6 +4824,10 @@ def initialize(index) @index = index end + def disasm(fmt) + fmt.instruction("setlocal_WC_1", [fmt.local(index, implicit: 1)]) + end + def to_a(iseq) [:setlocal_WC_1, iseq.parent_iseq.local_table.offset(index)] end @@ -4409,6 +4871,10 @@ def initialize(number) @number = number end + def disasm(fmt) + fmt.instruction("setn", [fmt.object(number)]) + end + def to_a(_iseq) [:setn, number] end @@ -4453,6 +4919,10 @@ def initialize(key) @key = key end + def disasm(fmt) + fmt.instruction("setspecial", [fmt.object(key)]) + end + def to_a(_iseq) [:setspecial, key] end @@ -4504,6 +4974,10 @@ def initialize(flag) @flag = flag end + def disasm(fmt) + fmt.instruction("splatarray", [fmt.object(flag)]) + end + def to_a(_iseq) [:splatarray, flag] end @@ -4544,6 +5018,10 @@ def call(vm) # ~~~ # class Swap + def disasm(fmt) + fmt.instruction("swap") + end + def to_a(_iseq) [:swap] end @@ -4599,6 +5077,10 @@ def initialize(type) @type = type end + def disasm(fmt) + fmt.instruction("throw", [fmt.object(type)]) + end + def to_a(_iseq) [:throw, type] end @@ -4645,6 +5127,10 @@ def initialize(number) @number = number end + def disasm(fmt) + fmt.instruction("topn", [fmt.object(number)]) + end + def to_a(_iseq) [:topn, number] end @@ -4689,6 +5175,10 @@ def initialize(options, length) @length = length end + def disasm(fmt) + fmt.instruction("toregexp", [fmt.object(options), fmt.object(length)]) + end + def to_a(_iseq) [:toregexp, options, length] end diff --git a/lib/syntax_tree/yarv/legacy.rb b/lib/syntax_tree/yarv/legacy.rb index 93c4e4c3..30a95437 100644 --- a/lib/syntax_tree/yarv/legacy.rb +++ b/lib/syntax_tree/yarv/legacy.rb @@ -26,6 +26,10 @@ def initialize(name) @name = name end + def disasm(fmt) + fmt.instruction("getclassvariable", [fmt.object(name)]) + end + def to_a(_iseq) [:getclassvariable, name] end @@ -67,6 +71,13 @@ def initialize(label, cache) @cache = cache end + def disasm(fmt) + fmt.instruction( + "opt_getinlinecache", + [fmt.label(label), fmt.inline_storage(cache)] + ) + end + def to_a(_iseq) [:opt_getinlinecache, label.name, cache] end @@ -110,6 +121,10 @@ def initialize(cache) @cache = cache end + def disasm(fmt) + fmt.instruction("opt_setinlinecache", [fmt.inline_storage(cache)]) + end + def to_a(_iseq) [:opt_setinlinecache, cache] end @@ -152,6 +167,10 @@ def initialize(name) @name = name end + def disasm(fmt) + fmt.instruction("setclassvariable", [fmt.object(name)]) + end + def to_a(_iseq) [:setclassvariable, name] end diff --git a/lib/syntax_tree/yarv/local_table.rb b/lib/syntax_tree/yarv/local_table.rb index 5eac346c..54cc55ad 100644 --- a/lib/syntax_tree/yarv/local_table.rb +++ b/lib/syntax_tree/yarv/local_table.rb @@ -44,6 +44,10 @@ def initialize @locals = [] end + def empty? + locals.empty? + end + def find(name, level = 0) index = locals.index { |local| local.name == name } Lookup.new(locals[index], index, level) if index @@ -57,6 +61,10 @@ def names locals.map(&:name) end + def name_at(index) + locals[index].name + end + def size locals.length end diff --git a/test/compiler_test.rb b/test/compiler_test.rb index 1f4a5299..1922f8c6 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -449,6 +449,10 @@ class CompilerTest < Minitest::Test define_method(:"test_loads_#{source}_(#{suffix})") do assert_loads(source, options) end + + define_method(:"test_disasms_#{source}_(#{suffix})") do + assert_disasms(source, options) + end end end @@ -507,6 +511,13 @@ def assert_loads(source, options) ) end + # Check that we can successfully disasm the compiled instruction sequence. + def assert_disasms(source, options) + compiled = RubyVM::InstructionSequence.compile(source, **options) + yarv = YARV::InstructionSequence.from(compiled.to_a, options) + assert_kind_of String, yarv.disasm + end + def assert_evaluates(expected, source) assert_equal expected, YARV.compile(source).eval end From 67463fb645bfa2df195662df583d366e4a220fd7 Mon Sep 17 00:00:00 2001 From: Andy Waite Date: Wed, 7 Dec 2022 13:31:52 -0500 Subject: [PATCH 290/536] Some README typo fixes --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 0f1b626a..0d9e8856 100644 --- a/README.md +++ b/README.md @@ -328,7 +328,7 @@ Syntax Tree can be used as a library to access the syntax tree underlying Ruby s ### SyntaxTree.read(filepath) -This function takes a filepath and returns a string associated with the content of that file. It is similar in functionality to `File.read`, except htat it takes into account Ruby-level file encoding (through magic comments at the top of the file). +This function takes a filepath and returns a string associated with the content of that file. It is similar in functionality to `File.read`, except that it takes into account Ruby-level file encoding (through magic comments at the top of the file). ### SyntaxTree.parse(source) @@ -570,7 +570,7 @@ SyntaxTree::Formatter.format(source, program.accept(visitor)) ### WithEnvironment The `WithEnvironment` module can be included in visitors to automatically keep track of local variables and arguments -defined inside each environment. A `current_environment` accessor is made availble to the request, allowing it to find +defined inside each environment. A `current_environment` accessor is made available to the request, allowing it to find all usages and definitions of a local. ```ruby @@ -611,7 +611,7 @@ The language server also responds to the relatively new inlay hints request. Thi 1 + 2 * 3 ``` -Implicity, the `2 * 3` is going to be executed first because the `*` operator has higher precedence than the `+` operator. To ease mental overhead, our language server includes small parentheses to make this explicit, as in: +Implicitly, the `2 * 3` is going to be executed first because the `*` operator has higher precedence than the `+` operator. To ease mental overhead, our language server includes small parentheses to make this explicit, as in: ```ruby 1 + ₍2 * 3₎ @@ -686,7 +686,7 @@ Below are listed all of the "official" language plugins hosted under the same Gi ## Integration -Syntax Tree's goal is to seemlessly integrate into your workflow. To this end, it provides a couple of additional tools beyond the CLI and the Ruby library. +Syntax Tree's goal is to seamlessly integrate into your workflow. To this end, it provides a couple of additional tools beyond the CLI and the Ruby library. ### Rake From ad671c49fea40240fc25c49546acc7f1f0b0945f Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 7 Dec 2022 22:24:36 -0500 Subject: [PATCH 291/536] Ruby 3.2 argument forwarding --- lib/syntax_tree/yarv/compiler.rb | 54 ++++++++++++++++++++++++++------ 1 file changed, 44 insertions(+), 10 deletions(-) diff --git a/lib/syntax_tree/yarv/compiler.rb b/lib/syntax_tree/yarv/compiler.rb index 3ea6d22a..046fb438 100644 --- a/lib/syntax_tree/yarv/compiler.rb +++ b/lib/syntax_tree/yarv/compiler.rb @@ -650,14 +650,36 @@ def visit_call(node) flag |= CallData::CALL_ARGS_SPLAT visit(arg_part) when ArgsForward - flag |= CallData::CALL_ARGS_SPLAT - flag |= CallData::CALL_ARGS_BLOCKARG flag |= CallData::CALL_TAILCALL if options.tailcall_optimization? - lookup = iseq.local_table.find(:*) - iseq.getlocal(lookup.index, lookup.level) - iseq.splatarray(arg_parts.length != 1) + if RUBY_VERSION < "3.2" + flag |= CallData::CALL_ARGS_SPLAT + lookup = iseq.local_table.find(:*) + iseq.getlocal(lookup.index, lookup.level) + iseq.splatarray(arg_parts.length != 1) + else + flag |= CallData::CALL_ARGS_SPLAT + lookup = iseq.local_table.find(:*) + iseq.getlocal(lookup.index, lookup.level) + iseq.splatarray(true) + + flag |= CallData::CALL_KW_SPLAT + iseq.putspecialobject(PutSpecialObject::OBJECT_VMCORE) + iseq.newhash(0) + lookup = iseq.local_table.find(:**) + iseq.getlocal(lookup.index, lookup.level) + iseq.send( + YARV.calldata( + :"core#hash_merge_kwd", + 2, + CallData::CALL_ARGS_SIMPLE + ) + ) + iseq.newarray(1) + iseq.concatarray + end + flag |= CallData::CALL_ARGS_BLOCKARG lookup = iseq.local_table.find(:&) iseq.getblockparamproxy(lookup.index, lookup.level) when BareAssocHash @@ -1304,13 +1326,25 @@ def visit_params(node) end if node.keyword_rest.is_a?(ArgsForward) - iseq.local_table.plain(:*) - iseq.local_table.plain(:&) + if RUBY_VERSION >= "3.2" + iseq.local_table.plain(:*) + iseq.local_table.plain(:**) + iseq.local_table.plain(:&) + + iseq.argument_options[:rest_start] = iseq.argument_size + iseq.argument_options[:block_start] = iseq.argument_size + 2 + iseq.argument_options[:kwrest] = iseq.argument_size + 1 - iseq.argument_options[:rest_start] = iseq.argument_size - iseq.argument_options[:block_start] = iseq.argument_size + 1 + iseq.argument_size += 3 + else + iseq.local_table.plain(:*) + iseq.local_table.plain(:&) + + iseq.argument_options[:rest_start] = iseq.argument_size + iseq.argument_options[:block_start] = iseq.argument_size + 1 - iseq.argument_size += 2 + iseq.argument_size += 2 + end elsif node.keyword_rest visit(node.keyword_rest) end From 8dcb19cafcb17f58ede60cd0b08c3bed24df9f49 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 8 Dec 2022 17:07:06 +0000 Subject: [PATCH 292/536] Bump rubocop from 1.39.0 to 1.40.0 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.39.0 to 1.40.0. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.39.0...v1.40.0) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 0e81e5ff..05e482bf 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -9,17 +9,17 @@ GEM specs: ast (2.4.2) docile (1.4.0) - json (2.6.2) + json (2.6.3) minitest (5.16.3) parallel (1.22.1) - parser (3.1.2.1) + parser (3.1.3.0) ast (~> 2.4.1) prettier_print (1.1.0) rainbow (3.1.1) rake (13.0.6) - regexp_parser (2.6.0) + regexp_parser (2.6.1) rexml (3.2.5) - rubocop (1.39.0) + rubocop (1.40.0) json (~> 2.3) parallel (~> 1.10) parser (>= 3.1.2.1) @@ -29,7 +29,7 @@ GEM rubocop-ast (>= 1.23.0, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 1.4.0, < 3.0) - rubocop-ast (1.23.0) + rubocop-ast (1.24.0) parser (>= 3.1.1.0) ruby-progressbar (1.11.0) simplecov (0.21.2) From 24b62e256b57271b6c21c62bb0f0fb509fae2442 Mon Sep 17 00:00:00 2001 From: Andy Waite Date: Mon, 12 Dec 2022 15:42:43 -0500 Subject: [PATCH 293/536] Add link to docs --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 0f1b626a..70050619 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,8 @@ Syntax Tree is a suite of tools built on top of the internal CRuby parser. It pr It is built with only standard library dependencies. It additionally ships with a plugin system so that you can build your own syntax trees from other languages and incorporate these tools. +[RDoc Documentation](https://ruby-syntax-tree.github.io/syntax_tree/) + - [Installation](#installation) - [CLI](#cli) - [ast](#ast) From 9136254c32f726f17ccf7c60a92afcc7a3e6b621 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 13 Dec 2022 22:54:45 -0500 Subject: [PATCH 294/536] Move documentation --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index fb3556c9..7a943ca8 100644 --- a/README.md +++ b/README.md @@ -11,8 +11,6 @@ Syntax Tree is a suite of tools built on top of the internal CRuby parser. It pr It is built with only standard library dependencies. It additionally ships with a plugin system so that you can build your own syntax trees from other languages and incorporate these tools. -[RDoc Documentation](https://ruby-syntax-tree.github.io/syntax_tree/) - - [Installation](#installation) - [CLI](#cli) - [ast](#ast) @@ -326,7 +324,7 @@ stree write "**/{[!schema]*,*}.rb" ## Library -Syntax Tree can be used as a library to access the syntax tree underlying Ruby source code. +Syntax Tree can be used as a library to access the syntax tree underlying Ruby source code. The API is described below. For the full library documentation, see the [RDoc documentation](https://ruby-syntax-tree.github.io/syntax_tree/). ### SyntaxTree.read(filepath) From aeafc84aae49687ea2607dfad648a41132f913cb Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 14 Dec 2022 15:09:18 -0500 Subject: [PATCH 295/536] Rename YARV classes for consistency --- lib/syntax_tree.rb | 2 +- lib/syntax_tree/yarv/decompiler.rb | 254 ++++++++++++ lib/syntax_tree/yarv/disasm_formatter.rb | 211 ---------- lib/syntax_tree/yarv/disassembler.rb | 389 +++++++++---------- lib/syntax_tree/yarv/instruction_sequence.rb | 6 +- test/yarv_test.rb | 8 +- 6 files changed, 435 insertions(+), 435 deletions(-) create mode 100644 lib/syntax_tree/yarv/decompiler.rb delete mode 100644 lib/syntax_tree/yarv/disasm_formatter.rb diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index eadb485d..2e2d2a42 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -30,7 +30,7 @@ require_relative "syntax_tree/yarv" require_relative "syntax_tree/yarv/bf" require_relative "syntax_tree/yarv/compiler" -require_relative "syntax_tree/yarv/disasm_formatter" +require_relative "syntax_tree/yarv/decompiler" require_relative "syntax_tree/yarv/disassembler" require_relative "syntax_tree/yarv/instruction_sequence" require_relative "syntax_tree/yarv/instructions" diff --git a/lib/syntax_tree/yarv/decompiler.rb b/lib/syntax_tree/yarv/decompiler.rb new file mode 100644 index 00000000..a6a567fb --- /dev/null +++ b/lib/syntax_tree/yarv/decompiler.rb @@ -0,0 +1,254 @@ +# frozen_string_literal: true + +module SyntaxTree + module YARV + # This class is responsible for taking a compiled instruction sequence and + # walking through it to generate equivalent Ruby code. + class Decompiler + # When we're decompiling, we use a looped case statement to emulate + # jumping around in the same way the virtual machine would. This class + # provides convenience methods for generating the AST nodes that have to + # do with that label. + class BlockLabel + include DSL + attr_reader :name + + def initialize(name) + @name = name + end + + def field + VarField(Ident(name)) + end + + def ref + VarRef(Ident(name)) + end + end + + include DSL + attr_reader :iseq, :block_label + + def initialize(iseq) + @iseq = iseq + @block_label = BlockLabel.new("__block_label") + end + + def to_ruby + Program(decompile(iseq)) + end + + private + + def node_for(value) + case value + when Integer + Int(value.to_s) + when Symbol + SymbolLiteral(Ident(value.to_s)) + end + end + + def decompile(iseq) + label = :label_0 + clauses = {} + clause = [] + + iseq.insns.each do |insn| + case insn + when InstructionSequence::Label + unless clause.last.is_a?(Next) + clause << Assign(block_label.field, node_for(insn.name)) + end + + clauses[label] = clause + clause = [] + label = insn.name + when BranchUnless + body = [ + Assign(block_label.field, node_for(insn.label.name)), + Next(Args([])) + ] + + clause << IfNode(clause.pop, Statements(body), nil) + when Dup + clause << clause.last + when DupHash + assocs = + insn.object.map do |key, value| + Assoc(node_for(key), node_for(value)) + end + + clause << HashLiteral(LBrace("{"), assocs) + when GetGlobal + clause << VarRef(GVar(insn.name.to_s)) + when GetLocalWC0 + local = iseq.local_table.locals[insn.index] + clause << VarRef(Ident(local.name.to_s)) + when Jump + clause << Assign(block_label.field, node_for(insn.label.name)) + clause << Next(Args([])) + when Leave + value = Args([clause.pop]) + clause << (iseq.type == :top ? Break(value) : ReturnNode(value)) + when OptAnd, OptDiv, OptEq, OptGE, OptGT, OptLE, OptLT, OptLTLT, + OptMinus, OptMod, OptMult, OptOr, OptPlus + left, right = clause.pop(2) + clause << Binary(left, insn.calldata.method, right) + when OptAref + collection, arg = clause.pop(2) + clause << ARef(collection, Args([arg])) + when OptAset + collection, arg, value = clause.pop(3) + + clause << if value.is_a?(Binary) && value.left.is_a?(ARef) && + collection === value.left.collection && + arg === value.left.index.parts[0] + OpAssign( + ARefField(collection, Args([arg])), + Op("#{value.operator}="), + value.right + ) + else + Assign(ARefField(collection, Args([arg])), value) + end + when OptNEq + left, right = clause.pop(2) + clause << Binary(left, :"!=", right) + when OptSendWithoutBlock + method = insn.calldata.method.to_s + argc = insn.calldata.argc + + if insn.calldata.flag?(CallData::CALL_FCALL) + if argc == 0 + clause.pop + clause << CallNode(nil, nil, Ident(method), Args([])) + elsif argc == 1 && method.end_with?("=") + _receiver, argument = clause.pop(2) + clause << Assign( + CallNode(nil, nil, Ident(method[0..-2]), nil), + argument + ) + else + _receiver, *arguments = clause.pop(argc + 1) + clause << CallNode( + nil, + nil, + Ident(method), + ArgParen(Args(arguments)) + ) + end + else + if argc == 0 + clause << CallNode(clause.pop, Period("."), Ident(method), nil) + elsif argc == 1 && method.end_with?("=") + receiver, argument = clause.pop(2) + clause << Assign( + CallNode(receiver, Period("."), Ident(method[0..-2]), nil), + argument + ) + else + receiver, *arguments = clause.pop(argc + 1) + clause << CallNode( + receiver, + Period("."), + Ident(method), + ArgParen(Args(arguments)) + ) + end + end + when PutObject + case insn.object + when Float + clause << FloatLiteral(insn.object.inspect) + when Integer + clause << Int(insn.object.inspect) + else + raise "Unknown object type: #{insn.object.class.name}" + end + when PutObjectInt2Fix0 + clause << Int("0") + when PutObjectInt2Fix1 + clause << Int("1") + when PutSelf + clause << VarRef(Kw("self")) + when SetGlobal + target = GVar(insn.name.to_s) + value = clause.pop + + clause << if value.is_a?(Binary) && VarRef(target) === value.left + OpAssign(VarField(target), Op("#{value.operator}="), value.right) + else + Assign(VarField(target), value) + end + when SetLocalWC0 + target = Ident(local_name(insn.index, 0)) + value = clause.pop + + clause << if value.is_a?(Binary) && VarRef(target) === value.left + OpAssign(VarField(target), Op("#{value.operator}="), value.right) + else + Assign(VarField(target), value) + end + else + raise "Unknown instruction #{insn}" + end + end + + # If there's only one clause, then we don't need a case statement, and + # we can just disassemble the first clause. + clauses[label] = clause + return Statements(clauses.values.first) if clauses.size == 1 + + # Here we're going to build up a big case statement that will handle all + # of the different labels. + current = nil + clauses.reverse_each do |current_label, current_clause| + current = + When( + Args([node_for(current_label)]), + Statements(current_clause), + current + ) + end + switch = Case(Kw("case"), block_label.ref, current) + + # Here we're going to make sure that any locals that were established in + # the label_0 block are initialized so that scoping rules work + # correctly. + stack = [] + locals = [block_label.name] + + clauses[:label_0].each do |node| + if node.is_a?(Assign) && node.target.is_a?(VarField) && + node.target.value.is_a?(Ident) + value = node.target.value.value + next if locals.include?(value) + + stack << Assign(node.target, VarRef(Kw("nil"))) + locals << value + end + end + + # Finally, we'll set up the initial label and loop the entire case + # statement. + stack << Assign(block_label.field, node_for(:label_0)) + stack << MethodAddBlock( + CallNode(nil, nil, Ident("loop"), Args([])), + BlockNode( + Kw("do"), + nil, + BodyStmt(Statements([switch]), nil, nil, nil, nil) + ) + ) + Statements(stack) + end + + def local_name(index, level) + current = iseq + level.times { current = current.parent_iseq } + current.local_table.locals[index].name.to_s + end + end + end +end diff --git a/lib/syntax_tree/yarv/disasm_formatter.rb b/lib/syntax_tree/yarv/disasm_formatter.rb deleted file mode 100644 index 566bc8fd..00000000 --- a/lib/syntax_tree/yarv/disasm_formatter.rb +++ /dev/null @@ -1,211 +0,0 @@ -# frozen_string_literal: true - -module SyntaxTree - module YARV - class DisasmFormatter - attr_reader :output, :queue - attr_reader :current_prefix, :current_iseq - - def initialize - @output = StringIO.new - @queue = [] - - @current_prefix = "" - @current_iseq = nil - end - - ######################################################################## - # Helpers for various instructions - ######################################################################## - - def calldata(value) - flag_names = [] - flag_names << :ARGS_SPLAT if value.flag?(CallData::CALL_ARGS_SPLAT) - if value.flag?(CallData::CALL_ARGS_BLOCKARG) - flag_names << :ARGS_BLOCKARG - end - flag_names << :FCALL if value.flag?(CallData::CALL_FCALL) - flag_names << :VCALL if value.flag?(CallData::CALL_VCALL) - flag_names << :ARGS_SIMPLE if value.flag?(CallData::CALL_ARGS_SIMPLE) - flag_names << :BLOCKISEQ if value.flag?(CallData::CALL_BLOCKISEQ) - flag_names << :KWARG if value.flag?(CallData::CALL_KWARG) - flag_names << :KW_SPLAT if value.flag?(CallData::CALL_KW_SPLAT) - flag_names << :TAILCALL if value.flag?(CallData::CALL_TAILCALL) - flag_names << :SUPER if value.flag?(CallData::CALL_SUPER) - flag_names << :ZSUPER if value.flag?(CallData::CALL_ZSUPER) - flag_names << :OPT_SEND if value.flag?(CallData::CALL_OPT_SEND) - flag_names << :KW_SPLAT_MUT if value.flag?(CallData::CALL_KW_SPLAT_MUT) - - parts = [] - parts << "mid:#{value.method}" if value.method - parts << "argc:#{value.argc}" - parts << "kw:[#{value.kw_arg.join(", ")}]" if value.kw_arg - parts << flag_names.join("|") if flag_names.any? - - "" - end - - def enqueue(iseq) - queue << iseq - end - - def event(name) - case name - when :RUBY_EVENT_B_CALL - "Bc" - when :RUBY_EVENT_B_RETURN - "Br" - when :RUBY_EVENT_CALL - "Ca" - when :RUBY_EVENT_CLASS - "Cl" - when :RUBY_EVENT_END - "En" - when :RUBY_EVENT_LINE - "Li" - when :RUBY_EVENT_RETURN - "Re" - else - raise "Unknown event: #{name}" - end - end - - def inline_storage(cache) - "" - end - - def instruction(name, operands = []) - operands.empty? ? name : "%-38s %s" % [name, operands.join(", ")] - end - - def label(value) - value.name["label_".length..] - end - - def local(index, explicit: nil, implicit: nil) - current = current_iseq - (explicit || implicit).times { current = current.parent_iseq } - - value = "#{current.local_table.name_at(index)}@#{index}" - value << ", #{explicit}" if explicit - value - end - - def object(value) - value.inspect - end - - ######################################################################## - # Main entrypoint - ######################################################################## - - def format! - while (@current_iseq = queue.shift) - output << "\n" if output.pos > 0 - format_iseq(@current_iseq) - end - - output.string - end - - private - - def format_iseq(iseq) - output << "#{current_prefix}== disasm: " - output << "#:1 " - - location = iseq.location - output << "(#{location.start_line},#{location.start_column})-" - output << "(#{location.end_line},#{location.end_column})" - output << "> " - - if iseq.catch_table.any? - output << "(catch: TRUE)\n" - output << "#{current_prefix}== catch table\n" - - with_prefix("#{current_prefix}| ") do - iseq.catch_table.each do |entry| - case entry - when InstructionSequence::CatchBreak - output << "#{current_prefix}catch type: break\n" - format_iseq(entry.iseq) - when InstructionSequence::CatchNext - output << "#{current_prefix}catch type: next\n" - when InstructionSequence::CatchRedo - output << "#{current_prefix}catch type: redo\n" - when InstructionSequence::CatchRescue - output << "#{current_prefix}catch type: rescue\n" - format_iseq(entry.iseq) - end - end - end - - output << "#{current_prefix}|#{"-" * 72}\n" - else - output << "(catch: FALSE)\n" - end - - if (local_table = iseq.local_table) && !local_table.empty? - output << "#{current_prefix}local table (size: #{local_table.size})\n" - - locals = - local_table.locals.each_with_index.map do |local, index| - "[%2d] %s@%d" % [local_table.offset(index), local.name, index] - end - - output << "#{current_prefix}#{locals.join(" ")}\n" - end - - length = 0 - events = [] - lines = [] - - iseq.insns.each do |insn| - case insn - when Integer - lines << insn - when Symbol - events << event(insn) - when InstructionSequence::Label - # skip - else - output << "#{current_prefix}%04d " % length - - disasm = insn.disasm(self) - output << disasm - - if lines.any? - output << " " * (65 - disasm.length) if disasm.length < 65 - elsif events.any? - output << " " * (39 - disasm.length) if disasm.length < 39 - end - - if lines.any? - output << "(%4d)" % lines.last - lines.clear - end - - if events.any? - output << "[#{events.join}]" - events.clear - end - - output << "\n" - length += insn.length - end - end - end - - def with_prefix(value) - previous = @current_prefix - - begin - @current_prefix = value - yield - ensure - @current_prefix = previous - end - end - end - end -end diff --git a/lib/syntax_tree/yarv/disassembler.rb b/lib/syntax_tree/yarv/disassembler.rb index af325c31..033b6d3d 100644 --- a/lib/syntax_tree/yarv/disassembler.rb +++ b/lib/syntax_tree/yarv/disassembler.rb @@ -2,252 +2,209 @@ module SyntaxTree module YARV - # This class is responsible for taking a compiled instruction sequence and - # walking through it to generate equivalent Ruby code. class Disassembler - # When we're disassmebling, we use a looped case statement to emulate - # jumping around in the same way the virtual machine would. This class - # provides convenience methods for generating the AST nodes that have to - # do with that label. - class DisasmLabel - include DSL - attr_reader :name - - def initialize(name) - @name = name - end + attr_reader :output, :queue + attr_reader :current_prefix, :current_iseq + + def initialize + @output = StringIO.new + @queue = [] + + @current_prefix = "" + @current_iseq = nil + end + + ######################################################################## + # Helpers for various instructions + ######################################################################## - def field - VarField(Ident(name)) + def calldata(value) + flag_names = [] + flag_names << :ARGS_SPLAT if value.flag?(CallData::CALL_ARGS_SPLAT) + if value.flag?(CallData::CALL_ARGS_BLOCKARG) + flag_names << :ARGS_BLOCKARG end + flag_names << :FCALL if value.flag?(CallData::CALL_FCALL) + flag_names << :VCALL if value.flag?(CallData::CALL_VCALL) + flag_names << :ARGS_SIMPLE if value.flag?(CallData::CALL_ARGS_SIMPLE) + flag_names << :BLOCKISEQ if value.flag?(CallData::CALL_BLOCKISEQ) + flag_names << :KWARG if value.flag?(CallData::CALL_KWARG) + flag_names << :KW_SPLAT if value.flag?(CallData::CALL_KW_SPLAT) + flag_names << :TAILCALL if value.flag?(CallData::CALL_TAILCALL) + flag_names << :SUPER if value.flag?(CallData::CALL_SUPER) + flag_names << :ZSUPER if value.flag?(CallData::CALL_ZSUPER) + flag_names << :OPT_SEND if value.flag?(CallData::CALL_OPT_SEND) + flag_names << :KW_SPLAT_MUT if value.flag?(CallData::CALL_KW_SPLAT_MUT) + + parts = [] + parts << "mid:#{value.method}" if value.method + parts << "argc:#{value.argc}" + parts << "kw:[#{value.kw_arg.join(", ")}]" if value.kw_arg + parts << flag_names.join("|") if flag_names.any? + + "" + end - def ref - VarRef(Ident(name)) + def enqueue(iseq) + queue << iseq + end + + def event(name) + case name + when :RUBY_EVENT_B_CALL + "Bc" + when :RUBY_EVENT_B_RETURN + "Br" + when :RUBY_EVENT_CALL + "Ca" + when :RUBY_EVENT_CLASS + "Cl" + when :RUBY_EVENT_END + "En" + when :RUBY_EVENT_LINE + "Li" + when :RUBY_EVENT_RETURN + "Re" + else + raise "Unknown event: #{name}" end end - include DSL - attr_reader :iseq, :disasm_label + def inline_storage(cache) + "" + end - def initialize(iseq) - @iseq = iseq - @disasm_label = DisasmLabel.new("__disasm_label") + def instruction(name, operands = []) + operands.empty? ? name : "%-38s %s" % [name, operands.join(", ")] end - def to_ruby - Program(disassemble(iseq)) + def label(value) + value.name["label_".length..] end - private + def local(index, explicit: nil, implicit: nil) + current = current_iseq + (explicit || implicit).times { current = current.parent_iseq } + + value = "#{current.local_table.name_at(index)}@#{index}" + value << ", #{explicit}" if explicit + value + end - def node_for(value) - case value - when Integer - Int(value.to_s) - when Symbol - SymbolLiteral(Ident(value.to_s)) + def object(value) + value.inspect + end + + ######################################################################## + # Main entrypoint + ######################################################################## + + def format! + while (@current_iseq = queue.shift) + output << "\n" if output.pos > 0 + format_iseq(@current_iseq) end + + output.string end - def disassemble(iseq) - label = :label_0 - clauses = {} - clause = [] + private + + def format_iseq(iseq) + output << "#{current_prefix}== disasm: " + output << "#:1 " + + location = iseq.location + output << "(#{location.start_line},#{location.start_column})-" + output << "(#{location.end_line},#{location.end_column})" + output << "> " + + if iseq.catch_table.any? + output << "(catch: TRUE)\n" + output << "#{current_prefix}== catch table\n" + + with_prefix("#{current_prefix}| ") do + iseq.catch_table.each do |entry| + case entry + when InstructionSequence::CatchBreak + output << "#{current_prefix}catch type: break\n" + format_iseq(entry.iseq) + when InstructionSequence::CatchNext + output << "#{current_prefix}catch type: next\n" + when InstructionSequence::CatchRedo + output << "#{current_prefix}catch type: redo\n" + when InstructionSequence::CatchRescue + output << "#{current_prefix}catch type: rescue\n" + format_iseq(entry.iseq) + end + end + end + + output << "#{current_prefix}|#{"-" * 72}\n" + else + output << "(catch: FALSE)\n" + end + + if (local_table = iseq.local_table) && !local_table.empty? + output << "#{current_prefix}local table (size: #{local_table.size})\n" + + locals = + local_table.locals.each_with_index.map do |local, index| + "[%2d] %s@%d" % [local_table.offset(index), local.name, index] + end + + output << "#{current_prefix}#{locals.join(" ")}\n" + end + + length = 0 + events = [] + lines = [] iseq.insns.each do |insn| case insn + when Integer + lines << insn + when Symbol + events << event(insn) when InstructionSequence::Label - unless clause.last.is_a?(Next) - clause << Assign(disasm_label.field, node_for(insn.name)) - end + # skip + else + output << "#{current_prefix}%04d " % length - clauses[label] = clause - clause = [] - label = insn.name - when BranchUnless - body = [ - Assign(disasm_label.field, node_for(insn.label.name)), - Next(Args([])) - ] - - clause << IfNode(clause.pop, Statements(body), nil) - when Dup - clause << clause.last - when DupHash - assocs = - insn.object.map do |key, value| - Assoc(node_for(key), node_for(value)) - end + disasm = insn.disasm(self) + output << disasm - clause << HashLiteral(LBrace("{"), assocs) - when GetGlobal - clause << VarRef(GVar(insn.name.to_s)) - when GetLocalWC0 - local = iseq.local_table.locals[insn.index] - clause << VarRef(Ident(local.name.to_s)) - when Jump - clause << Assign(disasm_label.field, node_for(insn.label.name)) - clause << Next(Args([])) - when Leave - value = Args([clause.pop]) - clause << (iseq.type == :top ? Break(value) : ReturnNode(value)) - when OptAnd, OptDiv, OptEq, OptGE, OptGT, OptLE, OptLT, OptLTLT, - OptMinus, OptMod, OptMult, OptOr, OptPlus - left, right = clause.pop(2) - clause << Binary(left, insn.calldata.method, right) - when OptAref - collection, arg = clause.pop(2) - clause << ARef(collection, Args([arg])) - when OptAset - collection, arg, value = clause.pop(3) - - clause << if value.is_a?(Binary) && value.left.is_a?(ARef) && - collection === value.left.collection && - arg === value.left.index.parts[0] - OpAssign( - ARefField(collection, Args([arg])), - Op("#{value.operator}="), - value.right - ) - else - Assign(ARefField(collection, Args([arg])), value) - end - when OptNEq - left, right = clause.pop(2) - clause << Binary(left, :"!=", right) - when OptSendWithoutBlock - method = insn.calldata.method.to_s - argc = insn.calldata.argc - - if insn.calldata.flag?(CallData::CALL_FCALL) - if argc == 0 - clause.pop - clause << CallNode(nil, nil, Ident(method), Args([])) - elsif argc == 1 && method.end_with?("=") - _receiver, argument = clause.pop(2) - clause << Assign( - CallNode(nil, nil, Ident(method[0..-2]), nil), - argument - ) - else - _receiver, *arguments = clause.pop(argc + 1) - clause << CallNode( - nil, - nil, - Ident(method), - ArgParen(Args(arguments)) - ) - end - else - if argc == 0 - clause << CallNode(clause.pop, Period("."), Ident(method), nil) - elsif argc == 1 && method.end_with?("=") - receiver, argument = clause.pop(2) - clause << Assign( - CallNode(receiver, Period("."), Ident(method[0..-2]), nil), - argument - ) - else - receiver, *arguments = clause.pop(argc + 1) - clause << CallNode( - receiver, - Period("."), - Ident(method), - ArgParen(Args(arguments)) - ) - end - end - when PutObject - case insn.object - when Float - clause << FloatLiteral(insn.object.inspect) - when Integer - clause << Int(insn.object.inspect) - else - raise "Unknown object type: #{insn.object.class.name}" + if lines.any? + output << " " * (65 - disasm.length) if disasm.length < 65 + elsif events.any? + output << " " * (39 - disasm.length) if disasm.length < 39 end - when PutObjectInt2Fix0 - clause << Int("0") - when PutObjectInt2Fix1 - clause << Int("1") - when PutSelf - clause << VarRef(Kw("self")) - when SetGlobal - target = GVar(insn.name.to_s) - value = clause.pop - - clause << if value.is_a?(Binary) && VarRef(target) === value.left - OpAssign(VarField(target), Op("#{value.operator}="), value.right) - else - Assign(VarField(target), value) + + if lines.any? + output << "(%4d)" % lines.last + lines.clear end - when SetLocalWC0 - target = Ident(local_name(insn.index, 0)) - value = clause.pop - - clause << if value.is_a?(Binary) && VarRef(target) === value.left - OpAssign(VarField(target), Op("#{value.operator}="), value.right) - else - Assign(VarField(target), value) + + if events.any? + output << "[#{events.join}]" + events.clear end - else - raise "Unknown instruction #{insn}" - end - end - # If there's only one clause, then we don't need a case statement, and - # we can just disassemble the first clause. - clauses[label] = clause - return Statements(clauses.values.first) if clauses.size == 1 - - # Here we're going to build up a big case statement that will handle all - # of the different labels. - current = nil - clauses.reverse_each do |current_label, current_clause| - current = - When( - Args([node_for(current_label)]), - Statements(current_clause), - current - ) - end - switch = Case(Kw("case"), disasm_label.ref, current) - - # Here we're going to make sure that any locals that were established in - # the label_0 block are initialized so that scoping rules work - # correctly. - stack = [] - locals = [disasm_label.name] - - clauses[:label_0].each do |node| - if node.is_a?(Assign) && node.target.is_a?(VarField) && - node.target.value.is_a?(Ident) - value = node.target.value.value - next if locals.include?(value) - - stack << Assign(node.target, VarRef(Kw("nil"))) - locals << value + output << "\n" + length += insn.length end end - - # Finally, we'll set up the initial label and loop the entire case - # statement. - stack << Assign(disasm_label.field, node_for(:label_0)) - stack << MethodAddBlock( - CallNode(nil, nil, Ident("loop"), Args([])), - BlockNode( - Kw("do"), - nil, - BodyStmt(Statements([switch]), nil, nil, nil, nil) - ) - ) - Statements(stack) end - def local_name(index, level) - current = iseq - level.times { current = current.parent_iseq } - current.local_table.locals[index].name.to_s + def with_prefix(value) + previous = @current_prefix + + begin + @current_prefix = value + yield + ensure + @current_prefix = previous + end end end end diff --git a/lib/syntax_tree/yarv/instruction_sequence.rb b/lib/syntax_tree/yarv/instruction_sequence.rb index ee5390a1..93b5018e 100644 --- a/lib/syntax_tree/yarv/instruction_sequence.rb +++ b/lib/syntax_tree/yarv/instruction_sequence.rb @@ -272,9 +272,9 @@ def to_a end def disasm - formatter = DisasmFormatter.new - formatter.enqueue(self) - formatter.format! + disassembler = Disassembler.new + disassembler.enqueue(self) + disassembler.format! end # This method converts our linked list of instructions into a final array diff --git a/test/yarv_test.rb b/test/yarv_test.rb index 02514a93..f8e0ffdb 100644 --- a/test/yarv_test.rb +++ b/test/yarv_test.rb @@ -31,7 +31,7 @@ class YARVTest < Minitest::Test CASES.each do |source, expected| define_method("test_disassemble_#{source}") do - assert_disassembles(expected, source) + assert_decompiles(expected, source) end end @@ -41,13 +41,13 @@ def test_bf ">>.>---.+++++++..+++.>>.<-.<.+++.------.--------.>>+.>++." iseq = YARV::Bf.new(hello_world).compile - Formatter.format(hello_world, YARV::Disassembler.new(iseq).to_ruby) + Formatter.format(hello_world, YARV::Decompiler.new(iseq).to_ruby) end private - def assert_disassembles(expected, source) - ruby = YARV::Disassembler.new(YARV.compile(source)).to_ruby + def assert_decompiles(expected, source) + ruby = YARV::Decompiler.new(YARV.compile(source)).to_ruby actual = Formatter.format(source, ruby) assert_equal expected, actual end From 9d57b6a7b8592e4a00a5a1b90db89fa2988b45b1 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 16 Dec 2022 08:23:05 -0500 Subject: [PATCH 296/536] Assembler --- lib/syntax_tree.rb | 1 + lib/syntax_tree/yarv/assembler.rb | 244 +++++++++++++++++++ lib/syntax_tree/yarv/compiler.rb | 13 +- lib/syntax_tree/yarv/instruction_sequence.rb | 4 +- 4 files changed, 259 insertions(+), 3 deletions(-) create mode 100644 lib/syntax_tree/yarv/assembler.rb diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index 2e2d2a42..41a33a78 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -30,6 +30,7 @@ require_relative "syntax_tree/yarv" require_relative "syntax_tree/yarv/bf" require_relative "syntax_tree/yarv/compiler" +require_relative "syntax_tree/yarv/assembler" require_relative "syntax_tree/yarv/decompiler" require_relative "syntax_tree/yarv/disassembler" require_relative "syntax_tree/yarv/instruction_sequence" diff --git a/lib/syntax_tree/yarv/assembler.rb b/lib/syntax_tree/yarv/assembler.rb new file mode 100644 index 00000000..b5df37b8 --- /dev/null +++ b/lib/syntax_tree/yarv/assembler.rb @@ -0,0 +1,244 @@ +# frozen_string_literal: true + +module SyntaxTree + module YARV + class Assembler + class ObjectVisitor < Compiler::RubyVisitor + def visit_dyna_symbol(node) + if node.parts.empty? + :"" + else + raise CompilationError + end + end + + def visit_string_literal(node) + case node.parts.length + when 0 + "" + when 1 + raise CompilationError unless node.parts.first.is_a?(TStringContent) + node.parts.first.value + else + raise CompilationError + end + end + end + + attr_reader :filepath + + def initialize(filepath) + @filepath = filepath + end + + def assemble + iseq = InstructionSequence.new(:top, "
", nil, Location.default) + labels = {} + + File.foreach(filepath, chomp: true) do |line| + case line.strip + when "" + # skip over blank lines + next + when /^;/ + # skip over comments + next + when /^(\w+):$/ + # create labels + iseq.push(labels[$1] = iseq.label) + next + end + + insn, operands = line.split(" ", 2) + + case insn + when "adjuststack" + iseq.adjuststack(parse_number(operands)) + when "anytostring" + iseq.anytostring + when "checkmatch" + iseq.checkmatch(parse_number(operands)) + when "checktype" + iseq.checktype(parse_number(operands)) + when "concatarray" + iseq.concatarray + when "concatstrings" + iseq.concatstrings(parse_number(operands)) + when "dup" + iseq.dup + when "dupn" + iseq.dupn(parse_number(operands)) + when "duparray" + object = parse(operands) + raise unless object.is_a?(Array) + + iseq.duparray(object) + when "duphash" + object = parse(operands) + raise unless object.is_a?(Hash) + + iseq.duphash(object) + when "getinstancevariable" + object = parse(operands) + raise unless object.is_a?(Symbol) + + iseq.getinstancevariable(object) + when "intern" + iseq.intern + when "leave" + iseq.leave + when "newarray" + iseq.newarray(parse_number(operands)) + when "newrange" + object = parse(operands) + raise if object != 0 && object != 1 + + iseq.newrange(operands.to_i) + when "nop" + iseq.nop + when "objtostring" + iseq.objtostring( + YARV.calldata( + :to_s, + 0, + CallData::CALL_ARGS_SIMPLE | CallData::CALL_FCALL + ) + ) + when "opt_and" + iseq.send(YARV.calldata(:&, 1)) + when "opt_aref" + iseq.send(YARV.calldata(:[], 1)) + when "opt_aref_with" + object = parse(operands) + raise unless object.is_a?(String) + + iseq.opt_aref_with(object, YARV.calldata(:[], 1)) + when "opt_div" + iseq.send(YARV.calldata(:/, 1)) + when "opt_empty_p" + iseq.send( + YARV.calldata( + :empty?, + 0, + CallData::CALL_ARGS_SIMPLE | CallData::CALL_FCALL + ) + ) + when "opt_eqeq" + iseq.send(YARV.calldata(:==, 1)) + when "opt_ge" + iseq.send(YARV.calldata(:>=, 1)) + when "opt_getconstant_path" + object = parse(operands) + raise unless object.is_a?(Array) + + iseq.opt_getconstant_path(object) + when "opt_ltlt" + iseq.send(YARV.calldata(:<<, 1)) + when "opt_minus" + iseq.send(YARV.calldata(:-, 1)) + when "opt_mult" + iseq.send(YARV.calldata(:*, 1)) + when "opt_or" + iseq.send(YARV.calldata(:|, 1)) + when "opt_plus" + iseq.send(YARV.calldata(:+, 1)) + when "pop" + iseq.pop + when "putnil" + iseq.putnil + when "putobject" + iseq.putobject(parse(operands)) + when "putself" + iseq.putself + when "putstring" + object = parse(operands) + raise unless object.is_a?(String) + + iseq.putstring(object) + when "send" + iseq.send(calldata(operands)) + when "setinstancevariable" + object = parse(operands) + raise unless object.is_a?(Symbol) + + iseq.setinstancevariable(object) + when "swap" + iseq.swap + when "toregexp" + options, length = operands.split(", ") + iseq.toregexp(parse_number(options), parse_number(length)) + else + raise "Could not understand: #{line}" + end + end + + iseq.compile! + iseq + end + + def self.assemble(filepath) + new(filepath).assemble + end + + private + + def parse(value) + program = SyntaxTree.parse(value) + raise if program.statements.body.length != 1 + + program.statements.body.first.accept(ObjectVisitor.new) + end + + def parse_number(value) + object = parse(value) + raise unless object.is_a?(Integer) + + object + end + + def calldata(value) + message, argc_value, flags_value = value.split + flags = + if flags_value + flags_value + .split("|") + .map do |flag| + case flag + when "ARGS_SPLAT" + CallData::CALL_ARGS_SPLAT + when "ARGS_BLOCKARG" + CallData::CALL_ARGS_BLOCKARG + when "FCALL" + CallData::CALL_FCALL + when "VCALL" + CallData::CALL_VCALL + when "ARGS_SIMPLE" + CallData::CALL_ARGS_SIMPLE + when "BLOCKISEQ" + CallData::CALL_BLOCKISEQ + when "KWARG" + CallData::CALL_KWARG + when "KW_SPLAT" + CallData::CALL_KW_SPLAT + when "TAILCALL" + CallData::CALL_TAILCALL + when "SUPER" + CallData::CALL_SUPER + when "ZSUPER" + CallData::CALL_ZSUPER + when "OPT_SEND" + CallData::CALL_OPT_SEND + when "KW_SPLAT_MUT" + CallData::CALL_KW_SPLAT_MUT + end + end + .inject(:|) + else + CallData::CALL_ARGS_SIMPLE + end + + YARV.calldata(message.to_sym, argc_value&.to_i || 0, flags) + end + end + end +end diff --git a/lib/syntax_tree/yarv/compiler.rb b/lib/syntax_tree/yarv/compiler.rb index 046fb438..4bb5d654 100644 --- a/lib/syntax_tree/yarv/compiler.rb +++ b/lib/syntax_tree/yarv/compiler.rb @@ -148,7 +148,18 @@ def visit_imaginary(node) end def visit_int(node) - node.value.to_i + case (value = node.value) + when /^0b/ + value[2..].to_i(2) + when /^0o/ + value[2..].to_i(8) + when /^0d/ + value[2..].to_i + when /^0x/ + value[2..].to_i(16) + else + value.to_i + end end def visit_label(node) diff --git a/lib/syntax_tree/yarv/instruction_sequence.rb b/lib/syntax_tree/yarv/instruction_sequence.rb index 93b5018e..0f1eadd0 100644 --- a/lib/syntax_tree/yarv/instruction_sequence.rb +++ b/lib/syntax_tree/yarv/instruction_sequence.rb @@ -223,8 +223,8 @@ def eval compiled = to_a # Temporary hack until we get these working. - compiled[4][:node_id] = 11 - compiled[4][:node_ids] = [1, 0, 3, 2, 6, 7, 9, -1] + compiled[4][:node_id] = -1 + compiled[4][:node_ids] = [-1] * insns.length Fiddle.dlunwrap(ISEQ_LOAD.call(Fiddle.dlwrap(compiled), 0, nil)).eval end From 7e1b2a8176c37786323e334e477da8bd216f6ad6 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 16 Dec 2022 09:12:21 -0500 Subject: [PATCH 297/536] Fix Ruby head build --- lib/syntax_tree/yarv/compiler.rb | 37 ++++++-------------------------- 1 file changed, 7 insertions(+), 30 deletions(-) diff --git a/lib/syntax_tree/yarv/compiler.rb b/lib/syntax_tree/yarv/compiler.rb index 4bb5d654..496c2075 100644 --- a/lib/syntax_tree/yarv/compiler.rb +++ b/lib/syntax_tree/yarv/compiler.rb @@ -663,32 +663,10 @@ def visit_call(node) when ArgsForward flag |= CallData::CALL_TAILCALL if options.tailcall_optimization? - if RUBY_VERSION < "3.2" - flag |= CallData::CALL_ARGS_SPLAT - lookup = iseq.local_table.find(:*) - iseq.getlocal(lookup.index, lookup.level) - iseq.splatarray(arg_parts.length != 1) - else - flag |= CallData::CALL_ARGS_SPLAT - lookup = iseq.local_table.find(:*) - iseq.getlocal(lookup.index, lookup.level) - iseq.splatarray(true) - - flag |= CallData::CALL_KW_SPLAT - iseq.putspecialobject(PutSpecialObject::OBJECT_VMCORE) - iseq.newhash(0) - lookup = iseq.local_table.find(:**) - iseq.getlocal(lookup.index, lookup.level) - iseq.send( - YARV.calldata( - :"core#hash_merge_kwd", - 2, - CallData::CALL_ARGS_SIMPLE - ) - ) - iseq.newarray(1) - iseq.concatarray - end + flag |= CallData::CALL_ARGS_SPLAT + lookup = iseq.local_table.find(:*) + iseq.getlocal(lookup.index, lookup.level) + iseq.splatarray(arg_parts.length != 1) flag |= CallData::CALL_ARGS_BLOCKARG lookup = iseq.local_table.find(:&) @@ -1339,14 +1317,13 @@ def visit_params(node) if node.keyword_rest.is_a?(ArgsForward) if RUBY_VERSION >= "3.2" iseq.local_table.plain(:*) - iseq.local_table.plain(:**) iseq.local_table.plain(:&) + iseq.local_table.plain(:"...") iseq.argument_options[:rest_start] = iseq.argument_size - iseq.argument_options[:block_start] = iseq.argument_size + 2 - iseq.argument_options[:kwrest] = iseq.argument_size + 1 + iseq.argument_options[:block_start] = iseq.argument_size + 1 - iseq.argument_size += 3 + iseq.argument_size += 2 else iseq.local_table.plain(:*) iseq.local_table.plain(:&) From 83d21fde2fb5f87e219d689486ea5edd911338b0 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 16 Dec 2022 18:22:52 -0500 Subject: [PATCH 298/536] Assemble other instructions --- lib/syntax_tree.rb | 2 +- lib/syntax_tree/yarv/assembler.rb | 361 ++++++++++++++----- lib/syntax_tree/yarv/compiler.rb | 2 +- lib/syntax_tree/yarv/instruction_sequence.rb | 12 +- lib/syntax_tree/yarv/instructions.rb | 2 +- 5 files changed, 269 insertions(+), 110 deletions(-) diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index 41a33a78..1357e95f 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -30,13 +30,13 @@ require_relative "syntax_tree/yarv" require_relative "syntax_tree/yarv/bf" require_relative "syntax_tree/yarv/compiler" -require_relative "syntax_tree/yarv/assembler" require_relative "syntax_tree/yarv/decompiler" require_relative "syntax_tree/yarv/disassembler" require_relative "syntax_tree/yarv/instruction_sequence" require_relative "syntax_tree/yarv/instructions" require_relative "syntax_tree/yarv/legacy" require_relative "syntax_tree/yarv/local_table" +require_relative "syntax_tree/yarv/assembler" # Syntax Tree is a suite of tools built on top of the internal CRuby parser. It # provides the ability to generate a syntax tree from source, as well as the diff --git a/lib/syntax_tree/yarv/assembler.rb b/lib/syntax_tree/yarv/assembler.rb index b5df37b8..c3a874e9 100644 --- a/lib/syntax_tree/yarv/assembler.rb +++ b/lib/syntax_tree/yarv/assembler.rb @@ -33,20 +33,37 @@ def initialize(filepath) def assemble iseq = InstructionSequence.new(:top, "
", nil, Location.default) - labels = {} + assemble_iseq(iseq, File.readlines(filepath, chomp: true)) + + iseq.compile! + iseq + end + + def self.assemble(filepath) + new(filepath).assemble + end + + private + + def assemble_iseq(iseq, lines) + labels = Hash.new { |hash, name| hash[name] = iseq.label } + line_index = 0 + + while line_index < lines.length + line = lines[line_index] + line_index += 1 - File.foreach(filepath, chomp: true) do |line| case line.strip - when "" - # skip over blank lines - next - when /^;/ - # skip over comments + when "", /^;/ + # skip over blank lines and comments next when /^(\w+):$/ # create labels - iseq.push(labels[$1] = iseq.label) + iseq.push(labels[$1]) next + when /^__END__/ + # skip over the rest of the file when we hit __END__ + return end insn, operands = line.split(" ", 2) @@ -56,6 +73,12 @@ def assemble iseq.adjuststack(parse_number(operands)) when "anytostring" iseq.anytostring + when "branchif" + iseq.branchif(labels[operands]) + when "branchnil" + iseq.branchnil(labels[operands]) + when "branchunless" + iseq.branchunless(labels[operands]) when "checkmatch" iseq.checkmatch(parse_number(operands)) when "checktype" @@ -64,84 +87,200 @@ def assemble iseq.concatarray when "concatstrings" iseq.concatstrings(parse_number(operands)) + when "defineclass" + body = parse_nested(lines[line_index..]) + line_index += body.length + + name_value, flags_value = operands.split(/,\s*/) + name = parse_symbol(name_value) + flags = parse_number(flags_value) + + class_iseq = iseq.class_child_iseq(name.to_s, Location.default) + assemble_iseq(class_iseq, body) + iseq.defineclass(name, class_iseq, flags) + when "definemethod" + body = parse_nested(lines[line_index..]) + line_index += body.length + + name = parse_symbol(operands) + method_iseq = iseq.method_child_iseq(name.to_s, Location.default) + assemble_iseq(method_iseq, body) + + iseq.definemethod(name, method_iseq) + when "definesmethod" + body = parse_nested(lines[line_index..]) + line_index += body.length + + name = parse_symbol(operands) + method_iseq = iseq.method_child_iseq(name.to_s, Location.default) + + assemble_iseq(method_iseq, body) + iseq.definesmethod(name, method_iseq) when "dup" iseq.dup when "dupn" iseq.dupn(parse_number(operands)) when "duparray" - object = parse(operands) - raise unless object.is_a?(Array) - - iseq.duparray(object) + iseq.duparray(parse_type(operands, Array)) when "duphash" - object = parse(operands) - raise unless object.is_a?(Hash) - - iseq.duphash(object) + iseq.duphash(parse_type(operands, Hash)) + when "expandarray" + number, flags = operands.split(/,\s*/) + iseq.expandarray(parse_number(number), parse_number(flags)) + when "getclassvariable" + iseq.getclassvariable(parse_symbol(operands)) + when "getconstant" + iseq.getconstant(parse_symbol(operands)) + when "getglobal" + iseq.getglobal(parse_symbol(operands)) when "getinstancevariable" - object = parse(operands) - raise unless object.is_a?(Symbol) + iseq.getinstancevariable(parse_symbol(operands)) + when "getlocal" + name_string, level_string = operands.split(/,\s*/) + name = name_string.to_sym + level = level_string&.to_i || 0 - iseq.getinstancevariable(object) + iseq.local_table.plain(name) + lookup = iseq.local_table.find(name, level) + iseq.getlocal(lookup.index, lookup.level) + when "getspecial" + key, type = operands.split(/,\s*/) + iseq.getspecial(parse_number(key), parse_number(type)) when "intern" iseq.intern + when "invokesuper" + cdata = + if operands + calldata(operands) + else + YARV.calldata( + nil, + 0, + CallData::CALL_FCALL | CallData::CALL_ARGS_SIMPLE | + CallData::CALL_SUPER + ) + end + + block_iseq = + if lines[line_index].start_with?(" ") + body = parse_nested(lines[line_index..]) + line_index += body.length + + block_iseq = iseq.block_child_iseq(Location.default) + assemble_iseq(block_iseq, body) + block_iseq + end + + iseq.invokesuper(cdata, block_iseq) + when "jump" + iseq.jump(labels[operands]) when "leave" iseq.leave when "newarray" iseq.newarray(parse_number(operands)) + when "newarraykwsplat" + iseq.newarraykwsplat(parse_number(operands)) + when "newhash" + iseq.newhash(parse_number(operands)) when "newrange" - object = parse(operands) - raise if object != 0 && object != 1 - - iseq.newrange(operands.to_i) + iseq.newrange(parse_options(operands, [0, 1])) when "nop" iseq.nop when "objtostring" - iseq.objtostring( - YARV.calldata( - :to_s, - 0, - CallData::CALL_ARGS_SIMPLE | CallData::CALL_FCALL - ) - ) + iseq.objtostring(YARV.calldata(:to_s)) + when "once" + block_iseq = + if lines[line_index].start_with?(" ") + body = parse_nested(lines[line_index..]) + line_index += body.length + + block_iseq = iseq.block_child_iseq(Location.default) + assemble_iseq(block_iseq, body) + block_iseq + end + + iseq.once(block_iseq, iseq.inline_storage) when "opt_and" iseq.send(YARV.calldata(:&, 1)) when "opt_aref" iseq.send(YARV.calldata(:[], 1)) when "opt_aref_with" - object = parse(operands) - raise unless object.is_a?(String) + iseq.opt_aref_with(parse_string(operands), YARV.calldata(:[], 1)) + when "opt_aset" + iseq.send(YARV.calldata(:[]=, 2)) + when "opt_aset_with" + iseq.opt_aset_with(parse_string(operands), YARV.calldata(:[]=, 2)) + when "opt_case_dispatch" + cdhash_value, else_label_value = operands.split(/\s*\},\s*/) + cdhash_value.sub!(/\A\{/, "") + + pairs = + cdhash_value + .split(/\s*,\s*/) + .map! { |pair| pair.split(/\s*=>\s*/) } + + cdhash = pairs.to_h { |value, nm| [parse(value), labels[nm]] } + else_label = labels[else_label_value] - iseq.opt_aref_with(object, YARV.calldata(:[], 1)) + iseq.opt_case_dispatch(cdhash, else_label) when "opt_div" iseq.send(YARV.calldata(:/, 1)) when "opt_empty_p" - iseq.send( - YARV.calldata( - :empty?, - 0, - CallData::CALL_ARGS_SIMPLE | CallData::CALL_FCALL - ) - ) - when "opt_eqeq" + iseq.send(YARV.calldata(:empty?)) + when "opt_eq" iseq.send(YARV.calldata(:==, 1)) when "opt_ge" iseq.send(YARV.calldata(:>=, 1)) + when "opt_gt" + iseq.send(YARV.calldata(:>, 1)) when "opt_getconstant_path" - object = parse(operands) - raise unless object.is_a?(Array) - - iseq.opt_getconstant_path(object) + iseq.opt_getconstant_path(parse_type(operands, Array)) + when "opt_le" + iseq.send(YARV.calldata(:<=, 1)) + when "opt_length" + iseq.send(YARV.calldata(:length)) + when "opt_lt" + iseq.send(YARV.calldata(:<, 1)) when "opt_ltlt" iseq.send(YARV.calldata(:<<, 1)) when "opt_minus" iseq.send(YARV.calldata(:-, 1)) + when "opt_mod" + iseq.send(YARV.calldata(:%, 1)) when "opt_mult" iseq.send(YARV.calldata(:*, 1)) + when "opt_neq" + iseq.send(YARV.calldata(:!=, 1)) + when "opt_newarray_max" + iseq.newarray(parse_number(operands)) + iseq.send(YARV.calldata(:max)) + when "opt_newarray_min" + iseq.newarray(parse_number(operands)) + iseq.send(YARV.calldata(:min)) + when "opt_nil_p" + iseq.send(YARV.calldata(:nil?)) + when "opt_not" + iseq.send(YARV.calldata(:!)) when "opt_or" iseq.send(YARV.calldata(:|, 1)) when "opt_plus" iseq.send(YARV.calldata(:+, 1)) + when "opt_regexpmatch2" + iseq.send(YARV.calldata(:=~, 1)) + when "opt_reverse" + iseq.send(YARV.calldata(:reverse)) + when "opt_send_without_block" + iseq.send(calldata(operands)) + when "opt_size" + iseq.send(YARV.calldata(:size)) + when "opt_str_freeze" + iseq.putstring(parse_string(operands)) + iseq.send(YARV.calldata(:freeze)) + when "opt_str_uminus" + iseq.putstring(parse_string(operands)) + iseq.send(YARV.calldata(:-@)) + when "opt_succ" + iseq.send(YARV.calldata(:succ)) when "pop" iseq.pop when "putnil" @@ -150,38 +289,60 @@ def assemble iseq.putobject(parse(operands)) when "putself" iseq.putself + when "putspecialobject" + iseq.putspecialobject(parse_options(operands, [1, 2, 3])) when "putstring" - object = parse(operands) - raise unless object.is_a?(String) - - iseq.putstring(object) + iseq.putstring(parse_string(operands)) when "send" - iseq.send(calldata(operands)) - when "setinstancevariable" - object = parse(operands) - raise unless object.is_a?(Symbol) + block_iseq = + if lines[line_index].start_with?(" ") + body = parse_nested(lines[line_index..]) + line_index += body.length + + block_iseq = iseq.block_child_iseq(Location.default) + assemble_iseq(block_iseq, body) + block_iseq + end + + iseq.send(calldata(operands), block_iseq) + when "setconstant" + iseq.setconstant(parse_symbol(operands)) + when "setglobal" + iseq.setglobal(parse_symbol(operands)) + when "setlocal" + name_string, level_string = operands.split(/,\s*/) + name = name_string.to_sym + level = level_string&.to_i || 0 - iseq.setinstancevariable(object) + iseq.local_table.plain(name) + lookup = iseq.local_table.find(name, level) + iseq.setlocal(lookup.index, lookup.level) + when "setn" + iseq.setn(parse_number(operands)) + when "setclassvariable" + iseq.setclassvariable(parse_symbol(operands)) + when "setinstancevariable" + iseq.setinstancevariable(parse_symbol(operands)) + when "setspecial" + iseq.setspecial(parse_number(operands)) + when "splatarray" + iseq.splatarray(parse_options(operands, [true, false])) when "swap" iseq.swap + when "topn" + iseq.topn(parse_number(operands)) when "toregexp" options, length = operands.split(", ") iseq.toregexp(parse_number(options), parse_number(length)) + when "ARG_REQ" + iseq.argument_size += 1 + iseq.local_table.plain(operands.to_sym) else raise "Could not understand: #{line}" end end - - iseq.compile! - iseq end - def self.assemble(filepath) - new(filepath).assemble - end - - private - def parse(value) program = SyntaxTree.parse(value) raise if program.statements.body.length != 1 @@ -189,50 +350,52 @@ def parse(value) program.statements.body.first.accept(ObjectVisitor.new) end + def parse_options(value, options) + parse(value).tap { raise unless options.include?(_1) } + end + + def parse_type(value, type) + parse(value).tap { raise unless _1.is_a?(type) } + end + def parse_number(value) - object = parse(value) - raise unless object.is_a?(Integer) + parse_type(value, Integer) + end + + def parse_string(value) + parse_type(value, String) + end - object + def parse_symbol(value) + parse_type(value, Symbol) end + def parse_nested(lines) + body = lines.take_while { |line| line.match?(/^($|;| )/) } + body.map! { |line| line.delete_prefix!(" ") || +"" } + end + + CALLDATA_FLAGS = { + "ARGS_SPLAT" => CallData::CALL_ARGS_SPLAT, + "ARGS_BLOCKARG" => CallData::CALL_ARGS_BLOCKARG, + "FCALL" => CallData::CALL_FCALL, + "VCALL" => CallData::CALL_VCALL, + "ARGS_SIMPLE" => CallData::CALL_ARGS_SIMPLE, + "BLOCKISEQ" => CallData::CALL_BLOCKISEQ, + "KWARG" => CallData::CALL_KWARG, + "KW_SPLAT" => CallData::CALL_KW_SPLAT, + "TAILCALL" => CallData::CALL_TAILCALL, + "SUPER" => CallData::CALL_SUPER, + "ZSUPER" => CallData::CALL_ZSUPER, + "OPT_SEND" => CallData::CALL_OPT_SEND, + "KW_SPLAT_MUT" => CallData::CALL_KW_SPLAT_MUT + }.freeze + def calldata(value) message, argc_value, flags_value = value.split flags = if flags_value - flags_value - .split("|") - .map do |flag| - case flag - when "ARGS_SPLAT" - CallData::CALL_ARGS_SPLAT - when "ARGS_BLOCKARG" - CallData::CALL_ARGS_BLOCKARG - when "FCALL" - CallData::CALL_FCALL - when "VCALL" - CallData::CALL_VCALL - when "ARGS_SIMPLE" - CallData::CALL_ARGS_SIMPLE - when "BLOCKISEQ" - CallData::CALL_BLOCKISEQ - when "KWARG" - CallData::CALL_KWARG - when "KW_SPLAT" - CallData::CALL_KW_SPLAT - when "TAILCALL" - CallData::CALL_TAILCALL - when "SUPER" - CallData::CALL_SUPER - when "ZSUPER" - CallData::CALL_ZSUPER - when "OPT_SEND" - CallData::CALL_OPT_SEND - when "KW_SPLAT_MUT" - CallData::CALL_KW_SPLAT_MUT - end - end - .inject(:|) + flags_value.split("|").map(&CALLDATA_FLAGS).inject(:|) else CallData::CALL_ARGS_SIMPLE end diff --git a/lib/syntax_tree/yarv/compiler.rb b/lib/syntax_tree/yarv/compiler.rb index 496c2075..4af5d6f0 100644 --- a/lib/syntax_tree/yarv/compiler.rb +++ b/lib/syntax_tree/yarv/compiler.rb @@ -125,7 +125,7 @@ def self.compile(node) end def visit_array(node) - visit_all(node.contents.parts) + node.contents ? visit_all(node.contents.parts) : [] end def visit_bare_assoc_hash(node) diff --git a/lib/syntax_tree/yarv/instruction_sequence.rb b/lib/syntax_tree/yarv/instruction_sequence.rb index 0f1eadd0..48305be6 100644 --- a/lib/syntax_tree/yarv/instruction_sequence.rb +++ b/lib/syntax_tree/yarv/instruction_sequence.rb @@ -220,13 +220,7 @@ def length def eval raise "Unsupported platform" if ISEQ_LOAD.nil? - compiled = to_a - - # Temporary hack until we get these working. - compiled[4][:node_id] = -1 - compiled[4][:node_ids] = [-1] * insns.length - - Fiddle.dlunwrap(ISEQ_LOAD.call(Fiddle.dlwrap(compiled), 0, nil)).eval + Fiddle.dlunwrap(ISEQ_LOAD.call(Fiddle.dlwrap(to_a), 0, nil)).eval end def to_a @@ -257,7 +251,9 @@ def to_a { arg_size: argument_size, local_size: local_table.size, - stack_max: stack.maximum_size + stack_max: stack.maximum_size, + node_id: -1, + node_ids: [-1] * insns.length }, name, "", diff --git a/lib/syntax_tree/yarv/instructions.rb b/lib/syntax_tree/yarv/instructions.rb index 772f1bb3..288edb16 100644 --- a/lib/syntax_tree/yarv/instructions.rb +++ b/lib/syntax_tree/yarv/instructions.rb @@ -2695,7 +2695,7 @@ def to_a(_iseq) [ :opt_case_dispatch, case_dispatch_hash.flat_map { |key, value| [key, value.name] }, - else_label + else_label.name ] end From 13c07cfaf67a2fc9a26ae477c0382bcad7773855 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Sat, 17 Dec 2022 13:13:51 -0500 Subject: [PATCH 299/536] Assemble other instructions --- lib/syntax_tree/yarv/assembler.rb | 49 +++++++++++++++++++++++-------- 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/lib/syntax_tree/yarv/assembler.rb b/lib/syntax_tree/yarv/assembler.rb index c3a874e9..7d8a712f 100644 --- a/lib/syntax_tree/yarv/assembler.rb +++ b/lib/syntax_tree/yarv/assembler.rb @@ -79,6 +79,12 @@ def assemble_iseq(iseq, lines) iseq.branchnil(labels[operands]) when "branchunless" iseq.branchunless(labels[operands]) + when "checkkeyword" + kwbits_index, keyword_index = operands.split(/,\s*/) + iseq.checkkeyword( + parse_number(kwbits_index), + parse_number(keyword_index) + ) when "checkmatch" iseq.checkmatch(parse_number(operands)) when "checktype" @@ -98,6 +104,8 @@ def assemble_iseq(iseq, lines) class_iseq = iseq.class_child_iseq(name.to_s, Location.default) assemble_iseq(class_iseq, body) iseq.defineclass(name, class_iseq, flags) + when "defined" + raise NotImplementedError when "definemethod" body = parse_nested(lines[line_index..]) line_index += body.length @@ -127,6 +135,12 @@ def assemble_iseq(iseq, lines) when "expandarray" number, flags = operands.split(/,\s*/) iseq.expandarray(parse_number(number), parse_number(flags)) + when "getblockparam" + lookup = find_local(iseq, operands) + iseq.getblockparam(lookup.index, lookup.level) + when "getblockparamproxy" + lookup = find_local(iseq, operands) + iseq.getblockparamproxy(lookup.index, lookup.level) when "getclassvariable" iseq.getclassvariable(parse_symbol(operands)) when "getconstant" @@ -136,18 +150,16 @@ def assemble_iseq(iseq, lines) when "getinstancevariable" iseq.getinstancevariable(parse_symbol(operands)) when "getlocal" - name_string, level_string = operands.split(/,\s*/) - name = name_string.to_sym - level = level_string&.to_i || 0 - - iseq.local_table.plain(name) - lookup = iseq.local_table.find(name, level) + lookup = find_local(iseq, operands) iseq.getlocal(lookup.index, lookup.level) when "getspecial" key, type = operands.split(/,\s*/) iseq.getspecial(parse_number(key), parse_number(type)) when "intern" iseq.intern + when "invokeblock" + cdata = operands ? calldata(operands) : YARV.calldata(nil, 0) + iseq.invokeblock(cdata) when "invokesuper" cdata = if operands @@ -305,17 +317,15 @@ def assemble_iseq(iseq, lines) end iseq.send(calldata(operands), block_iseq) + when "setblockparam" + lookup = find_local(iseq, operands) + iseq.setblockparam(lookup.index, lookup.level) when "setconstant" iseq.setconstant(parse_symbol(operands)) when "setglobal" iseq.setglobal(parse_symbol(operands)) when "setlocal" - name_string, level_string = operands.split(/,\s*/) - name = name_string.to_sym - level = level_string&.to_i || 0 - - iseq.local_table.plain(name) - lookup = iseq.local_table.find(name, level) + lookup = find_local(iseq, operands) iseq.setlocal(lookup.index, lookup.level) when "setn" iseq.setn(parse_number(operands)) @@ -329,6 +339,8 @@ def assemble_iseq(iseq, lines) iseq.splatarray(parse_options(operands, [true, false])) when "swap" iseq.swap + when "throw" + iseq.throw(parse_number(operands)) when "topn" iseq.topn(parse_number(operands)) when "toregexp" @@ -337,12 +349,25 @@ def assemble_iseq(iseq, lines) when "ARG_REQ" iseq.argument_size += 1 iseq.local_table.plain(operands.to_sym) + when "ARG_BLOCK" + iseq.argument_options[:block_start] = iseq.argument_size + iseq.local_table.block(operands.to_sym) + iseq.argument_size += 1 else raise "Could not understand: #{line}" end end end + def find_local(iseq, operands) + name_string, level_string = operands.split(/,\s*/) + name = name_string.to_sym + level = level_string&.to_i || 0 + + iseq.local_table.plain(name) + iseq.local_table.find(name, level) + end + def parse(value) program = SyntaxTree.parse(value) raise if program.statements.body.length != 1 From 9749bc02afca46e3616df88d069d1c5af58bb5ec Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Sun, 18 Dec 2022 15:00:41 -0500 Subject: [PATCH 300/536] Assemble the defined instruction --- lib/syntax_tree/yarv/assembler.rb | 77 +++++++++++++++++++++---------- 1 file changed, 52 insertions(+), 25 deletions(-) diff --git a/lib/syntax_tree/yarv/assembler.rb b/lib/syntax_tree/yarv/assembler.rb index 7d8a712f..efb179c1 100644 --- a/lib/syntax_tree/yarv/assembler.rb +++ b/lib/syntax_tree/yarv/assembler.rb @@ -25,6 +25,43 @@ def visit_string_literal(node) end end + CALLDATA_FLAGS = { + "ARGS_SPLAT" => CallData::CALL_ARGS_SPLAT, + "ARGS_BLOCKARG" => CallData::CALL_ARGS_BLOCKARG, + "FCALL" => CallData::CALL_FCALL, + "VCALL" => CallData::CALL_VCALL, + "ARGS_SIMPLE" => CallData::CALL_ARGS_SIMPLE, + "BLOCKISEQ" => CallData::CALL_BLOCKISEQ, + "KWARG" => CallData::CALL_KWARG, + "KW_SPLAT" => CallData::CALL_KW_SPLAT, + "TAILCALL" => CallData::CALL_TAILCALL, + "SUPER" => CallData::CALL_SUPER, + "ZSUPER" => CallData::CALL_ZSUPER, + "OPT_SEND" => CallData::CALL_OPT_SEND, + "KW_SPLAT_MUT" => CallData::CALL_KW_SPLAT_MUT + }.freeze + + DEFINED_TYPES = [ + nil, + "nil", + "instance-variable", + "local-variable", + "global-variable", + "class variable", + "constant", + "method", + "yield", + "super", + "self", + "true", + "false", + "assignment", + "expression", + "ref", + "func", + "constant-from" + ].freeze + attr_reader :filepath def initialize(filepath) @@ -105,7 +142,12 @@ def assemble_iseq(iseq, lines) assemble_iseq(class_iseq, body) iseq.defineclass(name, class_iseq, flags) when "defined" - raise NotImplementedError + type, object, message = operands.split(/,\s*/) + iseq.defined( + DEFINED_TYPES.index(type), + parse_symbol(object), + parse_string(message) + ) when "definemethod" body = parse_nested(lines[line_index..]) line_index += body.length @@ -158,12 +200,13 @@ def assemble_iseq(iseq, lines) when "intern" iseq.intern when "invokeblock" - cdata = operands ? calldata(operands) : YARV.calldata(nil, 0) - iseq.invokeblock(cdata) + iseq.invokeblock( + operands ? parse_calldata(operands) : YARV.calldata(nil, 0) + ) when "invokesuper" - cdata = + calldata = if operands - calldata(operands) + parse_calldata(operands) else YARV.calldata( nil, @@ -183,7 +226,7 @@ def assemble_iseq(iseq, lines) block_iseq end - iseq.invokesuper(cdata, block_iseq) + iseq.invokesuper(calldata, block_iseq) when "jump" iseq.jump(labels[operands]) when "leave" @@ -282,7 +325,7 @@ def assemble_iseq(iseq, lines) when "opt_reverse" iseq.send(YARV.calldata(:reverse)) when "opt_send_without_block" - iseq.send(calldata(operands)) + iseq.send(parse_calldata(operands)) when "opt_size" iseq.send(YARV.calldata(:size)) when "opt_str_freeze" @@ -316,7 +359,7 @@ def assemble_iseq(iseq, lines) block_iseq end - iseq.send(calldata(operands), block_iseq) + iseq.send(parse_calldata(operands), block_iseq) when "setblockparam" lookup = find_local(iseq, operands) iseq.setblockparam(lookup.index, lookup.level) @@ -400,23 +443,7 @@ def parse_nested(lines) body.map! { |line| line.delete_prefix!(" ") || +"" } end - CALLDATA_FLAGS = { - "ARGS_SPLAT" => CallData::CALL_ARGS_SPLAT, - "ARGS_BLOCKARG" => CallData::CALL_ARGS_BLOCKARG, - "FCALL" => CallData::CALL_FCALL, - "VCALL" => CallData::CALL_VCALL, - "ARGS_SIMPLE" => CallData::CALL_ARGS_SIMPLE, - "BLOCKISEQ" => CallData::CALL_BLOCKISEQ, - "KWARG" => CallData::CALL_KWARG, - "KW_SPLAT" => CallData::CALL_KW_SPLAT, - "TAILCALL" => CallData::CALL_TAILCALL, - "SUPER" => CallData::CALL_SUPER, - "ZSUPER" => CallData::CALL_ZSUPER, - "OPT_SEND" => CallData::CALL_OPT_SEND, - "KW_SPLAT_MUT" => CallData::CALL_KW_SPLAT_MUT - }.freeze - - def calldata(value) + def parse_calldata(value) message, argc_value, flags_value = value.split flags = if flags_value From dc82220cb4239ca28c703f26ac07c089b7f7c911 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 20 Dec 2022 17:04:47 +0000 Subject: [PATCH 301/536] Bump rubocop from 1.40.0 to 1.41.0 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.40.0 to 1.41.0. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.40.0...v1.41.0) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 05e482bf..4f8dfc06 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -19,7 +19,7 @@ GEM rake (13.0.6) regexp_parser (2.6.1) rexml (3.2.5) - rubocop (1.40.0) + rubocop (1.41.0) json (~> 2.3) parallel (~> 1.10) parser (>= 3.1.2.1) From 14a1f5bdfaef1fb64abf636c51c83ec5c8f9619b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 22 Dec 2022 17:05:18 +0000 Subject: [PATCH 302/536] Bump rubocop from 1.41.0 to 1.41.1 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.41.0 to 1.41.1. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.41.0...v1.41.1) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 4f8dfc06..cddd3f21 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -19,7 +19,7 @@ GEM rake (13.0.6) regexp_parser (2.6.1) rexml (3.2.5) - rubocop (1.41.0) + rubocop (1.41.1) json (~> 2.3) parallel (~> 1.10) parser (>= 3.1.2.1) From b9ec70019d0628c8836e087d828b0a4439d7a69c Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 23 Dec 2022 12:31:56 -0500 Subject: [PATCH 303/536] BodyStmt location We were previously relying on #bind to set up the bounds for bodystmt but that isn't sufficient. This PR fixes that. --- lib/syntax_tree/parser.rb | 9 +++++---- test/node_test.rb | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index 5b093a87..85f6661e 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -820,13 +820,13 @@ def on_begin(bodystmt) end bodystmt.bind( - keyword.location.end_char, + find_next_statement_start(keyword.location.end_char), keyword.location.end_column, end_location.end_char, end_location.end_column ) - location = keyword.location.to(bodystmt.location) + location = keyword.location.to(end_location) Begin.new(bodystmt: bodystmt, location: location) end end @@ -905,14 +905,15 @@ def on_blockarg(name) # (nil | Ensure) ensure_clause # ) -> BodyStmt def on_bodystmt(statements, rescue_clause, else_clause, ensure_clause) + parts = [statements, rescue_clause, else_clause, ensure_clause].compact + BodyStmt.new( statements: statements, rescue_clause: rescue_clause, else_keyword: else_clause && consume_keyword(:else), else_clause: else_clause, ensure_clause: ensure_clause, - location: - Location.fixed(line: lineno, char: char_pos, column: current_column) + location: parts.first.location.to(parts.last.location) ) end diff --git a/test/node_test.rb b/test/node_test.rb index 15826be0..3d700e73 100644 --- a/test/node_test.rb +++ b/test/node_test.rb @@ -268,7 +268,7 @@ def test_bodystmt end SOURCE - at = location(lines: 9..9, chars: 5..64) + at = location(lines: 2..9, chars: 5..64) assert_node(BodyStmt, source, at: at, &:bodystmt) end From bedf6348601e7bdb43f3e15af82ba663d42fea12 Mon Sep 17 00:00:00 2001 From: Wei Zhe Heng Date: Sun, 25 Dec 2022 02:20:16 +0800 Subject: [PATCH 304/536] Add and ignore textDocument/documentColor --- lib/syntax_tree/language_server.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/syntax_tree/language_server.rb b/lib/syntax_tree/language_server.rb index c2265c32..a7b23664 100644 --- a/lib/syntax_tree/language_server.rb +++ b/lib/syntax_tree/language_server.rb @@ -111,6 +111,8 @@ def run write(id: request[:id], result: PP.pp(SyntaxTree.parse(store[uri]), +"")) when Request[method: %r{\$/.+}] # ignored + when Request[method: "textDocument/documentColor", params: { textDocument: { uri: :any } }] + # ignored else raise ArgumentError, "Unhandled: #{request}" end From 11cbcc9fdbd9123844eba3f66acada66b5b09d31 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Dec 2022 17:13:24 +0000 Subject: [PATCH 305/536] Bump simplecov from 0.21.2 to 0.22.0 Bumps [simplecov](https://github.com/simplecov-ruby/simplecov) from 0.21.2 to 0.22.0. - [Release notes](https://github.com/simplecov-ruby/simplecov/releases) - [Changelog](https://github.com/simplecov-ruby/simplecov/blob/main/CHANGELOG.md) - [Commits](https://github.com/simplecov-ruby/simplecov/compare/v0.21.2...v0.22.0) --- updated-dependencies: - dependency-name: simplecov dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index cddd3f21..206f71d9 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -32,7 +32,7 @@ GEM rubocop-ast (1.24.0) parser (>= 3.1.1.0) ruby-progressbar (1.11.0) - simplecov (0.21.2) + simplecov (0.22.0) docile (~> 1.1) simplecov-html (~> 0.11) simplecov_json_formatter (~> 0.1) From c68552ccc472c95248ea005924041f173c68951b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Dec 2022 17:13:36 +0000 Subject: [PATCH 306/536] Bump prettier_print from 1.1.0 to 1.2.0 Bumps [prettier_print](https://github.com/ruby-syntax-tree/prettier_print) from 1.1.0 to 1.2.0. - [Release notes](https://github.com/ruby-syntax-tree/prettier_print/releases) - [Changelog](https://github.com/ruby-syntax-tree/prettier_print/blob/main/CHANGELOG.md) - [Commits](https://github.com/ruby-syntax-tree/prettier_print/compare/v1.1.0...v1.2.0) --- updated-dependencies: - dependency-name: prettier_print dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index cddd3f21..638f83ca 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -14,7 +14,7 @@ GEM parallel (1.22.1) parser (3.1.3.0) ast (~> 2.4.1) - prettier_print (1.1.0) + prettier_print (1.2.0) rainbow (3.1.1) rake (13.0.6) regexp_parser (2.6.1) From e5dde653f9cbe9ede5f183d9adb5f0efb01e8816 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 28 Dec 2022 14:04:18 -0500 Subject: [PATCH 307/536] Fix #234 --- lib/syntax_tree/node.rb | 10 +++++----- test/fixtures/if.rb | 7 +++++++ 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 53fb3905..e5b09044 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -6160,7 +6160,7 @@ def call(q, node) # want to force it to not be a ternary, like if the predicate is an # assignment because it's hard to read. case node.predicate - when Assign, Command, CommandCall, MAssign, OpAssign + when Assign, Binary, Command, CommandCall, MAssign, OpAssign return false when Not return false unless node.predicate.parentheses? @@ -6183,10 +6183,10 @@ def call(q, node) # and default instead to breaking them into multiple lines. def ternaryable?(statement) case statement - when AliasNode, Assign, Break, Command, CommandCall, Heredoc, IfNode, - IfOp, Lambda, MAssign, Next, OpAssign, RescueMod, ReturnNode, - Super, Undef, UnlessNode, UntilNode, VoidStmt, WhileNode, - YieldNode, ZSuper + when AliasNode, Assign, Break, Command, CommandCall, Defined, Heredoc, + IfNode, IfOp, Lambda, MAssign, Next, OpAssign, RescueMod, + ReturnNode, Super, Undef, UnlessNode, UntilNode, VoidStmt, + WhileNode, YieldNode, ZSuper # This is a list of nodes that should not be allowed to be a part of a # ternary clause. false diff --git a/test/fixtures/if.rb b/test/fixtures/if.rb index cfd6a882..b25386b9 100644 --- a/test/fixtures/if.rb +++ b/test/fixtures/if.rb @@ -67,3 +67,10 @@ if true # comment1 # comment2 end +% +result = + if false && val = 1 + "A" + else + "B" + end From 6a65136a8875339f1514ba1310c78bbb8ed6d1ce Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 28 Dec 2022 13:48:19 -0500 Subject: [PATCH 308/536] Fix for #235 --- lib/syntax_tree/parser.rb | 30 ++++++++++++++---------------- test/fixtures/rassign.rb | 6 ++++++ 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index 85f6661e..fcefed30 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -995,22 +995,11 @@ def on_call(receiver, operator, message) # :call-seq: # on_case: (untyped value, untyped consequent) -> Case | RAssign def on_case(value, consequent) - if (keyword = find_keyword(:case)) - tokens.delete(keyword) - - Case.new( - keyword: keyword, - value: value, - consequent: consequent, - location: keyword.location.to(consequent.location) - ) - else - operator = - if (keyword = find_keyword(:in)) - tokens.delete(keyword) - else - consume_operator(:"=>") - end + if value && (operator = find_keyword(:in) || find_operator(:"=>")) && + (value.location.end_char...consequent.location.start_char).cover?( + operator.location.start_char + ) + tokens.delete(operator) node = RAssign.new( @@ -1022,6 +1011,15 @@ def on_case(value, consequent) PinVisitor.visit(node, tokens) node + else + keyword = consume_keyword(:case) + + Case.new( + keyword: keyword, + value: value, + consequent: consequent, + location: keyword.location.to(consequent.location) + ) end end diff --git a/test/fixtures/rassign.rb b/test/fixtures/rassign.rb index 3db52b18..3d357351 100644 --- a/test/fixtures/rassign.rb +++ b/test/fixtures/rassign.rb @@ -23,3 +23,9 @@ % a in Integer b => [Integer => c] +% +case [0] +when 0 + { a: 0 } => { a: } + puts a +end From 73761bb701e7ea98eb15ad97fb1cd214a6ca4adb Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 28 Dec 2022 20:41:51 -0500 Subject: [PATCH 309/536] Update CHANGELOG --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20808e3b..b6b854d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,9 +6,17 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +### Added + +- An experiment in working with instruction sequences has been added to Syntax Tree. This is subject to change, so it is not well documented or tested at the moment. It does not impact other functionality. + ### Changed - Support forwarding anonymous keyword arguments with `**`. +- The `BodyStmt` node now has a more correct location information. +- Ignore the `textDocument/documentColor` request coming into the language server to support clients that require that request be received. +- Do not attempt to convert `if..else` into ternaries if the predicate has a `Binary` node. +- Properly handle nested pattern matching when a rightward assignment is inside a `when` clause. ## [5.0.1] - 2022-11-10 From 83675f9bd5cc240ba70afc901312347971b1c38c Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 28 Dec 2022 20:44:36 -0500 Subject: [PATCH 310/536] This branch has no conflicts with the base branch. --- Gemfile.lock | 4 ++-- syntax_tree.gemspec | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index ffbdc5d1..5f7d8754 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -2,7 +2,7 @@ PATH remote: . specs: syntax_tree (5.0.1) - prettier_print (>= 1.1.0) + prettier_print (>= 1.2.0) GEM remote: https://rubygems.org/ @@ -14,7 +14,7 @@ GEM parallel (1.22.1) parser (3.1.2.1) ast (~> 2.4.1) - prettier_print (1.1.0) + prettier_print (1.2.0) rainbow (3.1.1) rake (13.0.6) regexp_parser (2.6.0) diff --git a/syntax_tree.gemspec b/syntax_tree.gemspec index 19f4ee97..f6c4a734 100644 --- a/syntax_tree.gemspec +++ b/syntax_tree.gemspec @@ -25,7 +25,7 @@ Gem::Specification.new do |spec| spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } spec.require_paths = %w[lib] - spec.add_dependency "prettier_print", ">= 1.1.0" + spec.add_dependency "prettier_print", ">= 1.2.0" spec.add_development_dependency "bundler" spec.add_development_dependency "minitest" From 640f64127251068d08843fd469d2dea26f379a0e Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 28 Dec 2022 20:50:21 -0500 Subject: [PATCH 311/536] Update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6b854d3..71e66403 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ### Added - An experiment in working with instruction sequences has been added to Syntax Tree. This is subject to change, so it is not well documented or tested at the moment. It does not impact other functionality. +- You can now format at a different base layer of indentation. This is an optional third argument to `SyntaxTree::format`. ### Changed From 8ebd8da9fb023f2eb687b9b5ab125303d72a5247 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 28 Dec 2022 20:55:23 -0500 Subject: [PATCH 312/536] Bump to version 5.1.0 --- CHANGELOG.md | 5 ++++- Gemfile.lock | 2 +- lib/syntax_tree/version.rb | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 71e66403..557fdf5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +## [5.1.0] - 2022-12-28 + ### Added - An experiment in working with instruction sequences has been added to Syntax Tree. This is subject to change, so it is not well documented or tested at the moment. It does not impact other functionality. @@ -469,7 +471,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - 🎉 Initial release! 🎉 -[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v5.0.1...HEAD +[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v5.1.0...HEAD +[5.1.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v5.0.1...v5.1.0 [5.0.1]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v5.0.0...v5.0.1 [5.0.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v4.3.0...v5.0.0 [4.3.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v4.2.0...v4.3.0 diff --git a/Gemfile.lock b/Gemfile.lock index 995fa74e..47d0c66b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - syntax_tree (5.0.1) + syntax_tree (5.1.0) prettier_print (>= 1.2.0) GEM diff --git a/lib/syntax_tree/version.rb b/lib/syntax_tree/version.rb index 340bbbdf..d9bbdfa4 100644 --- a/lib/syntax_tree/version.rb +++ b/lib/syntax_tree/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SyntaxTree - VERSION = "5.0.1" + VERSION = "5.1.0" end From ddda2d440f3ed0040ce7a462be24aac8c916437e Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 6 Dec 2022 10:46:34 -0500 Subject: [PATCH 313/536] Evaluate YARV bytecode --- .gitmodules | 6 + .rubocop.yml | 5 +- Rakefile | 7 + exe/yarv | 63 ++ lib/syntax_tree.rb | 1 + lib/syntax_tree/yarv.rb | 269 -------- lib/syntax_tree/yarv/assembler.rb | 14 +- lib/syntax_tree/yarv/bf.rb | 29 +- lib/syntax_tree/yarv/compiler.rb | 37 +- lib/syntax_tree/yarv/decompiler.rb | 9 + lib/syntax_tree/yarv/disassembler.rb | 5 +- lib/syntax_tree/yarv/instruction_sequence.rb | 276 +++++--- lib/syntax_tree/yarv/instructions.rb | 253 ++++++-- lib/syntax_tree/yarv/legacy.rb | 29 +- lib/syntax_tree/yarv/vm.rb | 624 +++++++++++++++++++ spec/mspec | 1 + spec/ruby | 1 + test/yarv_test.rb | 280 +++++++++ 18 files changed, 1470 insertions(+), 439 deletions(-) create mode 100644 .gitmodules create mode 100755 exe/yarv create mode 100644 lib/syntax_tree/yarv/vm.rb create mode 160000 spec/mspec create mode 160000 spec/ruby diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..f5477ea3 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,6 @@ +[submodule "mspec"] + path = spec/mspec + url = git@github.com:ruby/mspec.git +[submodule "spec"] + path = spec/ruby + url = git@github.com:ruby/spec.git diff --git a/.rubocop.yml b/.rubocop.yml index daf5a824..1e3e2f83 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -7,7 +7,7 @@ AllCops: SuggestExtensions: false TargetRubyVersion: 2.7 Exclude: - - '{.git,.github,bin,coverage,pkg,test/fixtures,vendor,tmp}/**/*' + - '{.git,.github,bin,coverage,pkg,spec,test/fixtures,vendor,tmp}/**/*' - test.rb Layout/LineLength: @@ -43,6 +43,9 @@ Lint/NonLocalExitFromIterator: Lint/RedundantRequireStatement: Enabled: false +Lint/RescueException: + Enabled: false + Lint/SuppressedException: Enabled: false diff --git a/Rakefile b/Rakefile index 4973d45e..f06d8cf8 100644 --- a/Rakefile +++ b/Rakefile @@ -26,3 +26,10 @@ end SyntaxTree::Rake::CheckTask.new(&configure) SyntaxTree::Rake::WriteTask.new(&configure) + +desc "Run mspec tests using YARV emulation" +task :spec do + Dir["./spec/ruby/language/**/*_spec.rb"].each do |filepath| + sh "exe/yarv ./spec/mspec/bin/mspec-tag #{filepath}" + end +end diff --git a/exe/yarv b/exe/yarv new file mode 100755 index 00000000..3efb23ff --- /dev/null +++ b/exe/yarv @@ -0,0 +1,63 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +$:.unshift(File.expand_path("../lib", __dir__)) + +require "syntax_tree" + +# Require these here so that we can run binding.irb without having them require +# anything that we've already patched. +require "irb" +require "irb/completion" +require "irb/color_printer" +require "readline" + +# First, create an instance of our virtual machine. +events = + if ENV["DEBUG"] + SyntaxTree::YARV::VM::STDOUTEvents.new + else + SyntaxTree::YARV::VM::NullEvents.new + end + +vm = SyntaxTree::YARV::VM.new(events) + +# Next, set up a bunch of aliases for methods that we're going to hook into in +# order to set up our virtual machine. +class << Kernel + alias yarv_require require + alias yarv_require_relative require_relative + alias yarv_load load + alias yarv_eval eval + alias yarv_throw throw + alias yarv_catch catch +end + +# Next, patch the methods that we just aliased so that they use our virtual +# machine's versions instead. This allows us to load Ruby files and have them +# execute in our virtual machine instead of the runtime environment. +[Kernel, Kernel.singleton_class].each do |klass| + klass.define_method(:require) { |filepath| vm.require(filepath) } + + klass.define_method(:load) { |filepath| vm.load(filepath) } + + # klass.define_method(:require_relative) do |filepath| + # vm.require_relative(filepath) + # end + + # klass.define_method(:eval) do | + # source, + # binding = TOPLEVEL_BINDING, + # filename = "(eval)", + # lineno = 1 + # | + # vm.eval(source, binding, filename, lineno) + # end + + # klass.define_method(:throw) { |tag, value = nil| vm.throw(tag, value) } + + # klass.define_method(:catch) { |tag, &block| vm.catch(tag, &block) } +end + +# Finally, require the file that we want to execute. +vm.require_resolved(ARGV.shift) diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index 1357e95f..39b55372 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -37,6 +37,7 @@ require_relative "syntax_tree/yarv/legacy" require_relative "syntax_tree/yarv/local_table" require_relative "syntax_tree/yarv/assembler" +require_relative "syntax_tree/yarv/vm" # Syntax Tree is a suite of tools built on top of the internal CRuby parser. It # provides the ability to generate a syntax tree from source, as well as the diff --git a/lib/syntax_tree/yarv.rb b/lib/syntax_tree/yarv.rb index 97592d4d..7e4da7bb 100644 --- a/lib/syntax_tree/yarv.rb +++ b/lib/syntax_tree/yarv.rb @@ -1,277 +1,8 @@ # frozen_string_literal: true -require "forwardable" - module SyntaxTree # This module provides an object representation of the YARV bytecode. module YARV - class VM - class Jump - attr_reader :name - - def initialize(name) - @name = name - end - end - - class Leave - attr_reader :value - - def initialize(value) - @value = value - end - end - - class Frame - attr_reader :iseq, :parent, :stack_index, :_self, :nesting, :svars - - def initialize(iseq, parent, stack_index, _self, nesting) - @iseq = iseq - @parent = parent - @stack_index = stack_index - @_self = _self - @nesting = nesting - @svars = {} - end - end - - class TopFrame < Frame - def initialize(iseq) - super(iseq, nil, 0, TOPLEVEL_BINDING.eval("self"), [Object]) - end - end - - class BlockFrame < Frame - def initialize(iseq, parent, stack_index) - super(iseq, parent, stack_index, parent._self, parent.nesting) - end - end - - class MethodFrame < Frame - attr_reader :name, :block - - def initialize(iseq, parent, stack_index, _self, name, block) - super(iseq, parent, stack_index, _self, parent.nesting) - @name = name - @block = block - end - end - - class ClassFrame < Frame - def initialize(iseq, parent, stack_index, _self) - super(iseq, parent, stack_index, _self, parent.nesting + [_self]) - end - end - - class FrozenCore - define_method("core#hash_merge_kwd") { |left, right| left.merge(right) } - - define_method("core#hash_merge_ptr") do |hash, *values| - hash.merge(values.each_slice(2).to_h) - end - - define_method("core#set_method_alias") do |clazz, new_name, old_name| - clazz.alias_method(new_name, old_name) - end - - define_method("core#set_variable_alias") do |new_name, old_name| - # Using eval here since there isn't a reflection API to be able to - # alias global variables. - eval("alias #{new_name} #{old_name}", binding, __FILE__, __LINE__) - end - - define_method("core#set_postexe") { |&block| END { block.call } } - - define_method("core#undef_method") do |clazz, name| - clazz.undef_method(name) - end - end - - FROZEN_CORE = FrozenCore.new.freeze - - extend Forwardable - - attr_reader :stack - def_delegators :stack, :push, :pop - - attr_reader :frame - def_delegators :frame, :_self - - def initialize - @stack = [] - @frame = nil - end - - ########################################################################## - # Helper methods for frames - ########################################################################## - - def run_frame(frame) - # First, set the current frame to the given value. - @frame = frame - - # Next, set up the local table for the frame. This is actually incorrect - # as it could use the values already on the stack, but for now we're - # just doing this for simplicity. - frame.iseq.local_table.size.times { push(nil) } - - # Yield so that some frame-specific setup can be done. - yield if block_given? - - # This hash is going to hold a mapping of label names to their - # respective indices in our instruction list. - labels = {} - - # This array is going to hold our instructions. - insns = [] - - # Here we're going to preprocess the instruction list from the - # instruction sequence to set up the labels hash and the insns array. - frame.iseq.insns.each do |insn| - case insn - when Integer, Symbol - # skip - when InstructionSequence::Label - labels[insn.name] = insns.length - else - insns << insn - end - end - - # Finally we can execute the instructions one at a time. If they return - # jumps or leaves we will handle those appropriately. - pc = 0 - while pc < insns.length - insn = insns[pc] - pc += 1 - - case (result = insn.call(self)) - when Jump - pc = labels[result.name] - when Leave - return result.value - end - end - ensure - @stack = stack[0...frame.stack_index] - @frame = frame.parent - end - - def run_top_frame(iseq) - run_frame(TopFrame.new(iseq)) - end - - def run_block_frame(iseq, *args, &block) - run_frame(BlockFrame.new(iseq, frame, stack.length)) do - locals = [*args, block] - iseq.local_table.size.times do |index| - local_set(index, 0, locals.shift) - end - end - end - - def run_class_frame(iseq, clazz) - run_frame(ClassFrame.new(iseq, frame, stack.length, clazz)) - end - - def run_method_frame(name, iseq, _self, *args, **kwargs, &block) - run_frame( - MethodFrame.new(iseq, frame, stack.length, _self, name, block) - ) do - locals = [*args, block] - - if iseq.argument_options[:keyword] - # First, set up the keyword bits array. - keyword_bits = - iseq.argument_options[:keyword].map do |config| - kwargs.key?(config.is_a?(Array) ? config[0] : config) - end - - iseq.local_table.locals.each_with_index do |local, index| - # If this is the keyword bits local, then set it appropriately. - if local.name == 2 - locals.insert(index, keyword_bits) - next - end - - # First, find the configuration for this local in the keywords - # list if it exists. - name = local.name - config = - iseq.argument_options[:keyword].find do |keyword| - keyword.is_a?(Array) ? keyword[0] == name : keyword == name - end - - # If the configuration doesn't exist, then the local is not a - # keyword local. - next unless config - - if !config.is_a?(Array) - # required keyword - locals.insert(index, kwargs.fetch(name)) - elsif !config[1].nil? - # optional keyword with embedded default value - locals.insert(index, kwargs.fetch(name, config[1])) - else - # optional keyword with expression default value - locals.insert(index, nil) - end - end - end - - iseq.local_table.size.times do |index| - local_set(index, 0, locals.shift) - end - end - end - - ########################################################################## - # Helper methods for instructions - ########################################################################## - - def const_base - frame.nesting.last - end - - def frame_at(level) - current = frame - level.times { current = current.parent } - current - end - - def frame_svar - current = frame - current = current.parent while current.is_a?(BlockFrame) - current - end - - def frame_yield - current = frame - current = current.parent until current.is_a?(MethodFrame) - current - end - - def frozen_core - FROZEN_CORE - end - - def jump(label) - Jump.new(label.name) - end - - def leave - Leave.new(pop) - end - - def local_get(index, level) - stack[frame_at(level).stack_index + index] - end - - def local_set(index, level, value) - stack[frame_at(level).stack_index + index] = value - end - end - # Compile the given source into a YARV instruction sequence. def self.compile(source, options = Compiler::Options.new) SyntaxTree.parse(source).accept(Compiler.new(options)) diff --git a/lib/syntax_tree/yarv/assembler.rb b/lib/syntax_tree/yarv/assembler.rb index efb179c1..ec467b58 100644 --- a/lib/syntax_tree/yarv/assembler.rb +++ b/lib/syntax_tree/yarv/assembler.rb @@ -69,7 +69,7 @@ def initialize(filepath) end def assemble - iseq = InstructionSequence.new(:top, "
", nil, Location.default) + iseq = InstructionSequence.new("
", "", 1, :top) assemble_iseq(iseq, File.readlines(filepath, chomp: true)) iseq.compile! @@ -138,7 +138,7 @@ def assemble_iseq(iseq, lines) name = parse_symbol(name_value) flags = parse_number(flags_value) - class_iseq = iseq.class_child_iseq(name.to_s, Location.default) + class_iseq = iseq.class_child_iseq(name.to_s, 1) assemble_iseq(class_iseq, body) iseq.defineclass(name, class_iseq, flags) when "defined" @@ -153,7 +153,7 @@ def assemble_iseq(iseq, lines) line_index += body.length name = parse_symbol(operands) - method_iseq = iseq.method_child_iseq(name.to_s, Location.default) + method_iseq = iseq.method_child_iseq(name.to_s, 1) assemble_iseq(method_iseq, body) iseq.definemethod(name, method_iseq) @@ -162,7 +162,7 @@ def assemble_iseq(iseq, lines) line_index += body.length name = parse_symbol(operands) - method_iseq = iseq.method_child_iseq(name.to_s, Location.default) + method_iseq = iseq.method_child_iseq(name.to_s, 1) assemble_iseq(method_iseq, body) iseq.definesmethod(name, method_iseq) @@ -221,7 +221,7 @@ def assemble_iseq(iseq, lines) body = parse_nested(lines[line_index..]) line_index += body.length - block_iseq = iseq.block_child_iseq(Location.default) + block_iseq = iseq.block_child_iseq(1) assemble_iseq(block_iseq, body) block_iseq end @@ -249,7 +249,7 @@ def assemble_iseq(iseq, lines) body = parse_nested(lines[line_index..]) line_index += body.length - block_iseq = iseq.block_child_iseq(Location.default) + block_iseq = iseq.block_child_iseq(1) assemble_iseq(block_iseq, body) block_iseq end @@ -354,7 +354,7 @@ def assemble_iseq(iseq, lines) body = parse_nested(lines[line_index..]) line_index += body.length - block_iseq = iseq.block_child_iseq(Location.default) + block_iseq = iseq.block_child_iseq(1) assemble_iseq(block_iseq, body) block_iseq end diff --git a/lib/syntax_tree/yarv/bf.rb b/lib/syntax_tree/yarv/bf.rb index f642fb2f..21bc2982 100644 --- a/lib/syntax_tree/yarv/bf.rb +++ b/lib/syntax_tree/yarv/bf.rb @@ -13,7 +13,7 @@ def initialize(source) def compile # Set up the top-level instruction sequence that will be returned. - iseq = InstructionSequence.new(:top, "", nil, location) + iseq = InstructionSequence.new("", "", 1, :top) # Set up the $tape global variable that will hold our state. iseq.duphash({ 0 => 0 }) @@ -80,19 +80,6 @@ def compile private - # This is the location of the top instruction sequence, derived from the - # source string. - def location - Location.new( - start_line: 1, - start_char: 0, - start_column: 0, - end_line: source.count("\n") + 1, - end_char: source.size, - end_column: source.size - (source.rindex("\n") || 0) - 1 - ) - end - # $tape[$cursor] += value def change_by(iseq, value) iseq.getglobal(:$tape) @@ -111,6 +98,7 @@ def change_by(iseq, value) end iseq.send(YARV.calldata(:[]=, 2)) + iseq.pop end # $cursor += value @@ -138,6 +126,7 @@ def output_char(iseq) iseq.send(YARV.calldata(:chr)) iseq.send(YARV.calldata(:putc, 1)) + iseq.pop end # $tape[$cursor] = $stdin.getc.ord @@ -150,6 +139,7 @@ def input_char(iseq) iseq.send(YARV.calldata(:ord)) iseq.send(YARV.calldata(:[]=, 2)) + iseq.pop end # unless $tape[$cursor] == 0 @@ -164,14 +154,21 @@ def loop_start(iseq) iseq.putobject(0) iseq.send(YARV.calldata(:==, 1)) - iseq.branchunless(end_label) + iseq.branchif(end_label) [start_label, end_label] end # Jump back to the start of the loop. def loop_end(iseq, start_label, end_label) - iseq.jump(start_label) + iseq.getglobal(:$tape) + iseq.getglobal(:$cursor) + iseq.send(YARV.calldata(:[], 1)) + + iseq.putobject(0) + iseq.send(YARV.calldata(:==, 1)) + iseq.branchunless(start_label) + iseq.push(end_label) end end diff --git a/lib/syntax_tree/yarv/compiler.rb b/lib/syntax_tree/yarv/compiler.rb index 4af5d6f0..4c9a4d50 100644 --- a/lib/syntax_tree/yarv/compiler.rb +++ b/lib/syntax_tree/yarv/compiler.rb @@ -304,10 +304,11 @@ def visit_CHAR(node) end def visit_END(node) + start_line = node.location.start_line once_iseq = - with_child_iseq(iseq.block_child_iseq(node.location)) do + with_child_iseq(iseq.block_child_iseq(start_line)) do postexe_iseq = - with_child_iseq(iseq.block_child_iseq(node.location)) do + with_child_iseq(iseq.block_child_iseq(start_line)) do iseq.event(:RUBY_EVENT_B_CALL) *statements, last_statement = node.statements.body @@ -567,7 +568,7 @@ def visit_binary(node) end def visit_block(node) - with_child_iseq(iseq.block_child_iseq(node.location)) do + with_child_iseq(iseq.block_child_iseq(node.location.start_line)) do iseq.event(:RUBY_EVENT_B_CALL) visit(node.block_var) visit(node.bodystmt) @@ -751,7 +752,9 @@ def visit_case(node) def visit_class(node) name = node.constant.constant.value.to_sym class_iseq = - with_child_iseq(iseq.class_child_iseq(name, node.location)) do + with_child_iseq( + iseq.class_child_iseq(name, node.location.start_line) + ) do iseq.event(:RUBY_EVENT_CLASS) visit(node.bodystmt) iseq.event(:RUBY_EVENT_END) @@ -818,7 +821,8 @@ def visit_const_path_ref(node) def visit_def(node) name = node.name.value.to_sym - method_iseq = iseq.method_child_iseq(name.to_s, node.location) + method_iseq = + iseq.method_child_iseq(name.to_s, node.location.start_line) with_child_iseq(method_iseq) do visit(node.params) if node.params @@ -939,7 +943,9 @@ def visit_for(node) iseq.local_table.plain(name) block_iseq = - with_child_iseq(iseq.block_child_iseq(node.statements.location)) do + with_child_iseq( + iseq.block_child_iseq(node.statements.location.start_line) + ) do iseq.argument_options[:lead_num] ||= 0 iseq.argument_options[:lead_num] += 1 iseq.argument_options[:ambiguous_param0] = true @@ -1076,7 +1082,7 @@ def visit_label(node) def visit_lambda(node) lambda_iseq = - with_child_iseq(iseq.block_child_iseq(node.location)) do + with_child_iseq(iseq.block_child_iseq(node.location.start_line)) do iseq.event(:RUBY_EVENT_B_CALL) visit(node.params) visit(node.statements) @@ -1127,7 +1133,9 @@ def visit_mlhs(node) def visit_module(node) name = node.constant.constant.value.to_sym module_iseq = - with_child_iseq(iseq.module_child_iseq(name, node.location)) do + with_child_iseq( + iseq.module_child_iseq(name, node.location.start_line) + ) do iseq.event(:RUBY_EVENT_CLASS) visit(node.bodystmt) iseq.event(:RUBY_EVENT_END) @@ -1375,10 +1383,11 @@ def visit_program(node) top_iseq = InstructionSequence.new( - :top, "", + "", + 1, + :top, nil, - node.location, options ) @@ -1543,7 +1552,9 @@ def visit_sclass(node) iseq.putnil singleton_iseq = - with_child_iseq(iseq.singleton_class_child_iseq(node.location)) do + with_child_iseq( + iseq.singleton_class_child_iseq(node.location.start_line) + ) do iseq.event(:RUBY_EVENT_CLASS) visit(node.bodystmt) iseq.event(:RUBY_EVENT_END) @@ -2018,7 +2029,7 @@ def visit_pattern(node, end_label) if node.constant iseq.dup visit(node.constant) - iseq.checkmatch(CheckMatch::TYPE_CASE) + iseq.checkmatch(CheckMatch::VM_CHECKMATCH_TYPE_CASE) iseq.branchunless(match_failure_label) end @@ -2078,7 +2089,7 @@ def visit_pattern(node, end_label) iseq.setlocal(lookup.index, lookup.level) else visit(required) - iseq.checkmatch(CheckMatch::TYPE_CASE) + iseq.checkmatch(CheckMatch::VM_CHECKMATCH_TYPE_CASE) iseq.branchunless(match_failure_label) end diff --git a/lib/syntax_tree/yarv/decompiler.rb b/lib/syntax_tree/yarv/decompiler.rb index a6a567fb..47d2a2df 100644 --- a/lib/syntax_tree/yarv/decompiler.rb +++ b/lib/syntax_tree/yarv/decompiler.rb @@ -64,6 +64,13 @@ def decompile(iseq) clauses[label] = clause clause = [] label = insn.name + when BranchIf + body = [ + Assign(block_label.field, node_for(insn.label.name)), + Next(Args([])) + ] + + clause << UnlessNode(clause.pop, Statements(body), nil) when BranchUnless body = [ Assign(block_label.field, node_for(insn.label.name)), @@ -157,6 +164,8 @@ def decompile(iseq) ) end end + when Pop + # skip when PutObject case insn.object when Float diff --git a/lib/syntax_tree/yarv/disassembler.rb b/lib/syntax_tree/yarv/disassembler.rb index 033b6d3d..d303bcb7 100644 --- a/lib/syntax_tree/yarv/disassembler.rb +++ b/lib/syntax_tree/yarv/disassembler.rb @@ -4,7 +4,8 @@ module SyntaxTree module YARV class Disassembler attr_reader :output, :queue - attr_reader :current_prefix, :current_iseq + attr_reader :current_prefix + attr_accessor :current_iseq def initialize @output = StringIO.new @@ -114,7 +115,7 @@ def format_iseq(iseq) output << "#{current_prefix}== disasm: " output << "#:1 " - location = iseq.location + location = Location.fixed(line: iseq.line, char: 0, column: 0) output << "(#{location.start_line},#{location.start_column})-" output << "(#{location.end_line},#{location.end_column})" output << "> " diff --git a/lib/syntax_tree/yarv/instruction_sequence.rb b/lib/syntax_tree/yarv/instruction_sequence.rb index 48305be6..c284221b 100644 --- a/lib/syntax_tree/yarv/instruction_sequence.rb +++ b/lib/syntax_tree/yarv/instruction_sequence.rb @@ -116,18 +116,18 @@ def inspect end end - # The type of the instruction sequence. - attr_reader :type - # The name of the instruction sequence. attr_reader :name + # The source location of the instruction sequence. + attr_reader :file, :line + + # The type of the instruction sequence. + attr_reader :type + # The parent instruction sequence, if there is one. attr_reader :parent_iseq - # The location of the root node of this instruction sequence. - attr_reader :location - # This is the list of information about the arguments to this # instruction sequence. attr_accessor :argument_size @@ -157,16 +157,18 @@ def inspect attr_reader :options def initialize( - type, name, - parent_iseq, - location, + file, + line, + type, + parent_iseq = nil, options = Compiler::Options.new ) - @type = type @name = name + @file = file + @line = line + @type = type @parent_iseq = parent_iseq - @location = location @argument_size = 0 @argument_options = {} @@ -256,9 +258,9 @@ def to_a node_ids: [-1] * insns.length }, name, + file, "", - "", - location.start_line, + line, type, local_table.names, dumped_options, @@ -278,6 +280,12 @@ def disasm def compile! specialize_instructions! if options.specialized_instruction? + catch_table.each do |catch_entry| + if !catch_entry.is_a?(CatchBreak) && catch_entry.iseq + catch_entry.iseq.compile! + end + end + length = 0 insns.each do |insn| case insn @@ -416,30 +424,30 @@ def specialize_instructions! # Child instruction sequence methods ########################################################################## - def child_iseq(type, name, location) - InstructionSequence.new(type, name, self, location, options) + def child_iseq(name, line, type) + InstructionSequence.new(name, file, line, type, self, options) end - def block_child_iseq(location) + def block_child_iseq(line) current = self current = current.parent_iseq while current.type == :block - child_iseq(:block, "block in #{current.name}", location) + child_iseq("block in #{current.name}", line, :block) end - def class_child_iseq(name, location) - child_iseq(:class, "", location) + def class_child_iseq(name, line) + child_iseq("", line, :class) end - def method_child_iseq(name, location) - child_iseq(:method, name, location) + def method_child_iseq(name, line) + child_iseq(name, line, :method) end - def module_child_iseq(name, location) - child_iseq(:class, "", location) + def module_child_iseq(name, line) + child_iseq("", line, :class) end - def singleton_class_child_iseq(location) - child_iseq(:class, "singleton class", location) + def singleton_class_child_iseq(line) + child_iseq("singleton class", line, :class) end ########################################################################## @@ -447,19 +455,39 @@ def singleton_class_child_iseq(location) ########################################################################## class CatchEntry - attr_reader :iseq, :begin_label, :end_label, :exit_label + attr_reader :iseq, :begin_label, :end_label, :exit_label, :restore_sp - def initialize(iseq, begin_label, end_label, exit_label) + def initialize(iseq, begin_label, end_label, exit_label, restore_sp) @iseq = iseq @begin_label = begin_label @end_label = end_label @exit_label = exit_label + @restore_sp = restore_sp end end class CatchBreak < CatchEntry def to_a - [:break, iseq.to_a, begin_label.name, end_label.name, exit_label.name] + [ + :break, + iseq.to_a, + begin_label.name, + end_label.name, + exit_label.name, + restore_sp + ] + end + end + + class CatchEnsure < CatchEntry + def to_a + [ + :ensure, + iseq.to_a, + begin_label.name, + end_label.name, + exit_label.name + ] end end @@ -493,24 +521,64 @@ def to_a end end - def catch_break(iseq, begin_label, end_label, exit_label) - catch_table << CatchBreak.new(iseq, begin_label, end_label, exit_label) - end - - def catch_next(begin_label, end_label, exit_label) - catch_table << CatchNext.new(nil, begin_label, end_label, exit_label) - end - - def catch_redo(begin_label, end_label, exit_label) - catch_table << CatchRedo.new(nil, begin_label, end_label, exit_label) - end - - def catch_rescue(iseq, begin_label, end_label, exit_label) - catch_table << CatchRescue.new(iseq, begin_label, end_label, exit_label) - end - - def catch_retry(begin_label, end_label, exit_label) - catch_table << CatchRetry.new(nil, begin_label, end_label, exit_label) + def catch_break(iseq, begin_label, end_label, exit_label, restore_sp) + catch_table << CatchBreak.new( + iseq, + begin_label, + end_label, + exit_label, + restore_sp + ) + end + + def catch_ensure(iseq, begin_label, end_label, exit_label, restore_sp) + catch_table << CatchEnsure.new( + iseq, + begin_label, + end_label, + exit_label, + restore_sp + ) + end + + def catch_next(begin_label, end_label, exit_label, restore_sp) + catch_table << CatchNext.new( + nil, + begin_label, + end_label, + exit_label, + restore_sp + ) + end + + def catch_redo(begin_label, end_label, exit_label, restore_sp) + catch_table << CatchRedo.new( + nil, + begin_label, + end_label, + exit_label, + restore_sp + ) + end + + def catch_rescue(iseq, begin_label, end_label, exit_label, restore_sp) + catch_table << CatchRescue.new( + iseq, + begin_label, + end_label, + exit_label, + restore_sp + ) + end + + def catch_retry(begin_label, end_label, exit_label, restore_sp) + catch_table << CatchRetry.new( + nil, + begin_label, + end_label, + exit_label, + restore_sp + ) end ########################################################################## @@ -895,7 +963,8 @@ def toregexp(options, length) # This method will create a new instruction sequence from a serialized # RubyVM::InstructionSequence object. def self.from(source, options = Compiler::Options.new, parent_iseq = nil) - iseq = new(source[9], source[5], parent_iseq, Location.default, options) + iseq = + new(source[5], source[6], source[8], source[9], parent_iseq, options) # set up the labels object so that the labels are shared between the # location in the instruction sequence and the instructions that @@ -914,45 +983,9 @@ def self.from(source, options = Compiler::Options.new, parent_iseq = nil) iseq.argument_options[:opt].map! { |opt| labels[opt] } end - # set up the catch table - source[12].each do |entry| - case entry[0] - when :break - iseq.catch_break( - from(entry[1]), - labels[entry[2]], - labels[entry[3]], - labels[entry[4]] - ) - when :next - iseq.catch_next( - labels[entry[2]], - labels[entry[3]], - labels[entry[4]] - ) - when :rescue - iseq.catch_rescue( - from(entry[1]), - labels[entry[2]], - labels[entry[3]], - labels[entry[4]] - ) - when :redo - iseq.catch_redo( - labels[entry[2]], - labels[entry[3]], - labels[entry[4]] - ) - when :retry - iseq.catch_retry( - labels[entry[2]], - labels[entry[3]], - labels[entry[4]] - ) - else - raise "unknown catch type: #{entry[0]}" - end - end + # track the child block iseqs so that our catch table can point to the + # correctly created iseqs + block_iseqs = [] # set up all of the instructions source[13].each do |insn| @@ -1135,6 +1168,7 @@ def self.from(source, options = Compiler::Options.new, parent_iseq = nil) iseq.putspecialobject(opnds[0]) when :send block_iseq = opnds[1] ? from(opnds[1], options, iseq) : nil + block_iseqs << block_iseq if block_iseq iseq.send(CallData.from(opnds[0]), block_iseq) when :setclassvariable iseq.push(SetClassVariable.new(opnds[0], opnds[1])) @@ -1163,6 +1197,76 @@ def self.from(source, options = Compiler::Options.new, parent_iseq = nil) end end + # set up the catch table + source[12].each do |entry| + case entry[0] + when :break + if entry[1] + break_iseq = + block_iseqs.find do |block_iseq| + block_iseq.name == entry[1][5] && + block_iseq.file == entry[1][6] && + block_iseq.line == entry[1][8] + end + + iseq.catch_break( + break_iseq || from(entry[1], options, iseq), + labels[entry[2]], + labels[entry[3]], + labels[entry[4]], + entry[5] + ) + else + iseq.catch_break( + nil, + labels[entry[2]], + labels[entry[3]], + labels[entry[4]], + entry[5] + ) + end + when :ensure + iseq.catch_ensure( + from(entry[1], options, iseq), + labels[entry[2]], + labels[entry[3]], + labels[entry[4]], + entry[5] + ) + when :next + iseq.catch_next( + labels[entry[2]], + labels[entry[3]], + labels[entry[4]], + entry[5] + ) + when :rescue + iseq.catch_rescue( + from(entry[1], options, iseq), + labels[entry[2]], + labels[entry[3]], + labels[entry[4]], + entry[5] + ) + when :redo + iseq.catch_redo( + labels[entry[2]], + labels[entry[3]], + labels[entry[4]], + entry[5] + ) + when :retry + iseq.catch_retry( + labels[entry[2]], + labels[entry[3]], + labels[entry[4]], + entry[5] + ) + else + raise "unknown catch type: #{entry[0]}" + end + end + iseq.compile! if iseq.type == :top iseq end diff --git a/lib/syntax_tree/yarv/instructions.rb b/lib/syntax_tree/yarv/instructions.rb index 288edb16..5e1d116b 100644 --- a/lib/syntax_tree/yarv/instructions.rb +++ b/lib/syntax_tree/yarv/instructions.rb @@ -399,9 +399,11 @@ def call(vm) # ~~~ # class CheckMatch - TYPE_WHEN = 1 - TYPE_CASE = 2 - TYPE_RESCUE = 3 + VM_CHECKMATCH_TYPE_WHEN = 1 + VM_CHECKMATCH_TYPE_CASE = 2 + VM_CHECKMATCH_TYPE_RESCUE = 3 + VM_CHECKMATCH_TYPE_MASK = 0x03 + VM_CHECKMATCH_ARRAY = 0x04 attr_reader :type @@ -434,7 +436,32 @@ def canonical end def call(vm) - raise NotImplementedError, "checkmatch" + target, pattern = vm.pop(2) + + vm.push( + if type & VM_CHECKMATCH_ARRAY > 0 + pattern.any? { |item| check?(item, target) } + else + check?(pattern, target) + end + ) + end + + private + + def check?(pattern, target) + case type & VM_CHECKMATCH_TYPE_MASK + when VM_CHECKMATCH_TYPE_WHEN + pattern + when VM_CHECKMATCH_TYPE_CASE + pattern === target + when VM_CHECKMATCH_TYPE_RESCUE + unless pattern.is_a?(Module) + raise TypeError, "class or module required for rescue clause" + end + + pattern === target + end end end @@ -762,12 +789,26 @@ def canonical def call(vm) object, superclass = vm.pop(2) - iseq = class_iseq - clazz = Class.new(superclass || Object) - vm.push(vm.run_class_frame(iseq, clazz)) + if name == :singletonclass + vm.push(vm.run_class_frame(class_iseq, object.singleton_class)) + elsif object.const_defined?(name) + vm.push(vm.run_class_frame(class_iseq, object.const_get(name))) + elsif flags & TYPE_MODULE > 0 + clazz = Module.new + object.const_set(name, clazz) + vm.push(vm.run_class_frame(class_iseq, clazz)) + else + clazz = + if flags & FLAG_HAS_SUPERCLASS > 0 + Class.new(superclass) + else + Class.new + end - object.const_set(name, clazz) + object.const_set(name, clazz) + vm.push(vm.run_class_frame(class_iseq, clazz)) + end end end @@ -882,17 +923,19 @@ def call(vm) when TYPE_NIL, TYPE_SELF, TYPE_TRUE, TYPE_FALSE, TYPE_ASGN, TYPE_EXPR message when TYPE_IVAR - message if vm._self.instance_variable_defined?(name) + message if vm.frame._self.instance_variable_defined?(name) when TYPE_LVAR raise NotImplementedError, "defined TYPE_LVAR" when TYPE_GVAR message if global_variables.include?(name) when TYPE_CVAR - clazz = vm._self + clazz = vm.frame._self clazz = clazz.singleton_class unless clazz.is_a?(Module) message if clazz.class_variable_defined?(name) when TYPE_CONST - raise NotImplementedError, "defined TYPE_CONST" + clazz = vm.frame._self + clazz = clazz.singleton_class unless clazz.is_a?(Module) + message if clazz.const_defined?(name) when TYPE_METHOD raise NotImplementedError, "defined TYPE_METHOD" when TYPE_YIELD @@ -904,7 +947,9 @@ def call(vm) when TYPE_FUNC message if object.respond_to?(name, true) when TYPE_CONST_FROM - raise NotImplementedError, "defined TYPE_CONST_FROM" + defined = + vm.frame.nesting.any? { |scope| scope.const_defined?(name, true) } + message if defined end vm.push(result) @@ -962,12 +1007,22 @@ def canonical def call(vm) name = method_name + nesting = vm.frame.nesting iseq = method_iseq vm + .frame ._self .__send__(:define_method, name) do |*args, **kwargs, &block| - vm.run_method_frame(name, iseq, self, *args, **kwargs, &block) + vm.run_method_frame( + name, + nesting, + iseq, + self, + *args, + **kwargs, + &block + ) end end end @@ -1024,12 +1079,22 @@ def canonical def call(vm) name = method_name + nesting = vm.frame.nesting iseq = method_iseq vm + .frame ._self .__send__(:define_singleton_method, name) do |*args, **kwargs, &block| - vm.run_method_frame(name, iseq, self, *args, **kwargs, &block) + vm.run_method_frame( + name, + nesting, + iseq, + self, + *args, + **kwargs, + &block + ) end end end @@ -1259,7 +1324,42 @@ def canonical end def call(vm) - raise NotImplementedError, "expandarray" + object = vm.pop + object = + if Array === object + object.dup + elsif object.respond_to?(:to_ary, true) + object.to_ary + else + [object] + end + + splat_flag = flags & 0x01 > 0 + postarg_flag = flags & 0x02 > 0 + + if number == 0 && splat_flag == 0 + # no space left on stack + elsif postarg_flag + values = [] + + if number > object.size + (number - object.size).times { values.push(nil) } + end + [number, object.size].min.times { values.push(object.pop) } + values.push(object.to_a) if splat_flag + + values.each { |item| vm.push(item) } + else + values = [] + + [number, object.size].min.times { values.push(object.shift) } + if number > values.size + (number - values.size).times { values.push(nil) } + end + values.push(object.to_a) if splat_flag + + values.reverse_each { |item| vm.push(item) } + end end end @@ -1424,7 +1524,7 @@ def canonical end def call(vm) - clazz = vm._self + clazz = vm.frame._self clazz = clazz.class unless clazz.is_a?(Class) vm.push(clazz.class_variable_get(name)) end @@ -1474,14 +1574,20 @@ def canonical end def call(vm) - # const_base, allow_nil = - vm.pop(2) + const_base, allow_nil = vm.pop(2) - vm.frame.nesting.reverse_each do |clazz| - if clazz.const_defined?(name) - vm.push(clazz.const_get(name)) + if const_base + if const_base.const_defined?(name) + vm.push(const_base.const_get(name)) return end + elsif const_base.nil? && allow_nil + vm.frame.nesting.reverse_each do |clazz| + if clazz.const_defined?(name) + vm.push(clazz.const_get(name)) + return + end + end end raise NameError, "uninitialized constant #{name}" @@ -1590,7 +1696,7 @@ def canonical def call(vm) method = Object.instance_method(:instance_variable_get) - vm.push(method.bind(vm._self).call(name)) + vm.push(method.bind(vm.frame._self).call(name)) end end @@ -1948,8 +2054,9 @@ def canonical def call(vm) block = if (iseq = block_iseq) + frame = vm.frame ->(*args, **kwargs, &blk) do - vm.run_block_frame(iseq, *args, **kwargs, &blk) + vm.run_block_frame(iseq, frame, *args, **kwargs, &blk) end end @@ -2396,7 +2503,7 @@ def canonical def call(vm) return if @executed - vm.push(vm.run_block_frame(iseq)) + vm.push(vm.run_block_frame(iseq, vm.frame)) @executed = true end end @@ -2960,7 +3067,7 @@ def canonical end def call(vm) - current = vm._self + current = vm.frame._self current = current.class unless current.is_a?(Class) names.each do |name| @@ -4254,7 +4361,7 @@ def canonical end def call(vm) - vm.push(vm._self) + vm.push(vm.frame._self) end end @@ -4310,7 +4417,7 @@ def call(vm) when OBJECT_VMCORE vm.push(vm.frozen_core) when OBJECT_CBASE - value = vm._self + value = vm.frame._self value = value.singleton_class unless value.is_a?(Class) vm.push(value) when OBJECT_CONST_BASE @@ -4418,9 +4525,12 @@ def canonical def call(vm) block = if (iseq = block_iseq) + frame = vm.frame ->(*args, **kwargs, &blk) do - vm.run_block_frame(iseq, *args, **kwargs, &blk) + vm.run_block_frame(iseq, frame, *args, **kwargs, &blk) end + elsif calldata.flag?(CallData::CALL_ARGS_BLOCKARG) + vm.pop end keywords = @@ -4542,7 +4652,7 @@ def canonical end def call(vm) - clazz = vm._self + clazz = vm.frame._self clazz = clazz.class unless clazz.is_a?(Class) clazz.class_variable_set(name, vm.pop) end @@ -4698,7 +4808,7 @@ def canonical def call(vm) method = Object.instance_method(:instance_variable_set) - method.bind(vm._self).call(name, vm.pop) + method.bind(vm.frame._self).call(name, vm.pop) end end @@ -4946,7 +5056,7 @@ def canonical def call(vm) case key when GetSpecial::SVAR_LASTLINE - raise NotImplementedError, "svar SVAR_LASTLINE" + raise NotImplementedError, "setspecial SVAR_LASTLINE" when GetSpecial::SVAR_BACKREF raise NotImplementedError, "setspecial SVAR_BACKREF" when GetSpecial::SVAR_FLIPFLOP_START @@ -4999,7 +5109,27 @@ def canonical end def call(vm) - vm.push(*vm.pop) + value = vm.pop + + vm.push( + if Array === value + value.instance_of?(Array) ? value.dup : Array[*value] + elsif value.nil? + value.to_a + else + if value.respond_to?(:to_a, true) + result = value.to_a + + if result.nil? + [value] + elsif !result.is_a?(Array) + raise TypeError, "expected to_a to return an Array" + end + else + [value] + end + end + ) end end @@ -5061,15 +5191,18 @@ def call(vm) # ~~~ # class Throw - TAG_NONE = 0x0 - TAG_RETURN = 0x1 - TAG_BREAK = 0x2 - TAG_NEXT = 0x3 - TAG_RETRY = 0x4 - TAG_REDO = 0x5 - TAG_RAISE = 0x6 - TAG_THROW = 0x7 - TAG_FATAL = 0x8 + RUBY_TAG_NONE = 0x0 + RUBY_TAG_RETURN = 0x1 + RUBY_TAG_BREAK = 0x2 + RUBY_TAG_NEXT = 0x3 + RUBY_TAG_RETRY = 0x4 + RUBY_TAG_REDO = 0x5 + RUBY_TAG_RAISE = 0x6 + RUBY_TAG_THROW = 0x7 + RUBY_TAG_FATAL = 0x8 + + VM_THROW_NO_ESCAPE_FLAG = 0x8000 + VM_THROW_STATE_MASK = 0xff attr_reader :type @@ -5102,7 +5235,43 @@ def canonical end def call(vm) - raise NotImplementedError, "throw" + state = type & VM_THROW_STATE_MASK + value = vm.pop + + case state + when RUBY_TAG_NONE + case value + when nil + # do nothing + when Exception + raise value + else + raise NotImplementedError + end + when RUBY_TAG_RETURN + raise VM::ReturnError.new(value, error_backtrace(vm)) + when RUBY_TAG_BREAK + raise VM::BreakError.new(value, error_backtrace(vm)) + when RUBY_TAG_NEXT + raise VM::NextError.new(value, error_backtrace(vm)) + else + raise NotImplementedError, "Unknown throw kind #{state}" + end + end + + private + + def error_backtrace(vm) + backtrace = [] + current = vm.frame + + while current + backtrace << "#{current.iseq.file}:#{current.line}:in" \ + "`#{current.iseq.name}'" + current = current.parent + end + + [*backtrace, *caller] end end diff --git a/lib/syntax_tree/yarv/legacy.rb b/lib/syntax_tree/yarv/legacy.rb index 30a95437..b2e33290 100644 --- a/lib/syntax_tree/yarv/legacy.rb +++ b/lib/syntax_tree/yarv/legacy.rb @@ -45,6 +45,14 @@ def pops def pushes 1 end + + def canonical + YARV::GetClassVariable.new(name, nil) + end + + def call(vm) + canonical.call(vm) + end end # ### Summary @@ -94,6 +102,10 @@ def pushes 1 end + def canonical + self + end + def call(vm) vm.push(nil) end @@ -102,8 +114,8 @@ def call(vm) # ### Summary # # `opt_setinlinecache` sets an inline cache for a constant lookup. It pops - # the value it should set off the top of the stack. It then pushes that - # value back onto the top of the stack. + # the value it should set off the top of the stack. It uses this value to + # set the cache. It then pushes that value back onto the top of the stack. # # This instruction is no longer used since in Ruby 3.2 it was replaced by # the consolidated `opt_getconstant_path` instruction. @@ -141,8 +153,11 @@ def pushes 1 end + def canonical + self + end + def call(vm) - vm.push(vm.pop) end end @@ -186,6 +201,14 @@ def pops def pushes 0 end + + def canonical + YARV::SetClassVariable.new(name, nil) + end + + def call(vm) + canonical.call(vm) + end end end end diff --git a/lib/syntax_tree/yarv/vm.rb b/lib/syntax_tree/yarv/vm.rb new file mode 100644 index 00000000..1bbb82ed --- /dev/null +++ b/lib/syntax_tree/yarv/vm.rb @@ -0,0 +1,624 @@ +# frozen_string_literal: true + +require "forwardable" + +module SyntaxTree + # This module provides an object representation of the YARV bytecode. + module YARV + class VM + class Jump + attr_reader :label + + def initialize(label) + @label = label + end + end + + class Leave + attr_reader :value + + def initialize(value) + @value = value + end + end + + class Frame + attr_reader :iseq, :parent, :stack_index, :_self, :nesting, :svars + attr_accessor :line, :pc + + def initialize(iseq, parent, stack_index, _self, nesting) + @iseq = iseq + @parent = parent + @stack_index = stack_index + @_self = _self + @nesting = nesting + + @svars = {} + @line = iseq.line + @pc = 0 + end + end + + class TopFrame < Frame + def initialize(iseq) + super(iseq, nil, 0, TOPLEVEL_BINDING.eval("self"), [Object]) + end + end + + class BlockFrame < Frame + def initialize(iseq, parent, stack_index) + super(iseq, parent, stack_index, parent._self, parent.nesting) + end + end + + class MethodFrame < Frame + attr_reader :name, :block + + def initialize(iseq, nesting, parent, stack_index, _self, name, block) + super(iseq, parent, stack_index, _self, nesting) + @name = name + @block = block + end + end + + class ClassFrame < Frame + def initialize(iseq, parent, stack_index, _self) + super(iseq, parent, stack_index, _self, parent.nesting + [_self]) + end + end + + class RescueFrame < Frame + def initialize(iseq, parent, stack_index) + super(iseq, parent, stack_index, parent._self, parent.nesting) + end + end + + class ThrownError < StandardError + attr_reader :value + + def initialize(value, backtrace) + super("This error was thrown by the Ruby VM.") + @value = value + set_backtrace(backtrace) + end + end + + class ReturnError < ThrownError + end + + class BreakError < ThrownError + end + + class NextError < ThrownError + end + + class FrozenCore + define_method("core#hash_merge_kwd") { |left, right| left.merge(right) } + + define_method("core#hash_merge_ptr") do |hash, *values| + hash.merge(values.each_slice(2).to_h) + end + + define_method("core#set_method_alias") do |clazz, new_name, old_name| + clazz.alias_method(new_name, old_name) + end + + define_method("core#set_variable_alias") do |new_name, old_name| + # Using eval here since there isn't a reflection API to be able to + # alias global variables. + eval("alias #{new_name} #{old_name}", binding, __FILE__, __LINE__) + end + + define_method("core#set_postexe") { |&block| END { block.call } } + + define_method("core#undef_method") do |clazz, name| + clazz.undef_method(name) + nil + end + end + + # This is the main entrypoint for events firing in the VM, which allows + # us to implement tracing. + class NullEvents + def publish_frame_change(frame) + end + + def publish_instruction(iseq, insn) + end + + def publish_stack_change(stack) + end + + def publish_tracepoint(event) + end + end + + # This is a simple implementation of tracing that prints to STDOUT. + class STDOUTEvents + attr_reader :disassembler + + def initialize + @disassembler = Disassembler.new + end + + def publish_frame_change(frame) + puts "%-16s %s" % ["frame-change", "#{frame.iseq.file}@#{frame.line}"] + end + + def publish_instruction(iseq, insn) + disassembler.current_iseq = iseq + puts "%-16s %s" % ["instruction", insn.disasm(disassembler)] + end + + def publish_stack_change(stack) + puts "%-16s %s" % ["stack-change", stack.values.inspect] + end + + def publish_tracepoint(event) + puts "%-16s %s" % ["tracepoint", event.inspect] + end + end + + # This represents the global VM stack. It effectively is an array, but + # wraps mutating functions with instrumentation. + class Stack + attr_reader :events, :values + + def initialize(events) + @events = events + @values = [] + end + + def concat(...) + values.concat(...).tap { events.publish_stack_change(self) } + end + + def last + values.last + end + + def length + values.length + end + + def push(...) + values.push(...).tap { events.publish_stack_change(self) } + end + + def pop(...) + values.pop(...).tap { events.publish_stack_change(self) } + end + + def slice!(...) + values.slice!(...).tap { events.publish_stack_change(self) } + end + + def [](...) + values.[](...) + end + + def []=(...) + values.[]=(...).tap { events.publish_stack_change(self) } + end + end + + FROZEN_CORE = FrozenCore.new.freeze + + extend Forwardable + + attr_reader :events + + attr_reader :stack + def_delegators :stack, :push, :pop + + attr_reader :frame + + def initialize(events = NullEvents.new) + @events = events + @stack = Stack.new(events) + @frame = nil + end + + ########################################################################## + # Helper methods for frames + ########################################################################## + + def run_frame(frame) + # First, set the current frame to the given value. + previous = @frame + @frame = frame + events.publish_frame_change(@frame) + + # Next, set up the local table for the frame. This is actually incorrect + # as it could use the values already on the stack, but for now we're + # just doing this for simplicity. + stack.concat(Array.new(frame.iseq.local_table.size)) + + # Yield so that some frame-specific setup can be done. + start_label = yield if block_given? + frame.pc = frame.iseq.insns.index(start_label) if start_label + + # Finally we can execute the instructions one at a time. If they return + # jumps or leaves we will handle those appropriately. + loop do + case (insn = frame.iseq.insns[frame.pc]) + when Integer + frame.line = insn + frame.pc += 1 + when Symbol + events.publish_tracepoint(insn) + frame.pc += 1 + when InstructionSequence::Label + # skip labels + frame.pc += 1 + else + begin + events.publish_instruction(frame.iseq, insn) + result = insn.call(self) + rescue ReturnError => error + raise if frame.iseq.type != :method + + stack.slice!(frame.stack_index..) + @frame = frame.parent + events.publish_frame_change(@frame) + + return error.value + rescue BreakError => error + raise if frame.iseq.type != :block + + catch_entry = + find_catch_entry(frame, InstructionSequence::CatchBreak) + raise unless catch_entry + + stack.slice!( + ( + frame.stack_index + frame.iseq.local_table.size + + catch_entry.restore_sp + ).. + ) + @frame = frame + events.publish_frame_change(@frame) + + frame.pc = frame.iseq.insns.index(catch_entry.exit_label) + push(result = error.value) + rescue NextError => error + raise if frame.iseq.type != :block + + catch_entry = + find_catch_entry(frame, InstructionSequence::CatchNext) + raise unless catch_entry + + stack.slice!( + ( + frame.stack_index + frame.iseq.local_table.size + + catch_entry.restore_sp + ).. + ) + @frame = frame + events.publish_frame_change(@frame) + + frame.pc = frame.iseq.insns.index(catch_entry.exit_label) + push(result = error.value) + rescue Exception => error + catch_entry = + find_catch_entry(frame, InstructionSequence::CatchRescue) + raise unless catch_entry + + stack.slice!( + ( + frame.stack_index + frame.iseq.local_table.size + + catch_entry.restore_sp + ).. + ) + @frame = frame + events.publish_frame_change(@frame) + + frame.pc = frame.iseq.insns.index(catch_entry.exit_label) + push(result = run_rescue_frame(catch_entry.iseq, frame, error)) + end + + case result + when Jump + frame.pc = frame.iseq.insns.index(result.label) + 1 + when Leave + # this shouldn't be necessary, but is because we're not handling + # the stack correctly at the moment + stack.slice!(frame.stack_index..) + + # restore the previous frame + @frame = previous || frame.parent + events.publish_frame_change(@frame) if @frame + + return result.value + else + frame.pc += 1 + end + end + end + end + + def find_catch_entry(frame, type) + iseq = frame.iseq + iseq.catch_table.find do |catch_entry| + next unless catch_entry.is_a?(type) + + begin_pc = iseq.insns.index(catch_entry.begin_label) + end_pc = iseq.insns.index(catch_entry.end_label) + + (begin_pc...end_pc).cover?(frame.pc) + end + end + + def run_top_frame(iseq) + run_frame(TopFrame.new(iseq)) + end + + def run_block_frame(iseq, frame, *args, **kwargs, &block) + run_frame(BlockFrame.new(iseq, frame, stack.length)) do + setup_arguments(iseq, args, kwargs, block) + end + end + + def run_class_frame(iseq, clazz) + run_frame(ClassFrame.new(iseq, frame, stack.length, clazz)) + end + + def run_method_frame(name, nesting, iseq, _self, *args, **kwargs, &block) + run_frame( + MethodFrame.new( + iseq, + nesting, + frame, + stack.length, + _self, + name, + block + ) + ) { setup_arguments(iseq, args, kwargs, block) } + end + + def run_rescue_frame(iseq, frame, error) + run_frame(RescueFrame.new(iseq, frame, stack.length)) do + local_set(0, 0, error) + nil + end + end + + def setup_arguments(iseq, args, kwargs, block) + locals = [*args] + local_index = 0 + start_label = nil + + # First, set up all of the leading arguments. These are positional and + # required arguments at the start of the argument list. + if (lead_num = iseq.argument_options[:lead_num]) + lead_num.times do + local_set(local_index, 0, locals.shift) + local_index += 1 + end + end + + # Next, set up all of the optional arguments. The opt array contains + # the labels that the frame should start at if the optional is + # present. The last element of the array is the label that the frame + # should start at if all of the optional arguments are present. + if (opt = iseq.argument_options[:opt]) + opt[0...-1].each do |label| + if locals.empty? + start_label = label + break + else + local_set(local_index, 0, locals.shift) + local_index += 1 + end + + start_label = opt.last if start_label.nil? + end + end + + # If there is a splat argument, then we'll set that up here. It will + # grab up all of the remaining positional arguments. + if (rest_start = iseq.argument_options[:rest_start]) + if (post_start = iseq.argument_options[:post_start]) + length = post_start - rest_start + local_set(local_index, 0, locals[0...length]) + locals = locals[length..] + else + local_set(local_index, 0, locals.dup) + locals.clear + end + local_index += 1 + end + + # Next, set up any post arguments. These are positional arguments that + # come after the splat argument. + if (post_num = iseq.argument_options[:post_num]) + post_num.times do + local_set(local_index, 0, locals.shift) + local_index += 1 + end + end + + if (keyword_option = iseq.argument_options[:keyword]) + # First, set up the keyword bits array. + keyword_bits = + keyword_option.map do |config| + kwargs.key?(config.is_a?(Array) ? config[0] : config) + end + + iseq.local_table.locals.each_with_index do |local, index| + # If this is the keyword bits local, then set it appropriately. + if local.name.is_a?(Integer) + local_set(index, 0, keyword_bits) + next + end + + # First, find the configuration for this local in the keywords + # list if it exists. + name = local.name + config = + keyword_option.find do |keyword| + keyword.is_a?(Array) ? keyword[0] == name : keyword == name + end + + # If the configuration doesn't exist, then the local is not a + # keyword local. + next unless config + + if !config.is_a?(Array) + # required keyword + local_set(index, 0, kwargs.fetch(name)) + elsif !config[1].nil? + # optional keyword with embedded default value + local_set(index, 0, kwargs.fetch(name, config[1])) + else + # optional keyword with expression default value + local_set(index, 0, kwargs[name]) + end + end + end + + local_set(local_index, 0, block) if iseq.argument_options[:block_start] + + start_label + end + + ########################################################################## + # Helper methods for instructions + ########################################################################## + + def const_base + frame.nesting.last + end + + def frame_at(level) + current = frame + level.times { current = current.parent } + current + end + + def frame_svar + current = frame + current = current.parent while current.is_a?(BlockFrame) + current + end + + def frame_yield + current = frame + current = current.parent until current.is_a?(MethodFrame) + current + end + + def frozen_core + FROZEN_CORE + end + + def jump(label) + Jump.new(label) + end + + def leave + Leave.new(pop) + end + + def local_get(index, level) + stack[frame_at(level).stack_index + index] + end + + def local_set(index, level, value) + stack[frame_at(level).stack_index + index] = value + end + + ########################################################################## + # Methods for overriding runtime behavior + ########################################################################## + + DLEXT = ".#{RbConfig::CONFIG["DLEXT"]}" + SOEXT = ".#{RbConfig::CONFIG["SOEXT"]}" + + def require_resolved(filepath) + $LOADED_FEATURES << filepath + iseq = RubyVM::InstructionSequence.compile_file(filepath) + run_top_frame(InstructionSequence.from(iseq.to_a)) + end + + def require_internal(filepath, loading: false) + case (extname = File.extname(filepath)) + when "" + # search for all the extensions + searching = filepath + extensions = ["", ".rb", DLEXT, SOEXT] + when ".rb", DLEXT, SOEXT + # search only for the given extension name + searching = File.basename(filepath, extname) + extensions = [extname] + else + # we don't handle these extensions, raise a load error + raise LoadError, "cannot load such file -- #{filepath}" + end + + if filepath.start_with?("/") + # absolute path, search only in the given directory + directories = [File.dirname(searching)] + searching = File.basename(searching) + else + # relative path, search in the load path + directories = $LOAD_PATH + end + + directories.each do |directory| + extensions.each do |extension| + absolute_path = File.join(directory, "#{searching}#{extension}") + next unless File.exist?(absolute_path) + + if !loading && $LOADED_FEATURES.include?(absolute_path) + return false + elsif extension == ".rb" + require_resolved(absolute_path) + return true + elsif loading + return Kernel.send(:yarv_load, filepath) + else + return Kernel.send(:yarv_require, filepath) + end + end + end + + if loading + Kernel.send(:yarv_load, filepath) + else + Kernel.send(:yarv_require, filepath) + end + end + + def require(filepath) + require_internal(filepath, loading: false) + end + + def require_relative(filepath) + Kernel.yarv_require_relative(filepath) + end + + def load(filepath) + require_internal(filepath, loading: true) + end + + def eval( + source, + binding = TOPLEVEL_BINDING, + filename = "(eval)", + lineno = 1 + ) + Kernel.yarv_eval(source, binding, filename, lineno) + end + + def throw(tag, value = nil) + Kernel.throw(tag, value) + end + + def catch(tag, &block) + Kernel.catch(tag, &block) + end + end + end +end diff --git a/spec/mspec b/spec/mspec new file mode 160000 index 00000000..4877d58d --- /dev/null +++ b/spec/mspec @@ -0,0 +1 @@ +Subproject commit 4877d58dff577641bc1ecd1bf3d3c3daa93b423f diff --git a/spec/ruby b/spec/ruby new file mode 160000 index 00000000..71873ae4 --- /dev/null +++ b/spec/ruby @@ -0,0 +1 @@ +Subproject commit 71873ae4421f5b551a5af0f3427e901414736835 diff --git a/test/yarv_test.rb b/test/yarv_test.rb index f8e0ffdb..6f60d74e 100644 --- a/test/yarv_test.rb +++ b/test/yarv_test.rb @@ -41,9 +41,253 @@ def test_bf ">>.>---.+++++++..+++.>>.<-.<.+++.------.--------.>>+.>++." iseq = YARV::Bf.new(hello_world).compile + stdout, = capture_io { iseq.eval } + assert_equal "Hello World!\n", stdout + Formatter.format(hello_world, YARV::Decompiler.new(iseq).to_ruby) end + # rubocop:disable Layout/LineLength + EMULATION_CASES = { + # adjuststack + "x = [true]; x[0] ||= nil; x[0]" => true, + # anytostring + "\"\#{5}\"" => "5", + "class A2Str; def to_s; 1; end; end; \"\#{A2Str.new}\"" => + "#", + # branchif + "x = true; x ||= \"foo\"; x" => true, + # branchnil + "x = nil; if x&.to_s; 'hi'; else; 'bye'; end" => "bye", + # branchunless + "if 2 + 3; 'hi'; else; 'bye'; end" => "hi", + # checkkeyword + # "def evaluate(value: rand); value.floor; end; evaluate" => 0, + # checkmatch + "'foo' in String" => true, + "case 1; when *[1, 2, 3]; true; end" => true, + # checktype + "['foo'] in [String]" => true, + # concatarray + "[1, *2]" => [1, 2], + # concatstrings + "\"\#{7}\"" => "7", + # defineclass + "class DefineClass; def bar; end; end" => :bar, + "module DefineModule; def bar; end; end" => :bar, + "class << self; self; end" => + TOPLEVEL_BINDING.eval("self").singleton_class, + # defined + "defined?(1)" => "expression", + "defined?(foo = 1)" => "assignment", + "defined?(Object)" => "constant", + # definemethod + "def definemethod = 5; definemethod" => 5, + # definesmethod + "def self.definesmethod = 5; self.definesmethod" => 5, + # dup + "$global = 5" => 5, + # duparray + "[true]" => [true], + # duphash + "{ a: 1 }" => { + a: 1 + }, + # dupn + "Object::X ||= true" => true, + # expandarray + "x, = [true, false, nil]" => [true, false, nil], + "*, x = [true, false, nil]" => [true, false, nil], + # getblockparam + "def getblockparam(&block); block; end; getblockparam { 1 }.call" => 1, + # getblockparamproxy + "def getblockparamproxy(&block); block.call; end; getblockparamproxy { 1 }" => + 1, + # getclassvariable + "class CVar; @@foo = 5; end; class << CVar; @@foo; end" => 5, + # getconstant + "Object" => Object, + # getglobal + "$$" => $$, + # getinstancevariable + "@foo = 5; @foo" => 5, + # getlocal + "value = 5; self.then { self.then { self.then { value } } }" => 5, + # getlocalwc0 + "value = 5; value" => 5, + # getlocalwc1 + "value = 5; self.then { value }" => 5, + # getspecial + "1 if (2 == 2) .. (3 == 3)" => 1, + # intern + ":\"foo\#{1}\"" => :foo1, + # invokeblock + "def invokeblock = yield; invokeblock { 1 }" => 1, + # invokesuper + <<~RUBY => 2, + class Parent + def value + 1 + end + end + + class Child < Parent + def value + super + 1 + end + end + + Child.new.value + RUBY + # jump + "x = 0; if x == 0 then 1 else 2 end" => 1, + # newarray + "[\"value\"]" => ["value"], + # newarraykwsplat + "[\"string\", **{ foo: \"bar\" }]" => ["string", { foo: "bar" }], + # newhash + "def newhash(key, value) = { key => value }; newhash(1, 2)" => { + 1 => 2 + }, + # newrange + "x = 0; y = 1; (x..y).to_a" => [0, 1], + # nop + # objtostring + "\"\#{6}\"" => "6", + # once + "/\#{1}/o" => /1/o, + # opt_and + "0b0110 & 0b1011" => 0b0010, + # opt_aref + "x = [1, 2, 3]; x[1]" => 2, + # opt_aref_with + "x = { \"a\" => 1 }; x[\"a\"]" => 1, + # opt_aset + "x = [1, 2, 3]; x[1] = 4; x" => [1, 4, 3], + # opt_aset_with + "x = { \"a\" => 1 }; x[\"a\"] = 2; x" => { + "a" => 2 + }, + # opt_case_dispatch + <<~RUBY => "foo", + case 1 + when 1 + "foo" + else + "bar" + end + RUBY + # opt_div + "5 / 2" => 2, + # opt_empty_p + "[].empty?" => true, + # opt_eq + "1 == 1" => true, + # opt_ge + "1 >= 1" => true, + # opt_getconstant_path + "::Object" => Object, + # opt_gt + "1 > 1" => false, + # opt_le + "1 <= 1" => true, + # opt_length + "[1, 2, 3].length" => 3, + # opt_lt + "1 < 1" => false, + # opt_ltlt + "\"\" << 2" => "\u0002", + # opt_minus + "1 - 1" => 0, + # opt_mod + "5 % 2" => 1, + # opt_mult + "5 * 2" => 10, + # opt_neq + "1 != 1" => false, + # opt_newarray_max + "def opt_newarray_max(a, b, c) = [a, b, c].max; opt_newarray_max(1, 2, 3)" => + 3, + # opt_newarray_min + "def opt_newarray_min(a, b, c) = [a, b, c].min; opt_newarray_min(1, 2, 3)" => + 1, + # opt_nil_p + "nil.nil?" => true, + # opt_not + "!true" => false, + # opt_or + "0b0110 | 0b1011" => 0b1111, + # opt_plus + "1 + 1" => 2, + # opt_regexpmatch2 + "/foo/ =~ \"~~~foo\"" => 3, + # opt_send_without_block + "5.to_s" => "5", + # opt_size + "[1, 2, 3].size" => 3, + # opt_str_freeze + "\"foo\".freeze" => "foo", + # opt_str_uminus + "-\"foo\"" => -"foo", + # opt_succ + "1.succ" => 2, + # pop + "a ||= 2; a" => 2, + # putnil + "[nil]" => [nil], + # putobject + "2" => 2, + # putobject_INT2FIX_0_ + "0" => 0, + # putobject_INT2FIX_1_ + "1" => 1, + # putself + "self" => TOPLEVEL_BINDING.eval("self"), + # putspecialobject + "[class Undef; def foo = 1; undef foo; end]" => [nil], + # putstring + "\"foo\"" => "foo", + # send + "\"hello\".then { |value| value }" => "hello", + # setblockparam + "def setblockparam(&bar); bar = -> { 1 }; bar.call; end; setblockparam" => + 1, + # setclassvariable + "class CVarSet; @@foo = 1; end; class << CVarSet; @@foo = 10; end" => 10, + # setconstant + "SetConstant = 1" => 1, + # setglobal + "$global = 10" => 10, + # setinstancevariable + "@ivar = 5" => 5, + # setlocal + "x = 5; tap { tap { tap { x = 10 } } }; x" => 10, + # setlocal_WC_0 + "x = 5; x" => 5, + # setlocal_WC_1 + "x = 5; tap { x = 10 }; x" => 10, + # setn + "{}[:key] = 'value'" => "value", + # setspecial + "1 if (1 == 1) .. (2 == 2)" => 1, + # splatarray + "x = *(5)" => [5], + # swap + "!!defined?([[]])" => true, + # throw + # topn + "case 3; when 1..5; 'foo'; end" => "foo", + # toregexp + "/abc \#{1 + 2} def/" => /abc 3 def/ + }.freeze + # rubocop:enable Layout/LineLength + + EMULATION_CASES.each do |source, expected| + define_method("test_emulate_#{source}") do + assert_emulates(expected, source) + end + end + private def assert_decompiles(expected, source) @@ -51,5 +295,41 @@ def assert_decompiles(expected, source) actual = Formatter.format(source, ruby) assert_equal expected, actual end + + def assert_emulates(expected, source) + ruby_iseq = RubyVM::InstructionSequence.compile(source) + yarv_iseq = YARV::InstructionSequence.from(ruby_iseq.to_a) + + exercise_iseq(yarv_iseq) + result = SyntaxTree::YARV::VM.new.run_top_frame(yarv_iseq) + assert_equal(expected, result) + end + + def exercise_iseq(iseq) + iseq.disasm + iseq.to_a + + iseq.insns.each do |insn| + case insn + when YARV::InstructionSequence::Label, Integer, Symbol + next + end + + insn.pushes + insn.pops + insn.canonical + + case insn + when YARV::DefineClass + exercise_iseq(insn.class_iseq) + when YARV::DefineMethod, YARV::DefineSMethod + exercise_iseq(insn.method_iseq) + when YARV::InvokeSuper, YARV::Send + exercise_iseq(insn.block_iseq) if insn.block_iseq + when YARV::Once + exercise_iseq(insn.iseq) + end + end + end end end From 9a5d228f58b96ba05e689f3d625ba572fed4304d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Jan 2023 17:05:22 +0000 Subject: [PATCH 314/536] Bump minitest from 5.16.3 to 5.17.0 Bumps [minitest](https://github.com/seattlerb/minitest) from 5.16.3 to 5.17.0. - [Release notes](https://github.com/seattlerb/minitest/releases) - [Changelog](https://github.com/minitest/minitest/blob/master/History.rdoc) - [Commits](https://github.com/seattlerb/minitest/compare/v5.16.3...v5.17.0) --- updated-dependencies: - dependency-name: minitest dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 47d0c66b..93608ae1 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -10,7 +10,7 @@ GEM ast (2.4.2) docile (1.4.0) json (2.6.3) - minitest (5.16.3) + minitest (5.17.0) parallel (1.22.1) parser (3.1.3.0) ast (~> 2.4.1) From e4ff8fe8db557a2325276dc4b94aca2f17770879 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 3 Jan 2023 23:15:01 -0500 Subject: [PATCH 315/536] Various config updates --- .github/dependabot.yml | 4 ++++ .github/workflows/main.yml | 7 +++++-- .rubocop.yml | 3 +++ Gemfile.lock | 10 +++++----- lib/syntax_tree/parser.rb | 2 +- 5 files changed, 18 insertions(+), 8 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 46959146..9f77688a 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,3 +4,7 @@ updates: directory: "/" schedule: interval: "daily" + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9f95cc9d..3f811317 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,7 +1,9 @@ name: Main + on: - push - pull_request + jobs: ci: strategy: @@ -11,13 +13,14 @@ jobs: - '2.7.0' - '3.0' - '3.1' + - '3.2' - head - truffleruby-head name: CI runs-on: ubuntu-latest env: CI: true - TESTOPTS: --verbose + # TESTOPTS: --verbose steps: - uses: actions/checkout@master - uses: ruby/setup-ruby@v1 @@ -37,7 +40,7 @@ jobs: - uses: ruby/setup-ruby@v1 with: bundler-cache: true - ruby-version: '3.1' + ruby-version: '3.2' - name: Check run: | bundle exec rake stree:check diff --git a/.rubocop.yml b/.rubocop.yml index 1e3e2f83..069041bd 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -147,3 +147,6 @@ Style/SpecialGlobalVars: Style/StructInheritance: Enabled: false + +Style/YodaExpression: + Enabled: false diff --git a/Gemfile.lock b/Gemfile.lock index 93608ae1..94a088c9 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -12,24 +12,24 @@ GEM json (2.6.3) minitest (5.17.0) parallel (1.22.1) - parser (3.1.3.0) + parser (3.2.0.0) ast (~> 2.4.1) prettier_print (1.2.0) rainbow (3.1.1) rake (13.0.6) regexp_parser (2.6.1) rexml (3.2.5) - rubocop (1.41.1) + rubocop (1.42.0) json (~> 2.3) parallel (~> 1.10) parser (>= 3.1.2.1) rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 1.8, < 3.0) rexml (>= 3.2.5, < 4.0) - rubocop-ast (>= 1.23.0, < 2.0) + rubocop-ast (>= 1.24.1, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 1.4.0, < 3.0) - rubocop-ast (1.24.0) + rubocop-ast (1.24.1) parser (>= 3.1.1.0) ruby-progressbar (1.11.0) simplecov (0.22.0) @@ -38,7 +38,7 @@ GEM simplecov_json_formatter (~> 0.1) simplecov-html (0.12.3) simplecov_json_formatter (0.1.4) - unicode-display_width (2.3.0) + unicode-display_width (2.4.1) PLATFORMS arm64-darwin-21 diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index fcefed30..602bb98f 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -53,7 +53,7 @@ def initialize(start, line) # there's a BOM at the beginning of the file, which is the reason we need # to compare it to 0 here. def [](byteindex) - indices[byteindex < 0 ? 0 : byteindex] + indices[[byteindex, 0].max] end end From e53ea80ef72e37c5c373e8e72bff2e7873283fb9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 4 Jan 2023 04:21:09 +0000 Subject: [PATCH 316/536] Bump dependabot/fetch-metadata from 1.3.3 to 1.3.5 Bumps [dependabot/fetch-metadata](https://github.com/dependabot/fetch-metadata) from 1.3.3 to 1.3.5. - [Release notes](https://github.com/dependabot/fetch-metadata/releases) - [Commits](https://github.com/dependabot/fetch-metadata/compare/v1.3.3...v1.3.5) --- updated-dependencies: - dependency-name: dependabot/fetch-metadata dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/auto-merge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/auto-merge.yml b/.github/workflows/auto-merge.yml index 9b28abf4..514ac27a 100644 --- a/.github/workflows/auto-merge.yml +++ b/.github/workflows/auto-merge.yml @@ -12,7 +12,7 @@ jobs: steps: - name: Dependabot metadata id: metadata - uses: dependabot/fetch-metadata@v1.3.3 + uses: dependabot/fetch-metadata@v1.3.5 with: github-token: "${{ secrets.GITHUB_TOKEN }}" - name: Enable auto-merge for Dependabot PRs From 11ead3e61ae3570f20abf70c63c35e2f6693b696 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 4 Jan 2023 12:55:53 -0500 Subject: [PATCH 317/536] Leave parentheses in place on method calls Note that this explicitly leaves parentheses in place even if they are empty. There are two reasons we would need to do this. The first is if we're calling something that looks like a constant, as in: Foo() In this case if we remove the parentheses then this becomes a constant reference and not a method call. The second is if we're calling a method that is the same name as a local variable that is in scope, as in: foo = foo() In this case we have to keep the parentheses or else it treats this like assigning nil to the local variable. Note that we could attempt to be smarter about this by tracking the local variables that are in scope, but for now it's simpler and more efficient to just leave the parentheses in place. --- lib/syntax_tree/node.rb | 29 +++++++++++++++++++---------- test/fixtures/arg_paren.rb | 2 -- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index e5b09044..f19cfb2c 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -3001,16 +3001,25 @@ def format(q) else q.format(message) - if arguments.is_a?(ArgParen) && arguments.arguments.nil? && - !message.is_a?(Const) - # If you're using an explicit set of parentheses on something that - # looks like a constant, then we need to match that in order to - # maintain valid Ruby. For example, you could do something like Foo(), - # on which we would need to keep the parentheses to make it look like - # a method call. - else - q.format(arguments) - end + # Note that this explicitly leaves parentheses in place even if they are + # empty. There are two reasons we would need to do this. The first is if + # we're calling something that looks like a constant, as in: + # + # Foo() + # + # In this case if we remove the parentheses then this becomes a constant + # reference and not a method call. The second is if we're calling a + # method that is the same name as a local variable that is in scope, as + # in: + # + # foo = foo() + # + # In this case we have to keep the parentheses or else it treats this + # like assigning nil to the local variable. Note that we could attempt + # to be smarter about this by tracking the local variables that are in + # scope, but for now it's simpler and more efficient to just leave the + # parentheses in place. + q.format(arguments) if arguments end end diff --git a/test/fixtures/arg_paren.rb b/test/fixtures/arg_paren.rb index 0e01e208..0816af6a 100644 --- a/test/fixtures/arg_paren.rb +++ b/test/fixtures/arg_paren.rb @@ -2,8 +2,6 @@ foo(bar) % foo() -- -foo % foo(barrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr) - From e1d89eee8d1af63e91db3becd705ced89e16e0d0 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 4 Jan 2023 14:02:55 -0500 Subject: [PATCH 318/536] Bump to version 5.2.0 --- CHANGELOG.md | 13 ++++++++++++- Gemfile.lock | 2 +- lib/syntax_tree/version.rb | 2 +- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 557fdf5c..4b29fcbb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,16 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +## [5.2.0] - 2023-01-04 + +### Added + +- An experiment in evaluating compiled instruction sequences has been added to Syntax Tree. This is subject to change, so it will not be well documented or testing at the moment. It does not impact other functionality. + +### Changed + +- Empty parentheses on method calls will now be left in place. Previously they were left in place if the method being called looked like a constant. Now they are left in place for all method calls since the method name can mirror the name of a local variable, in which case the parentheses are required. + ## [5.1.0] - 2022-12-28 ### Added @@ -471,7 +481,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - 🎉 Initial release! 🎉 -[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v5.1.0...HEAD +[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v5.2.0...HEAD +[5.2.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v5.1.0...v5.2.0 [5.1.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v5.0.1...v5.1.0 [5.0.1]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v5.0.0...v5.0.1 [5.0.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v4.3.0...v5.0.0 diff --git a/Gemfile.lock b/Gemfile.lock index 94a088c9..bb5e3663 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - syntax_tree (5.1.0) + syntax_tree (5.2.0) prettier_print (>= 1.2.0) GEM diff --git a/lib/syntax_tree/version.rb b/lib/syntax_tree/version.rb index d9bbdfa4..a97f5e43 100644 --- a/lib/syntax_tree/version.rb +++ b/lib/syntax_tree/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SyntaxTree - VERSION = "5.1.0" + VERSION = "5.2.0" end From 85f61349c94fa27b4f6bddcdf80071e57c5ae001 Mon Sep 17 00:00:00 2001 From: David Taylor Date: Sat, 7 Jan 2023 13:25:11 +0000 Subject: [PATCH 319/536] Only write files when content changes Previously the CLI would call `File.write` for every file, even if the contents was unchanged. This unnecessary filesystem churn can have a knock-on effect on other tools which may be watching directories for changes (e.g. IDEs). This commit updates the `stree write` command so that it only performs a write when the file contents has changed. --- lib/syntax_tree/cli.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/syntax_tree/cli.rb b/lib/syntax_tree/cli.rb index 392dd627..7e6f4067 100644 --- a/lib/syntax_tree/cli.rb +++ b/lib/syntax_tree/cli.rb @@ -303,10 +303,11 @@ def run(item) options.print_width, options: options.formatter_options ) + changed = source != formatted - File.write(filepath, formatted) if item.writable? + File.write(filepath, formatted) if item.writable? && changed - color = source == formatted ? Color.gray(filepath) : filepath + color = changed ? filepath : Color.gray(filepath) delta = ((Time.now - start) * 1000).round puts "#{color} #{delta}ms" From 9c8198969b8a8b4701a6bb487806d51745a39b74 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Jan 2023 17:51:41 +0000 Subject: [PATCH 320/536] Bump rubocop from 1.42.0 to 1.43.0 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.42.0 to 1.43.0. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.42.0...v1.43.0) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index bb5e3663..b691d5e9 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -19,16 +19,16 @@ GEM rake (13.0.6) regexp_parser (2.6.1) rexml (3.2.5) - rubocop (1.42.0) + rubocop (1.43.0) json (~> 2.3) parallel (~> 1.10) - parser (>= 3.1.2.1) + parser (>= 3.2.0.0) rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 1.8, < 3.0) rexml (>= 3.2.5, < 4.0) rubocop-ast (>= 1.24.1, < 2.0) ruby-progressbar (~> 1.7) - unicode-display_width (>= 1.4.0, < 3.0) + unicode-display_width (>= 2.4.0, < 3.0) rubocop-ast (1.24.1) parser (>= 3.1.1.0) ruby-progressbar (1.11.0) @@ -38,7 +38,7 @@ GEM simplecov_json_formatter (~> 0.1) simplecov-html (0.12.3) simplecov_json_formatter (0.1.4) - unicode-display_width (2.4.1) + unicode-display_width (2.4.2) PLATFORMS arm64-darwin-21 From 6712db16c4bf7ad500ba9c653e0b876601134bd6 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 10 Jan 2023 16:13:39 -0500 Subject: [PATCH 321/536] Use ruby-syntax-fixtures --- .gitmodules | 3 +++ test/ruby-syntax-fixtures | 1 + test/ruby_syntax_fixtures_test.rb | 13 +++++++++++++ 3 files changed, 17 insertions(+) create mode 160000 test/ruby-syntax-fixtures create mode 100644 test/ruby_syntax_fixtures_test.rb diff --git a/.gitmodules b/.gitmodules index f5477ea3..1a2c45cc 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "spec"] path = spec/ruby url = git@github.com:ruby/spec.git +[submodule "test/ruby-syntax-fixtures"] + path = test/ruby-syntax-fixtures + url = https://github.com/ruby-syntax-tree/ruby-syntax-fixtures diff --git a/test/ruby-syntax-fixtures b/test/ruby-syntax-fixtures new file mode 160000 index 00000000..5b333f5a --- /dev/null +++ b/test/ruby-syntax-fixtures @@ -0,0 +1 @@ +Subproject commit 5b333f5a34d6fb08f88acc93b69c7d19b3fee8e7 diff --git a/test/ruby_syntax_fixtures_test.rb b/test/ruby_syntax_fixtures_test.rb new file mode 100644 index 00000000..9aae8cc8 --- /dev/null +++ b/test/ruby_syntax_fixtures_test.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +require_relative "test_helper" + +module SyntaxTree + class RubySyntaxFixturesTest < Minitest::Test + Dir[File.expand_path("ruby-syntax-fixtures/**/*.rb", __dir__)].each do |file| + define_method "test_ruby_syntax_fixtures_#{file}" do + refute_nil(SyntaxTree.parse(SyntaxTree.read(file))) + end + end + end +end From a116e97dc81b59370e62885c6ab3875bf54dc522 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 11 Jan 2023 10:19:15 -0500 Subject: [PATCH 322/536] Fix up formatting on main --- test/ruby_syntax_fixtures_test.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/ruby_syntax_fixtures_test.rb b/test/ruby_syntax_fixtures_test.rb index 9aae8cc8..0cf89310 100644 --- a/test/ruby_syntax_fixtures_test.rb +++ b/test/ruby_syntax_fixtures_test.rb @@ -4,7 +4,9 @@ module SyntaxTree class RubySyntaxFixturesTest < Minitest::Test - Dir[File.expand_path("ruby-syntax-fixtures/**/*.rb", __dir__)].each do |file| + Dir[ + File.expand_path("ruby-syntax-fixtures/**/*.rb", __dir__) + ].each do |file| define_method "test_ruby_syntax_fixtures_#{file}" do refute_nil(SyntaxTree.parse(SyntaxTree.read(file))) end From 46c0e00025c79339dd60c46dd32eba8430836f24 Mon Sep 17 00:00:00 2001 From: Nanashi Date: Sat, 14 Jan 2023 18:21:46 +0900 Subject: [PATCH 323/536] Fix: Handle Fiddle::DLError --- lib/syntax_tree/yarv/instruction_sequence.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/syntax_tree/yarv/instruction_sequence.rb b/lib/syntax_tree/yarv/instruction_sequence.rb index c284221b..6aa7279e 100644 --- a/lib/syntax_tree/yarv/instruction_sequence.rb +++ b/lib/syntax_tree/yarv/instruction_sequence.rb @@ -70,7 +70,7 @@ def push(instruction) [Fiddle::TYPE_VOIDP] * 3, Fiddle::TYPE_VOIDP ) - rescue NameError + rescue NameError, Fiddle::DLError end # This object is used to track the size of the stack at any given time. It From e789ead77c274f3d1a9fa43a915ceb78f56804a6 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 18 Jan 2023 12:59:14 -0500 Subject: [PATCH 324/536] Add interface test for instructions --- lib/syntax_tree/yarv/instructions.rb | 822 +++++++++++++++++++++++++++ lib/syntax_tree/yarv/legacy.rb | 34 ++ test/yarv_test.rb | 35 ++ 3 files changed, 891 insertions(+) diff --git a/lib/syntax_tree/yarv/instructions.rb b/lib/syntax_tree/yarv/instructions.rb index 5e1d116b..20068eac 100644 --- a/lib/syntax_tree/yarv/instructions.rb +++ b/lib/syntax_tree/yarv/instructions.rb @@ -91,6 +91,14 @@ def to_a(_iseq) [:adjuststack, number] end + def deconstruct_keys(keys) + { number: number } + end + + def ==(other) + other.is_a?(AdjustStack) && other.number == number + end + def length 2 end @@ -139,6 +147,14 @@ def to_a(_iseq) [:anytostring] end + def deconstruct_keys(keys) + {} + end + + def ==(other) + other.is_a?(AnyToString) + end + def length 1 end @@ -197,6 +213,14 @@ def to_a(_iseq) [:branchif, label.name] end + def deconstruct_keys(keys) + { label: label } + end + + def ==(other) + other.is_a?(BranchIf) && other.label == label + end + def length 2 end @@ -250,6 +274,14 @@ def to_a(_iseq) [:branchnil, label.name] end + def deconstruct_keys(keys) + { label: label } + end + + def ==(other) + other.is_a?(BranchNil) && other.label == label + end + def length 2 end @@ -302,6 +334,14 @@ def to_a(_iseq) [:branchunless, label.name] end + def deconstruct_keys(keys) + { label: label } + end + + def ==(other) + other.is_a?(BranchUnless) && other.label == label + end + def length 2 end @@ -365,6 +405,16 @@ def to_a(iseq) ] end + def deconstruct_keys(keys) + { keyword_bits_index: keyword_bits_index, keyword_index: keyword_index } + end + + def ==(other) + other.is_a?(CheckKeyword) && + other.keyword_bits_index == keyword_bits_index && + other.keyword_index == keyword_index + end + def length 3 end @@ -419,6 +469,14 @@ def to_a(_iseq) [:checkmatch, type] end + def deconstruct_keys(keys) + { type: type } + end + + def ==(other) + other.is_a?(CheckMatch) && other.type == type + end + def length 2 end @@ -561,6 +619,14 @@ def to_a(_iseq) [:checktype, type] end + def deconstruct_keys(keys) + { type: type } + end + + def ==(other) + other.is_a?(CheckType) && other.type == type + end + def length 2 end @@ -656,6 +722,14 @@ def to_a(_iseq) [:concatarray] end + def deconstruct_keys(keys) + {} + end + + def ==(other) + other.is_a?(ConcatArray) + end + def length 1 end @@ -708,6 +782,14 @@ def to_a(_iseq) [:concatstrings, number] end + def deconstruct_keys(keys) + { number: number } + end + + def ==(other) + other.is_a?(ConcatStrings) && other.number == number + end + def length 2 end @@ -771,6 +853,17 @@ def to_a(_iseq) [:defineclass, name, class_iseq.to_a, flags] end + def deconstruct_keys(keys) + { name: name, class_iseq: class_iseq, flags: flags } + end + + def ==(other) + other.is_a?(DefineClass) && + other.name == name && + other.class_iseq == class_iseq && + other.flags == flags + end + def length 4 end @@ -899,6 +992,17 @@ def to_a(_iseq) [:defined, type, name, message] end + def deconstruct_keys(keys) + { type: type, name: name, message: message } + end + + def ==(other) + other.is_a?(Defined) && + other.type == type && + other.name == name && + other.message == message + end + def length 4 end @@ -989,6 +1093,16 @@ def to_a(_iseq) [:definemethod, method_name, method_iseq.to_a] end + def deconstruct_keys(keys) + { method_name: method_name, method_iseq: method_iseq } + end + + def ==(other) + other.is_a?(DefineMethod) && + other.method_name == method_name && + other.method_iseq == method_iseq + end + def length 3 end @@ -1061,6 +1175,16 @@ def to_a(_iseq) [:definesmethod, method_name, method_iseq.to_a] end + def deconstruct_keys(keys) + { method_name: method_name, method_iseq: method_iseq } + end + + def ==(other) + other.is_a?(DefineSMethod) && + other.method_name == method_name && + other.method_iseq == method_iseq + end + def length 3 end @@ -1118,6 +1242,14 @@ def to_a(_iseq) [:dup] end + def deconstruct_keys(keys) + {} + end + + def ==(other) + other.is_a?(Dup) + end + def length 1 end @@ -1164,6 +1296,14 @@ def to_a(_iseq) [:duparray, object] end + def deconstruct_keys(keys) + { object: object } + end + + def ==(other) + other.is_a?(DupArray) && other.object == object + end + def length 2 end @@ -1210,6 +1350,14 @@ def to_a(_iseq) [:duphash, object] end + def deconstruct_keys(keys) + { object: object } + end + + def ==(other) + other.is_a?(DupHash) && other.object == object + end + def length 2 end @@ -1256,6 +1404,14 @@ def to_a(_iseq) [:dupn, number] end + def deconstruct_keys(keys) + { number: number } + end + + def ==(other) + other.is_a?(DupN) && other.number == number + end + def length 2 end @@ -1307,6 +1463,16 @@ def to_a(_iseq) [:expandarray, number, flags] end + def deconstruct_keys(keys) + { number: number, flags: flags } + end + + def ==(other) + other.is_a?(ExpandArray) && + other.number == number && + other.flags == flags + end + def length 3 end @@ -1398,6 +1564,16 @@ def to_a(iseq) [:getblockparam, current.local_table.offset(index), level] end + def deconstruct_keys(keys) + { index: index, level: level } + end + + def ==(other) + other.is_a?(GetBlockParam) && + other.index == index && + other.level == level + end + def length 3 end @@ -1455,6 +1631,16 @@ def to_a(iseq) [:getblockparamproxy, current.local_table.offset(index), level] end + def deconstruct_keys(keys) + { index: index, level: level } + end + + def ==(other) + other.is_a?(GetBlockParamProxy) && + other.index == index && + other.level == level + end + def length 3 end @@ -1507,6 +1693,16 @@ def to_a(_iseq) [:getclassvariable, name, cache] end + def deconstruct_keys(keys) + { name: name, cache: cache } + end + + def ==(other) + other.is_a?(GetClassVariable) && + other.name == name && + other.cache == cache + end + def length 3 end @@ -1557,6 +1753,14 @@ def to_a(_iseq) [:getconstant, name] end + def deconstruct_keys(keys) + { name: name } + end + + def ==(other) + other.is_a?(GetConstant) && other.name == name + end + def length 2 end @@ -1619,6 +1823,14 @@ def to_a(_iseq) [:getglobal, name] end + def deconstruct_keys(keys) + { name: name } + end + + def ==(other) + other.is_a?(GetGlobal) && other.name == name + end + def length 2 end @@ -1678,6 +1890,16 @@ def to_a(_iseq) [:getinstancevariable, name, cache] end + def deconstruct_keys(keys) + { name: name, cache: cache } + end + + def ==(other) + other.is_a?(GetInstanceVariable) && + other.name == name && + other.cache == cache + end + def length 3 end @@ -1732,6 +1954,14 @@ def to_a(iseq) [:getlocal, current.local_table.offset(index), level] end + def deconstruct_keys(keys) + { index: index, level: level } + end + + def ==(other) + other.is_a?(GetLocal) && other.index == index && other.level == level + end + def length 3 end @@ -1781,6 +2011,14 @@ def to_a(iseq) [:getlocal_WC_0, iseq.local_table.offset(index)] end + def deconstruct_keys(keys) + { index: index } + end + + def ==(other) + other.is_a?(GetLocalWC0) && other.index == index + end + def length 2 end @@ -1830,6 +2068,14 @@ def to_a(iseq) [:getlocal_WC_1, iseq.parent_iseq.local_table.offset(index)] end + def deconstruct_keys(keys) + { index: index } + end + + def ==(other) + other.is_a?(GetLocalWC1) && other.index == index + end + def length 2 end @@ -1881,6 +2127,14 @@ def to_a(_iseq) [:getspecial, key, type] end + def deconstruct_keys(keys) + { key: key, type: type } + end + + def ==(other) + other.is_a?(GetSpecial) && other.key == key && other.type == type + end + def length 3 end @@ -1929,6 +2183,14 @@ def to_a(_iseq) [:intern] end + def deconstruct_keys(keys) + {} + end + + def ==(other) + other.is_a?(Intern) + end + def length 1 end @@ -1979,6 +2241,14 @@ def to_a(_iseq) [:invokeblock, calldata.to_h] end + def deconstruct_keys(keys) + { calldata: calldata } + end + + def ==(other) + other.is_a?(InvokeBlock) && other.calldata == calldata + end + def length 2 end @@ -2034,6 +2304,16 @@ def to_a(_iseq) [:invokesuper, calldata.to_h, block_iseq&.to_a] end + def deconstruct_keys(keys) + { calldata: calldata, block_iseq: block_iseq } + end + + def ==(other) + other.is_a?(InvokeSuper) && + other.calldata == calldata && + other.block_iseq == block_iseq + end + def length 1 end @@ -2105,6 +2385,14 @@ def to_a(_iseq) [:jump, label.name] end + def deconstruct_keys(keys) + { label: label } + end + + def ==(other) + other.is_a?(Jump) && other.label == label + end + def length 2 end @@ -2145,6 +2433,14 @@ def to_a(_iseq) [:leave] end + def deconstruct_keys(keys) + {} + end + + def ==(other) + other.is_a?(Leave) + end + def length 1 end @@ -2195,6 +2491,14 @@ def to_a(_iseq) [:newarray, number] end + def deconstruct_keys(keys) + { number: number } + end + + def ==(other) + other.is_a?(NewArray) && other.number == number + end + def length 2 end @@ -2243,6 +2547,14 @@ def to_a(_iseq) [:newarraykwsplat, number] end + def deconstruct_keys(keys) + { number: number } + end + + def ==(other) + other.is_a?(NewArrayKwSplat) && other.number == number + end + def length 2 end @@ -2293,6 +2605,14 @@ def to_a(_iseq) [:newhash, number] end + def deconstruct_keys(keys) + { number: number } + end + + def ==(other) + other.is_a?(NewHash) && other.number == number + end + def length 2 end @@ -2344,6 +2664,14 @@ def to_a(_iseq) [:newrange, exclude_end] end + def deconstruct_keys(keys) + { exclude_end: exclude_end } + end + + def ==(other) + other.is_a?(NewRange) && other.exclude_end == exclude_end + end + def length 2 end @@ -2385,6 +2713,14 @@ def to_a(_iseq) [:nop] end + def deconstruct_keys(keys) + {} + end + + def ==(other) + other.is_a?(Nop) + end + def length 1 end @@ -2434,6 +2770,14 @@ def to_a(_iseq) [:objtostring, calldata.to_h] end + def deconstruct_keys(keys) + { calldata: calldata } + end + + def ==(other) + other.is_a?(ObjToString) && other.calldata == calldata + end + def length 2 end @@ -2485,6 +2829,14 @@ def to_a(_iseq) [:once, iseq.to_a, cache] end + def deconstruct_keys(keys) + { iseq: iseq, cache: cache } + end + + def ==(other) + other.is_a?(Once) && other.iseq == iseq && other.cache == cache + end + def length 3 end @@ -2536,6 +2888,14 @@ def to_a(_iseq) [:opt_and, calldata.to_h] end + def deconstruct_keys(keys) + { calldata: calldata } + end + + def ==(other) + other.is_a?(OptAnd) && other.calldata == calldata + end + def length 2 end @@ -2584,6 +2944,14 @@ def to_a(_iseq) [:opt_aref, calldata.to_h] end + def deconstruct_keys(keys) + { calldata: calldata } + end + + def ==(other) + other.is_a?(OptAref) && other.calldata == calldata + end + def length 2 end @@ -2637,6 +3005,16 @@ def to_a(_iseq) [:opt_aref_with, object, calldata.to_h] end + def deconstruct_keys(keys) + { object: object, calldata: calldata } + end + + def ==(other) + other.is_a?(OptArefWith) && + other.object == object && + other.calldata == calldata + end + def length 3 end @@ -2686,6 +3064,14 @@ def to_a(_iseq) [:opt_aset, calldata.to_h] end + def deconstruct_keys(keys) + { calldata: calldata } + end + + def ==(other) + other.is_a?(OptAset) && other.calldata == calldata + end + def length 2 end @@ -2738,6 +3124,16 @@ def to_a(_iseq) [:opt_aset_with, object, calldata.to_h] end + def deconstruct_keys(keys) + { object: object, calldata: calldata } + end + + def ==(other) + other.is_a?(OptAsetWith) && + other.object == object && + other.calldata == calldata + end + def length 3 end @@ -2806,6 +3202,16 @@ def to_a(_iseq) ] end + def deconstruct_keys(keys) + { case_dispatch_hash: case_dispatch_hash, else_label: else_label } + end + + def ==(other) + other.is_a?(OptCaseDispatch) && + other.case_dispatch_hash == case_dispatch_hash && + other.else_label == else_label + end + def length 3 end @@ -2855,6 +3261,14 @@ def to_a(_iseq) [:opt_div, calldata.to_h] end + def deconstruct_keys(keys) + { calldata: calldata } + end + + def ==(other) + other.is_a?(OptDiv) && other.calldata == calldata + end + def length 2 end @@ -2903,6 +3317,14 @@ def to_a(_iseq) [:opt_empty_p, calldata.to_h] end + def deconstruct_keys(keys) + { calldata: calldata } + end + + def ==(other) + other.is_a?(OptEmptyP) && other.calldata == calldata + end + def length 2 end @@ -2952,6 +3374,14 @@ def to_a(_iseq) [:opt_eq, calldata.to_h] end + def deconstruct_keys(keys) + { calldata: calldata } + end + + def ==(other) + other.is_a?(OptEq) && other.calldata == calldata + end + def length 2 end @@ -3001,6 +3431,14 @@ def to_a(_iseq) [:opt_ge, calldata.to_h] end + def deconstruct_keys(keys) + { calldata: calldata } + end + + def ==(other) + other.is_a?(OptGE) && other.calldata == calldata + end + def length 2 end @@ -3050,6 +3488,14 @@ def to_a(_iseq) [:opt_getconstant_path, names] end + def deconstruct_keys(keys) + { names: names } + end + + def ==(other) + other.is_a?(OptGetConstantPath) && other.names == names + end + def length 2 end @@ -3106,6 +3552,14 @@ def to_a(_iseq) [:opt_gt, calldata.to_h] end + def deconstruct_keys(keys) + { calldata: calldata } + end + + def ==(other) + other.is_a?(OptGT) && other.calldata == calldata + end + def length 2 end @@ -3155,6 +3609,14 @@ def to_a(_iseq) [:opt_le, calldata.to_h] end + def deconstruct_keys(keys) + { calldata: calldata } + end + + def ==(other) + other.is_a?(OptLE) && other.calldata == calldata + end + def length 2 end @@ -3204,6 +3666,14 @@ def to_a(_iseq) [:opt_length, calldata.to_h] end + def deconstruct_keys(keys) + { calldata: calldata } + end + + def ==(other) + other.is_a?(OptLength) && other.calldata == calldata + end + def length 2 end @@ -3253,6 +3723,14 @@ def to_a(_iseq) [:opt_lt, calldata.to_h] end + def deconstruct_keys(keys) + { calldata: calldata } + end + + def ==(other) + other.is_a?(OptLT) && other.calldata == calldata + end + def length 2 end @@ -3302,6 +3780,14 @@ def to_a(_iseq) [:opt_ltlt, calldata.to_h] end + def deconstruct_keys(keys) + { calldata: calldata } + end + + def ==(other) + other.is_a?(OptLTLT) && other.calldata == calldata + end + def length 2 end @@ -3352,6 +3838,14 @@ def to_a(_iseq) [:opt_minus, calldata.to_h] end + def deconstruct_keys(keys) + { calldata: calldata } + end + + def ==(other) + other.is_a?(OptMinus) && other.calldata == calldata + end + def length 2 end @@ -3401,6 +3895,14 @@ def to_a(_iseq) [:opt_mod, calldata.to_h] end + def deconstruct_keys(keys) + { calldata: calldata } + end + + def ==(other) + other.is_a?(OptMod) && other.calldata == calldata + end + def length 2 end @@ -3450,6 +3952,14 @@ def to_a(_iseq) [:opt_mult, calldata.to_h] end + def deconstruct_keys(keys) + { calldata: calldata } + end + + def ==(other) + other.is_a?(OptMult) && other.calldata == calldata + end + def length 2 end @@ -3505,6 +4015,16 @@ def to_a(_iseq) [:opt_neq, eq_calldata.to_h, neq_calldata.to_h] end + def deconstruct_keys(keys) + { eq_calldata: eq_calldata, neq_calldata: neq_calldata } + end + + def ==(other) + other.is_a?(OptNEq) && + other.eq_calldata == eq_calldata && + other.neq_calldata == neq_calldata + end + def length 3 end @@ -3554,6 +4074,14 @@ def to_a(_iseq) [:opt_newarray_max, number] end + def deconstruct_keys(keys) + { number: number } + end + + def ==(other) + other.is_a?(OptNewArrayMax) && other.number == number + end + def length 2 end @@ -3602,6 +4130,14 @@ def to_a(_iseq) [:opt_newarray_min, number] end + def deconstruct_keys(keys) + { number: number } + end + + def ==(other) + other.is_a?(OptNewArrayMin) && other.number == number + end + def length 2 end @@ -3651,6 +4187,14 @@ def to_a(_iseq) [:opt_nil_p, calldata.to_h] end + def deconstruct_keys(keys) + { calldata: calldata } + end + + def ==(other) + other.is_a?(OptNilP) && other.calldata == calldata + end + def length 2 end @@ -3698,6 +4242,14 @@ def to_a(_iseq) [:opt_not, calldata.to_h] end + def deconstruct_keys(keys) + { calldata: calldata } + end + + def ==(other) + other.is_a?(OptNot) && other.calldata == calldata + end + def length 2 end @@ -3747,6 +4299,14 @@ def to_a(_iseq) [:opt_or, calldata.to_h] end + def deconstruct_keys(keys) + { calldata: calldata } + end + + def ==(other) + other.is_a?(OptOr) && other.calldata == calldata + end + def length 2 end @@ -3796,6 +4356,14 @@ def to_a(_iseq) [:opt_plus, calldata.to_h] end + def deconstruct_keys(keys) + { calldata: calldata } + end + + def ==(other) + other.is_a?(OptPlus) && other.calldata == calldata + end + def length 2 end @@ -3844,6 +4412,14 @@ def to_a(_iseq) [:opt_regexpmatch2, calldata.to_h] end + def deconstruct_keys(keys) + { calldata: calldata } + end + + def ==(other) + other.is_a?(OptRegExpMatch2) && other.calldata == calldata + end + def length 2 end @@ -3892,6 +4468,14 @@ def to_a(_iseq) [:opt_send_without_block, calldata.to_h] end + def deconstruct_keys(keys) + { calldata: calldata } + end + + def ==(other) + other.is_a?(OptSendWithoutBlock) && other.calldata == calldata + end + def length 2 end @@ -3941,6 +4525,14 @@ def to_a(_iseq) [:opt_size, calldata.to_h] end + def deconstruct_keys(keys) + { calldata: calldata } + end + + def ==(other) + other.is_a?(OptSize) && other.calldata == calldata + end + def length 2 end @@ -3993,6 +4585,16 @@ def to_a(_iseq) [:opt_str_freeze, object, calldata.to_h] end + def deconstruct_keys(keys) + { object: object, calldata: calldata } + end + + def ==(other) + other.is_a?(OptStrFreeze) && + other.object == object && + other.calldata == calldata + end + def length 3 end @@ -4045,6 +4647,16 @@ def to_a(_iseq) [:opt_str_uminus, object, calldata.to_h] end + def deconstruct_keys(keys) + { object: object, calldata: calldata } + end + + def ==(other) + other.is_a?(OptStrUMinus) && + other.object == object && + other.calldata == calldata + end + def length 3 end @@ -4094,6 +4706,14 @@ def to_a(_iseq) [:opt_succ, calldata.to_h] end + def deconstruct_keys(keys) + { calldata: calldata } + end + + def ==(other) + other.is_a?(OptSucc) && other.calldata == calldata + end + def length 2 end @@ -4134,6 +4754,14 @@ def to_a(_iseq) [:pop] end + def deconstruct_keys(keys) + {} + end + + def ==(other) + other.is_a?(Pop) + end + def length 1 end @@ -4174,6 +4802,14 @@ def to_a(_iseq) [:putnil] end + def deconstruct_keys(keys) + {} + end + + def ==(other) + other.is_a?(PutNil) + end + def length 1 end @@ -4220,6 +4856,14 @@ def to_a(_iseq) [:putobject, object] end + def deconstruct_keys(keys) + { object: object } + end + + def ==(other) + other.is_a?(PutObject) && other.object == object + end + def length 2 end @@ -4262,6 +4906,14 @@ def to_a(_iseq) [:putobject_INT2FIX_0_] end + def deconstruct_keys(keys) + {} + end + + def ==(other) + other.is_a?(PutObjectInt2Fix0) + end + def length 1 end @@ -4304,6 +4956,14 @@ def to_a(_iseq) [:putobject_INT2FIX_1_] end + def deconstruct_keys(keys) + {} + end + + def ==(other) + other.is_a?(PutObjectInt2Fix1) + end + def length 1 end @@ -4344,6 +5004,14 @@ def to_a(_iseq) [:putself] end + def deconstruct_keys(keys) + {} + end + + def ==(other) + other.is_a?(PutSelf) + end + def length 1 end @@ -4396,6 +5064,14 @@ def to_a(_iseq) [:putspecialobject, object] end + def deconstruct_keys(keys) + { object: object } + end + + def ==(other) + other.is_a?(PutSpecialObject) && other.object == object + end + def length 2 end @@ -4451,6 +5127,14 @@ def to_a(_iseq) [:putstring, object] end + def deconstruct_keys(keys) + { object: object } + end + + def ==(other) + other.is_a?(PutString) && other.object == object + end + def length 2 end @@ -4505,6 +5189,16 @@ def to_a(_iseq) [:send, calldata.to_h, block_iseq&.to_a] end + def deconstruct_keys(keys) + { calldata: calldata, block_iseq: block_iseq } + end + + def ==(other) + other.is_a?(Send) && + other.calldata == calldata && + other.block_iseq == block_iseq + end + def length 3 end @@ -4582,6 +5276,16 @@ def to_a(iseq) [:setblockparam, current.local_table.offset(index), level] end + def deconstruct_keys(keys) + { index: index, level: level } + end + + def ==(other) + other.is_a?(SetBlockParam) && + other.index == index && + other.level == level + end + def length 3 end @@ -4635,6 +5339,16 @@ def to_a(_iseq) [:setclassvariable, name, cache] end + def deconstruct_keys(keys) + { name: name, cache: cache } + end + + def ==(other) + other.is_a?(SetClassVariable) && + other.name == name && + other.cache == cache + end + def length 3 end @@ -4684,6 +5398,14 @@ def to_a(_iseq) [:setconstant, name] end + def deconstruct_keys(keys) + { name: name } + end + + def ==(other) + other.is_a?(SetConstant) && other.name == name + end + def length 2 end @@ -4732,6 +5454,14 @@ def to_a(_iseq) [:setglobal, name] end + def deconstruct_keys(keys) + { name: name } + end + + def ==(other) + other.is_a?(SetGlobal) && other.name == name + end + def length 2 end @@ -4790,6 +5520,16 @@ def to_a(_iseq) [:setinstancevariable, name, cache] end + def deconstruct_keys(keys) + { name: name, cache: cache } + end + + def ==(other) + other.is_a?(SetInstanceVariable) && + other.name == name && + other.cache == cache + end + def length 3 end @@ -4844,6 +5584,14 @@ def to_a(iseq) [:setlocal, current.local_table.offset(index), level] end + def deconstruct_keys(keys) + { index: index, level: level } + end + + def ==(other) + other.is_a?(SetLocal) && other.index == index && other.level == level + end + def length 3 end @@ -4893,6 +5641,14 @@ def to_a(iseq) [:setlocal_WC_0, iseq.local_table.offset(index)] end + def deconstruct_keys(keys) + { index: index } + end + + def ==(other) + other.is_a?(SetLocalWC0) && other.index == index + end + def length 2 end @@ -4942,6 +5698,14 @@ def to_a(iseq) [:setlocal_WC_1, iseq.parent_iseq.local_table.offset(index)] end + def deconstruct_keys(keys) + { index: index } + end + + def ==(other) + other.is_a?(SetLocalWC1) && other.index == index + end + def length 2 end @@ -4989,6 +5753,14 @@ def to_a(_iseq) [:setn, number] end + def deconstruct_keys(keys) + { number: number } + end + + def ==(other) + other.is_a?(SetN) && other.number == number + end + def length 2 end @@ -5037,6 +5809,14 @@ def to_a(_iseq) [:setspecial, key] end + def deconstruct_keys(keys) + { key: key } + end + + def ==(other) + other.is_a?(SetSpecial) && other.key == key + end + def length 2 end @@ -5092,6 +5872,14 @@ def to_a(_iseq) [:splatarray, flag] end + def deconstruct_keys(keys) + { flag: flag } + end + + def ==(other) + other.is_a?(SplatArray) && other.flag == flag + end + def length 2 end @@ -5156,6 +5944,14 @@ def to_a(_iseq) [:swap] end + def deconstruct_keys(keys) + {} + end + + def ==(other) + other.is_a?(Swap) + end + def length 1 end @@ -5218,6 +6014,14 @@ def to_a(_iseq) [:throw, type] end + def deconstruct_keys(keys) + { type: type } + end + + def ==(other) + other.is_a?(Throw) && other.type == type + end + def length 2 end @@ -5304,6 +6108,14 @@ def to_a(_iseq) [:topn, number] end + def deconstruct_keys(keys) + { number: number } + end + + def ==(other) + other.is_a?(TopN) && other.number == number + end + def length 2 end @@ -5352,6 +6164,16 @@ def to_a(_iseq) [:toregexp, options, length] end + def deconstruct_keys(keys) + { options: options, length: length } + end + + def ==(other) + other.is_a?(ToRegExp) && + other.options == options && + other.length == length + end + def pops length end diff --git a/lib/syntax_tree/yarv/legacy.rb b/lib/syntax_tree/yarv/legacy.rb index b2e33290..1ee8e0d5 100644 --- a/lib/syntax_tree/yarv/legacy.rb +++ b/lib/syntax_tree/yarv/legacy.rb @@ -34,6 +34,14 @@ def to_a(_iseq) [:getclassvariable, name] end + def deconstruct_keys(keys) + { name: name } + end + + def ==(other) + other.is_a?(GetClassVariable) && other.name == name + end + def length 2 end @@ -90,6 +98,16 @@ def to_a(_iseq) [:opt_getinlinecache, label.name, cache] end + def deconstruct_keys(keys) + { label: label, cache: cache } + end + + def ==(other) + other.is_a?(OptGetInlineCache) && + other.label == label && + other.cache == cache + end + def length 3 end @@ -141,6 +159,14 @@ def to_a(_iseq) [:opt_setinlinecache, cache] end + def deconstruct_keys(keys) + { cache: cache } + end + + def ==(other) + other.is_a?(OptSetInlineCache) && other.cache == cache + end + def length 2 end @@ -190,6 +216,14 @@ def to_a(_iseq) [:setclassvariable, name] end + def deconstruct_keys(keys) + { name: name } + end + + def ==(other) + other.is_a?(SetClassVariable) && other.name == name + end + def length 2 end diff --git a/test/yarv_test.rb b/test/yarv_test.rb index 6f60d74e..4efeae25 100644 --- a/test/yarv_test.rb +++ b/test/yarv_test.rb @@ -288,6 +288,41 @@ def value end end + instructions = + YARV.constants.map { YARV.const_get(_1) } + + YARV::Legacy.constants.map { YARV::Legacy.const_get(_1) } - + [ + YARV::Assembler, + YARV::Bf, + YARV::CallData, + YARV::Compiler, + YARV::Decompiler, + YARV::Disassembler, + YARV::InstructionSequence, + YARV::Legacy, + YARV::LocalTable, + YARV::VM + ] + + interface = %i[ + disasm + to_a + deconstruct_keys + length + pops + pushes + canonical + call + == + ] + + instructions.each do |instruction| + define_method("test_instruction_interface_#{instruction.name}") do + instance_methods = instruction.instance_methods(false) + assert_empty(interface - instance_methods) + end + end + private def assert_decompiles(expected, source) From b47eb46be8cb47fca1474d4c532a99484f59217c Mon Sep 17 00:00:00 2001 From: Vinicius Stock Date: Wed, 18 Jan 2023 14:22:02 -0300 Subject: [PATCH 325/536] Add arity to Params, DefNode and BlockNode --- CHANGELOG.md | 4 + lib/syntax_tree/node.rb | 37 +++++++++ test/node_test.rb | 163 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 204 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b29fcbb..f71e5d21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +### Added + +- Arity has been added to DefNode, BlockNode and Params. The method returns a range where the lower bound is the minimum and the upper bound is the maximum number of arguments that can be used to invoke that block/method definition. + ## [5.2.0] - 2023-01-04 ### Added diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index f19cfb2c..3e35bf41 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -4175,6 +4175,17 @@ def ===(other) def endless? !bodystmt.is_a?(BodyStmt) end + + def arity + case params + when Params + params.arity + when Paren + params.contents.arity + else + 0..0 + end + end end # Defined represents the use of the +defined?+ operator. It can be used with @@ -4362,6 +4373,15 @@ def keywords? opening.is_a?(Kw) end + def arity + case block_var + when BlockVar + block_var.params.arity + else + 0..0 + end + end + private # If this is nested anywhere inside certain nodes, then we can't change @@ -8325,6 +8345,23 @@ def ===(other) keyword_rest === other.keyword_rest && block === other.block end + # Returns a range representing the possible number of arguments accepted + # by this params node not including the block. For example: + # def foo(a, b = 1, c:, d: 2, &block) + # ... + # end + # has arity 2..4 + def arity + optional_keywords = keywords.count { |_label, value| value } + lower_bound = + requireds.length + posts.length + keywords.length - optional_keywords + + upper_bound = + lower_bound + optionals.length + + optional_keywords if keyword_rest.nil? && rest.nil? + lower_bound..upper_bound + end + private def format_contents(q, parts) diff --git a/test/node_test.rb b/test/node_test.rb index 3d700e73..8741f274 100644 --- a/test/node_test.rb +++ b/test/node_test.rb @@ -1058,6 +1058,169 @@ def test_root_class_raises_not_implemented_errors end end + def test_arity_no_args + source = <<~SOURCE + def foo + end + SOURCE + + at = location(chars: 0..11, columns: 0..3, lines: 1..2) + assert_node(DefNode, source, at: at) do |node| + assert_equal(0..0, node.arity) + node + end + end + + def test_arity_positionals + source = <<~SOURCE + def foo(a, b = 1) + end + SOURCE + + at = location(chars: 0..21, columns: 0..3, lines: 1..2) + assert_node(DefNode, source, at: at) do |node| + assert_equal(1..2, node.arity) + node + end + end + + def test_arity_rest + source = <<~SOURCE + def foo(a, *b) + end + SOURCE + + at = location(chars: 0..18, columns: 0..3, lines: 1..2) + assert_node(DefNode, source, at: at) do |node| + assert_equal(1.., node.arity) + node + end + end + + def test_arity_keyword_rest + source = <<~SOURCE + def foo(a, **b) + end + SOURCE + + at = location(chars: 0..19, columns: 0..3, lines: 1..2) + assert_node(DefNode, source, at: at) do |node| + assert_equal(1.., node.arity) + node + end + end + + def test_arity_keywords + source = <<~SOURCE + def foo(a:, b: 1) + end + SOURCE + + at = location(chars: 0..21, columns: 0..3, lines: 1..2) + assert_node(DefNode, source, at: at) do |node| + assert_equal(1..2, node.arity) + node + end + end + + def test_arity_mixed + source = <<~SOURCE + def foo(a, b = 1, c:, d: 2) + end + SOURCE + + at = location(chars: 0..31, columns: 0..3, lines: 1..2) + assert_node(DefNode, source, at: at) do |node| + assert_equal(2..4, node.arity) + node + end + end + + guard_version("2.7.3") do + def test_arity_arg_forward + source = <<~SOURCE + def foo(...) + end + SOURCE + + at = location(chars: 0..16, columns: 0..3, lines: 1..2) + assert_node(DefNode, source, at: at) do |node| + assert_equal(0.., node.arity) + node + end + end + end + + guard_version("3.0.0") do + def test_arity_positional_and_arg_forward + source = <<~SOURCE + def foo(a, ...) + end + SOURCE + + at = location(chars: 0..19, columns: 0..3, lines: 1..2) + assert_node(DefNode, source, at: at) do |node| + assert_equal(1.., node.arity) + node + end + end + end + + def test_arity_no_parenthesis + source = <<~SOURCE + def foo a, b = 1 + end + SOURCE + + at = location(chars: 0..20, columns: 0..3, lines: 1..2) + assert_node(DefNode, source, at: at) do |node| + assert_equal(1..2, node.arity) + node + end + end + + def test_block_arity_positionals + source = <<~SOURCE + [].each do |a, b, c| + end + SOURCE + + at = location(chars: 8..24, columns: 8..3, lines: 1..2) + assert_node(BlockNode, source, at: at) do |node| + block = node.block + assert_equal(3..3, block.arity) + block + end + end + + def test_block_arity_with_optional + source = <<~SOURCE + [].each do |a, b = 1| + end + SOURCE + + at = location(chars: 8..25, columns: 8..3, lines: 1..2) + assert_node(BlockNode, source, at: at) do |node| + block = node.block + assert_equal(1..2, block.arity) + block + end + end + + def test_block_arity_with_optional_keyword + source = <<~SOURCE + [].each do |a, b: 2| + end + SOURCE + + at = location(chars: 8..24, columns: 8..3, lines: 1..2) + assert_node(BlockNode, source, at: at) do |node| + block = node.block + assert_equal(1..2, block.arity) + block + end + end + private def location(lines: 1..1, chars: 0..0, columns: 0..0) From 2c12f9a55243b215a80660d64256c99b6e43ea7b Mon Sep 17 00:00:00 2001 From: Vinicius Stock Date: Wed, 18 Jan 2023 15:14:18 -0300 Subject: [PATCH 326/536] Add arity to CallNode, VCall, CommandCall and Command --- CHANGELOG.md | 1 + lib/syntax_tree/node.rb | 40 ++++++++++ test/node_test.rb | 173 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 214 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f71e5d21..cf347efb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ### Added - Arity has been added to DefNode, BlockNode and Params. The method returns a range where the lower bound is the minimum and the upper bound is the maximum number of arguments that can be used to invoke that block/method definition. +- Arity has been added to CallNode, Command, CommandCall and VCall nodes. The method returns the number of arguments included in the invocation. For splats, double splats or argument forwards, this method returns Float::INFINITY. ## [5.2.0] - 2023-01-04 diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 3e35bf41..d1d40154 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -775,6 +775,10 @@ def ===(other) other.is_a?(ArgParen) && arguments === other.arguments end + def arity + arguments&.arity || 0 + end + private def trailing_comma? @@ -848,6 +852,22 @@ def format(q) def ===(other) other.is_a?(Args) && ArrayMatch.call(parts, other.parts) end + + def arity + accepts_infinite_arguments? ? Float::INFINITY : parts.length + end + + private + + def accepts_infinite_arguments? + parts.any? do |part| + part.is_a?(ArgStar) || part.is_a?(ArgsForward) || + ( + part.is_a?(BareAssocHash) && + part.assocs.any? { |p| p.is_a?(AssocSplat) } + ) + end + end end # ArgBlock represents using a block operator on an expression. @@ -1008,6 +1028,10 @@ def format(q) def ===(other) other.is_a?(ArgsForward) end + + def arity + Float::INFINITY + end end # ArrayLiteral represents an array literal, which can optionally contain @@ -3068,6 +3092,10 @@ def format_contents(q) end end end + + def arity + arguments&.arity || 0 + end end # Case represents the beginning of a case chain. @@ -3481,6 +3509,10 @@ def ===(other) arguments === other.arguments && block === other.block end + def arity + arguments.arity + end + private def align(q, node, &block) @@ -3646,6 +3678,10 @@ def ===(other) arguments === other.arguments && block === other.block end + def arity + arguments&.arity || 0 + end + private def argument_alignment(q, doc) @@ -11631,6 +11667,10 @@ def ===(other) def access_control? @access_control ||= %w[private protected public].include?(value.value) end + + def arity + 0 + end end # VoidStmt represents an empty lexical block of code. diff --git a/test/node_test.rb b/test/node_test.rb index 8741f274..7254c086 100644 --- a/test/node_test.rb +++ b/test/node_test.rb @@ -1221,6 +1221,179 @@ def test_block_arity_with_optional_keyword end end + def test_call_node_arity_positional_arguments + source = <<~SOURCE + foo(1, 2, 3) + SOURCE + + at = location(chars: 0..12, columns: 0..3, lines: 1..1) + assert_node(CallNode, source, at: at) do |node| + assert_equal(3, node.arity) + node + end + end + + def test_call_node_arity_keyword_arguments + source = <<~SOURCE + foo(bar, something: 123) + SOURCE + + at = location(chars: 0..24, columns: 0..24, lines: 1..1) + assert_node(CallNode, source, at: at) do |node| + assert_equal(2, node.arity) + node + end + end + + def test_call_node_arity_splat_arguments + source = <<~SOURCE + foo(*bar) + SOURCE + + at = location(chars: 0..9, columns: 0..9, lines: 1..1) + assert_node(CallNode, source, at: at) do |node| + assert_equal(Float::INFINITY, node.arity) + node + end + end + + def test_call_node_arity_keyword_rest_arguments + source = <<~SOURCE + foo(**bar) + SOURCE + + at = location(chars: 0..10, columns: 0..10, lines: 1..1) + assert_node(CallNode, source, at: at) do |node| + assert_equal(Float::INFINITY, node.arity) + node + end + end + + guard_version("2.7.3") do + def test_call_node_arity_arg_forward_arguments + source = <<~SOURCE + def foo(...) + bar(...) + end + SOURCE + + at = location(chars: 15..23, columns: 2..10, lines: 2..2) + assert_node(CallNode, source, at: at) do |node| + call = node.bodystmt.statements.body.first + assert_equal(Float::INFINITY, call.arity) + call + end + end + end + + def test_command_arity_positional_arguments + source = <<~SOURCE + foo 1, 2, 3 + SOURCE + + at = location(chars: 0..11, columns: 0..3, lines: 1..1) + assert_node(Command, source, at: at) do |node| + assert_equal(3, node.arity) + node + end + end + + def test_command_arity_keyword_arguments + source = <<~SOURCE + foo bar, something: 123 + SOURCE + + at = location(chars: 0..23, columns: 0..23, lines: 1..1) + assert_node(Command, source, at: at) do |node| + assert_equal(2, node.arity) + node + end + end + + def test_command_arity_splat_arguments + source = <<~SOURCE + foo *bar + SOURCE + + at = location(chars: 0..8, columns: 0..8, lines: 1..1) + assert_node(Command, source, at: at) do |node| + assert_equal(Float::INFINITY, node.arity) + node + end + end + + def test_command_arity_keyword_rest_arguments + source = <<~SOURCE + foo **bar + SOURCE + + at = location(chars: 0..9, columns: 0..9, lines: 1..1) + assert_node(Command, source, at: at) do |node| + assert_equal(Float::INFINITY, node.arity) + node + end + end + + def test_command_call_arity_positional_arguments + source = <<~SOURCE + object.foo 1, 2, 3 + SOURCE + + at = location(chars: 0..18, columns: 0..3, lines: 1..1) + assert_node(CommandCall, source, at: at) do |node| + assert_equal(3, node.arity) + node + end + end + + def test_command_call_arity_keyword_arguments + source = <<~SOURCE + object.foo bar, something: 123 + SOURCE + + at = location(chars: 0..30, columns: 0..30, lines: 1..1) + assert_node(CommandCall, source, at: at) do |node| + assert_equal(2, node.arity) + node + end + end + + def test_command_call_arity_splat_arguments + source = <<~SOURCE + object.foo *bar + SOURCE + + at = location(chars: 0..15, columns: 0..15, lines: 1..1) + assert_node(CommandCall, source, at: at) do |node| + assert_equal(Float::INFINITY, node.arity) + node + end + end + + def test_command_call_arity_keyword_rest_arguments + source = <<~SOURCE + object.foo **bar + SOURCE + + at = location(chars: 0..16, columns: 0..16, lines: 1..1) + assert_node(CommandCall, source, at: at) do |node| + assert_equal(Float::INFINITY, node.arity) + node + end + end + + def test_vcall_arity + source = <<~SOURCE + foo + SOURCE + + at = location(chars: 0..3, columns: 0..3, lines: 1..1) + assert_node(VCall, source, at: at) do |node| + assert_equal(0, node.arity) + node + end + end + private def location(lines: 1..1, chars: 0..0, columns: 0..0) From 66613acd533b7c01f008b50741e0d3c8f7b308d9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Jan 2023 17:12:31 +0000 Subject: [PATCH 327/536] Bump actions/configure-pages from 2 to 3 Bumps [actions/configure-pages](https://github.com/actions/configure-pages) from 2 to 3. - [Release notes](https://github.com/actions/configure-pages/releases) - [Commits](https://github.com/actions/configure-pages/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/configure-pages dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/gh-pages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index fc02f2fe..6c64676d 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -27,7 +27,7 @@ jobs: - name: Checkout uses: actions/checkout@v3 - name: Setup Pages - uses: actions/configure-pages@v2 + uses: actions/configure-pages@v3 - name: Set up Ruby uses: ruby/setup-ruby@v1 with: From 8d9fa5a2bd87a6a883c18e18c975837447888ab6 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 18 Jan 2023 13:00:02 -0500 Subject: [PATCH 328/536] Enhance and test the interface for YARV instructions --- .rubocop.yml | 1 + lib/syntax_tree/yarv/instructions.rb | 258 ++++++++++++--------------- lib/syntax_tree/yarv/legacy.rb | 19 +- test/yarv_test.rb | 26 +-- 4 files changed, 141 insertions(+), 163 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index 069041bd..0212027b 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -8,6 +8,7 @@ AllCops: TargetRubyVersion: 2.7 Exclude: - '{.git,.github,bin,coverage,pkg,spec,test/fixtures,vendor,tmp}/**/*' + - test/ruby-syntax-fixtures/**/* - test.rb Layout/LineLength: diff --git a/lib/syntax_tree/yarv/instructions.rb b/lib/syntax_tree/yarv/instructions.rb index 20068eac..bba06f8d 100644 --- a/lib/syntax_tree/yarv/instructions.rb +++ b/lib/syntax_tree/yarv/instructions.rb @@ -91,7 +91,7 @@ def to_a(_iseq) [:adjuststack, number] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { number: number } end @@ -147,7 +147,7 @@ def to_a(_iseq) [:anytostring] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) {} end @@ -213,7 +213,7 @@ def to_a(_iseq) [:branchif, label.name] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { label: label } end @@ -274,7 +274,7 @@ def to_a(_iseq) [:branchnil, label.name] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { label: label } end @@ -334,7 +334,7 @@ def to_a(_iseq) [:branchunless, label.name] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { label: label } end @@ -405,7 +405,7 @@ def to_a(iseq) ] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { keyword_bits_index: keyword_bits_index, keyword_index: keyword_index } end @@ -469,7 +469,7 @@ def to_a(_iseq) [:checkmatch, type] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { type: type } end @@ -619,7 +619,7 @@ def to_a(_iseq) [:checktype, type] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { type: type } end @@ -722,7 +722,7 @@ def to_a(_iseq) [:concatarray] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) {} end @@ -782,7 +782,7 @@ def to_a(_iseq) [:concatstrings, number] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { number: number } end @@ -853,15 +853,13 @@ def to_a(_iseq) [:defineclass, name, class_iseq.to_a, flags] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { name: name, class_iseq: class_iseq, flags: flags } end def ==(other) - other.is_a?(DefineClass) && - other.name == name && - other.class_iseq == class_iseq && - other.flags == flags + other.is_a?(DefineClass) && other.name == name && + other.class_iseq == class_iseq && other.flags == flags end def length @@ -992,14 +990,12 @@ def to_a(_iseq) [:defined, type, name, message] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { type: type, name: name, message: message } end def ==(other) - other.is_a?(Defined) && - other.type == type && - other.name == name && + other.is_a?(Defined) && other.type == type && other.name == name && other.message == message end @@ -1093,13 +1089,12 @@ def to_a(_iseq) [:definemethod, method_name, method_iseq.to_a] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { method_name: method_name, method_iseq: method_iseq } end def ==(other) - other.is_a?(DefineMethod) && - other.method_name == method_name && + other.is_a?(DefineMethod) && other.method_name == method_name && other.method_iseq == method_iseq end @@ -1175,13 +1170,12 @@ def to_a(_iseq) [:definesmethod, method_name, method_iseq.to_a] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { method_name: method_name, method_iseq: method_iseq } end def ==(other) - other.is_a?(DefineSMethod) && - other.method_name == method_name && + other.is_a?(DefineSMethod) && other.method_name == method_name && other.method_iseq == method_iseq end @@ -1242,7 +1236,7 @@ def to_a(_iseq) [:dup] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) {} end @@ -1296,7 +1290,7 @@ def to_a(_iseq) [:duparray, object] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { object: object } end @@ -1350,7 +1344,7 @@ def to_a(_iseq) [:duphash, object] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { object: object } end @@ -1404,7 +1398,7 @@ def to_a(_iseq) [:dupn, number] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { number: number } end @@ -1463,13 +1457,12 @@ def to_a(_iseq) [:expandarray, number, flags] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { number: number, flags: flags } end def ==(other) - other.is_a?(ExpandArray) && - other.number == number && + other.is_a?(ExpandArray) && other.number == number && other.flags == flags end @@ -1564,13 +1557,12 @@ def to_a(iseq) [:getblockparam, current.local_table.offset(index), level] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { index: index, level: level } end def ==(other) - other.is_a?(GetBlockParam) && - other.index == index && + other.is_a?(GetBlockParam) && other.index == index && other.level == level end @@ -1631,13 +1623,12 @@ def to_a(iseq) [:getblockparamproxy, current.local_table.offset(index), level] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { index: index, level: level } end def ==(other) - other.is_a?(GetBlockParamProxy) && - other.index == index && + other.is_a?(GetBlockParamProxy) && other.index == index && other.level == level end @@ -1693,13 +1684,12 @@ def to_a(_iseq) [:getclassvariable, name, cache] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { name: name, cache: cache } end def ==(other) - other.is_a?(GetClassVariable) && - other.name == name && + other.is_a?(GetClassVariable) && other.name == name && other.cache == cache end @@ -1753,7 +1743,7 @@ def to_a(_iseq) [:getconstant, name] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { name: name } end @@ -1823,7 +1813,7 @@ def to_a(_iseq) [:getglobal, name] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { name: name } end @@ -1890,13 +1880,12 @@ def to_a(_iseq) [:getinstancevariable, name, cache] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { name: name, cache: cache } end def ==(other) - other.is_a?(GetInstanceVariable) && - other.name == name && + other.is_a?(GetInstanceVariable) && other.name == name && other.cache == cache end @@ -1954,7 +1943,7 @@ def to_a(iseq) [:getlocal, current.local_table.offset(index), level] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { index: index, level: level } end @@ -2011,7 +2000,7 @@ def to_a(iseq) [:getlocal_WC_0, iseq.local_table.offset(index)] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { index: index } end @@ -2068,7 +2057,7 @@ def to_a(iseq) [:getlocal_WC_1, iseq.parent_iseq.local_table.offset(index)] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { index: index } end @@ -2127,7 +2116,7 @@ def to_a(_iseq) [:getspecial, key, type] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { key: key, type: type } end @@ -2183,7 +2172,7 @@ def to_a(_iseq) [:intern] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) {} end @@ -2241,7 +2230,7 @@ def to_a(_iseq) [:invokeblock, calldata.to_h] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { calldata: calldata } end @@ -2304,13 +2293,12 @@ def to_a(_iseq) [:invokesuper, calldata.to_h, block_iseq&.to_a] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { calldata: calldata, block_iseq: block_iseq } end def ==(other) - other.is_a?(InvokeSuper) && - other.calldata == calldata && + other.is_a?(InvokeSuper) && other.calldata == calldata && other.block_iseq == block_iseq end @@ -2385,7 +2373,7 @@ def to_a(_iseq) [:jump, label.name] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { label: label } end @@ -2433,7 +2421,7 @@ def to_a(_iseq) [:leave] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) {} end @@ -2491,7 +2479,7 @@ def to_a(_iseq) [:newarray, number] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { number: number } end @@ -2547,7 +2535,7 @@ def to_a(_iseq) [:newarraykwsplat, number] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { number: number } end @@ -2605,7 +2593,7 @@ def to_a(_iseq) [:newhash, number] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { number: number } end @@ -2664,7 +2652,7 @@ def to_a(_iseq) [:newrange, exclude_end] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { exclude_end: exclude_end } end @@ -2713,7 +2701,7 @@ def to_a(_iseq) [:nop] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) {} end @@ -2770,7 +2758,7 @@ def to_a(_iseq) [:objtostring, calldata.to_h] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { calldata: calldata } end @@ -2829,7 +2817,7 @@ def to_a(_iseq) [:once, iseq.to_a, cache] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { iseq: iseq, cache: cache } end @@ -2888,7 +2876,7 @@ def to_a(_iseq) [:opt_and, calldata.to_h] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { calldata: calldata } end @@ -2944,7 +2932,7 @@ def to_a(_iseq) [:opt_aref, calldata.to_h] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { calldata: calldata } end @@ -3005,13 +2993,12 @@ def to_a(_iseq) [:opt_aref_with, object, calldata.to_h] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { object: object, calldata: calldata } end def ==(other) - other.is_a?(OptArefWith) && - other.object == object && + other.is_a?(OptArefWith) && other.object == object && other.calldata == calldata end @@ -3064,7 +3051,7 @@ def to_a(_iseq) [:opt_aset, calldata.to_h] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { calldata: calldata } end @@ -3124,13 +3111,12 @@ def to_a(_iseq) [:opt_aset_with, object, calldata.to_h] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { object: object, calldata: calldata } end def ==(other) - other.is_a?(OptAsetWith) && - other.object == object && + other.is_a?(OptAsetWith) && other.object == object && other.calldata == calldata end @@ -3202,7 +3188,7 @@ def to_a(_iseq) ] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { case_dispatch_hash: case_dispatch_hash, else_label: else_label } end @@ -3261,7 +3247,7 @@ def to_a(_iseq) [:opt_div, calldata.to_h] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { calldata: calldata } end @@ -3317,7 +3303,7 @@ def to_a(_iseq) [:opt_empty_p, calldata.to_h] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { calldata: calldata } end @@ -3374,7 +3360,7 @@ def to_a(_iseq) [:opt_eq, calldata.to_h] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { calldata: calldata } end @@ -3431,7 +3417,7 @@ def to_a(_iseq) [:opt_ge, calldata.to_h] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { calldata: calldata } end @@ -3488,7 +3474,7 @@ def to_a(_iseq) [:opt_getconstant_path, names] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { names: names } end @@ -3552,7 +3538,7 @@ def to_a(_iseq) [:opt_gt, calldata.to_h] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { calldata: calldata } end @@ -3609,7 +3595,7 @@ def to_a(_iseq) [:opt_le, calldata.to_h] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { calldata: calldata } end @@ -3666,7 +3652,7 @@ def to_a(_iseq) [:opt_length, calldata.to_h] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { calldata: calldata } end @@ -3723,7 +3709,7 @@ def to_a(_iseq) [:opt_lt, calldata.to_h] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { calldata: calldata } end @@ -3780,7 +3766,7 @@ def to_a(_iseq) [:opt_ltlt, calldata.to_h] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { calldata: calldata } end @@ -3838,7 +3824,7 @@ def to_a(_iseq) [:opt_minus, calldata.to_h] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { calldata: calldata } end @@ -3895,7 +3881,7 @@ def to_a(_iseq) [:opt_mod, calldata.to_h] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { calldata: calldata } end @@ -3952,7 +3938,7 @@ def to_a(_iseq) [:opt_mult, calldata.to_h] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { calldata: calldata } end @@ -4015,13 +4001,12 @@ def to_a(_iseq) [:opt_neq, eq_calldata.to_h, neq_calldata.to_h] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { eq_calldata: eq_calldata, neq_calldata: neq_calldata } end def ==(other) - other.is_a?(OptNEq) && - other.eq_calldata == eq_calldata && + other.is_a?(OptNEq) && other.eq_calldata == eq_calldata && other.neq_calldata == neq_calldata end @@ -4074,7 +4059,7 @@ def to_a(_iseq) [:opt_newarray_max, number] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { number: number } end @@ -4130,7 +4115,7 @@ def to_a(_iseq) [:opt_newarray_min, number] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { number: number } end @@ -4187,7 +4172,7 @@ def to_a(_iseq) [:opt_nil_p, calldata.to_h] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { calldata: calldata } end @@ -4242,7 +4227,7 @@ def to_a(_iseq) [:opt_not, calldata.to_h] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { calldata: calldata } end @@ -4299,7 +4284,7 @@ def to_a(_iseq) [:opt_or, calldata.to_h] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { calldata: calldata } end @@ -4356,7 +4341,7 @@ def to_a(_iseq) [:opt_plus, calldata.to_h] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { calldata: calldata } end @@ -4412,7 +4397,7 @@ def to_a(_iseq) [:opt_regexpmatch2, calldata.to_h] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { calldata: calldata } end @@ -4468,7 +4453,7 @@ def to_a(_iseq) [:opt_send_without_block, calldata.to_h] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { calldata: calldata } end @@ -4525,7 +4510,7 @@ def to_a(_iseq) [:opt_size, calldata.to_h] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { calldata: calldata } end @@ -4585,13 +4570,12 @@ def to_a(_iseq) [:opt_str_freeze, object, calldata.to_h] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { object: object, calldata: calldata } end def ==(other) - other.is_a?(OptStrFreeze) && - other.object == object && + other.is_a?(OptStrFreeze) && other.object == object && other.calldata == calldata end @@ -4647,13 +4631,12 @@ def to_a(_iseq) [:opt_str_uminus, object, calldata.to_h] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { object: object, calldata: calldata } end def ==(other) - other.is_a?(OptStrUMinus) && - other.object == object && + other.is_a?(OptStrUMinus) && other.object == object && other.calldata == calldata end @@ -4706,7 +4689,7 @@ def to_a(_iseq) [:opt_succ, calldata.to_h] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { calldata: calldata } end @@ -4754,7 +4737,7 @@ def to_a(_iseq) [:pop] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) {} end @@ -4802,7 +4785,7 @@ def to_a(_iseq) [:putnil] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) {} end @@ -4856,7 +4839,7 @@ def to_a(_iseq) [:putobject, object] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { object: object } end @@ -4906,7 +4889,7 @@ def to_a(_iseq) [:putobject_INT2FIX_0_] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) {} end @@ -4956,7 +4939,7 @@ def to_a(_iseq) [:putobject_INT2FIX_1_] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) {} end @@ -5004,7 +4987,7 @@ def to_a(_iseq) [:putself] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) {} end @@ -5064,7 +5047,7 @@ def to_a(_iseq) [:putspecialobject, object] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { object: object } end @@ -5127,7 +5110,7 @@ def to_a(_iseq) [:putstring, object] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { object: object } end @@ -5189,13 +5172,12 @@ def to_a(_iseq) [:send, calldata.to_h, block_iseq&.to_a] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { calldata: calldata, block_iseq: block_iseq } end def ==(other) - other.is_a?(Send) && - other.calldata == calldata && + other.is_a?(Send) && other.calldata == calldata && other.block_iseq == block_iseq end @@ -5276,13 +5258,12 @@ def to_a(iseq) [:setblockparam, current.local_table.offset(index), level] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { index: index, level: level } end def ==(other) - other.is_a?(SetBlockParam) && - other.index == index && + other.is_a?(SetBlockParam) && other.index == index && other.level == level end @@ -5339,13 +5320,12 @@ def to_a(_iseq) [:setclassvariable, name, cache] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { name: name, cache: cache } end def ==(other) - other.is_a?(SetClassVariable) && - other.name == name && + other.is_a?(SetClassVariable) && other.name == name && other.cache == cache end @@ -5398,7 +5378,7 @@ def to_a(_iseq) [:setconstant, name] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { name: name } end @@ -5454,7 +5434,7 @@ def to_a(_iseq) [:setglobal, name] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { name: name } end @@ -5520,13 +5500,12 @@ def to_a(_iseq) [:setinstancevariable, name, cache] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { name: name, cache: cache } end def ==(other) - other.is_a?(SetInstanceVariable) && - other.name == name && + other.is_a?(SetInstanceVariable) && other.name == name && other.cache == cache end @@ -5584,7 +5563,7 @@ def to_a(iseq) [:setlocal, current.local_table.offset(index), level] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { index: index, level: level } end @@ -5641,7 +5620,7 @@ def to_a(iseq) [:setlocal_WC_0, iseq.local_table.offset(index)] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { index: index } end @@ -5698,7 +5677,7 @@ def to_a(iseq) [:setlocal_WC_1, iseq.parent_iseq.local_table.offset(index)] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { index: index } end @@ -5753,7 +5732,7 @@ def to_a(_iseq) [:setn, number] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { number: number } end @@ -5809,7 +5788,7 @@ def to_a(_iseq) [:setspecial, key] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { key: key } end @@ -5872,7 +5851,7 @@ def to_a(_iseq) [:splatarray, flag] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { flag: flag } end @@ -5944,7 +5923,7 @@ def to_a(_iseq) [:swap] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) {} end @@ -6014,7 +5993,7 @@ def to_a(_iseq) [:throw, type] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { type: type } end @@ -6108,7 +6087,7 @@ def to_a(_iseq) [:topn, number] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { number: number } end @@ -6164,13 +6143,12 @@ def to_a(_iseq) [:toregexp, options, length] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { options: options, length: length } end def ==(other) - other.is_a?(ToRegExp) && - other.options == options && + other.is_a?(ToRegExp) && other.options == options && other.length == length end diff --git a/lib/syntax_tree/yarv/legacy.rb b/lib/syntax_tree/yarv/legacy.rb index 1ee8e0d5..ab9b00df 100644 --- a/lib/syntax_tree/yarv/legacy.rb +++ b/lib/syntax_tree/yarv/legacy.rb @@ -34,10 +34,10 @@ def to_a(_iseq) [:getclassvariable, name] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { name: name } end - + def ==(other) other.is_a?(GetClassVariable) && other.name == name end @@ -98,13 +98,12 @@ def to_a(_iseq) [:opt_getinlinecache, label.name, cache] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { label: label, cache: cache } end - + def ==(other) - other.is_a?(OptGetInlineCache) && - other.label == label && + other.is_a?(OptGetInlineCache) && other.label == label && other.cache == cache end @@ -159,10 +158,10 @@ def to_a(_iseq) [:opt_setinlinecache, cache] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { cache: cache } end - + def ==(other) other.is_a?(OptSetInlineCache) && other.cache == cache end @@ -216,10 +215,10 @@ def to_a(_iseq) [:setclassvariable, name] end - def deconstruct_keys(keys) + def deconstruct_keys(_keys) { name: name } end - + def ==(other) other.is_a?(SetClassVariable) && other.name == name end diff --git a/test/yarv_test.rb b/test/yarv_test.rb index 4efeae25..be7c4c2d 100644 --- a/test/yarv_test.rb +++ b/test/yarv_test.rb @@ -290,19 +290,19 @@ def value instructions = YARV.constants.map { YARV.const_get(_1) } + - YARV::Legacy.constants.map { YARV::Legacy.const_get(_1) } - - [ - YARV::Assembler, - YARV::Bf, - YARV::CallData, - YARV::Compiler, - YARV::Decompiler, - YARV::Disassembler, - YARV::InstructionSequence, - YARV::Legacy, - YARV::LocalTable, - YARV::VM - ] + YARV::Legacy.constants.map { YARV::Legacy.const_get(_1) } - + [ + YARV::Assembler, + YARV::Bf, + YARV::CallData, + YARV::Compiler, + YARV::Decompiler, + YARV::Disassembler, + YARV::InstructionSequence, + YARV::Legacy, + YARV::LocalTable, + YARV::VM + ] interface = %i[ disasm From c1cd547451e04809c1a261e59c58c75641410baf Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 20 Jan 2023 09:48:08 -0500 Subject: [PATCH 329/536] A couple of convenience APIs --- lib/syntax_tree/yarv/assembler.rb | 16 +++++++----- lib/syntax_tree/yarv/compiler.rb | 2 +- lib/syntax_tree/yarv/decompiler.rb | 2 +- lib/syntax_tree/yarv/vm.rb | 4 +++ test/yarv_test.rb | 42 +++++++++++++++--------------- 5 files changed, 37 insertions(+), 29 deletions(-) diff --git a/lib/syntax_tree/yarv/assembler.rb b/lib/syntax_tree/yarv/assembler.rb index ec467b58..ac400506 100644 --- a/lib/syntax_tree/yarv/assembler.rb +++ b/lib/syntax_tree/yarv/assembler.rb @@ -62,22 +62,26 @@ def visit_string_literal(node) "constant-from" ].freeze - attr_reader :filepath + attr_reader :lines - def initialize(filepath) - @filepath = filepath + def initialize(lines) + @lines = lines end def assemble iseq = InstructionSequence.new("
", "", 1, :top) - assemble_iseq(iseq, File.readlines(filepath, chomp: true)) + assemble_iseq(iseq, lines) iseq.compile! iseq end - def self.assemble(filepath) - new(filepath).assemble + def self.assemble(source) + new(source.lines(chomp: true)).assemble + end + + def self.assemble_file(filepath) + new(File.readlines(filepath, chomp: true)).assemble end private diff --git a/lib/syntax_tree/yarv/compiler.rb b/lib/syntax_tree/yarv/compiler.rb index 4c9a4d50..c1b4d6dd 100644 --- a/lib/syntax_tree/yarv/compiler.rb +++ b/lib/syntax_tree/yarv/compiler.rb @@ -285,7 +285,7 @@ def visit_unsupported(_node) # if we need to return the value of the last statement. attr_reader :last_statement - def initialize(options) + def initialize(options = Options.new) @options = options @iseq = nil @last_statement = false diff --git a/lib/syntax_tree/yarv/decompiler.rb b/lib/syntax_tree/yarv/decompiler.rb index 47d2a2df..753ba80a 100644 --- a/lib/syntax_tree/yarv/decompiler.rb +++ b/lib/syntax_tree/yarv/decompiler.rb @@ -97,7 +97,7 @@ def decompile(iseq) clause << Next(Args([])) when Leave value = Args([clause.pop]) - clause << (iseq.type == :top ? Break(value) : ReturnNode(value)) + clause << (iseq.type != :top ? Break(value) : ReturnNode(value)) when OptAnd, OptDiv, OptEq, OptGE, OptGT, OptLE, OptLT, OptLTLT, OptMinus, OptMod, OptMult, OptOr, OptPlus left, right = clause.pop(2) diff --git a/lib/syntax_tree/yarv/vm.rb b/lib/syntax_tree/yarv/vm.rb index 1bbb82ed..b303944d 100644 --- a/lib/syntax_tree/yarv/vm.rb +++ b/lib/syntax_tree/yarv/vm.rb @@ -219,6 +219,10 @@ def initialize(events = NullEvents.new) @frame = nil end + def self.run(iseq) + new.run_top_frame(iseq) + end + ########################################################################## # Helper methods for frames ########################################################################## diff --git a/test/yarv_test.rb b/test/yarv_test.rb index be7c4c2d..e3995435 100644 --- a/test/yarv_test.rb +++ b/test/yarv_test.rb @@ -6,27 +6,27 @@ module SyntaxTree class YARVTest < Minitest::Test CASES = { - "0" => "break 0\n", - "1" => "break 1\n", - "2" => "break 2\n", - "1.0" => "break 1.0\n", - "1 + 2" => "break 1 + 2\n", - "1 - 2" => "break 1 - 2\n", - "1 * 2" => "break 1 * 2\n", - "1 / 2" => "break 1 / 2\n", - "1 % 2" => "break 1 % 2\n", - "1 < 2" => "break 1 < 2\n", - "1 <= 2" => "break 1 <= 2\n", - "1 > 2" => "break 1 > 2\n", - "1 >= 2" => "break 1 >= 2\n", - "1 == 2" => "break 1 == 2\n", - "1 != 2" => "break 1 != 2\n", - "1 & 2" => "break 1 & 2\n", - "1 | 2" => "break 1 | 2\n", - "1 << 2" => "break 1 << 2\n", - "1 >> 2" => "break 1.>>(2)\n", - "1 ** 2" => "break 1.**(2)\n", - "a = 1; a" => "a = 1\nbreak a\n" + "0" => "return 0\n", + "1" => "return 1\n", + "2" => "return 2\n", + "1.0" => "return 1.0\n", + "1 + 2" => "return 1 + 2\n", + "1 - 2" => "return 1 - 2\n", + "1 * 2" => "return 1 * 2\n", + "1 / 2" => "return 1 / 2\n", + "1 % 2" => "return 1 % 2\n", + "1 < 2" => "return 1 < 2\n", + "1 <= 2" => "return 1 <= 2\n", + "1 > 2" => "return 1 > 2\n", + "1 >= 2" => "return 1 >= 2\n", + "1 == 2" => "return 1 == 2\n", + "1 != 2" => "return 1 != 2\n", + "1 & 2" => "return 1 & 2\n", + "1 | 2" => "return 1 | 2\n", + "1 << 2" => "return 1 << 2\n", + "1 >> 2" => "return 1.>>(2)\n", + "1 ** 2" => "return 1.**(2)\n", + "a = 1; a" => "a = 1\nreturn a\n" }.freeze CASES.each do |source, expected| From 68fa0ad5987ded34d58e4f1c3c932cb75d6f5a04 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 24 Jan 2023 17:10:47 +0000 Subject: [PATCH 330/536] Bump dependabot/fetch-metadata from 1.3.5 to 1.3.6 Bumps [dependabot/fetch-metadata](https://github.com/dependabot/fetch-metadata) from 1.3.5 to 1.3.6. - [Release notes](https://github.com/dependabot/fetch-metadata/releases) - [Commits](https://github.com/dependabot/fetch-metadata/compare/v1.3.5...v1.3.6) --- updated-dependencies: - dependency-name: dependabot/fetch-metadata dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/auto-merge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/auto-merge.yml b/.github/workflows/auto-merge.yml index 514ac27a..e54c9100 100644 --- a/.github/workflows/auto-merge.yml +++ b/.github/workflows/auto-merge.yml @@ -12,7 +12,7 @@ jobs: steps: - name: Dependabot metadata id: metadata - uses: dependabot/fetch-metadata@v1.3.5 + uses: dependabot/fetch-metadata@v1.3.6 with: github-token: "${{ secrets.GITHUB_TOKEN }}" - name: Enable auto-merge for Dependabot PRs From bce6b87b0ab8b0c02de62b86da0c75f680ea5df6 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 25 Jan 2023 10:44:50 -0500 Subject: [PATCH 331/536] Handle invalid byte sequences in UTF-8 --- lib/syntax_tree/parser.rb | 21 +++++++++++++++++++-- test/parser_test.rb | 9 +++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index 602bb98f..99b703d0 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -1103,6 +1103,7 @@ def on_command_call(receiver, operator, message, arguments) # :call-seq: # on_comment: (String value) -> Comment def on_comment(value) + # char is the index of the # character in the source. char = char_pos location = Location.token( @@ -1112,8 +1113,24 @@ def on_comment(value) size: value.size - 1 ) - index = source.rindex(/[^\t ]/, char - 1) if char != 0 - inline = index && (source[index] != "\n") + # Loop backward in the source string, starting from the beginning of the + # comment, and find the first character that is not a space or a tab. If + # index is -1, this indicates that we've checked all of the characters + # back to the start of the source, so this comment must be at the + # beginning of the file. + # + # We are purposefully not using rindex or regular expressions here because + # they check if there are invalid characters, which is actually possible + # with the use of __END__ syntax. + index = char - 1 + while index > -1 && (source[index] == "\t" || source[index] == " ") + index -= 1 + end + + # If we found a character that was not a space or a tab before the comment + # and it's a newline, then this comment is inline. Otherwise, it stands on + # its own and can be attached as its own node in the tree. + inline = index != -1 && source[index] != "\n" comment = Comment.new(value: value.chomp, inline: inline, location: location) diff --git a/test/parser_test.rb b/test/parser_test.rb index 6048cf11..8d6c0a16 100644 --- a/test/parser_test.rb +++ b/test/parser_test.rb @@ -65,5 +65,14 @@ def foo end RUBY end + + def test_does_not_choke_on_invalid_characters_in_source_string + SyntaxTree.parse(<<~RUBY) + # comment + # comment + __END__ + \xC5 + RUBY + end end end From bc9e665798b68081c0cb14c75cb2fddc7c331d40 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 25 Jan 2023 11:38:38 -0500 Subject: [PATCH 332/536] Indexing functionality --- lib/syntax_tree.rb | 15 +++ lib/syntax_tree/index.rb | 223 +++++++++++++++++++++++++++++++++++++++ test/index_test.rb | 59 +++++++++++ 3 files changed, 297 insertions(+) create mode 100644 lib/syntax_tree/index.rb create mode 100644 test/index_test.rb diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index f1217ac3..f5c71aba 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -26,6 +26,7 @@ require_relative "syntax_tree/parser" require_relative "syntax_tree/pattern" require_relative "syntax_tree/search" +require_relative "syntax_tree/index" require_relative "syntax_tree/yarv" require_relative "syntax_tree/yarv/bf" @@ -116,4 +117,18 @@ def self.read(filepath) def self.search(source, query, &block) Search.new(Pattern.new(query).compile).scan(parse(source), &block) end + + # Indexes the given source code to return a list of all class, module, and + # method definitions. Used to quickly provide indexing capability for IDEs or + # documentation generation. + def self.index(source) + Index.index(source) + end + + # Indexes the given file to return a list of all class, module, and method + # definitions. Used to quickly provide indexing capability for IDEs or + # documentation generation. + def self.index_file(filepath) + Index.index_file(filepath) + end end diff --git a/lib/syntax_tree/index.rb b/lib/syntax_tree/index.rb new file mode 100644 index 00000000..60158314 --- /dev/null +++ b/lib/syntax_tree/index.rb @@ -0,0 +1,223 @@ +# frozen_string_literal: true + +module SyntaxTree + # This class can be used to build an index of the structure of Ruby files. We + # define an index as the list of constants and methods defined within a file. + # + # This index strives to be as fast as possible to better support tools like + # IDEs. Because of that, it has different backends depending on what + # functionality is available. + module Index + # This is a location for an index entry. + class Location + attr_reader :line, :column + + def initialize(line, column) + @line = line + @column = column + end + end + + # This entry represents a class definition using the class keyword. + class ClassDefinition + attr_reader :nesting, :name, :location + + def initialize(nesting, name, location) + @nesting = nesting + @name = name + @location = location + end + end + + # This entry represents a module definition using the module keyword. + class ModuleDefinition + attr_reader :nesting, :name, :location + + def initialize(nesting, name, location) + @nesting = nesting + @name = name + @location = location + end + end + + # This entry represents a method definition using the def keyword. + class MethodDefinition + attr_reader :nesting, :name, :location + + def initialize(nesting, name, location) + @nesting = nesting + @name = name + @location = location + end + end + + # This entry represents a singleton method definition using the def keyword + # with a specified target. + class SingletonMethodDefinition + attr_reader :nesting, :name, :location + + def initialize(nesting, name, location) + @nesting = nesting + @name = name + @location = location + end + end + + # This backend creates the index using RubyVM::InstructionSequence, which is + # faster than using the Syntax Tree parser, but is not available on all + # runtimes. + class ISeqBackend + VM_DEFINECLASS_TYPE_CLASS = 0x00 + VM_DEFINECLASS_TYPE_SINGLETON_CLASS = 0x01 + VM_DEFINECLASS_TYPE_MODULE = 0x02 + VM_DEFINECLASS_FLAG_SCOPED = 0x08 + VM_DEFINECLASS_FLAG_HAS_SUPERCLASS = 0x10 + + def index(source) + index_iseq(RubyVM::InstructionSequence.compile(source).to_a) + end + + def index_file(filepath) + index_iseq(RubyVM::InstructionSequence.compile_file(filepath).to_a) + end + + private + + def index_iseq(iseq) + results = [] + queue = [[iseq, []]] + + while (current_iseq, current_nesting = queue.shift) + current_iseq[13].each_with_index do |insn, index| + next unless insn.is_a?(Array) + + case insn[0] + when :defineclass + _, name, class_iseq, flags = insn + + if flags == VM_DEFINECLASS_TYPE_SINGLETON_CLASS + # At the moment, we don't support singletons that aren't + # defined on self. We could, but it would require more + # emulation. + if current_iseq[13][index - 2] != [:putself] + raise NotImplementedError, + "singleton class with non-self receiver" + end + elsif flags & VM_DEFINECLASS_TYPE_MODULE > 0 + code_location = class_iseq[4][:code_location] + location = Location.new(code_location[0], code_location[1]) + results << ModuleDefinition.new(current_nesting, name, location) + else + code_location = class_iseq[4][:code_location] + location = Location.new(code_location[0], code_location[1]) + results << ClassDefinition.new(current_nesting, name, location) + end + + queue << [class_iseq, current_nesting + [name]] + when :definemethod + _, name, method_iseq = insn + + code_location = method_iseq[4][:code_location] + location = Location.new(code_location[0], code_location[1]) + results << SingletonMethodDefinition.new( + current_nesting, + name, + location + ) + when :definesmethod + _, name, method_iseq = insn + + code_location = method_iseq[4][:code_location] + location = Location.new(code_location[0], code_location[1]) + results << MethodDefinition.new(current_nesting, name, location) + end + end + end + + results + end + end + + # This backend creates the index using the Syntax Tree parser and a visitor. + # It is not as fast as using the instruction sequences directly, but is + # supported on all runtimes. + class ParserBackend + class IndexVisitor < Visitor + attr_reader :results, :nesting + + def initialize + @results = [] + @nesting = [] + end + + def visit_class(node) + name = visit(node.constant).to_sym + location = + Location.new(node.location.start_line, node.location.start_column) + + results << ClassDefinition.new(nesting.dup, name, location) + nesting << name + + super + nesting.pop + end + + def visit_const_ref(node) + node.constant.value + end + + def visit_def(node) + name = node.name.value.to_sym + location = + Location.new(node.location.start_line, node.location.start_column) + + results << if node.target.nil? + MethodDefinition.new(nesting.dup, name, location) + else + SingletonMethodDefinition.new(nesting.dup, name, location) + end + end + + def visit_module(node) + name = visit(node.constant).to_sym + location = + Location.new(node.location.start_line, node.location.start_column) + + results << ModuleDefinition.new(nesting.dup, name, location) + nesting << name + + super + nesting.pop + end + + def visit_program(node) + super + results + end + end + + def index(source) + SyntaxTree.parse(source).accept(IndexVisitor.new) + end + + def index_file(filepath) + index(SyntaxTree.read(filepath)) + end + end + + # The class defined here is used to perform the indexing, depending on what + # functionality is available from the runtime. + INDEX_BACKEND = + defined?(RubyVM::InstructionSequence) ? ISeqBackend : ParserBackend + + # This method accepts source code and then indexes it. + def self.index(source) + INDEX_BACKEND.new.index(source) + end + + # This method accepts a filepath and then indexes it. + def self.index_file(filepath) + INDEX_BACKEND.new.index_file(filepath) + end + end +end diff --git a/test/index_test.rb b/test/index_test.rb new file mode 100644 index 00000000..3ea02a20 --- /dev/null +++ b/test/index_test.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +require_relative "test_helper" + +module SyntaxTree + class IndexTest < Minitest::Test + def test_module + index_each("module Foo; end") do |entry| + assert_equal :Foo, entry.name + assert_empty entry.nesting + end + end + + def test_module_nested + index_each("module Foo; module Bar; end; end") do |entry| + assert_equal :Bar, entry.name + assert_equal [:Foo], entry.nesting + end + end + + def test_class + index_each("class Foo; end") do |entry| + assert_equal :Foo, entry.name + assert_empty entry.nesting + end + end + + def test_class_nested + index_each("class Foo; class Bar; end; end") do |entry| + assert_equal :Bar, entry.name + assert_equal [:Foo], entry.nesting + end + end + + def test_method + index_each("def foo; end") do |entry| + assert_equal :foo, entry.name + assert_empty entry.nesting + end + end + + def test_method_nested + index_each("class Foo; def foo; end; end") do |entry| + assert_equal :foo, entry.name + assert_equal [:Foo], entry.nesting + end + end + + private + + def index_each(source) + yield SyntaxTree::Index::ParserBackend.new.index(source).last + + if defined?(RubyVM::InstructionSequence) + yield SyntaxTree::Index::ISeqBackend.new.index(source).last + end + end + end +end From 4d659883264ffd831572e84ef437e94e88b3b7a6 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 25 Jan 2023 13:19:05 -0500 Subject: [PATCH 333/536] Comments on index entries --- lib/syntax_tree/index.rb | 196 ++++++++++++++++++++++++++++++++++----- test/index_test.rb | 21 +++++ 2 files changed, 195 insertions(+), 22 deletions(-) diff --git a/lib/syntax_tree/index.rb b/lib/syntax_tree/index.rb index 60158314..6956ae9c 100644 --- a/lib/syntax_tree/index.rb +++ b/lib/syntax_tree/index.rb @@ -20,46 +20,128 @@ def initialize(line, column) # This entry represents a class definition using the class keyword. class ClassDefinition - attr_reader :nesting, :name, :location + attr_reader :nesting, :name, :location, :comments - def initialize(nesting, name, location) + def initialize(nesting, name, location, comments) @nesting = nesting @name = name @location = location + @comments = comments end end # This entry represents a module definition using the module keyword. class ModuleDefinition - attr_reader :nesting, :name, :location + attr_reader :nesting, :name, :location, :comments - def initialize(nesting, name, location) + def initialize(nesting, name, location, comments) @nesting = nesting @name = name @location = location + @comments = comments end end # This entry represents a method definition using the def keyword. class MethodDefinition - attr_reader :nesting, :name, :location + attr_reader :nesting, :name, :location, :comments - def initialize(nesting, name, location) + def initialize(nesting, name, location, comments) @nesting = nesting @name = name @location = location + @comments = comments end end # This entry represents a singleton method definition using the def keyword # with a specified target. class SingletonMethodDefinition - attr_reader :nesting, :name, :location + attr_reader :nesting, :name, :location, :comments - def initialize(nesting, name, location) + def initialize(nesting, name, location, comments) @nesting = nesting @name = name @location = location + @comments = comments + end + end + + # When you're using the instruction sequence backend, this class is used to + # lazily parse comments out of the source code. + class FileComments + # We use the ripper library to pull out source comments. + class Parser < Ripper + attr_reader :comments + + def initialize(*) + super + @comments = {} + end + + def on_comment(value) + comments[lineno] = value.chomp + end + end + + # This represents the Ruby source in the form of a file. When it needs to + # be read we'll read the file. + class FileSource + attr_reader :filepath + + def initialize(filepath) + @filepath = filepath + end + + def source + File.read(filepath) + end + end + + # This represents the Ruby source in the form of a string. When it needs + # to be read the string is returned. + class StringSource + attr_reader :source + + def initialize(source) + @source = source + end + end + + attr_reader :source + + def initialize(source) + @source = source + end + + def comments + @comments ||= Parser.new(source.source).tap(&:parse).comments + end + end + + # This class handles parsing comments from Ruby source code in the case that + # we use the instruction sequence backend. Because the instruction sequence + # backend doesn't provide comments (since they are dropped) we provide this + # interface to lazily parse them out. + class EntryComments + include Enumerable + attr_reader :file_comments, :location + + def initialize(file_comments, location) + @file_comments = file_comments + @location = location + end + + def each(&block) + line = location.line - 1 + result = [] + + while line >= 0 && (comment = file_comments.comments[line]) + result.unshift(comment) + line -= 1 + end + + result.each(&block) end end @@ -74,16 +156,22 @@ class ISeqBackend VM_DEFINECLASS_FLAG_HAS_SUPERCLASS = 0x10 def index(source) - index_iseq(RubyVM::InstructionSequence.compile(source).to_a) + index_iseq( + RubyVM::InstructionSequence.compile(source).to_a, + FileComments.new(FileComments::StringSource.new(source)) + ) end def index_file(filepath) - index_iseq(RubyVM::InstructionSequence.compile_file(filepath).to_a) + index_iseq( + RubyVM::InstructionSequence.compile_file(filepath).to_a, + FileComments.new(FileComments::FileSource.new(filepath)) + ) end private - def index_iseq(iseq) + def index_iseq(iseq, file_comments) results = [] queue = [[iseq, []]] @@ -106,11 +194,23 @@ def index_iseq(iseq) elsif flags & VM_DEFINECLASS_TYPE_MODULE > 0 code_location = class_iseq[4][:code_location] location = Location.new(code_location[0], code_location[1]) - results << ModuleDefinition.new(current_nesting, name, location) + + results << ModuleDefinition.new( + current_nesting, + name, + location, + EntryComments.new(file_comments, location) + ) else code_location = class_iseq[4][:code_location] location = Location.new(code_location[0], code_location[1]) - results << ClassDefinition.new(current_nesting, name, location) + + results << ClassDefinition.new( + current_nesting, + name, + location, + EntryComments.new(file_comments, location) + ) end queue << [class_iseq, current_nesting + [name]] @@ -122,14 +222,21 @@ def index_iseq(iseq) results << SingletonMethodDefinition.new( current_nesting, name, - location + location, + EntryComments.new(file_comments, location) ) when :definesmethod _, name, method_iseq = insn code_location = method_iseq[4][:code_location] location = Location.new(code_location[0], code_location[1]) - results << MethodDefinition.new(current_nesting, name, location) + + results << MethodDefinition.new( + current_nesting, + name, + location, + EntryComments.new(file_comments, location) + ) end end end @@ -143,11 +250,12 @@ def index_iseq(iseq) # supported on all runtimes. class ParserBackend class IndexVisitor < Visitor - attr_reader :results, :nesting + attr_reader :results, :nesting, :statements def initialize @results = [] @nesting = [] + @statements = nil end def visit_class(node) @@ -155,9 +263,14 @@ def visit_class(node) location = Location.new(node.location.start_line, node.location.start_column) - results << ClassDefinition.new(nesting.dup, name, location) - nesting << name + results << ClassDefinition.new( + nesting.dup, + name, + location, + comments_for(node) + ) + nesting << name super nesting.pop end @@ -172,9 +285,19 @@ def visit_def(node) Location.new(node.location.start_line, node.location.start_column) results << if node.target.nil? - MethodDefinition.new(nesting.dup, name, location) + MethodDefinition.new( + nesting.dup, + name, + location, + comments_for(node) + ) else - SingletonMethodDefinition.new(nesting.dup, name, location) + SingletonMethodDefinition.new( + nesting.dup, + name, + location, + comments_for(node) + ) end end @@ -183,9 +306,14 @@ def visit_module(node) location = Location.new(node.location.start_line, node.location.start_column) - results << ModuleDefinition.new(nesting.dup, name, location) - nesting << name + results << ModuleDefinition.new( + nesting.dup, + name, + location, + comments_for(node) + ) + nesting << name super nesting.pop end @@ -194,6 +322,30 @@ def visit_program(node) super results end + + def visit_statements(node) + @statements = node + super + end + + private + + def comments_for(node) + comments = [] + + body = statements.body + line = node.location.start_line - 1 + index = body.index(node) - 1 + + while index >= 0 && body[index].is_a?(Comment) && + (line - body[index].location.start_line < 2) + comments.unshift(body[index].value) + line = body[index].location.start_line + index -= 1 + end + + comments + end end def index(source) diff --git a/test/index_test.rb b/test/index_test.rb index 3ea02a20..91dfcc76 100644 --- a/test/index_test.rb +++ b/test/index_test.rb @@ -18,6 +18,13 @@ def test_module_nested end end + def test_module_comments + index_each("# comment1\n# comment2\nmodule Foo; end") do |entry| + assert_equal :Foo, entry.name + assert_equal ["# comment1", "# comment2"], entry.comments.to_a + end + end + def test_class index_each("class Foo; end") do |entry| assert_equal :Foo, entry.name @@ -32,6 +39,13 @@ def test_class_nested end end + def test_class_comments + index_each("# comment1\n# comment2\nclass Foo; end") do |entry| + assert_equal :Foo, entry.name + assert_equal ["# comment1", "# comment2"], entry.comments.to_a + end + end + def test_method index_each("def foo; end") do |entry| assert_equal :foo, entry.name @@ -46,6 +60,13 @@ def test_method_nested end end + def test_method_comments + index_each("# comment1\n# comment2\ndef foo; end") do |entry| + assert_equal :foo, entry.name + assert_equal ["# comment1", "# comment2"], entry.comments.to_a + end + end + private def index_each(source) From 7731c6d6721b3a733c6cd9fbf7725d3b3f257427 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 25 Jan 2023 20:16:58 -0500 Subject: [PATCH 334/536] More test coverage for indexing --- lib/syntax_tree/index.rb | 43 ++++++++++++++++++++-------------------- test/index_test.rb | 35 ++++++++++++++++++++++++++++++-- 2 files changed, 54 insertions(+), 24 deletions(-) diff --git a/lib/syntax_tree/index.rb b/lib/syntax_tree/index.rb index 6956ae9c..8b33f785 100644 --- a/lib/syntax_tree/index.rb +++ b/lib/syntax_tree/index.rb @@ -171,6 +171,11 @@ def index_file(filepath) private + def location_for(iseq) + code_location = iseq[4][:code_location] + Location.new(code_location[0], code_location[1]) + end + def index_iseq(iseq, file_comments) results = [] queue = [[iseq, []]] @@ -192,9 +197,7 @@ def index_iseq(iseq, file_comments) "singleton class with non-self receiver" end elsif flags & VM_DEFINECLASS_TYPE_MODULE > 0 - code_location = class_iseq[4][:code_location] - location = Location.new(code_location[0], code_location[1]) - + location = location_for(class_iseq) results << ModuleDefinition.new( current_nesting, name, @@ -202,9 +205,7 @@ def index_iseq(iseq, file_comments) EntryComments.new(file_comments, location) ) else - code_location = class_iseq[4][:code_location] - location = Location.new(code_location[0], code_location[1]) - + location = location_for(class_iseq) results << ClassDefinition.new( current_nesting, name, @@ -215,25 +216,23 @@ def index_iseq(iseq, file_comments) queue << [class_iseq, current_nesting + [name]] when :definemethod - _, name, method_iseq = insn - - code_location = method_iseq[4][:code_location] - location = Location.new(code_location[0], code_location[1]) - results << SingletonMethodDefinition.new( + location = location_for(insn[2]) + results << MethodDefinition.new( current_nesting, - name, + insn[1], location, EntryComments.new(file_comments, location) ) when :definesmethod - _, name, method_iseq = insn - - code_location = method_iseq[4][:code_location] - location = Location.new(code_location[0], code_location[1]) + if current_iseq[13][index - 1] != [:putself] + raise NotImplementedError, + "singleton method with non-self receiver" + end - results << MethodDefinition.new( + location = location_for(insn[2]) + results << SingletonMethodDefinition.new( current_nesting, - name, + insn[1], location, EntryComments.new(file_comments, location) ) @@ -363,13 +362,13 @@ def index_file(filepath) defined?(RubyVM::InstructionSequence) ? ISeqBackend : ParserBackend # This method accepts source code and then indexes it. - def self.index(source) - INDEX_BACKEND.new.index(source) + def self.index(source, backend: INDEX_BACKEND.new) + backend.index(source) end # This method accepts a filepath and then indexes it. - def self.index_file(filepath) - INDEX_BACKEND.new.index_file(filepath) + def self.index_file(filepath, backend: INDEX_BACKEND.new) + backend.index_file(filepath) end end end diff --git a/test/index_test.rb b/test/index_test.rb index 91dfcc76..6bb83881 100644 --- a/test/index_test.rb +++ b/test/index_test.rb @@ -67,13 +67,44 @@ def test_method_comments end end + def test_singleton_method + index_each("def self.foo; end") do |entry| + assert_equal :foo, entry.name + assert_empty entry.nesting + end + end + + def test_singleton_method_nested + index_each("class Foo; def self.foo; end; end") do |entry| + assert_equal :foo, entry.name + assert_equal [:Foo], entry.nesting + end + end + + def test_singleton_method_comments + index_each("# comment1\n# comment2\ndef self.foo; end") do |entry| + assert_equal :foo, entry.name + assert_equal ["# comment1", "# comment2"], entry.comments.to_a + end + end + + def test_this_file + entries = Index.index_file(__FILE__, backend: Index::ParserBackend.new) + + if defined?(RubyVM::InstructionSequence) + entries += Index.index_file(__FILE__, backend: Index::ISeqBackend.new) + end + + entries.map { |entry| entry.comments.to_a } + end + private def index_each(source) - yield SyntaxTree::Index::ParserBackend.new.index(source).last + yield Index.index(source, backend: Index::ParserBackend.new).last if defined?(RubyVM::InstructionSequence) - yield SyntaxTree::Index::ISeqBackend.new.index(source).last + yield Index.index(source, backend: Index::ISeqBackend.new).last end end end From 98a6b73017ace48f3e9d17aa189e73edaefbcf7c Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 26 Jan 2023 10:36:58 -0500 Subject: [PATCH 335/536] Bump deps --- .rubocop.yml | 3 +++ Gemfile.lock | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index 0212027b..bc98a43a 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -11,6 +11,9 @@ AllCops: - test/ruby-syntax-fixtures/**/* - test.rb +Gemspec/DevelopmentDependencies: + Enabled: false + Layout/LineLength: Max: 80 diff --git a/Gemfile.lock b/Gemfile.lock index b691d5e9..4ebe14d0 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -17,9 +17,9 @@ GEM prettier_print (1.2.0) rainbow (3.1.1) rake (13.0.6) - regexp_parser (2.6.1) + regexp_parser (2.6.2) rexml (3.2.5) - rubocop (1.43.0) + rubocop (1.44.1) json (~> 2.3) parallel (~> 1.10) parser (>= 3.2.0.0) From 5a9bf11e0060d9e583468a3fa5c8fd0abc4999e4 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 26 Jan 2023 11:06:24 -0500 Subject: [PATCH 336/536] Small arity refactor --- lib/syntax_tree/node.rb | 41 +++++++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index d1d40154..ee00940d 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -854,18 +854,17 @@ def ===(other) end def arity - accepts_infinite_arguments? ? Float::INFINITY : parts.length - end - - private - - def accepts_infinite_arguments? - parts.any? do |part| - part.is_a?(ArgStar) || part.is_a?(ArgsForward) || - ( - part.is_a?(BareAssocHash) && - part.assocs.any? { |p| p.is_a?(AssocSplat) } - ) + parts.sum do |part| + case part + when ArgStar, ArgsForward + Float::INFINITY + when BareAssocHash + part.assocs.sum do |assoc| + assoc.is_a?(AssocSplat) ? Float::INFINITY : 1 + end + else + 1 + end end end end @@ -8383,18 +8382,24 @@ def ===(other) # Returns a range representing the possible number of arguments accepted # by this params node not including the block. For example: - # def foo(a, b = 1, c:, d: 2, &block) - # ... - # end - # has arity 2..4 + # + # def foo(a, b = 1, c:, d: 2, &block) + # ... + # end + # + # has arity 2..4. + # def arity optional_keywords = keywords.count { |_label, value| value } + lower_bound = requireds.length + posts.length + keywords.length - optional_keywords upper_bound = - lower_bound + optionals.length + - optional_keywords if keyword_rest.nil? && rest.nil? + if keyword_rest.nil? && rest.nil? + lower_bound + optionals.length + optional_keywords + end + lower_bound..upper_bound end From a8ae2af6f6c54d48b1ba7033d4277c5adf6c89bb Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 26 Jan 2023 11:11:01 -0500 Subject: [PATCH 337/536] Remove VarRef array formatting --- lib/syntax_tree/node.rb | 140 +++++++++++---------------------- test/fixtures/array_literal.rb | 13 ++- 2 files changed, 54 insertions(+), 99 deletions(-) diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index ee00940d..b954152d 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -1103,58 +1103,6 @@ def format(q) end end - # Formats an array that contains only a list of variable references. To make - # things simpler, if there are a bunch, we format them all using the "fill" - # algorithm as opposed to breaking them into a ton of lines. For example, - # - # [foo, bar, baz] - # - # instead of becoming: - # - # [ - # foo, - # bar, - # baz - # ] - # - # would instead become: - # - # [ - # foo, bar, - # baz - # ] - # - # provided the line length was hit between `bar` and `baz`. - class VarRefsFormatter - # The separator for the fill algorithm. - class Separator - def call(q) - q.text(",") - q.fill_breakable - end - end - - # [Args] the contents of the array - attr_reader :contents - - def initialize(contents) - @contents = contents - end - - def format(q) - q.text("[") - q.group do - q.indent do - q.breakable_empty - q.seplist(contents.parts, Separator.new) { |part| q.format(part) } - q.if_break { q.text(",") } if q.trailing_comma? - end - q.breakable_empty - end - q.text("]") - end - end - # This is a special formatter used if the array literal contains no values # but _does_ contain comments. In this case we do some special formatting to # make sure the comments gets indented properly. @@ -1229,19 +1177,17 @@ def deconstruct_keys(_keys) end def format(q) - if qwords? - QWordsFormatter.new(contents).format(q) - return - end - - if qsymbols? - QSymbolsFormatter.new(contents).format(q) - return - end + if lbracket.comments.empty? && contents && contents.comments.empty? && + contents.parts.length > 1 + if qwords? + QWordsFormatter.new(contents).format(q) + return + end - if var_refs?(q) - VarRefsFormatter.new(contents).format(q) - return + if qsymbols? + QSymbolsFormatter.new(contents).format(q) + return + end end if empty_with_comments? @@ -1273,39 +1219,24 @@ def ===(other) private def qwords? - lbracket.comments.empty? && contents && contents.comments.empty? && - contents.parts.length > 1 && - contents.parts.all? do |part| - case part - when StringLiteral - part.comments.empty? && part.parts.length == 1 && - part.parts.first.is_a?(TStringContent) && - !part.parts.first.value.match?(/[\s\[\]\\]/) - when CHAR - !part.value.match?(/[\[\]\\]/) - else - false - end + contents.parts.all? do |part| + case part + when StringLiteral + part.comments.empty? && part.parts.length == 1 && + part.parts.first.is_a?(TStringContent) && + !part.parts.first.value.match?(/[\s\[\]\\]/) + when CHAR + !part.value.match?(/[\[\]\\]/) + else + false end + end end def qsymbols? - lbracket.comments.empty? && contents && contents.comments.empty? && - contents.parts.length > 1 && - contents.parts.all? do |part| - part.is_a?(SymbolLiteral) && part.comments.empty? - end - end - - def var_refs?(q) - lbracket.comments.empty? && contents && contents.comments.empty? && - contents.parts.all? do |part| - part.is_a?(VarRef) && part.comments.empty? - end && - ( - contents.parts.sum { |part| part.value.value.length + 2 } > - q.maxwidth * 2 - ) + contents.parts.all? do |part| + part.is_a?(SymbolLiteral) && part.comments.empty? + end end # If we have an empty array that contains only comments, then we're going @@ -6551,9 +6482,26 @@ def deconstruct_keys(_keys) def format(q) force_flat = [ - AliasNode, Assign, Break, Command, CommandCall, Heredoc, IfNode, IfOp, - Lambda, MAssign, Next, OpAssign, RescueMod, ReturnNode, Super, Undef, - UnlessNode, VoidStmt, YieldNode, ZSuper + AliasNode, + Assign, + Break, + Command, + CommandCall, + Heredoc, + IfNode, + IfOp, + Lambda, + MAssign, + Next, + OpAssign, + RescueMod, + ReturnNode, + Super, + Undef, + UnlessNode, + VoidStmt, + YieldNode, + ZSuper ] if q.parent.is_a?(Paren) || force_flat.include?(truthy.class) || diff --git a/test/fixtures/array_literal.rb b/test/fixtures/array_literal.rb index df807728..391d2eae 100644 --- a/test/fixtures/array_literal.rb +++ b/test/fixtures/array_literal.rb @@ -24,9 +24,16 @@ - fooooooooooooooooo = 1 [ - fooooooooooooooooo, fooooooooooooooooo, fooooooooooooooooo, - fooooooooooooooooo, fooooooooooooooooo, fooooooooooooooooo, - fooooooooooooooooo, fooooooooooooooooo, fooooooooooooooooo, fooooooooooooooooo + fooooooooooooooooo, + fooooooooooooooooo, + fooooooooooooooooo, + fooooooooooooooooo, + fooooooooooooooooo, + fooooooooooooooooo, + fooooooooooooooooo, + fooooooooooooooooo, + fooooooooooooooooo, + fooooooooooooooooo ] % [ From 82dc9b7ce91030dbf4ed2474853558d1bf05de1c Mon Sep 17 00:00:00 2001 From: David Taylor Date: Sat, 7 Jan 2023 13:46:14 +0000 Subject: [PATCH 338/536] Introduce `plugin/disable_ternary` This will prevent the automatic conversion of `if ... else` to ternary expressions. --- README.md | 1 + lib/syntax_tree/formatter.rb | 21 +++++++++++++-- lib/syntax_tree/node.rb | 1 + lib/syntax_tree/plugin/disable_ternary.rb | 7 +++++ test/plugin/disable_ternary_test.rb | 32 +++++++++++++++++++++++ 5 files changed, 60 insertions(+), 2 deletions(-) create mode 100644 lib/syntax_tree/plugin/disable_ternary.rb create mode 100644 test/plugin/disable_ternary_test.rb diff --git a/README.md b/README.md index 7a943ca8..7bb731e2 100644 --- a/README.md +++ b/README.md @@ -658,6 +658,7 @@ To register plugins, define a file somewhere in your load path named `syntax_tre * `plugin/single_quotes` - This will change all of your string literals to use single quotes instead of the default double quotes. * `plugin/trailing_comma` - This will put trailing commas into multiline array literals, hash literals, and method calls that can support trailing commas. +* `plugin/disable_ternary` - This will prevent the automatic conversion of `if ... else` to ternary expressions. If you're using Syntax Tree as a library, you can require those files directly or manually pass those options to the formatter initializer through the `SyntaxTree::Formatter::Options` class. diff --git a/lib/syntax_tree/formatter.rb b/lib/syntax_tree/formatter.rb index fddc06fe..067de7ee 100644 --- a/lib/syntax_tree/formatter.rb +++ b/lib/syntax_tree/formatter.rb @@ -21,11 +21,15 @@ def initialize(version) # that folks have become entrenched in their ways, we decided to provide a # small amount of configurability. class Options - attr_reader :quote, :trailing_comma, :target_ruby_version + attr_reader :quote, + :trailing_comma, + :disable_ternary, + :target_ruby_version def initialize( quote: :default, trailing_comma: :default, + disable_ternary: :default, target_ruby_version: :default ) @quote = @@ -50,6 +54,17 @@ def initialize( trailing_comma end + @disable_ternary = + if disable_ternary == :default + # We ship with a disable ternary plugin that will define this + # constant. That constant is responsible for determining the default + # disable ternary value. If it's defined, then we default to true. + # Otherwise we default to false. + defined?(DISABLE_TERNARY) + else + disable_ternary + end + @target_ruby_version = if target_ruby_version == :default # The default target Ruby version is the current version of Ruby. @@ -69,8 +84,9 @@ def initialize( # These options are overridden in plugins to we need to make sure they are # available here. - attr_reader :quote, :trailing_comma, :target_ruby_version + attr_reader :quote, :trailing_comma, :disable_ternary, :target_ruby_version alias trailing_comma? trailing_comma + alias disable_ternary? disable_ternary def initialize(source, *args, options: Options.new) super(*args) @@ -81,6 +97,7 @@ def initialize(source, *args, options: Options.new) # Memoizing these values to make access faster. @quote = options.quote @trailing_comma = options.trailing_comma + @disable_ternary = options.disable_ternary @target_ruby_version = options.target_ruby_version end diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index f19cfb2c..119180ec 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -6154,6 +6154,7 @@ module Ternaryable class << self def call(q, node) return false if ENV["STREE_FAST_FORMAT"] + return false if q.disable_ternary? # If this is a conditional inside of a parentheses as the only content, # then we don't want to transform it into a ternary. Presumably the user diff --git a/lib/syntax_tree/plugin/disable_ternary.rb b/lib/syntax_tree/plugin/disable_ternary.rb new file mode 100644 index 00000000..0cb48d84 --- /dev/null +++ b/lib/syntax_tree/plugin/disable_ternary.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +module SyntaxTree + class Formatter + DISABLE_TERNARY = true + end +end diff --git a/test/plugin/disable_ternary_test.rb b/test/plugin/disable_ternary_test.rb new file mode 100644 index 00000000..ac27ea5a --- /dev/null +++ b/test/plugin/disable_ternary_test.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +require_relative "../test_helper" + +module SyntaxTree + class DisableTernaryTest < Minitest::Test + def test_short_if_else_unchanged + assert_format(<<~RUBY) + if true + 1 + else + 2 + end + RUBY + end + + def test_short_ternary_unchanged + assert_format("true ? 1 : 2\n") + end + + private + + def assert_format(expected, source = expected) + options = Formatter::Options.new(disable_ternary: true) + formatter = Formatter.new(source, [], options: options) + SyntaxTree.parse(source).format(formatter) + + formatter.flush + assert_equal(expected, formatter.output.join) + end + end +end From ee84d7cb33b00e8bd4c9991df30a0a50cfe5cb8b Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 26 Jan 2023 11:27:28 -0500 Subject: [PATCH 339/536] Rename plugin/disable_ternary to plugin/disable_auto_ternary --- README.md | 2 +- lib/syntax_tree/formatter.rb | 20 ++++++++++++-------- lib/syntax_tree/node.rb | 3 +-- test/plugin/disable_ternary_test.rb | 2 +- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 7bb731e2..3c437947 100644 --- a/README.md +++ b/README.md @@ -658,7 +658,7 @@ To register plugins, define a file somewhere in your load path named `syntax_tre * `plugin/single_quotes` - This will change all of your string literals to use single quotes instead of the default double quotes. * `plugin/trailing_comma` - This will put trailing commas into multiline array literals, hash literals, and method calls that can support trailing commas. -* `plugin/disable_ternary` - This will prevent the automatic conversion of `if ... else` to ternary expressions. +* `plugin/disable_auto_ternary` - This will prevent the automatic conversion of `if ... else` to ternary expressions. If you're using Syntax Tree as a library, you can require those files directly or manually pass those options to the formatter initializer through the `SyntaxTree::Formatter::Options` class. diff --git a/lib/syntax_tree/formatter.rb b/lib/syntax_tree/formatter.rb index 067de7ee..c64cf7d1 100644 --- a/lib/syntax_tree/formatter.rb +++ b/lib/syntax_tree/formatter.rb @@ -23,13 +23,13 @@ def initialize(version) class Options attr_reader :quote, :trailing_comma, - :disable_ternary, + :disable_auto_ternary, :target_ruby_version def initialize( quote: :default, trailing_comma: :default, - disable_ternary: :default, + disable_auto_ternary: :default, target_ruby_version: :default ) @quote = @@ -54,15 +54,15 @@ def initialize( trailing_comma end - @disable_ternary = - if disable_ternary == :default + @disable_auto_ternary = + if disable_auto_ternary == :default # We ship with a disable ternary plugin that will define this # constant. That constant is responsible for determining the default # disable ternary value. If it's defined, then we default to true. # Otherwise we default to false. defined?(DISABLE_TERNARY) else - disable_ternary + disable_auto_ternary end @target_ruby_version = @@ -84,9 +84,13 @@ def initialize( # These options are overridden in plugins to we need to make sure they are # available here. - attr_reader :quote, :trailing_comma, :disable_ternary, :target_ruby_version + attr_reader :quote, + :trailing_comma, + :disable_auto_ternary, + :target_ruby_version + alias trailing_comma? trailing_comma - alias disable_ternary? disable_ternary + alias disable_auto_ternary? disable_auto_ternary def initialize(source, *args, options: Options.new) super(*args) @@ -97,7 +101,7 @@ def initialize(source, *args, options: Options.new) # Memoizing these values to make access faster. @quote = options.quote @trailing_comma = options.trailing_comma - @disable_ternary = options.disable_ternary + @disable_auto_ternary = options.disable_auto_ternary @target_ruby_version = options.target_ruby_version end diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 7ddfc710..fc5517cf 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -6139,8 +6139,7 @@ def self.call(parent) module Ternaryable class << self def call(q, node) - return false if ENV["STREE_FAST_FORMAT"] - return false if q.disable_ternary? + return false if ENV["STREE_FAST_FORMAT"] || q.disable_auto_ternary? # If this is a conditional inside of a parentheses as the only content, # then we don't want to transform it into a ternary. Presumably the user diff --git a/test/plugin/disable_ternary_test.rb b/test/plugin/disable_ternary_test.rb index ac27ea5a..b2af9d35 100644 --- a/test/plugin/disable_ternary_test.rb +++ b/test/plugin/disable_ternary_test.rb @@ -21,7 +21,7 @@ def test_short_ternary_unchanged private def assert_format(expected, source = expected) - options = Formatter::Options.new(disable_ternary: true) + options = Formatter::Options.new(disable_auto_ternary: true) formatter = Formatter.new(source, [], options: options) SyntaxTree.parse(source).format(formatter) From c497724afdd34ed00a4ac4804b1d26da973b4c95 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 26 Jan 2023 11:58:30 -0500 Subject: [PATCH 340/536] Bump to version 5.3.0 --- CHANGELOG.md | 18 +++++++++++++++--- Gemfile.lock | 2 +- lib/syntax_tree/version.rb | 2 +- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf347efb..c39bed36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,10 +6,21 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +## [5.3.0] - 2023-01-26 + ### Added -- Arity has been added to DefNode, BlockNode and Params. The method returns a range where the lower bound is the minimum and the upper bound is the maximum number of arguments that can be used to invoke that block/method definition. -- Arity has been added to CallNode, Command, CommandCall and VCall nodes. The method returns the number of arguments included in the invocation. For splats, double splats or argument forwards, this method returns Float::INFINITY. +- `#arity` has been added to `DefNode`, `BlockNode`, and `Params`. The method returns a range where the lower bound is the minimum and the upper bound is the maximum number of arguments that can be used to invoke that block/method definition. +- `#arity` has been added to `CallNode`, `Command`, `CommandCall`, and `VCall` nodes. The method returns the number of arguments included in the invocation. For splats, double splats, or argument forwards, this method returns `Float::INFINITY`. +- `SyntaxTree::index` and `SyntaxTree::index_file` APIs have been added to collect a list of classes, modules, and methods defined in a given source string or file, respectively. These APIs are experimental and subject to change. +- A `plugin/disable_auto_ternary` plugin has been added the disables the formatted that automatically changes permissable `if/else` clauses into ternaries. + +### Changed + +- Files are now only written from the CLI if the content of them changes, which should match watching files less chaotic. +- In the case that `rb_iseq_load` cannot be found, `Fiddle::DLError` is now rescued. +- Previously if there were invalid UTF-8 byte sequences after the `__END__` keyword the parser could potentially have crashed when parsing comments. This has been fixed. +- Previously there was special formatting for array literals that contained only variable references (either locals, method calls, or constants). For consistency, this has been removed and all array literals are now formatted the same way. ## [5.2.0] - 2023-01-04 @@ -486,7 +497,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - 🎉 Initial release! 🎉 -[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v5.2.0...HEAD +[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v5.3.0...HEAD +[5.3.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v5.2.0...v5.3.0 [5.2.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v5.1.0...v5.2.0 [5.1.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v5.0.1...v5.1.0 [5.0.1]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v5.0.0...v5.0.1 diff --git a/Gemfile.lock b/Gemfile.lock index 4ebe14d0..799bd891 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - syntax_tree (5.2.0) + syntax_tree (5.3.0) prettier_print (>= 1.2.0) GEM diff --git a/lib/syntax_tree/version.rb b/lib/syntax_tree/version.rb index a97f5e43..6cb1fccf 100644 --- a/lib/syntax_tree/version.rb +++ b/lib/syntax_tree/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SyntaxTree - VERSION = "5.2.0" + VERSION = "5.3.0" end From d96dad1984a695ca2171bd06b8562fb657127fe9 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 27 Jan 2023 10:43:37 -0500 Subject: [PATCH 341/536] Include parser translation --- .github/workflows/main.yml | 2 + .gitmodules | 3 + .rubocop.yml | 1 + Rakefile | 10 +- lib/syntax_tree.rb | 2 + lib/syntax_tree/translation.rb | 20 + lib/syntax_tree/translation/parser.rb | 1426 +++++++++++++++++++++++++ test/ruby_syntax_fixtures_test.rb | 4 + test/suites/helper.rb | 3 + test/suites/parse_helper.rb | 149 +++ test/suites/parser | 1 + 11 files changed, 1620 insertions(+), 1 deletion(-) create mode 100644 lib/syntax_tree/translation.rb create mode 100644 lib/syntax_tree/translation/parser.rb create mode 100644 test/suites/helper.rb create mode 100644 test/suites/parse_helper.rb create mode 160000 test/suites/parser diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3f811317..8bca2fc4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,6 +23,8 @@ jobs: # TESTOPTS: --verbose steps: - uses: actions/checkout@master + with: + submodules: true - uses: ruby/setup-ruby@v1 with: bundler-cache: true diff --git a/.gitmodules b/.gitmodules index 1a2c45cc..8287c5e3 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,3 +7,6 @@ [submodule "test/ruby-syntax-fixtures"] path = test/ruby-syntax-fixtures url = https://github.com/ruby-syntax-tree/ruby-syntax-fixtures +[submodule "test/suites/parser"] + path = test/suites/parser + url = https://github.com/whitequark/parser diff --git a/.rubocop.yml b/.rubocop.yml index bc98a43a..381d7a27 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -9,6 +9,7 @@ AllCops: Exclude: - '{.git,.github,bin,coverage,pkg,spec,test/fixtures,vendor,tmp}/**/*' - test/ruby-syntax-fixtures/**/* + - test/suites/parser/**/* - test.rb Gemspec/DevelopmentDependencies: diff --git a/Rakefile b/Rakefile index f06d8cf8..cb96e7bf 100644 --- a/Rakefile +++ b/Rakefile @@ -6,8 +6,16 @@ require "syntax_tree/rake_tasks" Rake::TestTask.new(:test) do |t| t.libs << "test" + t.libs << "test/suites" t.libs << "lib" - t.test_files = FileList["test/**/*_test.rb"] + + # These are our own tests. + test_files = FileList["test/**/*_test.rb"] + + # This is a big test file from the parser gem that tests its functionality. + test_files << "test/suites/parser/test/test_parser.rb" + + t.test_files = test_files end task default: :test diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index f5c71aba..73add469 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -40,6 +40,8 @@ require_relative "syntax_tree/yarv/assembler" require_relative "syntax_tree/yarv/vm" +require_relative "syntax_tree/translation" + # Syntax Tree is a suite of tools built on top of the internal CRuby parser. It # provides the ability to generate a syntax tree from source, as well as the # tools necessary to inspect and manipulate that syntax tree. It can be used to diff --git a/lib/syntax_tree/translation.rb b/lib/syntax_tree/translation.rb new file mode 100644 index 00000000..37785ea2 --- /dev/null +++ b/lib/syntax_tree/translation.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module SyntaxTree + # This module is responsible for translating the Syntax Tree syntax tree into + # other representations. + module Translation + # This method translates the given node into the representation defined by + # the whitequark/parser gem. We don't explicitly list it as a dependency + # because it's not required for the core functionality of Syntax Tree. + def self.to_parser(node, source) + require "parser" + require_relative "translation/parser" + + buffer = ::Parser::Source::Buffer.new("(string)") + buffer.source = source + + node.accept(Parser.new(buffer)) + end + end +end diff --git a/lib/syntax_tree/translation/parser.rb b/lib/syntax_tree/translation/parser.rb new file mode 100644 index 00000000..3443df37 --- /dev/null +++ b/lib/syntax_tree/translation/parser.rb @@ -0,0 +1,1426 @@ +# frozen_string_literal: true + +module SyntaxTree + module Translation + class Parser < BasicVisitor + attr_reader :buffer, :stack + + def initialize(buffer) + @buffer = buffer + @stack = [] + end + + # For each node that we visit, we keep track of it in a stack as we + # descend into its children. We do this so that child nodes can reflect on + # their parents if they need additional information about their context. + def visit(node) + stack << node + result = super + stack.pop + result + end + + # Visit an AliasNode node. + def visit_alias(node) + s(:alias, [visit(node.left), visit(node.right)]) + end + + # Visit an ARefNode. + def visit_aref(node) + if ::Parser::Builders::Default.emit_index + if node.index.nil? + s(:index, [visit(node.collection)]) + else + s(:index, [visit(node.collection), *visit_all(node.index.parts)]) + end + else + if node.index.nil? + s(:send, [visit(node.collection), :[], nil]) + else + s( + :send, + [visit(node.collection), :[], *visit_all(node.index.parts)] + ) + end + end + end + + # Visit an ARefField node. + def visit_aref_field(node) + if ::Parser::Builders::Default.emit_index + if node.index.nil? + s(:indexasgn, [visit(node.collection), nil]) + else + s( + :indexasgn, + [visit(node.collection), *visit_all(node.index.parts)] + ) + end + else + if node.index.nil? + s(:send, [visit(node.collection), :[]=, nil]) + else + s( + :send, + [visit(node.collection), :[]=, *visit_all(node.index.parts)] + ) + end + end + end + + # Visit an ArgBlock node. + def visit_arg_block(node) + s(:block_pass, [visit(node.value)]) + end + + # Visit an ArgStar node. + def visit_arg_star(node) + if stack[-3].is_a?(MLHSParen) && stack[-3].contents.is_a?(MLHS) + case node.value + when nil + s(:restarg) + when Ident + s(:restarg, [node.value.value.to_sym]) + else + s(:restarg, [node.value.value.value.to_sym]) + end + else + node.value.nil? ? s(:splat) : s(:splat, [visit(node.value)]) + end + end + + # Visit an ArgsForward node. + def visit_args_forward(_node) + s(:forwarded_args) + end + + # Visit an ArrayLiteral node. + def visit_array(node) + if node.contents.nil? + s(:array) + else + s(:array, visit_all(node.contents.parts)) + end + end + + # Visit an AryPtn node. + def visit_aryptn(node) + type = :array_pattern + children = visit_all(node.requireds) + + if node.rest.is_a?(VarField) + if !node.rest.value.nil? + children << s(:match_rest, [visit(node.rest)]) + elsif node.posts.empty? && + node.rest.location.start_char == node.rest.location.end_char + # Here we have an implicit rest, as in [foo,]. parser has a specific + # type for these patterns. + type = :array_pattern_with_tail + else + children << s(:match_rest) + end + end + + inner = s(type, children + visit_all(node.posts)) + node.constant ? s(:const_pattern, [visit(node.constant), inner]) : inner + end + + # Visit an Assign node. + def visit_assign(node) + target = visit(node.target) + s(target.type, target.children + [visit(node.value)]) + end + + # Visit an Assoc node. + def visit_assoc(node) + if node.value.nil? + type = node.key.value.start_with?(/[A-Z]/) ? :const : :send + s( + :pair, + [visit(node.key), s(type, [nil, node.key.value.chomp(":").to_sym])] + ) + else + s(:pair, [visit(node.key), visit(node.value)]) + end + end + + # Visit an AssocSplat node. + def visit_assoc_splat(node) + s(:kwsplat, [visit(node.value)]) + end + + # Visit a Backref node. + def visit_backref(node) + if node.value.match?(/^\$\d+$/) + s(:nth_ref, [node.value[1..].to_i]) + else + s(:back_ref, [node.value.to_sym]) + end + end + + # Visit a BareAssocHash node. + def visit_bare_assoc_hash(node) + type = + if ::Parser::Builders::Default.emit_kwargs && + !stack[-2].is_a?(ArrayLiteral) + :kwargs + else + :hash + end + + s(type, visit_all(node.assocs)) + end + + # Visit a BEGINBlock node. + def visit_BEGIN(node) + s(:preexe, [visit(node.statements)]) + end + + # Visit a Begin node. + def visit_begin(node) + if node.bodystmt.empty? + s(:kwbegin) + elsif node.bodystmt.rescue_clause.nil? && + node.bodystmt.ensure_clause.nil? && node.bodystmt.else_clause.nil? + visited = visit(node.bodystmt.statements) + s(:kwbegin, visited.type == :begin ? visited.children : [visited]) + else + s(:kwbegin, [visit(node.bodystmt)]) + end + end + + # Visit a Binary node. + def visit_binary(node) + case node.operator + when :| + current = -2 + current -= 1 while stack[current].is_a?(Binary) && + stack[current].operator == :| + + if stack[current].is_a?(In) + s(:match_alt, [visit(node.left), visit(node.right)]) + else + s(:send, [visit(node.left), node.operator, visit(node.right)]) + end + when :"=>" + s(:match_as, [visit(node.left), visit(node.right)]) + when :"&&", :and + s(:and, [visit(node.left), visit(node.right)]) + when :"||", :or + s(:or, [visit(node.left), visit(node.right)]) + when :=~ + if node.left.is_a?(RegexpLiteral) && node.left.parts.length == 1 && + node.left.parts.first.is_a?(TStringContent) + s(:match_with_lvasgn, [visit(node.left), visit(node.right)]) + else + s(:send, [visit(node.left), node.operator, visit(node.right)]) + end + else + s(:send, [visit(node.left), node.operator, visit(node.right)]) + end + end + + # Visit a BlockArg node. + def visit_blockarg(node) + if node.name.nil? + s(:blockarg, [nil]) + else + s(:blockarg, [node.name.value.to_sym]) + end + end + + # Visit a BlockVar node. + def visit_block_var(node) + shadowargs = + node.locals.map { |local| s(:shadowarg, [local.value.to_sym]) } + + # There is a special node type in the parser gem for when a single + # required parameter to a block would potentially be expanded + # automatically. We handle that case here. + if ::Parser::Builders::Default.emit_procarg0 + params = node.params + + if params.requireds.length == 1 && params.optionals.empty? && + params.rest.nil? && params.posts.empty? && + params.keywords.empty? && params.keyword_rest.nil? && + params.block.nil? + required = params.requireds.first + + procarg0 = + if ::Parser::Builders::Default.emit_arg_inside_procarg0 && + required.is_a?(Ident) + s(:procarg0, [s(:arg, [required.value.to_sym])]) + else + s(:procarg0, visit(required).children) + end + + return s(:args, [procarg0] + shadowargs) + end + end + + s(:args, visit(node.params).children + shadowargs) + end + + # Visit a BodyStmt node. + def visit_bodystmt(node) + inner = visit(node.statements) + + if node.rescue_clause + children = [inner] + visit(node.rescue_clause).children + + if node.else_clause + children.pop + children << visit(node.else_clause) + end + + inner = s(:rescue, children) + end + + if node.ensure_clause + inner = s(:ensure, [inner] + visit(node.ensure_clause).children) + end + + inner + end + + # Visit a Break node. + def visit_break(node) + s(:break, visit_all(node.arguments.parts)) + end + + # Visit a CallNode node. + def visit_call(node) + if node.receiver.nil? + children = [nil, node.message.value.to_sym] + + if node.arguments.is_a?(ArgParen) + case node.arguments.arguments + when nil + # skip + when ArgsForward + children << s(:forwarded_args) + else + children += visit_all(node.arguments.arguments.parts) + end + end + + s(:send, children) + elsif node.message == :call + children = [visit(node.receiver), :call] + + unless node.arguments.arguments.nil? + children += visit_all(node.arguments.arguments.parts) + end + + s(send_type(node.operator), children) + else + children = [visit(node.receiver), node.message.value.to_sym] + + case node.arguments + when Args + children += visit_all(node.arguments.parts) + when ArgParen + unless node.arguments.arguments.nil? + children += visit_all(node.arguments.arguments.parts) + end + end + + s(send_type(node.operator), children) + end + end + + # Visit a Case node. + def visit_case(node) + clauses = [node.consequent] + while clauses.last && !clauses.last.is_a?(Else) + clauses << clauses.last.consequent + end + + type = node.consequent.is_a?(In) ? :case_match : :case + s(type, [visit(node.value)] + clauses.map { |clause| visit(clause) }) + end + + # Visit a CHAR node. + def visit_CHAR(node) + s(:str, [node.value[1..]]) + end + + # Visit a ClassDeclaration node. + def visit_class(node) + s( + :class, + [visit(node.constant), visit(node.superclass), visit(node.bodystmt)] + ) + end + + # Visit a Command node. + def visit_command(node) + call = + s( + :send, + [nil, node.message.value.to_sym, *visit_all(node.arguments.parts)] + ) + + if node.block + type, arguments = block_children(node.block) + s(type, [call, arguments, visit(node.block.bodystmt)]) + else + call + end + end + + # Visit a CommandCall node. + def visit_command_call(node) + children = [visit(node.receiver), node.message.value.to_sym] + + case node.arguments + when Args + children += visit_all(node.arguments.parts) + when ArgParen + children += visit_all(node.arguments.arguments.parts) + end + + call = s(send_type(node.operator), children) + + if node.block + type, arguments = block_children(node.block) + s(type, [call, arguments, visit(node.block.bodystmt)]) + else + call + end + end + + # Visit a Const node. + def visit_const(node) + s(:const, [nil, node.value.to_sym]) + end + + # Visit a ConstPathField node. + def visit_const_path_field(node) + if node.parent.is_a?(VarRef) && node.parent.value.is_a?(Kw) && + node.parent.value.value == "self" && node.constant.is_a?(Ident) + s(:send, [visit(node.parent), :"#{node.constant.value}="]) + else + s(:casgn, [visit(node.parent), node.constant.value.to_sym]) + end + end + + # Visit a ConstPathRef node. + def visit_const_path_ref(node) + s(:const, [visit(node.parent), node.constant.value.to_sym]) + end + + # Visit a ConstRef node. + def visit_const_ref(node) + s(:const, [nil, node.constant.value.to_sym]) + end + + # Visit a CVar node. + def visit_cvar(node) + s(:cvar, [node.value.to_sym]) + end + + # Visit a DefNode node. + def visit_def(node) + name = node.name.value.to_sym + args = + case node.params + when Params + visit(node.params) + when Paren + visit(node.params.contents) + else + s(:args) + end + + if node.target + target = node.target.is_a?(Paren) ? node.target.contents : node.target + s(:defs, [visit(target), name, args, visit(node.bodystmt)]) + else + s(:def, [name, args, visit(node.bodystmt)]) + end + end + + # Visit a Defined node. + def visit_defined(node) + s(:defined?, [visit(node.value)]) + end + + # Visit a DynaSymbol node. + def visit_dyna_symbol(node) + if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) + s(:sym, ["\"#{node.parts.first.value}\"".undump.to_sym]) + else + s(:dsym, visit_all(node.parts)) + end + end + + # Visit an Else node. + def visit_else(node) + if node.statements.empty? && stack[-2].is_a?(Case) + s(:empty_else) + else + visit(node.statements) + end + end + + # Visit an Elsif node. + def visit_elsif(node) + s( + :if, + [ + visit(node.predicate), + visit(node.statements), + visit(node.consequent) + ] + ) + end + + # Visit an ENDBlock node. + def visit_END(node) + s(:postexe, [visit(node.statements)]) + end + + # Visit an Ensure node. + def visit_ensure(node) + s(:ensure, [visit(node.statements)]) + end + + # Visit a Field node. + def visit_field(node) + case stack[-2] + when Assign, MLHS + s( + send_type(node.operator), + [visit(node.parent), :"#{node.name.value}="] + ) + else + s( + send_type(node.operator), + [visit(node.parent), node.name.value.to_sym] + ) + end + end + + # Visit a FloatLiteral node. + def visit_float(node) + s(:float, [node.value.to_f]) + end + + # Visit a FndPtn node. + def visit_fndptn(node) + make_match_rest = ->(child) do + if child.is_a?(VarField) && child.value.nil? + s(:match_rest, []) + else + s(:match_rest, [visit(child)]) + end + end + + inner = + s( + :find_pattern, + [ + make_match_rest[node.left], + *visit_all(node.values), + make_match_rest[node.right] + ] + ) + node.constant ? s(:const_pattern, [visit(node.constant), inner]) : inner + end + + # Visit a For node. + def visit_for(node) + s( + :for, + [visit(node.index), visit(node.collection), visit(node.statements)] + ) + end + + # Visit a GVar node. + def visit_gvar(node) + s(:gvar, [node.value.to_sym]) + end + + # Visit a HashLiteral node. + def visit_hash(node) + s(:hash, visit_all(node.assocs)) + end + + # Heredocs are represented _very_ differently in the parser gem from how + # they are represented in the Syntax Tree AST. This class is responsible + # for handling the translation. + class HeredocSegments + HeredocLine = Struct.new(:value, :segments) + + attr_reader :node, :segments + + def initialize(node) + @node = node + @segments = [] + end + + def <<(segment) + if segment.type == :str && segments.last && + segments.last.type == :str && + !segments.last.children.first.end_with?("\n") + segments.last.children.first << segment.children.first + else + segments << segment + end + end + + def trim! + return unless node.beginning.value[2] == "~" + lines = [HeredocLine.new(+"", [])] + + segments.each do |segment| + lines.last.segments << segment + + if segment.type == :str + lines.last.value << segment.children.first + + if lines.last.value.end_with?("\n") + lines << HeredocLine.new(+"", []) + end + end + end + + lines.pop if lines.last.value.empty? + return if lines.empty? + + segments.clear + lines.each do |line| + remaining = node.dedent + + line.segments.each do |segment| + if segment.type == :str + if remaining > 0 + whitespace = segment.children.first[/^\s{0,#{remaining}}/] + segment.children.first.sub!(/^#{whitespace}/, "") + remaining -= whitespace.length + end + + if node.beginning.value[3] != "'" && segments.any? && + segments.last.type == :str && + segments.last.children.first.end_with?("\\\n") + segments.last.children.first.gsub!(/\\\n\z/, "") + segments.last.children.first.concat(segment.children.first) + elsif !segment.children.first.empty? + segments << segment + end + else + segments << segment + end + end + end + end + end + + # Visit a Heredoc node. + def visit_heredoc(node) + heredoc_segments = HeredocSegments.new(node) + + node.parts.each do |part| + if part.is_a?(TStringContent) && part.value.count("\n") > 1 + part + .value + .split("\n") + .each { |line| heredoc_segments << s(:str, ["#{line}\n"]) } + else + heredoc_segments << visit(part) + end + end + + heredoc_segments.trim! + + if node.beginning.value.match?(/`\w+`\z/) + s(:xstr, heredoc_segments.segments) + elsif heredoc_segments.segments.length > 1 + s(:dstr, heredoc_segments.segments) + elsif heredoc_segments.segments.empty? + s(:dstr) + else + heredoc_segments.segments.first + end + end + + # Visit a HshPtn node. + def visit_hshptn(node) + children = + node.keywords.map do |(keyword, value)| + next s(:pair, [visit(keyword), visit(value)]) if value + + case keyword + when Label + s(:match_var, [keyword.value.chomp(":").to_sym]) + when StringContent + raise if keyword.parts.length > 1 + s(:match_var, [keyword.parts.first.value.to_sym]) + end + end + + if node.keyword_rest.is_a?(VarField) + children << if node.keyword_rest.value.nil? + s(:match_rest) + elsif node.keyword_rest.value == :nil + s(:match_nil_pattern) + else + s(:match_rest, [visit(node.keyword_rest)]) + end + end + + inner = s(:hash_pattern, children) + node.constant ? s(:const_pattern, [visit(node.constant), inner]) : inner + end + + # Visit an Ident node. + def visit_ident(node) + s(:lvar, [node.value.to_sym]) + end + + # Visit an IfNode node. + def visit_if(node) + predicate = + case node.predicate + when RangeNode + type = + node.predicate.operator.value == ".." ? :iflipflop : :eflipflop + s(type, visit(node.predicate).children) + when RegexpLiteral + s(:match_current_line, [visit(node.predicate)]) + when Unary + if node.predicate.operator.value == "!" && + node.predicate.statement.is_a?(RegexpLiteral) + s( + :send, + [s(:match_current_line, [visit(node.predicate.statement)]), :!] + ) + else + visit(node.predicate) + end + else + visit(node.predicate) + end + + s(:if, [predicate, visit(node.statements), visit(node.consequent)]) + end + + # Visit an IfOp node. + def visit_if_op(node) + s(:if, [visit(node.predicate), visit(node.truthy), visit(node.falsy)]) + end + + # Visit an Imaginary node. + def visit_imaginary(node) + # We have to do an eval here in order to get the value in case it's + # something like 42ri. to_c will not give the right value in that case. + # Maybe there's an API for this but I can't find it. + s(:complex, [eval(node.value)]) + end + + # Visit an In node. + def visit_in(node) + case node.pattern + when IfNode + s( + :in_pattern, + [ + visit(node.pattern.statements), + s(:if_guard, [visit(node.pattern.predicate)]), + visit(node.statements) + ] + ) + when UnlessNode + s( + :in_pattern, + [ + visit(node.pattern.statements), + s(:unless_guard, [visit(node.pattern.predicate)]), + visit(node.statements) + ] + ) + else + s(:in_pattern, [visit(node.pattern), nil, visit(node.statements)]) + end + end + + # Visit an Int node. + def visit_int(node) + s(:int, [node.value.to_i]) + end + + # Visit an IVar node. + def visit_ivar(node) + s(:ivar, [node.value.to_sym]) + end + + # Visit a Kw node. + def visit_kw(node) + case node.value + when "__FILE__" + s(:str, [buffer.name]) + when "__LINE__" + s(:int, [node.location.start_line + buffer.first_line - 1]) + when "__ENCODING__" + if ::Parser::Builders::Default.emit_encoding + s(:__ENCODING__) + else + s(:const, [s(:const, [nil, :Encoding]), :UTF_8]) + end + else + s(node.value.to_sym) + end + end + + # Visit a KwRestParam node. + def visit_kwrest_param(node) + node.name.nil? ? s(:kwrestarg) : s(:kwrestarg, [node.name.value.to_sym]) + end + + # Visit a Label node. + def visit_label(node) + s(:sym, [node.value.chomp(":").to_sym]) + end + + # Visit a Lambda node. + def visit_lambda(node) + args = node.params.is_a?(LambdaVar) ? node.params : node.params.contents + + arguments = visit(args) + child = + if ::Parser::Builders::Default.emit_lambda + s(:lambda) + else + s(:send, [nil, :lambda]) + end + + type = :block + if args.empty? && (maximum = num_block_type(node.statements)) + type = :numblock + arguments = maximum + end + + s(type, [child, arguments, visit(node.statements)]) + end + + # Visit a LambdaVar node. + def visit_lambda_var(node) + shadowargs = + node.locals.map { |local| s(:shadowarg, [local.value.to_sym]) } + + s(:args, visit(node.params).children + shadowargs) + end + + # Visit an MAssign node. + def visit_massign(node) + s(:masgn, [visit(node.target), visit(node.value)]) + end + + # Visit a MethodAddBlock node. + def visit_method_add_block(node) + type, arguments = block_children(node.block) + + case node.call + when Break, Next, ReturnNode + call = visit(node.call) + s( + call.type, + [s(type, [*call.children, arguments, visit(node.block.bodystmt)])] + ) + else + s(type, [visit(node.call), arguments, visit(node.block.bodystmt)]) + end + end + + # Visit an MLHS node. + def visit_mlhs(node) + s( + :mlhs, + node.parts.map do |part| + part.is_a?(Ident) ? s(:arg, [part.value.to_sym]) : visit(part) + end + ) + end + + # Visit an MLHSParen node. + def visit_mlhs_paren(node) + visit(node.contents) + end + + # Visit a ModuleDeclaration node. + def visit_module(node) + s(:module, [visit(node.constant), visit(node.bodystmt)]) + end + + # Visit an MRHS node. + def visit_mrhs(node) + s(:array, visit_all(node.parts)) + end + + # Visit a Next node. + def visit_next(node) + s(:next, visit_all(node.arguments.parts)) + end + + # Visit a Not node. + def visit_not(node) + if node.statement.nil? + s(:send, [s(:begin), :!]) + else + s(:send, [visit(node.statement), :!]) + end + end + + # Visit an OpAssign node. + def visit_opassign(node) + case node.operator.value + when "||=" + s(:or_asgn, [visit(node.target), visit(node.value)]) + when "&&=" + s(:and_asgn, [visit(node.target), visit(node.value)]) + else + s( + :op_asgn, + [ + visit(node.target), + node.operator.value.chomp("=").to_sym, + visit(node.value) + ] + ) + end + end + + # Visit a Params node. + def visit_params(node) + children = [] + + children += + node.requireds.map do |required| + case required + when MLHSParen + visit(required) + else + s(:arg, [required.value.to_sym]) + end + end + + children += + node.optionals.map do |(name, value)| + s(:optarg, [name.value.to_sym, visit(value)]) + end + if node.rest && !node.rest.is_a?(ExcessedComma) + children << visit(node.rest) + end + children += node.posts.map { |post| s(:arg, [post.value.to_sym]) } + children += + node.keywords.map do |(name, value)| + key = name.value.chomp(":").to_sym + value ? s(:kwoptarg, [key, visit(value)]) : s(:kwarg, [key]) + end + + case node.keyword_rest + when nil, ArgsForward + # do nothing + when :nil + children << s(:kwnilarg) + else + children << visit(node.keyword_rest) + end + + children << visit(node.block) if node.block + + if node.keyword_rest.is_a?(ArgsForward) + if children.empty? && !::Parser::Builders::Default.emit_forward_arg + return s(:forward_args) + end + + children.insert( + node.requireds.length + node.optionals.length + + node.keywords.length, + s(:forward_arg) + ) + end + + s(:args, children) + end + + # Visit a Paren node. + def visit_paren(node) + if node.contents.nil? || + ( + node.contents.is_a?(Statements) && + node.contents.body.length == 1 && + node.contents.body.first.is_a?(VoidStmt) + ) + s(:begin) + elsif stack[-2].is_a?(DefNode) && stack[-2].target.nil? && + stack[-2].target == node + visit(node.contents) + else + visited = visit(node.contents) + visited.type == :begin ? visited : s(:begin, [visited]) + end + end + + # Visit a PinnedBegin node. + def visit_pinned_begin(node) + s(:pin, [s(:begin, [visit(node.statement)])]) + end + + # Visit a PinnedVarRef node. + def visit_pinned_var_ref(node) + s(:pin, [visit(node.value)]) + end + + # Visit a Program node. + def visit_program(node) + visit(node.statements) + end + + # Visit a QSymbols node. + def visit_qsymbols(node) + s( + :array, + node.elements.map { |element| s(:sym, [element.value.to_sym]) } + ) + end + + # Visit a QWords node. + def visit_qwords(node) + s(:array, visit_all(node.elements)) + end + + # Visit a RangeNode node. + def visit_range(node) + type = node.operator.value == ".." ? :irange : :erange + s(type, [visit(node.left), visit(node.right)]) + end + + # Visit an RAssign node. + def visit_rassign(node) + type = node.operator.value == "=>" ? :match_pattern : :match_pattern_p + s(type, [visit(node.value), visit(node.pattern)]) + end + + # Visit a Rational node. + def visit_rational(node) + s(:rational, [node.value.to_r]) + end + + # Visit a Redo node. + def visit_redo(_node) + s(:redo) + end + + # Visit a RegexpLiteral node. + def visit_regexp_literal(node) + s( + :regexp, + visit_all(node.parts) + + [s(:regopt, node.ending.scan(/[a-z]/).sort.map(&:to_sym))] + ) + end + + # Visit a Rescue node. + def visit_rescue(node) + exceptions = + case node.exception&.exceptions + when nil + nil + when VarRef + s(:array, [visit(node.exception.exceptions)]) + when MRHS + s(:array, visit_all(node.exception.exceptions.parts)) + else + s(:array, [visit(node.exception.exceptions)]) + end + + resbody = + if node.exception.nil? + s(:resbody, [nil, nil, visit(node.statements)]) + elsif node.exception.variable.nil? + s(:resbody, [exceptions, nil, visit(node.statements)]) + else + s( + :resbody, + [ + exceptions, + visit(node.exception.variable), + visit(node.statements) + ] + ) + end + + children = [resbody] + if node.consequent + children += visit(node.consequent).children + else + children << nil + end + + s(:rescue, children) + end + + # Visit a RescueMod node. + def visit_rescue_mod(node) + s( + :rescue, + [ + visit(node.statement), + s(:resbody, [nil, nil, visit(node.value)]), + nil + ] + ) + end + + # Visit a RestParam node. + def visit_rest_param(node) + s(:restarg, node.name ? [node.name.value.to_sym] : []) + end + + # Visit a Retry node. + def visit_retry(_node) + s(:retry) + end + + # Visit a ReturnNode node. + def visit_return(node) + s(:return, node.arguments ? visit_all(node.arguments.parts) : []) + end + + # Visit an SClass node. + def visit_sclass(node) + s(:sclass, [visit(node.target), visit(node.bodystmt)]) + end + + # Visit a Statements node. + def visit_statements(node) + children = + node.body.reject do |child| + child.is_a?(Comment) || child.is_a?(EmbDoc) || + child.is_a?(EndContent) || child.is_a?(VoidStmt) + end + + case children.length + when 0 + nil + when 1 + visit(children.first) + else + s(:begin, visit_all(children)) + end + end + + # Visit a StringConcat node. + def visit_string_concat(node) + s(:dstr, [visit(node.left), visit(node.right)]) + end + + # Visit a StringContent node. + def visit_string_content(node) + # Can get here if you're inside a hash pattern, e.g., in "a": 1 + s(:sym, [node.parts.first.value.to_sym]) + end + + # Visit a StringDVar node. + def visit_string_dvar(node) + visit(node.variable) + end + + # Visit a StringEmbExpr node. + def visit_string_embexpr(node) + child = visit(node.statements) + s(:begin, child ? [child] : []) + end + + # Visit a StringLiteral node. + def visit_string_literal(node) + if node.parts.empty? + s(:str, [""]) + elsif node.parts.length == 1 && node.parts.first.is_a?(TStringContent) + visit(node.parts.first) + else + s(:dstr, visit_all(node.parts)) + end + end + + # Visit a Super node. + def visit_super(node) + if node.arguments.is_a?(Args) + s(:super, visit_all(node.arguments.parts)) + else + case node.arguments.arguments + when nil + s(:super) + when ArgsForward + s(:super, [visit(node.arguments.arguments)]) + else + s(:super, visit_all(node.arguments.arguments.parts)) + end + end + end + + # Visit a SymbolLiteral node. + def visit_symbol_literal(node) + s(:sym, [node.value.value.to_sym]) + end + + # Visit a Symbols node. + def visit_symbols(node) + children = + node.elements.map do |element| + if element.parts.length > 1 || + !element.parts.first.is_a?(TStringContent) + s(:dsym, visit_all(element.parts)) + else + s(:sym, [element.parts.first.value.to_sym]) + end + end + + s(:array, children) + end + + # Visit a TopConstField node. + def visit_top_const_field(node) + s(:casgn, [s(:cbase), node.constant.value.to_sym]) + end + + # Visit a TopConstRef node. + def visit_top_const_ref(node) + s(:const, [s(:cbase), node.constant.value.to_sym]) + end + + # Visit a TStringContent node. + def visit_tstring_content(node) + value = node.value.gsub(/([^[:ascii:]])/) { $1.dump[1...-1] } + s(:str, ["\"#{value}\"".undump]) + end + + # Visit a Unary node. + def visit_unary(node) + # Special handling here for flipflops + if node.statement.is_a?(Paren) && + node.statement.contents.is_a?(Statements) && + node.statement.contents.body.length == 1 && + (range = node.statement.contents.body.first).is_a?(RangeNode) && + node.operator == "!" + type = range.operator.value == ".." ? :iflipflop : :eflipflop + return s(:send, [s(:begin, [s(type, visit(range).children)]), :!]) + end + + case node.operator + when "+" + case node.statement + when Int + s(:int, [node.statement.value.to_i]) + when FloatLiteral + s(:float, [node.statement.value.to_f]) + else + s(:send, [visit(node.statement), :+@]) + end + when "-" + case node.statement + when Int + s(:int, [-node.statement.value.to_i]) + when FloatLiteral + s(:float, [-node.statement.value.to_f]) + else + s(:send, [visit(node.statement), :-@]) + end + else + s(:send, [visit(node.statement), node.operator.to_sym]) + end + end + + # Visit an Undef node. + def visit_undef(node) + s(:undef, visit_all(node.symbols)) + end + + # Visit an UnlessNode node. + def visit_unless(node) + predicate = + case node.predicate + when RegexpLiteral + s(:match_current_line, [visit(node.predicate)]) + when Unary + if node.predicate.operator.value == "!" && + node.predicate.statement.is_a?(RegexpLiteral) + s( + :send, + [s(:match_current_line, [visit(node.predicate.statement)]), :!] + ) + else + visit(node.predicate) + end + else + visit(node.predicate) + end + + s(:if, [predicate, visit(node.consequent), visit(node.statements)]) + end + + # Visit an UntilNode node. + def visit_until(node) + type = + if node.modifier? && node.statements.is_a?(Statements) && + node.statements.body.length == 1 && + node.statements.body.first.is_a?(Begin) + :until_post + else + :until + end + + s(type, [visit(node.predicate), visit(node.statements)]) + end + + # Visit a VarField node. + def visit_var_field(node) + is_match_var = ->(parent) do + case parent + when AryPtn, FndPtn, HshPtn, In, RAssign + true + when Binary + parent.operator == :"=>" + else + false + end + end + + if [stack[-3], stack[-2]].any?(&is_match_var) + return s(:match_var, [node.value.value.to_sym]) + end + + case node.value + when Const + s(:casgn, [nil, node.value.value.to_sym]) + when CVar + s(:cvasgn, [node.value.value.to_sym]) + when GVar + s(:gvasgn, [node.value.value.to_sym]) + when Ident + s(:lvasgn, [node.value.value.to_sym]) + when IVar + s(:ivasgn, [node.value.value.to_sym]) + when VarRef + s(:lvasgn, [node.value.value.to_sym]) + else + s(:match_rest) + end + end + + # Visit a VarRef node. + def visit_var_ref(node) + visit(node.value) + end + + # Visit a VCall node. + def visit_vcall(node) + range = + ::Parser::Source::Range.new( + buffer, + node.location.start_char, + node.location.end_char + ) + location = ::Parser::Source::Map::Send.new(nil, range, nil, nil, range) + + s(:send, [nil, node.value.value.to_sym], location: location) + end + + # Visit a When node. + def visit_when(node) + s(:when, visit_all(node.arguments.parts) + [visit(node.statements)]) + end + + # Visit a WhileNode node. + def visit_while(node) + type = + if node.modifier? && node.statements.is_a?(Statements) && + node.statements.body.length == 1 && + node.statements.body.first.is_a?(Begin) + :while_post + else + :while + end + + s(type, [visit(node.predicate), visit(node.statements)]) + end + + # Visit a Word node. + def visit_word(node) + if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) + visit(node.parts.first) + else + s(:dstr, visit_all(node.parts)) + end + end + + # Visit a Words node. + def visit_words(node) + s(:array, visit_all(node.elements)) + end + + # Visit an XStringLiteral node. + def visit_xstring_literal(node) + s(:xstr, visit_all(node.parts)) + end + + def visit_yield(node) + case node.arguments + when nil + s(:yield) + when Args + s(:yield, visit_all(node.arguments.parts)) + else + s(:yield, visit_all(node.arguments.contents.parts)) + end + end + + # Visit a ZSuper node. + def visit_zsuper(_node) + s(:zsuper) + end + + private + + def block_children(node) + arguments = (node.block_var ? visit(node.block_var) : s(:args)) + + type = :block + if !node.block_var && (maximum = num_block_type(node.bodystmt)) + type = :numblock + arguments = maximum + end + + [type, arguments] + end + + # We need to find if we should transform this block into a numblock + # since there could be new numbered variables like _1. + def num_block_type(statements) + variables = [] + queue = [statements] + + while (child_node = queue.shift) + if child_node.is_a?(VarRef) && child_node.value.is_a?(Ident) && + child_node.value.value =~ /^_(\d+)$/ + variables << $1.to_i + end + + queue += child_node.child_nodes.compact + end + + variables.max + end + + def s(type, children = [], opts = {}) + ::Parser::AST::Node.new(type, children, opts) + end + + def send_type(operator) + operator.is_a?(Op) && operator.value == "&." ? :csend : :send + end + end + end +end diff --git a/test/ruby_syntax_fixtures_test.rb b/test/ruby_syntax_fixtures_test.rb index 0cf89310..c5c13b27 100644 --- a/test/ruby_syntax_fixtures_test.rb +++ b/test/ruby_syntax_fixtures_test.rb @@ -1,5 +1,9 @@ # frozen_string_literal: true +# The ruby-syntax-fixtures repository tests against the current Ruby syntax, so +# we don't execute this test unless we're running 3.2 or above. +return unless RUBY_VERSION >= "3.2" + require_relative "test_helper" module SyntaxTree diff --git a/test/suites/helper.rb b/test/suites/helper.rb new file mode 100644 index 00000000..b0f8c427 --- /dev/null +++ b/test/suites/helper.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +require "parser/current" diff --git a/test/suites/parse_helper.rb b/test/suites/parse_helper.rb new file mode 100644 index 00000000..685cd6d2 --- /dev/null +++ b/test/suites/parse_helper.rb @@ -0,0 +1,149 @@ +# frozen_string_literal: true + +module ParseHelper + include AST::Sexp + + CURRENT_VERSION = RUBY_VERSION.split(".")[0..1].join(".").freeze + ALL_VERSIONS = %w[1.8 1.9 2.0 2.1 2.2 2.3 2.4 2.5 2.6 2.7 3.0 3.1 3.2 mac ios] + + known_failures = [ + # I think this may be a bug in the parser gem's precedence calculation. + # Unary plus appears to be parsed as part of the number literal in CRuby, + # but parser is parsing it as a separate operator. + "test_unary_num_pow_precedence:3505", + + # Not much to be done about this. Basically, regular expressions with named + # capture groups that use the =~ operator inject local variables into the + # current scope. In the parser gem, it detects this and changes future + # references to that name to be a local variable instead of a potential + # method call. CRuby does not do this. + "test_lvar_injecting_match:3778", + + # This is failing because CRuby is not marking values captured in hash + # patterns as local variables, while the parser gem is. + "test_pattern_matching_hash:8971", + + # This is not actually allowed in the CRuby parser but the parser gem thinks + # it is allowed. + "test_pattern_matching_hash_with_string_keys:9016", + "test_pattern_matching_hash_with_string_keys:9027", + "test_pattern_matching_hash_with_string_keys:9038", + "test_pattern_matching_hash_with_string_keys:9060", + "test_pattern_matching_hash_with_string_keys:9071", + "test_pattern_matching_hash_with_string_keys:9082", + + # This happens with pattern matching where you're matching a literal value + # inside parentheses, which doesn't really do anything. Ripper doesn't + # capture that this value is inside a parentheses, so it's hard to translate + # properly. + "test_pattern_matching_expr_in_paren:9206", + + # These are also failing because of CRuby not marking values captured in + # hash patterns as local variables. + "test_pattern_matching_single_line_allowed_omission_of_parentheses:9205", + "test_pattern_matching_single_line_allowed_omission_of_parentheses:9581", + "test_pattern_matching_single_line_allowed_omission_of_parentheses:9611", + + # I'm not even sure what this is testing, because the code is invalid in + # CRuby. + "test_control_meta_escape_chars_in_regexp__since_31:*", + ] + + # These are failures that we need to take care of (or determine the reason + # that we're not going to handle them). + todo_failures = [ + "test_dedenting_heredoc:334", + "test_dedenting_heredoc:390", + "test_dedenting_heredoc:399", + "test_slash_newline_in_heredocs:7194", + "test_parser_slash_slash_n_escaping_in_literals:*", + "test_cond_match_current_line:4801", + "test_forwarded_restarg:*", + "test_forwarded_kwrestarg:*", + "test_forwarded_argument_with_restarg:*", + "test_forwarded_argument_with_kwrestarg:*" + ] + + if CURRENT_VERSION <= "2.7" + # I'm not sure why this is failing on 2.7.0, but we'll turn it off for now + # until we have more time to investigate. + todo_failures.push("test_pattern_matching_hash:*") + end + + if CURRENT_VERSION <= "3.0" + # In < 3.0, there are some changes to the way the parser gem handles + # forwarded args. We should eventually support this, but for now we're going + # to mark them as todo. + todo_failures.push( + "test_forward_arg:*", + "test_forward_args_legacy:*", + "test_endless_method_forwarded_args_legacy:*", + "test_trailing_forward_arg:*" + ) + end + + if CURRENT_VERSION == "3.1" + # This test actually fails on 3.1.0, even though it's marked as being since + # 3.1. So we're going to skip this test on 3.1, but leave it in for other + # versions. + known_failures.push( + "test_multiple_pattern_matches:11086", + "test_multiple_pattern_matches:11102" + ) + end + + # This is the list of all failures. + FAILURES = (known_failures + todo_failures).freeze + + private + + def assert_context(*) + end + + def assert_diagnoses(*) + end + + def assert_diagnoses_many(*) + end + + def refute_diagnoses(*) + end + + def with_versions(*) + end + + def assert_parses(_ast, code, _source_maps = "", versions = ALL_VERSIONS) + # We're going to skip any examples that aren't for the current version of + # Ruby. + return unless versions.include?(CURRENT_VERSION) + + # We're going to skip any examples that are for older Ruby versions that we + # do not support. + return if (versions & %w[3.1 3.2]).empty? + + caller(1, 3).each do |line| + _, lineno, name = *line.match(/(\d+):in `(.+)'/) + + # Return directly and don't do anything if it's a known failure. + return if FAILURES.include?("#{name}:#{lineno}") + return if FAILURES.include?("#{name}:*") + end + + expected = parse(code) + return if expected.nil? + + actual = SyntaxTree::Translation.to_parser(SyntaxTree.parse(code), code) + assert_equal(expected, actual) + end + + def parse(code) + parser = Parser::CurrentRuby.default_parser + parser.diagnostics.consumer = ->(*) {} + + buffer = Parser::Source::Buffer.new("(string)", 1) + buffer.source = code + + parser.parse(buffer) + rescue Parser::SyntaxError + end +end diff --git a/test/suites/parser b/test/suites/parser new file mode 160000 index 00000000..8de8b7fa --- /dev/null +++ b/test/suites/parser @@ -0,0 +1 @@ +Subproject commit 8de8b7fa7af471a2159860d6a0a5b615eac9c83c From 1155f851226b552e1ca7e435ab134783c997ac81 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 30 Jan 2023 10:34:35 -0500 Subject: [PATCH 342/536] BasicVisitor::visit_methods --- README.md | 21 ++++++++++++++ lib/syntax_tree/basic_visitor.rb | 49 ++++++++++++++++++++++++++++---- test/visitor_test.rb | 14 +++++++++ 3 files changed, 79 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 3c437947..6ca9b01a 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ It is built with only standard library dependencies. It additionally ships with - [construct_keys](#construct_keys) - [Visitor](#visitor) - [visit_method](#visit_method) + - [visit_methods](#visit_methods) - [BasicVisitor](#basicvisitor) - [MutationVisitor](#mutationvisitor) - [WithEnvironment](#withenvironment) @@ -517,6 +518,26 @@ Did you mean? visit_binary from bin/console:8:in `
' ``` +### visit_methods + +Similar to `visit_method`, `visit_methods` also checks that methods defined are valid visit methods. This variation however accepts a block and checks that all methods defined within that block are valid visit methods. It's meant to be used like: + +```ruby +class ArithmeticVisitor < SyntaxTree::Visitor + visit_methods do + def visit_binary(node) + # ... + end + + def visit_int(node) + # ... + end + end +end +``` + +This is only checked when the methods are defined and does not impose any kind of runtime overhead after that. It is very useful for upgrading versions of Syntax Tree in case these methods names change. + ### BasicVisitor When you're defining your own visitor, by default it will walk down the tree even if you don't define `visit_*` methods. This is to ensure you can define a subset of the necessary methods in order to only interact with the nodes you're interested in. If you'd like to change this default to instead raise an error if you visit a node you haven't explicitly handled, you can instead inherit from `BasicVisitor`. diff --git a/lib/syntax_tree/basic_visitor.rb b/lib/syntax_tree/basic_visitor.rb index 34b7876e..bd8ea5f2 100644 --- a/lib/syntax_tree/basic_visitor.rb +++ b/lib/syntax_tree/basic_visitor.rb @@ -29,7 +29,7 @@ def initialize(error) def corrections @corrections ||= DidYouMean::SpellChecker.new( - dictionary: Visitor.visit_methods + dictionary: BasicVisitor.valid_visit_methods ).correct(visit_method) end @@ -40,7 +40,40 @@ def corrections end end + # This module is responsible for checking all of the methods defined within + # a given block to ensure that they are valid visit methods. + class VisitMethodsChecker < Module + Status = Struct.new(:checking) + + # This is the status of the checker. It's used to determine whether or not + # we should be checking the methods that are defined. It is kept as an + # instance variable so that it can be disabled later. + attr_reader :status + + def initialize + # We need the status to be an instance variable so that it can be + # accessed by the disable! method, but also a local variable so that it + # can be captured by the define_method block. + status = @status = Status.new(true) + + define_method(:method_added) do |name| + BasicVisitor.visit_method(name) if status.checking + super(name) + end + end + + def disable! + status.checking = false + end + end + class << self + # This is the list of all of the valid visit methods. + def valid_visit_methods + @valid_visit_methods ||= + Visitor.instance_methods.grep(/^visit_(?!child_nodes)/) + end + # This method is here to help folks write visitors. # # It's not always easy to ensure you're writing the correct method name in @@ -51,15 +84,21 @@ class << self # name. It will raise an error if the visit method you're defining isn't # actually a method on the parent visitor. def visit_method(method_name) - return if visit_methods.include?(method_name) + return if valid_visit_methods.include?(method_name) raise VisitMethodError, method_name end - # This is the list of all of the valid visit methods. + # This method is here to help folks write visitors. + # + # Within the given block, every method that is defined will be checked to + # ensure it's a valid visit method using the BasicVisitor::visit_method + # method defined above. def visit_methods - @visit_methods ||= - Visitor.instance_methods.grep(/^visit_(?!child_nodes)/) + checker = VisitMethodsChecker.new + extend(checker) + yield + checker.disable! end end diff --git a/test/visitor_test.rb b/test/visitor_test.rb index 74f3df75..86ff1b01 100644 --- a/test/visitor_test.rb +++ b/test/visitor_test.rb @@ -53,5 +53,19 @@ def test_visit_method_correction assert_match(/visit_binary/, message) end end + + class VisitMethodsTestVisitor < BasicVisitor + end + + def test_visit_methods + VisitMethodsTestVisitor.visit_methods do + assert_raises(BasicVisitor::VisitMethodError) do + # In reality, this would be a method defined using the def keyword, + # but we're using method_added here to trigger the checker so that we + # aren't defining methods dynamically in the test suite. + VisitMethodsTestVisitor.method_added(:visit_foo) + end + end + end end end From db2979f87f1841719ff0cdd33e324d8a53631986 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 30 Jan 2023 15:09:26 -0500 Subject: [PATCH 343/536] Additionally provide parser gem location information --- bin/compare | 59 + lib/syntax_tree/translation.rb | 5 +- lib/syntax_tree/translation/parser.rb | 2092 ++++++++++++++++++++----- test/suites/parse_helper.rb | 28 +- 4 files changed, 1823 insertions(+), 361 deletions(-) create mode 100755 bin/compare diff --git a/bin/compare b/bin/compare new file mode 100755 index 00000000..bdca5a9a --- /dev/null +++ b/bin/compare @@ -0,0 +1,59 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "bundler/setup" +require "parser/current" + +$:.unshift(File.expand_path("../lib", __dir__)) +require "syntax_tree" + +# First, opt in to every AST feature. +# Parser::Builders::Default.modernize + +# Modify the source map == check so that it doesn't check against the node +# itself so we don't get into a recursive loop. +Parser::Source::Map.prepend( + Module.new { + def ==(other) + self.class == other.class && + (instance_variables - %i[@node]).map do |ivar| + instance_variable_get(ivar) == other.instance_variable_get(ivar) + end.reduce(:&) + end + } +) + +# Next, ensure that we're comparing the nodes and also comparing the source +# ranges so that we're getting all of the necessary information. +Parser::AST::Node.prepend( + Module.new { + def ==(other) + super && (location == other.location) + end + } +) + +source = ARGF.read + +parser = Parser::CurrentRuby.new +parser.diagnostics.all_errors_are_fatal = true + +buffer = Parser::Source::Buffer.new("(string)", 1) +buffer.source = source.dup.force_encoding(parser.default_encoding) + +stree = SyntaxTree::Translation.to_parser(SyntaxTree.parse(source), buffer) +ptree = parser.parse(buffer) + +if stree == ptree + puts "Syntax trees are equivalent." +else + warn "Syntax trees are different." + + warn "syntax_tree:" + pp stree + + warn "parser:" + pp ptree + + binding.irb +end diff --git a/lib/syntax_tree/translation.rb b/lib/syntax_tree/translation.rb index 37785ea2..d3f2e56f 100644 --- a/lib/syntax_tree/translation.rb +++ b/lib/syntax_tree/translation.rb @@ -7,13 +7,10 @@ module Translation # This method translates the given node into the representation defined by # the whitequark/parser gem. We don't explicitly list it as a dependency # because it's not required for the core functionality of Syntax Tree. - def self.to_parser(node, source) + def self.to_parser(node, buffer) require "parser" require_relative "translation/parser" - buffer = ::Parser::Source::Buffer.new("(string)") - buffer.source = source - node.accept(Parser.new(buffer)) end end diff --git a/lib/syntax_tree/translation/parser.rb b/lib/syntax_tree/translation/parser.rb index 3443df37..8a61ad94 100644 --- a/lib/syntax_tree/translation/parser.rb +++ b/lib/syntax_tree/translation/parser.rb @@ -2,6 +2,8 @@ module SyntaxTree module Translation + # This visitor is responsible for converting the syntax tree produced by + # Syntax Tree into the syntax tree produced by the whitequark/parser gem. class Parser < BasicVisitor attr_reader :buffer, :stack @@ -22,24 +24,81 @@ def visit(node) # Visit an AliasNode node. def visit_alias(node) - s(:alias, [visit(node.left), visit(node.right)]) + s( + :alias, + [visit(node.left), visit(node.right)], + source_map_keyword( + keyword: source_range_length(node.location.start_char, 5), + expression: source_range_node(node) + ) + ) end # Visit an ARefNode. def visit_aref(node) if ::Parser::Builders::Default.emit_index if node.index.nil? - s(:index, [visit(node.collection)]) + s( + :index, + [visit(node.collection)], + source_map_index( + begin_token: + source_range_find( + node.collection.location.end_char, + node.location.end_char, + "[" + ), + end_token: source_range_length(node.location.end_char, -1), + expression: source_range_node(node) + ) + ) else - s(:index, [visit(node.collection), *visit_all(node.index.parts)]) + s( + :index, + [visit(node.collection)].concat(visit_all(node.index.parts)), + source_map_index( + begin_token: + source_range_find( + node.collection.location.end_char, + node.index.location.start_char, + "[" + ), + end_token: source_range_length(node.location.end_char, -1), + expression: source_range_node(node) + ) + ) end else if node.index.nil? - s(:send, [visit(node.collection), :[], nil]) + s( + :send, + [visit(node.collection), :[]], + source_map_send( + selector: + source_range_find( + node.collection.location.end_char, + node.location.end_char, + "[]" + ), + expression: source_range_node(node) + ) + ) else s( :send, - [visit(node.collection), :[], *visit_all(node.index.parts)] + [visit(node.collection), :[], *visit_all(node.index.parts)], + source_map_send( + selector: + source_range( + source_range_find( + node.collection.location.end_char, + node.index.location.start_char, + "[" + ).begin_pos, + node.location.end_char + ), + expression: source_range_node(node) + ) ) end end @@ -49,20 +108,69 @@ def visit_aref(node) def visit_aref_field(node) if ::Parser::Builders::Default.emit_index if node.index.nil? - s(:indexasgn, [visit(node.collection), nil]) + s( + :indexasgn, + [visit(node.collection)], + source_map_index( + begin_token: + source_range_find( + node.collection.location.end_char, + node.location.end_char, + "[" + ), + end_token: source_range_length(node.location.end_char, -1), + expression: source_range_node(node) + ) + ) else s( :indexasgn, - [visit(node.collection), *visit_all(node.index.parts)] + [visit(node.collection)].concat(visit_all(node.index.parts)), + source_map_index( + begin_token: + source_range_find( + node.collection.location.end_char, + node.index.location.start_char, + "[" + ), + end_token: source_range_length(node.location.end_char, -1), + expression: source_range_node(node) + ) ) end else if node.index.nil? - s(:send, [visit(node.collection), :[]=, nil]) + s( + :send, + [visit(node.collection), :[]=], + source_map_send( + selector: + source_range_find( + node.collection.location.end_char, + node.location.end_char, + "[]" + ), + expression: source_range_node(node) + ) + ) else s( :send, - [visit(node.collection), :[]=, *visit_all(node.index.parts)] + [visit(node.collection), :[]=].concat( + visit_all(node.index.parts) + ), + source_map_send( + selector: + source_range( + source_range_find( + node.collection.location.end_char, + node.index.location.start_char, + "[" + ).begin_pos, + node.location.end_char + ), + expression: source_range_node(node) + ) ) end end @@ -70,7 +178,14 @@ def visit_aref_field(node) # Visit an ArgBlock node. def visit_arg_block(node) - s(:block_pass, [visit(node.value)]) + s( + :block_pass, + [visit(node.value)], + source_map_operator( + operator: source_range_length(node.location.start_char, 1), + expression: source_range_node(node) + ) + ) end # Visit an ArgStar node. @@ -78,29 +193,44 @@ def visit_arg_star(node) if stack[-3].is_a?(MLHSParen) && stack[-3].contents.is_a?(MLHS) case node.value when nil - s(:restarg) + s(:restarg, [], nil) when Ident - s(:restarg, [node.value.value.to_sym]) + s(:restarg, [node.value.value.to_sym], nil) else - s(:restarg, [node.value.value.value.to_sym]) + s(:restarg, [node.value.value.value.to_sym], nil) end else - node.value.nil? ? s(:splat) : s(:splat, [visit(node.value)]) + s( + :splat, + node.value.nil? ? [] : [visit(node.value)], + source_map_operator( + operator: source_range_length(node.location.start_char, 1), + expression: source_range_node(node) + ) + ) end end # Visit an ArgsForward node. def visit_args_forward(_node) - s(:forwarded_args) + s(:forwarded_args, [], nil) end # Visit an ArrayLiteral node. def visit_array(node) - if node.contents.nil? - s(:array) - else - s(:array, visit_all(node.contents.parts)) - end + s( + :array, + node.contents ? visit_all(node.contents.parts) : [], + if node.lbracket.nil? + source_map_collection(expression: source_range_node(node)) + else + source_map_collection( + begin_token: source_range_node(node.lbracket), + end_token: source_range_length(node.location.end_char, -1), + expression: source_range_node(node) + ) + end + ) end # Visit an AryPtn node. @@ -110,82 +240,142 @@ def visit_aryptn(node) if node.rest.is_a?(VarField) if !node.rest.value.nil? - children << s(:match_rest, [visit(node.rest)]) + children << s(:match_rest, [visit(node.rest)], nil) elsif node.posts.empty? && node.rest.location.start_char == node.rest.location.end_char # Here we have an implicit rest, as in [foo,]. parser has a specific # type for these patterns. type = :array_pattern_with_tail else - children << s(:match_rest) + children << s(:match_rest, [], nil) end end - inner = s(type, children + visit_all(node.posts)) - node.constant ? s(:const_pattern, [visit(node.constant), inner]) : inner + inner = s(type, children + visit_all(node.posts), nil) + if node.constant + s(:const_pattern, [visit(node.constant), inner], nil) + else + inner + end end # Visit an Assign node. def visit_assign(node) target = visit(node.target) - s(target.type, target.children + [visit(node.value)]) + location = + target + .location + .with_operator( + source_range_find( + node.target.location.end_char, + node.value.location.start_char, + "=" + ) + ) + .with_expression(source_range_node(node)) + + s(target.type, target.children + [visit(node.value)], location) end # Visit an Assoc node. def visit_assoc(node) if node.value.nil? type = node.key.value.start_with?(/[A-Z]/) ? :const : :send + s( :pair, - [visit(node.key), s(type, [nil, node.key.value.chomp(":").to_sym])] + [ + visit(node.key), + s(type, [nil, node.key.value.chomp(":").to_sym], nil) + ], + nil ) else - s(:pair, [visit(node.key), visit(node.value)]) + s( + :pair, + [visit(node.key), visit(node.value)], + source_map_operator( + operator: source_range_length(node.key.location.end_char, -1), + expression: source_range_node(node) + ) + ) end end # Visit an AssocSplat node. def visit_assoc_splat(node) - s(:kwsplat, [visit(node.value)]) + s( + :kwsplat, + [visit(node.value)], + source_map_operator( + operator: source_range_length(node.location.start_char, 2), + expression: source_range_node(node) + ) + ) end # Visit a Backref node. def visit_backref(node) + location = source_map(expression: source_range_node(node)) + if node.value.match?(/^\$\d+$/) - s(:nth_ref, [node.value[1..].to_i]) + s(:nth_ref, [node.value[1..].to_i], location) else - s(:back_ref, [node.value.to_sym]) + s(:back_ref, [node.value.to_sym], location) end end # Visit a BareAssocHash node. def visit_bare_assoc_hash(node) - type = + s( if ::Parser::Builders::Default.emit_kwargs && !stack[-2].is_a?(ArrayLiteral) :kwargs else :hash - end - - s(type, visit_all(node.assocs)) + end, + visit_all(node.assocs), + source_map_collection(expression: source_range_node(node)) + ) end # Visit a BEGINBlock node. def visit_BEGIN(node) - s(:preexe, [visit(node.statements)]) + s( + :preexe, + [visit(node.statements)], + source_map_keyword( + keyword: source_range_length(node.location.start_char, 5), + begin_token: + source_range_find( + node.location.start_char + 5, + node.statements.location.start_char, + "{" + ), + end_token: source_range_length(node.location.end_char, -1), + expression: source_range_node(node) + ) + ) end # Visit a Begin node. def visit_begin(node) + location = + source_map_collection( + begin_token: source_range_length(node.location.start_char, 5), + end_token: source_range_length(node.location.end_char, -3), + expression: source_range_node(node) + ) + if node.bodystmt.empty? - s(:kwbegin) + s(:kwbegin, [], location) elsif node.bodystmt.rescue_clause.nil? && node.bodystmt.ensure_clause.nil? && node.bodystmt.else_clause.nil? - visited = visit(node.bodystmt.statements) - s(:kwbegin, visited.type == :begin ? visited.children : [visited]) + child = visit(node.bodystmt.statements) + + s(:kwbegin, child.type == :begin ? child.children : [child], location) else - s(:kwbegin, [visit(node.bodystmt)]) + s(:kwbegin, [visit(node.bodystmt)], location) end end @@ -194,45 +384,80 @@ def visit_binary(node) case node.operator when :| current = -2 - current -= 1 while stack[current].is_a?(Binary) && - stack[current].operator == :| + while stack[current].is_a?(Binary) && stack[current].operator == :| + current -= 1 + end if stack[current].is_a?(In) - s(:match_alt, [visit(node.left), visit(node.right)]) + s(:match_alt, [visit(node.left), visit(node.right)], nil) else - s(:send, [visit(node.left), node.operator, visit(node.right)]) + visit(canonical_binary(node)) end - when :"=>" - s(:match_as, [visit(node.left), visit(node.right)]) - when :"&&", :and - s(:and, [visit(node.left), visit(node.right)]) - when :"||", :or - s(:or, [visit(node.left), visit(node.right)]) + when :"=>", :"&&", :and, :"||", :or + s( + { "=>": :match_as, "&&": :and, "||": :or }.fetch( + node.operator, + node.operator + ), + [visit(node.left), visit(node.right)], + source_map_operator( + operator: + source_range_find( + node.left.location.end_char, + node.right.location.start_char, + node.operator.to_s + ), + expression: source_range_node(node) + ) + ) when :=~ if node.left.is_a?(RegexpLiteral) && node.left.parts.length == 1 && node.left.parts.first.is_a?(TStringContent) - s(:match_with_lvasgn, [visit(node.left), visit(node.right)]) + s( + :match_with_lvasgn, + [visit(node.left), visit(node.right)], + source_map_operator( + operator: + source_range_find( + node.left.location.end_char, + node.right.location.start_char, + node.operator.to_s + ), + expression: source_range_node(node) + ) + ) else - s(:send, [visit(node.left), node.operator, visit(node.right)]) + visit(canonical_binary(node)) end else - s(:send, [visit(node.left), node.operator, visit(node.right)]) + visit(canonical_binary(node)) end end # Visit a BlockArg node. def visit_blockarg(node) if node.name.nil? - s(:blockarg, [nil]) + s( + :blockarg, + [nil], + source_map_variable(expression: source_range_node(node)) + ) else - s(:blockarg, [node.name.value.to_sym]) + s( + :blockarg, + [node.name.value.to_sym], + source_map_variable( + name: source_range_node(node.name), + expression: source_range_node(node) + ) + ) end end # Visit a BlockVar node. def visit_block_var(node) shadowargs = - node.locals.map { |local| s(:shadowarg, [local.value.to_sym]) } + node.locals.map { |local| s(:shadowarg, [local.value.to_sym], nil) } # There is a special node type in the parser gem for when a single # required parameter to a block would potentially be expanded @@ -249,16 +474,16 @@ def visit_block_var(node) procarg0 = if ::Parser::Builders::Default.emit_arg_inside_procarg0 && required.is_a?(Ident) - s(:procarg0, [s(:arg, [required.value.to_sym])]) + s(:procarg0, [s(:arg, [required.value.to_sym], nil)], nil) else - s(:procarg0, visit(required).children) + s(:procarg0, visit(required).children, nil) end - return s(:args, [procarg0] + shadowargs) + return s(:args, [procarg0] + shadowargs, nil) end end - s(:args, visit(node.params).children + shadowargs) + s(:args, visit(node.params).children + shadowargs, nil) end # Visit a BodyStmt node. @@ -273,11 +498,11 @@ def visit_bodystmt(node) children << visit(node.else_clause) end - inner = s(:rescue, children) + inner = s(:rescue, children, nil) end if node.ensure_clause - inner = s(:ensure, [inner] + visit(node.ensure_clause).children) + inner = s(:ensure, [inner] + visit(node.ensure_clause).children, nil) end inner @@ -285,48 +510,21 @@ def visit_bodystmt(node) # Visit a Break node. def visit_break(node) - s(:break, visit_all(node.arguments.parts)) + s(:break, visit_all(node.arguments.parts), nil) end # Visit a CallNode node. def visit_call(node) - if node.receiver.nil? - children = [nil, node.message.value.to_sym] - - if node.arguments.is_a?(ArgParen) - case node.arguments.arguments - when nil - # skip - when ArgsForward - children << s(:forwarded_args) - else - children += visit_all(node.arguments.arguments.parts) - end - end - - s(:send, children) - elsif node.message == :call - children = [visit(node.receiver), :call] - - unless node.arguments.arguments.nil? - children += visit_all(node.arguments.arguments.parts) - end - - s(send_type(node.operator), children) - else - children = [visit(node.receiver), node.message.value.to_sym] - - case node.arguments - when Args - children += visit_all(node.arguments.parts) - when ArgParen - unless node.arguments.arguments.nil? - children += visit_all(node.arguments.arguments.parts) - end - end - - s(send_type(node.operator), children) - end + visit_command_call( + CommandCall.new( + receiver: node.receiver, + operator: node.operator, + message: node.message, + arguments: node.arguments, + block: nil, + location: node.location + ) + ) end # Visit a Case node. @@ -336,55 +534,157 @@ def visit_case(node) clauses << clauses.last.consequent end - type = node.consequent.is_a?(In) ? :case_match : :case - s(type, [visit(node.value)] + clauses.map { |clause| visit(clause) }) + else_token = + if clauses.last.is_a?(Else) + source_range_length(clauses.last.location.start_char, 4) + end + + s( + node.consequent.is_a?(In) ? :case_match : :case, + [visit(node.value)] + clauses.map { |clause| visit(clause) }, + source_map_condition( + keyword: source_range_length(node.location.start_char, 4), + else_token: else_token, + end_token: source_range_length(node.location.end_char, -3), + expression: source_range_node(node) + ) + ) end # Visit a CHAR node. def visit_CHAR(node) - s(:str, [node.value[1..]]) + s( + :str, + [node.value[1..]], + source_map_collection( + begin_token: source_range_length(node.location.start_char, 1), + expression: source_range_node(node) + ) + ) end # Visit a ClassDeclaration node. def visit_class(node) + operator = + if node.superclass + source_range_find( + node.constant.location.end_char, + node.superclass.location.start_char, + "<" + ) + end + s( :class, - [visit(node.constant), visit(node.superclass), visit(node.bodystmt)] + [visit(node.constant), visit(node.superclass), visit(node.bodystmt)], + source_map_definition( + keyword: source_range_length(node.location.start_char, 5), + operator: operator, + name: source_range_node(node.constant), + end_token: source_range_length(node.location.end_char, -3) + ).with_expression(source_range_node(node)) ) end # Visit a Command node. def visit_command(node) - call = - s( - :send, - [nil, node.message.value.to_sym, *visit_all(node.arguments.parts)] + visit_command_call( + CommandCall.new( + receiver: nil, + operator: nil, + message: node.message, + arguments: node.arguments, + block: node.block, + location: node.location ) - - if node.block - type, arguments = block_children(node.block) - s(type, [call, arguments, visit(node.block.bodystmt)]) - else - call - end + ) end # Visit a CommandCall node. def visit_command_call(node) - children = [visit(node.receiver), node.message.value.to_sym] + children = [ + visit(node.receiver), + node.message == :call ? :call : node.message.value.to_sym + ] + begin_token = nil + end_token = nil case node.arguments when Args children += visit_all(node.arguments.parts) when ArgParen - children += visit_all(node.arguments.arguments.parts) + case node.arguments.arguments + when nil + # skip + when ArgsForward + children << visit(node.arguments.arguments) + else + children += visit_all(node.arguments.arguments.parts) + end + + begin_token = + source_range_length(node.arguments.location.start_char, 1) + end_token = source_range_length(node.arguments.location.end_char, -1) end - call = s(send_type(node.operator), children) + dot_bound = + if node.arguments + node.arguments.location.start_char + elsif node.block + node.block.location.start_char + else + node.location.end_char + end + + call = + s( + if node.operator.is_a?(Op) && node.operator.value == "&." + :csend + else + :send + end, + children, + source_map_send( + dot: + if node.operator == :"::" + source_range_find( + node.receiver.location.end_char, + ( + if node.message == :call + dot_bound + else + node.message.location.start_char + end + ), + "::" + ) + elsif node.operator + source_range_node(node.operator) + end, + begin_token: begin_token, + end_token: end_token, + selector: + node.message == :call ? nil : source_range_node(node.message), + expression: source_range_node(node) + ) + ) if node.block type, arguments = block_children(node.block) - s(type, [call, arguments, visit(node.block.bodystmt)]) + + s( + type, + [call, arguments, visit(node.block.bodystmt)], + source_map_collection( + begin_token: source_range_node(node.block.opening), + end_token: + source_range_length( + node.location.end_char, + node.block.opening.is_a?(Kw) ? -3 : -1 + ), + expression: source_range_node(node) + ) + ) else call end @@ -392,32 +692,79 @@ def visit_command_call(node) # Visit a Const node. def visit_const(node) - s(:const, [nil, node.value.to_sym]) + s( + :const, + [nil, node.value.to_sym], + source_map_constant( + name: source_range_node(node), + expression: source_range_node(node) + ) + ) end # Visit a ConstPathField node. def visit_const_path_field(node) if node.parent.is_a?(VarRef) && node.parent.value.is_a?(Kw) && node.parent.value.value == "self" && node.constant.is_a?(Ident) - s(:send, [visit(node.parent), :"#{node.constant.value}="]) + s(:send, [visit(node.parent), :"#{node.constant.value}="], nil) else - s(:casgn, [visit(node.parent), node.constant.value.to_sym]) + s( + :casgn, + [visit(node.parent), node.constant.value.to_sym], + source_map_constant( + double_colon: + source_range_find( + node.parent.location.end_char, + node.constant.location.start_char, + "::" + ), + name: source_range_node(node.constant), + expression: source_range_node(node) + ) + ) end end # Visit a ConstPathRef node. def visit_const_path_ref(node) - s(:const, [visit(node.parent), node.constant.value.to_sym]) + s( + :const, + [visit(node.parent), node.constant.value.to_sym], + source_map_constant( + double_colon: + source_range_find( + node.parent.location.end_char, + node.constant.location.start_char, + "::" + ), + name: source_range_node(node.constant), + expression: source_range_node(node) + ) + ) end # Visit a ConstRef node. def visit_const_ref(node) - s(:const, [nil, node.constant.value.to_sym]) + s( + :const, + [nil, node.constant.value.to_sym], + source_map_constant( + name: source_range_node(node.constant), + expression: source_range_node(node) + ) + ) end # Visit a CVar node. def visit_cvar(node) - s(:cvar, [node.value.to_sym]) + s( + :cvar, + [node.value.to_sym], + source_map_variable( + name: source_range_node(node), + expression: source_range_node(node) + ) + ) end # Visit a DefNode node. @@ -426,39 +773,110 @@ def visit_def(node) args = case node.params when Params - visit(node.params) + child = visit(node.params) + + s( + child.type, + child.children, + source_map_collection(expression: nil) + ) when Paren - visit(node.params.contents) + child = visit(node.params.contents) + + s( + child.type, + child.children, + source_map_collection( + begin_token: + source_range_length(node.params.location.start_char, 1), + end_token: + source_range_length(node.params.location.end_char, -1), + expression: source_range_node(node.params) + ) + ) else - s(:args) + s(:args, [], source_map_collection(expression: nil)) end if node.target target = node.target.is_a?(Paren) ? node.target.contents : node.target - s(:defs, [visit(target), name, args, visit(node.bodystmt)]) + + s( + :defs, + [visit(target), name, args, visit(node.bodystmt)], + source_map_method_definition( + keyword: source_range_length(node.location.start_char, 3), + operator: source_range_node(node.operator), + name: source_range_node(node.name), + end_token: source_range_length(node.location.end_char, -3), + expression: source_range_node(node) + ) + ) else - s(:def, [name, args, visit(node.bodystmt)]) + s( + :def, + [name, args, visit(node.bodystmt)], + source_map_method_definition( + keyword: source_range_length(node.location.start_char, 3), + name: source_range_node(node.name), + end_token: source_range_length(node.location.end_char, -3), + expression: source_range_node(node) + ) + ) end end # Visit a Defined node. def visit_defined(node) - s(:defined?, [visit(node.value)]) + paren_range = (node.location.start_char + 8)...node.location.end_char + begin_token, end_token = + if buffer.source[paren_range].include?("(") + [ + source_range_find(paren_range.begin, paren_range.end, "("), + source_range_length(node.location.end_char, -1) + ] + end + + s( + :defined?, + [visit(node.value)], + source_map_keyword( + keyword: source_range_length(node.location.start_char, 8), + begin_token: begin_token, + end_token: end_token, + expression: source_range_node(node) + ) + ) end # Visit a DynaSymbol node. def visit_dyna_symbol(node) + location = + if node.quote + source_map_collection( + begin_token: + source_range_length( + node.location.start_char, + node.quote.length + ), + end_token: source_range_length(node.location.end_char, -1), + expression: source_range_node(node) + ) + else + source_map_collection(expression: source_range_node(node)) + end + if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) - s(:sym, ["\"#{node.parts.first.value}\"".undump.to_sym]) + s(:sym, ["\"#{node.parts.first.value}\"".undump.to_sym], location) else - s(:dsym, visit_all(node.parts)) + s(:dsym, visit_all(node.parts), location) end end # Visit an Else node. def visit_else(node) if node.statements.empty? && stack[-2].is_a?(Case) - s(:empty_else) + s(:empty_else, [], nil) else visit(node.statements) end @@ -466,54 +884,108 @@ def visit_else(node) # Visit an Elsif node. def visit_elsif(node) + else_token = + case node.consequent + when Elsif + source_range_length(node.consequent.location.start_char, 5) + when Else + source_range_length(node.consequent.location.start_char, 4) + end + + expression = + source_range( + node.location.start_char, + node.statements.location.end_char - 1 + ) + s( :if, [ visit(node.predicate), visit(node.statements), visit(node.consequent) - ] + ], + source_map_condition( + keyword: source_range_length(node.location.start_char, 5), + else_token: else_token, + expression: expression + ) ) end # Visit an ENDBlock node. def visit_END(node) - s(:postexe, [visit(node.statements)]) + s( + :postexe, + [visit(node.statements)], + source_map_keyword( + keyword: source_range_length(node.location.start_char, 3), + begin_token: + source_range_find( + node.location.start_char + 3, + node.statements.location.start_char, + "{" + ), + end_token: source_range_length(node.location.end_char, -1), + expression: source_range_node(node) + ) + ) end # Visit an Ensure node. def visit_ensure(node) - s(:ensure, [visit(node.statements)]) + s(:ensure, [visit(node.statements)], nil) end # Visit a Field node. def visit_field(node) - case stack[-2] - when Assign, MLHS - s( - send_type(node.operator), - [visit(node.parent), :"#{node.name.value}="] - ) - else - s( - send_type(node.operator), - [visit(node.parent), node.name.value.to_sym] + message = + case stack[-2] + when Assign, MLHS + Ident.new( + value: :"#{node.name.value}=", + location: node.name.location + ) + else + node.name + end + + visit_command_call( + CommandCall.new( + receiver: node.parent, + operator: node.operator, + message: message, + arguments: nil, + block: nil, + location: node.location ) - end + ) end # Visit a FloatLiteral node. def visit_float(node) - s(:float, [node.value.to_f]) + operator = + if %w[+ -].include?(buffer.source[node.location.start_char]) + source_range_length(node.location.start_char, 1) + end + + s( + :float, + [node.value.to_f], + source_map_operator( + operator: operator, + expression: source_range_node(node) + ) + ) end # Visit a FndPtn node. def visit_fndptn(node) make_match_rest = ->(child) do if child.is_a?(VarField) && child.value.nil? - s(:match_rest, []) + s(:match_rest, [], nil) else - s(:match_rest, [visit(child)]) + s(:match_rest, [visit(child)], nil) end end @@ -524,27 +996,49 @@ def visit_fndptn(node) make_match_rest[node.left], *visit_all(node.values), make_match_rest[node.right] - ] + ], + nil ) - node.constant ? s(:const_pattern, [visit(node.constant), inner]) : inner + + if node.constant + s(:const_pattern, [visit(node.constant), inner], nil) + else + inner + end end # Visit a For node. def visit_for(node) s( :for, - [visit(node.index), visit(node.collection), visit(node.statements)] + [visit(node.index), visit(node.collection), visit(node.statements)], + nil ) end # Visit a GVar node. def visit_gvar(node) - s(:gvar, [node.value.to_sym]) + s( + :gvar, + [node.value.to_sym], + source_map_variable( + name: source_range_node(node), + expression: source_range_node(node) + ) + ) end # Visit a HashLiteral node. def visit_hash(node) - s(:hash, visit_all(node.assocs)) + s( + :hash, + visit_all(node.assocs), + source_map_collection( + begin_token: source_range_length(node.location.start_char, 1), + end_token: source_range_length(node.location.end_char, -1), + expression: source_range_node(node) + ) + ) end # Heredocs are represented _very_ differently in the parser gem from how @@ -626,7 +1120,7 @@ def visit_heredoc(node) part .value .split("\n") - .each { |line| heredoc_segments << s(:str, ["#{line}\n"]) } + .each { |line| heredoc_segments << s(:str, ["#{line}\n"], nil) } else heredoc_segments << visit(part) end @@ -635,11 +1129,11 @@ def visit_heredoc(node) heredoc_segments.trim! if node.beginning.value.match?(/`\w+`\z/) - s(:xstr, heredoc_segments.segments) + s(:xstr, heredoc_segments.segments, nil) elsif heredoc_segments.segments.length > 1 - s(:dstr, heredoc_segments.segments) + s(:dstr, heredoc_segments.segments, nil) elsif heredoc_segments.segments.empty? - s(:dstr) + s(:dstr, [], nil) else heredoc_segments.segments.first end @@ -649,34 +1143,45 @@ def visit_heredoc(node) def visit_hshptn(node) children = node.keywords.map do |(keyword, value)| - next s(:pair, [visit(keyword), visit(value)]) if value + next s(:pair, [visit(keyword), visit(value)], nil) if value case keyword when Label - s(:match_var, [keyword.value.chomp(":").to_sym]) + s(:match_var, [keyword.value.chomp(":").to_sym], nil) when StringContent raise if keyword.parts.length > 1 - s(:match_var, [keyword.parts.first.value.to_sym]) + s(:match_var, [keyword.parts.first.value.to_sym], nil) end end if node.keyword_rest.is_a?(VarField) children << if node.keyword_rest.value.nil? - s(:match_rest) + s(:match_rest, [], nil) elsif node.keyword_rest.value == :nil - s(:match_nil_pattern) + s(:match_nil_pattern, [], nil) else - s(:match_rest, [visit(node.keyword_rest)]) + s(:match_rest, [visit(node.keyword_rest)], nil) end end - inner = s(:hash_pattern, children) - node.constant ? s(:const_pattern, [visit(node.constant), inner]) : inner + inner = s(:hash_pattern, children, nil) + if node.constant + s(:const_pattern, [visit(node.constant), inner], nil) + else + inner + end end # Visit an Ident node. def visit_ident(node) - s(:lvar, [node.value.to_sym]) + s( + :lvar, + [node.value.to_sym], + source_map_variable( + name: source_range_node(node), + expression: source_range_node(node) + ) + ) end # Visit an IfNode node. @@ -686,15 +1191,16 @@ def visit_if(node) when RangeNode type = node.predicate.operator.value == ".." ? :iflipflop : :eflipflop - s(type, visit(node.predicate).children) + s(type, visit(node.predicate).children, nil) when RegexpLiteral - s(:match_current_line, [visit(node.predicate)]) + s(:match_current_line, [visit(node.predicate)], nil) when Unary if node.predicate.operator.value == "!" && node.predicate.statement.is_a?(RegexpLiteral) s( :send, - [s(:match_current_line, [visit(node.predicate.statement)]), :!] + [s(:match_current_line, [visit(node.predicate.statement)]), :!], + nil ) else visit(node.predicate) @@ -703,20 +1209,59 @@ def visit_if(node) visit(node.predicate) end - s(:if, [predicate, visit(node.statements), visit(node.consequent)]) + s( + :if, + [predicate, visit(node.statements), visit(node.consequent)], + if node.modifier? + source_map_keyword( + keyword: + source_range_find( + node.statements.location.end_char, + node.predicate.location.start_char, + "if" + ), + expression: source_range_node(node) + ) + else + else_token = + case node.consequent + when Elsif + source_range_length(node.consequent.location.start_char, 5) + when Else + source_range_length(node.consequent.location.start_char, 4) + end + + source_map_condition( + keyword: source_range_length(node.location.start_char, 2), + else_token: else_token, + end_token: source_range_length(node.location.end_char, -3), + expression: source_range_node(node) + ) + end + ) end # Visit an IfOp node. def visit_if_op(node) - s(:if, [visit(node.predicate), visit(node.truthy), visit(node.falsy)]) + s( + :if, + [visit(node.predicate), visit(node.truthy), visit(node.falsy)], + nil + ) end # Visit an Imaginary node. def visit_imaginary(node) - # We have to do an eval here in order to get the value in case it's - # something like 42ri. to_c will not give the right value in that case. - # Maybe there's an API for this but I can't find it. - s(:complex, [eval(node.value)]) + s( + :complex, + [ + # We have to do an eval here in order to get the value in case it's + # something like 42ri. to_c will not give the right value in that + # case. Maybe there's an API for this but I can't find it. + eval(node.value) + ], + source_map_operator(expression: source_range_node(node)) + ) end # Visit an In node. @@ -727,62 +1272,111 @@ def visit_in(node) :in_pattern, [ visit(node.pattern.statements), - s(:if_guard, [visit(node.pattern.predicate)]), + s(:if_guard, [visit(node.pattern.predicate)], nil), visit(node.statements) - ] + ], + nil ) when UnlessNode s( :in_pattern, [ visit(node.pattern.statements), - s(:unless_guard, [visit(node.pattern.predicate)]), + s(:unless_guard, [visit(node.pattern.predicate)], nil), visit(node.statements) - ] + ], + nil ) else - s(:in_pattern, [visit(node.pattern), nil, visit(node.statements)]) + s( + :in_pattern, + [visit(node.pattern), nil, visit(node.statements)], + nil + ) end end # Visit an Int node. def visit_int(node) - s(:int, [node.value.to_i]) + operator = + if %w[+ -].include?(buffer.source[node.location.start_char]) + source_range_length(node.location.start_char, 1) + end + + s( + :int, + [node.value.to_i], + source_map_operator( + operator: operator, + expression: source_range_node(node) + ) + ) end # Visit an IVar node. def visit_ivar(node) - s(:ivar, [node.value.to_sym]) + s( + :ivar, + [node.value.to_sym], + source_map_variable( + name: source_range_node(node), + expression: source_range_node(node) + ) + ) end # Visit a Kw node. def visit_kw(node) + location = source_map(expression: source_range_node(node)) + case node.value when "__FILE__" - s(:str, [buffer.name]) + s(:str, [buffer.name], location) when "__LINE__" - s(:int, [node.location.start_line + buffer.first_line - 1]) + s(:int, [node.location.start_line + buffer.first_line - 1], location) when "__ENCODING__" if ::Parser::Builders::Default.emit_encoding - s(:__ENCODING__) + s(:__ENCODING__, [], location) else - s(:const, [s(:const, [nil, :Encoding]), :UTF_8]) + s(:const, [s(:const, [nil, :Encoding], nil), :UTF_8], location) end else - s(node.value.to_sym) + s(node.value.to_sym, [], location) end end # Visit a KwRestParam node. def visit_kwrest_param(node) - node.name.nil? ? s(:kwrestarg) : s(:kwrestarg, [node.name.value.to_sym]) + if node.name.nil? + s( + :kwrestarg, + [], + source_map_variable(expression: source_range_node(node)) + ) + else + s( + :kwrestarg, + [node.name.value.to_sym], + source_map_variable( + name: source_range_node(node.name), + expression: source_range_node(node) + ) + ) + end end # Visit a Label node. def visit_label(node) - s(:sym, [node.value.chomp(":").to_sym]) - end - + s( + :sym, + [node.value.chomp(":").to_sym], + source_map_collection( + expression: + source_range(node.location.start_char, node.location.end_char - 1) + ) + ) + end + # Visit a Lambda node. def visit_lambda(node) args = node.params.is_a?(LambdaVar) ? node.params : node.params.contents @@ -790,9 +1384,9 @@ def visit_lambda(node) arguments = visit(args) child = if ::Parser::Builders::Default.emit_lambda - s(:lambda) + s(:lambda, [], nil) else - s(:send, [nil, :lambda]) + s(:send, [nil, :lambda], nil) end type = :block @@ -801,20 +1395,32 @@ def visit_lambda(node) arguments = maximum end - s(type, [child, arguments, visit(node.statements)]) + s(type, [child, arguments, visit(node.statements)], nil) end # Visit a LambdaVar node. def visit_lambda_var(node) shadowargs = - node.locals.map { |local| s(:shadowarg, [local.value.to_sym]) } + node.locals.map { |local| s(:shadowarg, [local.value.to_sym], nil) } - s(:args, visit(node.params).children + shadowargs) + s(:args, visit(node.params).children + shadowargs, nil) end # Visit an MAssign node. def visit_massign(node) - s(:masgn, [visit(node.target), visit(node.value)]) + s( + :masgn, + [visit(node.target), visit(node.value)], + source_map_operator( + operator: + source_range_find( + node.target.location.end_char, + node.value.location.start_char, + "=" + ), + expression: source_range_node(node) + ) + ) end # Visit a MethodAddBlock node. @@ -826,10 +1432,21 @@ def visit_method_add_block(node) call = visit(node.call) s( call.type, - [s(type, [*call.children, arguments, visit(node.block.bodystmt)])] + [ + s( + type, + [*call.children, arguments, visit(node.block.bodystmt)], + nil + ) + ], + nil ) else - s(type, [visit(node.call), arguments, visit(node.block.bodystmt)]) + s( + type, + [visit(node.call), arguments, visit(node.block.bodystmt)], + nil + ) end end @@ -838,8 +1455,9 @@ def visit_mlhs(node) s( :mlhs, node.parts.map do |part| - part.is_a?(Ident) ? s(:arg, [part.value.to_sym]) : visit(part) - end + part.is_a?(Ident) ? s(:arg, [part.value.to_sym], nil) : visit(part) + end, + source_map_collection(expression: source_range_node(node)) ) end @@ -850,35 +1468,104 @@ def visit_mlhs_paren(node) # Visit a ModuleDeclaration node. def visit_module(node) - s(:module, [visit(node.constant), visit(node.bodystmt)]) + s( + :module, + [visit(node.constant), visit(node.bodystmt)], + source_map_definition( + keyword: source_range_length(node.location.start_char, 6), + name: source_range_node(node.constant), + end_token: source_range_length(node.location.end_char, -3) + ).with_expression(source_range_node(node)) + ) end # Visit an MRHS node. def visit_mrhs(node) - s(:array, visit_all(node.parts)) + visit_array( + ArrayLiteral.new( + lbracket: nil, + contents: Args.new(parts: node.parts, location: node.location), + location: node.location + ) + ) end # Visit a Next node. def visit_next(node) - s(:next, visit_all(node.arguments.parts)) + s( + :next, + visit_all(node.arguments.parts), + source_map_keyword( + keyword: source_range_length(node.location.start_char, 4), + expression: source_range_node(node) + ) + ) end # Visit a Not node. def visit_not(node) if node.statement.nil? - s(:send, [s(:begin), :!]) + begin_token = source_range_find(node.location.start_char, nil, "(") + end_token = source_range_find(node.location.start_char, nil, ")") + + s( + :send, + [ + s( + :begin, + [], + source_map_collection( + begin_token: begin_token, + end_token: end_token, + expression: begin_token.join(end_token) + ) + ), + :! + ], + source_map_send( + selector: source_range_length(node.location.start_char, 3), + expression: source_range_node(node) + ) + ) else - s(:send, [visit(node.statement), :!]) + begin_token, end_token = + if node.parentheses? + [ + source_range_find( + node.location.start_char + 3, + node.statement.location.start_char, + "(" + ), + source_range_length(node.location.end_char, -1) + ] + end + + s( + :send, + [visit(node.statement), :!], + source_map_send( + begin_token: begin_token, + end_token: end_token, + selector: source_range_length(node.location.start_char, 3), + expression: source_range_node(node) + ) + ) end end # Visit an OpAssign node. def visit_opassign(node) + location = + source_map_variable( + name: source_range_node(node.target), + expression: source_range_node(node) + ).with_operator(source_range_node(node.operator)) + case node.operator.value when "||=" - s(:or_asgn, [visit(node.target), visit(node.value)]) + s(:or_asgn, [visit(node.target), visit(node.value)], location) when "&&=" - s(:and_asgn, [visit(node.target), visit(node.value)]) + s(:and_asgn, [visit(node.target), visit(node.value)], location) else s( :op_asgn, @@ -886,7 +1573,8 @@ def visit_opassign(node) visit(node.target), node.operator.value.chomp("=").to_sym, visit(node.value) - ] + ], + location ) end end @@ -901,29 +1589,91 @@ def visit_params(node) when MLHSParen visit(required) else - s(:arg, [required.value.to_sym]) + s( + :arg, + [required.value.to_sym], + source_map_variable( + name: source_range_node(required), + expression: source_range_node(required) + ) + ) end end children += node.optionals.map do |(name, value)| - s(:optarg, [name.value.to_sym, visit(value)]) + s( + :optarg, + [name.value.to_sym, visit(value)], + source_map_variable( + name: source_range_node(name), + expression: + source_range_node(name).join(source_range_node(value)) + ).with_operator( + source_range_find( + name.location.end_char, + value.location.start_char, + "=" + ) + ) + ) end + if node.rest && !node.rest.is_a?(ExcessedComma) children << visit(node.rest) end - children += node.posts.map { |post| s(:arg, [post.value.to_sym]) } + + children += + node.posts.map do |post| + s( + :arg, + [post.value.to_sym], + source_map_variable( + name: source_range_node(post), + expression: source_range_node(post) + ) + ) + end + children += node.keywords.map do |(name, value)| key = name.value.chomp(":").to_sym - value ? s(:kwoptarg, [key, visit(value)]) : s(:kwarg, [key]) + + if value + s( + :kwoptarg, + [key, visit(value)], + source_map_variable( + name: + source_range( + name.location.start_char, + name.location.end_char - 1 + ), + expression: + source_range_node(name).join(source_range_node(value)) + ) + ) + else + s( + :kwarg, + [key], + source_map_variable( + name: + source_range( + name.location.start_char, + name.location.end_char - 1 + ), + expression: source_range_node(name) + ) + ) + end end case node.keyword_rest when nil, ArgsForward # do nothing when :nil - children << s(:kwnilarg) + children << s(:kwnilarg, [], nil) else children << visit(node.keyword_rest) end @@ -932,17 +1682,17 @@ def visit_params(node) if node.keyword_rest.is_a?(ArgsForward) if children.empty? && !::Parser::Builders::Default.emit_forward_arg - return s(:forward_args) + return s(:forward_args, [], nil) end children.insert( node.requireds.length + node.optionals.length + node.keywords.length, - s(:forward_arg) + s(:forward_arg, [], nil) ) end - s(:args, children) + s(:args, children, nil) end # Visit a Paren node. @@ -953,24 +1703,36 @@ def visit_paren(node) node.contents.body.length == 1 && node.contents.body.first.is_a?(VoidStmt) ) - s(:begin) + s(:begin, [], nil) elsif stack[-2].is_a?(DefNode) && stack[-2].target.nil? && stack[-2].target == node visit(node.contents) else - visited = visit(node.contents) - visited.type == :begin ? visited : s(:begin, [visited]) + child = visit(node.contents) + if child.type == :begin + child + else + s( + :begin, + [child], + source_map_collection( + begin_token: source_range_length(node.location.start_char, 1), + end_token: source_range_length(node.location.end_char, -1), + expression: source_range_node(node) + ) + ) + end end end # Visit a PinnedBegin node. def visit_pinned_begin(node) - s(:pin, [s(:begin, [visit(node.statement)])]) + s(:pin, [s(:begin, [visit(node.statement)], nil)], nil) end # Visit a PinnedVarRef node. def visit_pinned_var_ref(node) - s(:pin, [visit(node.value)]) + s(:pin, [visit(node.value)], nil) end # Visit a Program node. @@ -980,45 +1742,106 @@ def visit_program(node) # Visit a QSymbols node. def visit_qsymbols(node) - s( - :array, - node.elements.map { |element| s(:sym, [element.value.to_sym]) } + parts = + node.elements.map do |element| + SymbolLiteral.new(value: element, location: element.location) + end + + visit_array( + ArrayLiteral.new( + lbracket: node.beginning, + contents: Args.new(parts: parts, location: node.location), + location: node.location + ) ) end # Visit a QWords node. def visit_qwords(node) - s(:array, visit_all(node.elements)) + visit_array( + ArrayLiteral.new( + lbracket: node.beginning, + contents: Args.new(parts: node.elements, location: node.location), + location: node.location + ) + ) end # Visit a RangeNode node. def visit_range(node) - type = node.operator.value == ".." ? :irange : :erange - s(type, [visit(node.left), visit(node.right)]) + s( + node.operator.value == ".." ? :irange : :erange, + [visit(node.left), visit(node.right)], + source_map_operator( + operator: source_range_node(node.operator), + expression: source_range_node(node) + ) + ) end # Visit an RAssign node. def visit_rassign(node) - type = node.operator.value == "=>" ? :match_pattern : :match_pattern_p - s(type, [visit(node.value), visit(node.pattern)]) + s( + node.operator.value == "=>" ? :match_pattern : :match_pattern_p, + [visit(node.value), visit(node.pattern)], + source_map_operator( + operator: source_range_node(node.operator), + expression: source_range_node(node) + ) + ) end # Visit a Rational node. def visit_rational(node) - s(:rational, [node.value.to_r]) + s( + :rational, + [node.value.to_r], + source_map_operator(expression: source_range_node(node)) + ) end # Visit a Redo node. - def visit_redo(_node) - s(:redo) + def visit_redo(node) + s( + :redo, + [], + source_map_keyword( + keyword: source_range_node(node), + expression: source_range_node(node) + ) + ) end # Visit a RegexpLiteral node. def visit_regexp_literal(node) s( :regexp, - visit_all(node.parts) + - [s(:regopt, node.ending.scan(/[a-z]/).sort.map(&:to_sym))] + visit_all(node.parts).push( + s( + :regopt, + node.ending.scan(/[a-z]/).sort.map(&:to_sym), + source_map( + expression: + source_range_length( + node.location.end_char, + -(node.ending.length - 1) + ) + ) + ) + ), + source_map_collection( + begin_token: + source_range_length( + node.location.start_char, + node.beginning.length + ), + end_token: + source_range_length( + node.location.end_char - node.ending.length, + 1 + ), + expression: source_range_node(node) + ) ) end @@ -1029,18 +1852,18 @@ def visit_rescue(node) when nil nil when VarRef - s(:array, [visit(node.exception.exceptions)]) + s(:array, [visit(node.exception.exceptions)], nil) when MRHS - s(:array, visit_all(node.exception.exceptions.parts)) + s(:array, visit_all(node.exception.exceptions.parts), nil) else - s(:array, [visit(node.exception.exceptions)]) + s(:array, [visit(node.exception.exceptions)], nil) end resbody = if node.exception.nil? - s(:resbody, [nil, nil, visit(node.statements)]) + s(:resbody, [nil, nil, visit(node.statements)], nil) elsif node.exception.variable.nil? - s(:resbody, [exceptions, nil, visit(node.statements)]) + s(:resbody, [exceptions, nil, visit(node.statements)], nil) else s( :resbody, @@ -1048,7 +1871,8 @@ def visit_rescue(node) exceptions, visit(node.exception.variable), visit(node.statements) - ] + ], + nil ) end @@ -1059,39 +1883,96 @@ def visit_rescue(node) children << nil end - s(:rescue, children) + s(:rescue, children, nil) end # Visit a RescueMod node. def visit_rescue_mod(node) + keyword = + source_range_find( + node.statement.location.end_char, + node.value.location.start_char, + "rescue" + ) + s( :rescue, [ visit(node.statement), - s(:resbody, [nil, nil, visit(node.value)]), + s( + :resbody, + [nil, nil, visit(node.value)], + source_map_rescue_body( + keyword: keyword, + expression: keyword.join(source_range_node(node.value)) + ) + ), nil - ] + ], + source_map_condition(expression: source_range_node(node)) ) end # Visit a RestParam node. def visit_rest_param(node) - s(:restarg, node.name ? [node.name.value.to_sym] : []) + if node.name + s( + :restarg, + [node.name.value.to_sym], + source_map_variable( + name: source_range_node(node.name), + expression: source_range_node(node) + ) + ) + else + s( + :restarg, + [], + source_map_variable(expression: source_range_node(node)) + ) + end end # Visit a Retry node. - def visit_retry(_node) - s(:retry) + def visit_retry(node) + s( + :retry, + [], + source_map_keyword( + keyword: source_range_node(node), + expression: source_range_node(node) + ) + ) end # Visit a ReturnNode node. def visit_return(node) - s(:return, node.arguments ? visit_all(node.arguments.parts) : []) + s( + :return, + node.arguments ? visit_all(node.arguments.parts) : [], + source_map_keyword( + keyword: source_range_length(node.location.start_char, 6), + expression: source_range_node(node) + ) + ) end # Visit an SClass node. def visit_sclass(node) - s(:sclass, [visit(node.target), visit(node.bodystmt)]) + s( + :sclass, + [visit(node.target), visit(node.bodystmt)], + source_map_definition( + keyword: source_range_length(node.location.start_char, 5), + operator: + source_range_find( + node.location.start_char + 5, + node.target.location.start_char, + "<<" + ), + end_token: source_range_length(node.location.end_char, -3) + ).with_expression(source_range_node(node)) + ) end # Visit a Statements node. @@ -1108,19 +1989,35 @@ def visit_statements(node) when 1 visit(children.first) else - s(:begin, visit_all(children)) + s( + :begin, + visit_all(children), + source_map_collection( + expression: + source_range( + children.first.location.start_char, + children.last.location.end_char + ) + ) + ) end end # Visit a StringConcat node. def visit_string_concat(node) - s(:dstr, [visit(node.left), visit(node.right)]) + visit_string_literal( + StringLiteral.new( + parts: [node.left, node.right], + quote: nil, + location: node.location + ) + ) end # Visit a StringContent node. def visit_string_content(node) # Can get here if you're inside a hash pattern, e.g., in "a": 1 - s(:sym, [node.parts.first.value.to_sym]) + s(:sym, [node.parts.first.value.to_sym], nil) end # Visit a StringDVar node. @@ -1130,71 +2027,187 @@ def visit_string_dvar(node) # Visit a StringEmbExpr node. def visit_string_embexpr(node) - child = visit(node.statements) - s(:begin, child ? [child] : []) + s( + :begin, + visit(node.statements).then { |child| child ? [child] : [] }, + source_map_collection( + begin_token: source_range_length(node.location.start_char, 2), + end_token: source_range_length(node.location.end_char, -1), + expression: source_range_node(node) + ) + ) end # Visit a StringLiteral node. def visit_string_literal(node) + location = + if node.quote + source_map_collection( + begin_token: source_range_length(node.location.start_char, 1), + end_token: source_range_length(node.location.end_char, -1), + expression: source_range_node(node) + ) + else + source_map_collection(expression: source_range_node(node)) + end + if node.parts.empty? - s(:str, [""]) + s(:str, [""], location) elsif node.parts.length == 1 && node.parts.first.is_a?(TStringContent) - visit(node.parts.first) + child = visit(node.parts.first) + s(child.type, child.children, location) else - s(:dstr, visit_all(node.parts)) + s(:dstr, visit_all(node.parts), location) end end # Visit a Super node. def visit_super(node) if node.arguments.is_a?(Args) - s(:super, visit_all(node.arguments.parts)) + s( + :super, + visit_all(node.arguments.parts), + source_map_keyword( + keyword: source_range_node(node), + expression: source_range_node(node) + ) + ) else case node.arguments.arguments when nil - s(:super) + s( + :super, + [], + source_map_keyword( + keyword: source_range_length(node.location.start_char, 5), + begin_token: + source_range_find( + node.location.start_char + 5, + node.location.end_char, + "(" + ), + end_token: source_range_length(node.location.end_char, -1), + expression: source_range_node(node) + ) + ) when ArgsForward - s(:super, [visit(node.arguments.arguments)]) + s(:super, [visit(node.arguments.arguments)], nil) else - s(:super, visit_all(node.arguments.arguments.parts)) + s( + :super, + visit_all(node.arguments.arguments.parts), + source_map_keyword( + keyword: source_range_length(node.location.start_char, 5), + begin_token: + source_range_find( + node.location.start_char + 5, + node.location.end_char, + "(" + ), + end_token: source_range_length(node.location.end_char, -1), + expression: source_range_node(node) + ) + ) end end end # Visit a SymbolLiteral node. def visit_symbol_literal(node) - s(:sym, [node.value.value.to_sym]) + begin_token = + if buffer.source[node.location.start_char] == ":" + source_range_length(node.location.start_char, 1) + end + + s( + :sym, + [node.value.value.to_sym], + source_map_collection( + begin_token: begin_token, + expression: source_range_node(node) + ) + ) end # Visit a Symbols node. def visit_symbols(node) - children = + parts = node.elements.map do |element| - if element.parts.length > 1 || - !element.parts.first.is_a?(TStringContent) - s(:dsym, visit_all(element.parts)) + part = element.parts.first + + if element.parts.length == 1 && part.is_a?(TStringContent) + SymbolLiteral.new(value: part, location: part.location) else - s(:sym, [element.parts.first.value.to_sym]) + DynaSymbol.new( + parts: element.parts, + quote: nil, + location: element.location + ) end end - s(:array, children) + visit_array( + ArrayLiteral.new( + lbracket: node.beginning, + contents: Args.new(parts: parts, location: node.location), + location: node.location + ) + ) end # Visit a TopConstField node. def visit_top_const_field(node) - s(:casgn, [s(:cbase), node.constant.value.to_sym]) + s( + :casgn, + [ + s( + :cbase, + [], + source_map( + expression: source_range_length(node.location.start_char, 2) + ) + ), + node.constant.value.to_sym + ], + source_map_constant( + double_colon: source_range_length(node.location.start_char, 2), + name: source_range_node(node.constant), + expression: source_range_node(node) + ) + ) end # Visit a TopConstRef node. def visit_top_const_ref(node) - s(:const, [s(:cbase), node.constant.value.to_sym]) + s( + :const, + [ + s( + :cbase, + [], + source_map( + expression: source_range_length(node.location.start_char, 2) + ) + ), + node.constant.value.to_sym + ], + source_map_constant( + double_colon: source_range_length(node.location.start_char, 2), + name: source_range_node(node.constant), + expression: source_range_node(node) + ) + ) end # Visit a TStringContent node. def visit_tstring_content(node) - value = node.value.gsub(/([^[:ascii:]])/) { $1.dump[1...-1] } - s(:str, ["\"#{value}\"".undump]) + dumped = node.value.gsub(/([^[:ascii:]])/) { $1.dump[1...-1] } + + s( + :str, + ["\"#{dumped}\"".undump], + source_map_collection(expression: source_range_node(node)) + ) end # Visit a Unary node. @@ -1206,36 +2219,28 @@ def visit_unary(node) (range = node.statement.contents.body.first).is_a?(RangeNode) && node.operator == "!" type = range.operator.value == ".." ? :iflipflop : :eflipflop - return s(:send, [s(:begin, [s(type, visit(range).children)]), :!]) + return( + s( + :send, + [s(:begin, [s(type, visit(range).children, nil)], nil), :!], + nil + ) + ) end - case node.operator - when "+" - case node.statement - when Int - s(:int, [node.statement.value.to_i]) - when FloatLiteral - s(:float, [node.statement.value.to_f]) - else - s(:send, [visit(node.statement), :+@]) - end - when "-" - case node.statement - when Int - s(:int, [-node.statement.value.to_i]) - when FloatLiteral - s(:float, [-node.statement.value.to_f]) - else - s(:send, [visit(node.statement), :-@]) - end - else - s(:send, [visit(node.statement), node.operator.to_sym]) - end + visit(canonical_unary(node)) end # Visit an Undef node. def visit_undef(node) - s(:undef, visit_all(node.symbols)) + s( + :undef, + visit_all(node.symbols), + source_map_keyword( + keyword: source_range_length(node.location.start_char, 5), + expression: source_range_node(node) + ) + ) end # Visit an UnlessNode node. @@ -1243,13 +2248,14 @@ def visit_unless(node) predicate = case node.predicate when RegexpLiteral - s(:match_current_line, [visit(node.predicate)]) + s(:match_current_line, [visit(node.predicate)], nil) when Unary if node.predicate.operator.value == "!" && node.predicate.statement.is_a?(RegexpLiteral) s( :send, - [s(:match_current_line, [visit(node.predicate.statement)]), :!] + [s(:match_current_line, [visit(node.predicate.statement)]), :!], + nil ) else visit(node.predicate) @@ -1258,21 +2264,52 @@ def visit_unless(node) visit(node.predicate) end - s(:if, [predicate, visit(node.consequent), visit(node.statements)]) + s( + :if, + [predicate, visit(node.consequent), visit(node.statements)], + if node.modifier? + source_map_keyword( + keyword: + source_range_find( + node.statements.location.end_char, + node.predicate.location.start_char, + "unless" + ), + expression: source_range_node(node) + ) + else + source_map_condition( + keyword: source_range_length(node.location.start_char, 6), + end_token: source_range_length(node.location.end_char, -3), + expression: source_range_node(node) + ) + end + ) end # Visit an UntilNode node. def visit_until(node) - type = - if node.modifier? && node.statements.is_a?(Statements) && - node.statements.body.length == 1 && - node.statements.body.first.is_a?(Begin) - :until_post + s( + loop_post?(node) ? :until_post : :until, + [visit(node.predicate), visit(node.statements)], + if node.modifier? + source_map_keyword( + keyword: + source_range_find( + node.statements.location.end_char, + node.predicate.location.start_char, + "until" + ), + expression: source_range_node(node) + ) else - :until + source_map_keyword( + keyword: source_range_length(node.location.start_char, 5), + end_token: source_range_length(node.location.end_char, -3), + expression: source_range_node(node) + ) end - - s(type, [visit(node.predicate), visit(node.statements)]) + ) end # Visit a VarField node. @@ -1289,24 +2326,47 @@ def visit_var_field(node) end if [stack[-3], stack[-2]].any?(&is_match_var) - return s(:match_var, [node.value.value.to_sym]) + return( + s( + :match_var, + [node.value.value.to_sym], + source_map_variable( + name: source_range_node(node), + expression: source_range_node(node) + ) + ) + ) end case node.value when Const - s(:casgn, [nil, node.value.value.to_sym]) - when CVar - s(:cvasgn, [node.value.value.to_sym]) - when GVar - s(:gvasgn, [node.value.value.to_sym]) - when Ident - s(:lvasgn, [node.value.value.to_sym]) - when IVar - s(:ivasgn, [node.value.value.to_sym]) - when VarRef - s(:lvasgn, [node.value.value.to_sym]) + s( + :casgn, + [nil, node.value.value.to_sym], + source_map_constant( + name: source_range_node(node.value), + expression: source_range_node(node) + ) + ) + when CVar, GVar, Ident, IVar, VarRef + s( + { + CVar => :cvasgn, + GVar => :gvasgn, + Ident => :lvasgn, + IVar => :ivasgn, + VarRef => :lvasgn + }[ + node.value.class + ], + [node.value.value.to_sym], + source_map_variable( + name: source_range_node(node), + expression: source_range_node(node) + ) + ) else - s(:match_rest) + s(:match_rest, [], nil) end end @@ -1317,75 +2377,147 @@ def visit_var_ref(node) # Visit a VCall node. def visit_vcall(node) - range = - ::Parser::Source::Range.new( - buffer, - node.location.start_char, - node.location.end_char + visit_command_call( + CommandCall.new( + receiver: nil, + operator: nil, + message: node.value, + arguments: nil, + block: nil, + location: node.location ) - location = ::Parser::Source::Map::Send.new(nil, range, nil, nil, range) - - s(:send, [nil, node.value.value.to_sym], location: location) + ) end # Visit a When node. def visit_when(node) - s(:when, visit_all(node.arguments.parts) + [visit(node.statements)]) + keyword = source_range_length(node.location.start_char, 4) + + s( + :when, + visit_all(node.arguments.parts) + [visit(node.statements)], + source_map_keyword( + keyword: keyword, + expression: + source_range( + keyword.begin_pos, + node.statements.location.end_char - 1 + ) + ) + ) end # Visit a WhileNode node. def visit_while(node) - type = - if node.modifier? && node.statements.is_a?(Statements) && - node.statements.body.length == 1 && - node.statements.body.first.is_a?(Begin) - :while_post + s( + loop_post?(node) ? :while_post : :while, + [visit(node.predicate), visit(node.statements)], + if node.modifier? + source_map_keyword( + keyword: + source_range_find( + node.statements.location.end_char, + node.predicate.location.start_char, + "while" + ), + expression: source_range_node(node) + ) else - :while + source_map_keyword( + keyword: source_range_length(node.location.start_char, 5), + end_token: source_range_length(node.location.end_char, -3), + expression: source_range_node(node) + ) end - - s(type, [visit(node.predicate), visit(node.statements)]) + ) end # Visit a Word node. def visit_word(node) - if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) - visit(node.parts.first) - else - s(:dstr, visit_all(node.parts)) - end + visit_string_literal( + StringLiteral.new( + parts: node.parts, + quote: nil, + location: node.location + ) + ) end # Visit a Words node. def visit_words(node) - s(:array, visit_all(node.elements)) + visit_array( + ArrayLiteral.new( + lbracket: node.beginning, + contents: Args.new(parts: node.elements, location: node.location), + location: node.location + ) + ) end # Visit an XStringLiteral node. def visit_xstring_literal(node) - s(:xstr, visit_all(node.parts)) + s( + :xstr, + visit_all(node.parts), + source_map_collection( + begin_token: source_range_length(node.location.start_char, 1), + end_token: source_range_length(node.location.end_char, -1), + expression: source_range_node(node) + ) + ) end def visit_yield(node) case node.arguments when nil - s(:yield) + s( + :yield, + [], + source_map_keyword( + keyword: source_range_length(node.location.start_char, 5), + expression: source_range_node(node) + ) + ) when Args - s(:yield, visit_all(node.arguments.parts)) + s( + :yield, + visit_all(node.arguments.parts), + source_map_keyword( + keyword: source_range_length(node.location.start_char, 5), + expression: source_range_node(node) + ) + ) else - s(:yield, visit_all(node.arguments.contents.parts)) + s( + :yield, + visit_all(node.arguments.contents.parts), + source_map_keyword( + keyword: source_range_length(node.location.start_char, 5), + begin_token: + source_range_length(node.arguments.location.start_char, 1), + end_token: source_range_length(node.location.end_char, -1), + expression: source_range_node(node) + ) + ) end end # Visit a ZSuper node. - def visit_zsuper(_node) - s(:zsuper) + def visit_zsuper(node) + s( + :zsuper, + [], + source_map_keyword( + keyword: source_range_length(node.location.start_char, 5), + expression: source_range_node(node) + ) + ) end private def block_children(node) - arguments = (node.block_var ? visit(node.block_var) : s(:args)) + arguments = (node.block_var ? visit(node.block_var) : s(:args, [], nil)) type = :block if !node.block_var && (maximum = num_block_type(node.bodystmt)) @@ -1396,6 +2528,89 @@ def block_children(node) [type, arguments] end + # Convert a Unary node into a canonical CommandCall node. + def canonical_unary(node) + # For integers and floats with a leading + or -, parser represents them + # as just their values with the signs attached. + if %w[+ -].include?(node.operator) && + (node.statement.is_a?(Int) || node.statement.is_a?(FloatLiteral)) + return( + node.statement.class.new( + value: "#{node.operator}#{node.statement.value}", + location: node.location + ) + ) + end + + value = { "+" => "+@", "-" => "-@" }.fetch(node.operator, node.operator) + length = node.operator.length + + CommandCall.new( + receiver: node.statement, + operator: nil, + message: + Op.new( + value: value, + location: + Location.new( + start_line: node.location.start_line, + start_char: node.location.start_char, + start_column: node.location.start_column, + end_line: node.location.start_line, + end_char: node.location.start_char + length, + end_column: node.location.start_column + length + ) + ), + arguments: nil, + block: nil, + location: node.location + ) + end + + # Convert a Binary node into a canonical CommandCall node. + def canonical_binary(node) + operator = node.operator.to_s + + start_char = node.left.location.end_char + end_char = node.right.location.start_char + + index = buffer.source[start_char...end_char].index(operator) + start_line = + node.location.start_line + + buffer.source[start_char...index].count("\n") + start_column = + index - (buffer.source[start_char...index].rindex("\n") || 0) + + op_location = + Location.new( + start_line: start_line, + start_column: start_column, + start_char: start_char + index, + end_line: start_line, + end_column: start_column + operator.length, + end_char: start_char + index + operator.length + ) + + CommandCall.new( + receiver: node.left, + operator: nil, + message: Op.new(value: operator, location: op_location), + arguments: + Args.new(parts: [node.right], location: node.right.location), + block: nil, + location: node.location + ) + end + + # When you have a begin..end while or begin..end until, it's a special + # kind of syntax that executes the block in a loop. In this case the + # parser gem has a special node type for it. + def loop_post?(node) + node.modifier? && node.statements.is_a?(Statements) && + node.statements.body.length == 1 && + node.statements.body.first.is_a?(Begin) + end + # We need to find if we should transform this block into a numblock # since there could be new numbered variables like _1. def num_block_type(statements) @@ -1414,12 +2629,177 @@ def num_block_type(statements) variables.max end - def s(type, children = [], opts = {}) - ::Parser::AST::Node.new(type, children, opts) + # This method comes almost directly from the parser gem and creates a new + # parser gem node from the given s-expression. type is expected to be a + # symbol, children is expected to be an array, and location is expected to + # be a source map. + def s(type, children, location) + ::Parser::AST::Node.new(type, children, location: location) + end + + # Constructs a plain source map just for an expression. + def source_map(expression:) + ::Parser::Source::Map.new(expression) + end + + # Constructs a new source map for a collection. + def source_map_collection(begin_token: nil, end_token: nil, expression:) + ::Parser::Source::Map::Collection.new( + begin_token, + end_token, + expression + ) + end + + # Constructs a new source map for a conditional expression. + def source_map_condition( + keyword: nil, + begin_token: nil, + else_token: nil, + end_token: nil, + expression: + ) + ::Parser::Source::Map::Condition.new( + keyword, + begin_token, + else_token, + end_token, + expression + ) + end + + # Constructs a new source map for a constant reference. + def source_map_constant(double_colon: nil, name: nil, expression:) + ::Parser::Source::Map::Constant.new(double_colon, name, expression) + end + + # Constructs a new source map for a class definition. + def source_map_definition( + keyword: nil, + operator: nil, + name: nil, + end_token: nil + ) + ::Parser::Source::Map::Definition.new( + keyword, + operator, + name, + end_token + ) + end + + # Construct a source map for an index operation. + def source_map_index(begin_token: nil, end_token: nil, expression:) + ::Parser::Source::Map::Index.new(begin_token, end_token, expression) + end + + # Constructs a new source map for the use of a keyword. + def source_map_keyword( + keyword: nil, + begin_token: nil, + end_token: nil, + expression: + ) + ::Parser::Source::Map::Keyword.new( + keyword, + begin_token, + end_token, + expression + ) + end + + # Constructs a new source map for a method definition. + def source_map_method_definition( + keyword: nil, + operator: nil, + name: nil, + end_token: nil, + assignment: nil, + expression: + ) + ::Parser::Source::Map::MethodDefinition.new( + keyword, + operator, + name, + end_token, + assignment, + expression + ) + end + + # Constructs a new source map for an operator. + def source_map_operator(operator: nil, expression:) + ::Parser::Source::Map::Operator.new(operator, expression) + end + + # Constructs a source map for the body of a rescue clause. + def source_map_rescue_body( + keyword: nil, + assoc: nil, + begin_token: nil, + expression: + ) + ::Parser::Source::Map::RescueBody.new( + keyword, + assoc, + begin_token, + expression + ) + end + + # Constructs a new source map for a method call. + def source_map_send( + dot: nil, + selector: nil, + begin_token: nil, + end_token: nil, + expression: + ) + ::Parser::Source::Map::Send.new( + dot, + selector, + begin_token, + end_token, + expression + ) + end + + # Constructs a new source map for a variable. + def source_map_variable(name: nil, expression:) + ::Parser::Source::Map::Variable.new(name, expression) + end + + # Constructs a new source range from the given start and end offsets. + def source_range(start_char, end_char) + ::Parser::Source::Range.new(buffer, start_char, end_char) + end + + # Constructs a new source range by finding the given needle in the given + # range of the source. + def source_range_find(start_char, end_char, needle) + index = buffer.source[start_char...end_char].index(needle) + unless index + slice = buffer.source[start_char...end_char].inspect + raise "Could not find #{needle.inspect} in #{slice}" + end + + offset = start_char + index + source_range(offset, offset + needle.length) + end + + # Constructs a new source range from the given start offset and length. + def source_range_length(start_char, length) + if length > 0 + source_range(start_char, start_char + length) + else + source_range(start_char + length, start_char) + end end - def send_type(operator) - operator.is_a?(Op) && operator.value == "&." ? :csend : :send + # Constructs a new source range using the given node's location. + def source_range_node(node) + location = node.location + source_range(location.start_char, location.end_char) end end end diff --git a/test/suites/parse_helper.rb b/test/suites/parse_helper.rb index 685cd6d2..04fe8123 100644 --- a/test/suites/parse_helper.rb +++ b/test/suites/parse_helper.rb @@ -132,7 +132,8 @@ def assert_parses(_ast, code, _source_maps = "", versions = ALL_VERSIONS) expected = parse(code) return if expected.nil? - actual = SyntaxTree::Translation.to_parser(SyntaxTree.parse(code), code) + buffer = expected.location.expression.source_buffer + actual = SyntaxTree::Translation.to_parser(SyntaxTree.parse(code), buffer) assert_equal(expected, actual) end @@ -147,3 +148,28 @@ def parse(code) rescue Parser::SyntaxError end end + +if ENV["PARSER_LOCATION"] + # Modify the source map == check so that it doesn't check against the node + # itself so we don't get into a recursive loop. + Parser::Source::Map.prepend( + Module.new do + def ==(other) + self.class == other.class && + (instance_variables - %i[@node]).map do |ivar| + instance_variable_get(ivar) == other.instance_variable_get(ivar) + end.reduce(:&) + end + end + ) + + # Next, ensure that we're comparing the nodes and also comparing the source + # ranges so that we're getting all of the necessary information. + Parser::AST::Node.prepend( + Module.new do + def ==(other) + super && (location == other.location) + end + end + ) +end From 5cc7e3d8bc23ad69279a60be81228aaa282db60e Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 1 Feb 2023 09:43:05 -0500 Subject: [PATCH 344/536] Even more locations --- bin/{compare => whitequark} | 34 +- lib/syntax_tree/node.rb | 8 + lib/syntax_tree/parser.rb | 88 ++-- lib/syntax_tree/translation/parser.rb | 687 ++++++++++++++++++++------ test/fixtures/break.rb | 6 + test/node_test.rb | 2 +- 6 files changed, 634 insertions(+), 191 deletions(-) rename bin/{compare => whitequark} (66%) diff --git a/bin/compare b/bin/whitequark similarity index 66% rename from bin/compare rename to bin/whitequark index bdca5a9a..121bcd53 100755 --- a/bin/compare +++ b/bin/whitequark @@ -8,7 +8,7 @@ $:.unshift(File.expand_path("../lib", __dir__)) require "syntax_tree" # First, opt in to every AST feature. -# Parser::Builders::Default.modernize +Parser::Builders::Default.modernize # Modify the source map == check so that it doesn't check against the node # itself so we don't get into a recursive loop. @@ -46,14 +46,34 @@ ptree = parser.parse(buffer) if stree == ptree puts "Syntax trees are equivalent." -else - warn "Syntax trees are different." +elsif stree.inspect == ptree.inspect + warn "Syntax tree locations are different." + + queue = [[stree, ptree]] + while (left, right = queue.shift) + if left.location != right.location + warn "Different node:" + pp left + + warn "Different location:" + + warn "Syntax Tree:" + pp left.location + + warn "whitequark/parser:" + pp right.location - warn "syntax_tree:" + exit + end + + left.children.zip(right.children).each do |left_child, right_child| + queue << [left_child, right_child] if left_child.is_a?(Parser::AST::Node) + end + end +else + warn "Syntax Tree:" pp stree - warn "parser:" + warn "whitequark/parser:" pp ptree - - binding.irb end diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index fc5517cf..b0d1b97a 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -2149,6 +2149,14 @@ def ===(other) other.is_a?(BlockVar) && params === other.params && ArrayMatch.call(locals, other.locals) end + + # When a single required parameter is declared for a block, it gets + # automatically expanded if the values being yielded into it are an array. + def arg0? + params.requireds.length == 1 && params.optionals.empty? && + params.rest.nil? && params.posts.empty? && params.keywords.empty? && + params.keyword_rest.nil? && params.block.nil? + end end # BlockArg represents declaring a block parameter on a method definition. diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index 99b703d0..75af65bf 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -670,18 +670,22 @@ def self.visit(node, tokens) # (nil | Array[untyped]) posts # ) -> AryPtn def on_aryptn(constant, requireds, rest, posts) - parts = [constant, *requireds, rest, *posts].compact + lbracket = find_token(LBracket) + lbracket ||= find_token(LParen) if constant - # If there aren't any parts (no constant, no positional arguments), then - # we're matching an empty array. In this case, we're going to look for the - # left and right brackets explicitly. Otherwise, we'll just use the bounds - # of the various parts. - location = - if parts.empty? - consume_token(LBracket).location.to(consume_token(RBracket).location) - else - parts[0].location.to(parts[-1].location) - end + rbracket = find_token(RBracket) + rbracket ||= find_token(RParen) if constant + + parts = [constant, lbracket, *requireds, rest, *posts, rbracket].compact + + # The location is going to be determined by the first part to the last + # part. This includes potential brackets. + location = parts[0].location.to(parts[-1].location) + + # Now that we have the location calculated, we can remove the brackets + # from the list of tokens. + tokens.delete(lbracket) if lbracket + tokens.delete(rbracket) if rbracket # If there is a plain *, then we're going to fix up the location of it # here because it currently doesn't have anything to use for its precise @@ -2353,23 +2357,30 @@ def on_method_add_arg(call, arguments) # :call-seq: # on_method_add_block: ( - # (Call | Command | CommandCall) call, + # (Break | Call | Command | CommandCall) call, # Block block - # ) -> MethodAddBlock + # ) -> Break | MethodAddBlock def on_method_add_block(call, block) location = call.location.to(block.location) case call + when Break + parts = call.arguments.parts + + node = parts.pop + copied = + node.copy(block: block, location: node.location.to(block.location)) + + copied.comments.concat(call.comments) + parts << copied + + call.copy(location: location) when Command, CommandCall node = call.copy(block: block, location: location) node.comments.concat(call.comments) node else - MethodAddBlock.new( - call: call, - block: block, - location: call.location.to(block.location) - ) + MethodAddBlock.new(call: call, block: block, location: location) end end @@ -2592,19 +2603,40 @@ def on_params( # have a `nil` for the value instead of a `false`. keywords&.map! { |(key, value)| [key, value || nil] } - parts = [ - *requireds, - *optionals&.flatten(1), - rest, - *posts, - *keywords&.flatten(1), - (keyword_rest if keyword_rest != :nil), - (block if block != :&) - ].compact + # Here we're going to build up a list of all of the params so that we can + # determine our location information. + parts = [] + + requireds&.each { |required| parts << required.location } + optionals&.each do |(key, value)| + parts << key.location + parts << value.location if value + end + + parts << rest.location if rest + posts&.each { |post| parts << post.location } + + keywords&.each do |(key, value)| + parts << key.location + parts << value.location if value + end + + if keyword_rest == :nil + # When we get a :nil here, it means that we have **nil syntax, which + # means this set of parameters accepts no more keyword arguments. In + # this case we need to go and find the location of these two tokens. + operator = consume_operator(:**) + parts << operator.location.to(consume_keyword(:nil).location) + elsif keyword_rest + parts << keyword_rest.location + end + + parts << block.location if block && block != :& + parts = parts.compact location = if parts.any? - parts[0].location.to(parts[-1].location) + parts[0].to(parts[-1]) else Location.fixed(line: lineno, char: char_pos, column: current_column) end diff --git a/lib/syntax_tree/translation/parser.rb b/lib/syntax_tree/translation/parser.rb index 8a61ad94..1e47b4e7 100644 --- a/lib/syntax_tree/translation/parser.rb +++ b/lib/syntax_tree/translation/parser.rb @@ -191,13 +191,21 @@ def visit_arg_block(node) # Visit an ArgStar node. def visit_arg_star(node) if stack[-3].is_a?(MLHSParen) && stack[-3].contents.is_a?(MLHS) - case node.value - when nil - s(:restarg, [], nil) - when Ident - s(:restarg, [node.value.value.to_sym], nil) + if node.value.nil? + s( + :restarg, + [], + source_map_variable(expression: source_range_node(node)) + ) else - s(:restarg, [node.value.value.value.to_sym], nil) + s( + :restarg, + [node.value.value.to_sym], + source_map_variable( + name: source_range_node(node.value), + expression: source_range_node(node) + ) + ) end else s( @@ -212,8 +220,8 @@ def visit_arg_star(node) end # Visit an ArgsForward node. - def visit_args_forward(_node) - s(:forwarded_args, [], nil) + def visit_args_forward(node) + s(:forwarded_args, [], source_map(expression: source_range_node(node))) end # Visit an ArrayLiteral node. @@ -251,11 +259,44 @@ def visit_aryptn(node) end end - inner = s(type, children + visit_all(node.posts), nil) if node.constant - s(:const_pattern, [visit(node.constant), inner], nil) + s( + :const_pattern, + [ + visit(node.constant), + s( + type, + children + visit_all(node.posts), + source_map_collection( + expression: + source_range( + node.constant.location.end_char + 1, + node.location.end_char - 1 + ) + ) + ) + ], + source_map_collection( + begin_token: + source_range_length(node.constant.location.end_char, 1), + end_token: source_range_length(node.location.end_char, -1), + expression: source_range_node(node) + ) + ) else - inner + s( + type, + children + visit_all(node.posts), + if buffer.source[node.location.start_char] == "[" + source_map_collection( + begin_token: source_range_length(node.location.start_char, 1), + end_token: source_range_length(node.location.end_char, -1), + expression: source_range_node(node) + ) + else + source_map_collection(expression: source_range_node(node)) + end + ) end end @@ -280,15 +321,23 @@ def visit_assign(node) # Visit an Assoc node. def visit_assoc(node) if node.value.nil? - type = node.key.value.start_with?(/[A-Z]/) ? :const : :send + expression = + source_range(node.location.start_char, node.location.end_char - 1) s( :pair, [ visit(node.key), - s(type, [nil, node.key.value.chomp(":").to_sym], nil) + s( + node.key.value.start_with?(/[A-Z]/) ? :const : :send, + [nil, node.key.value.chomp(":").to_sym], + source_map_send(selector: expression, expression: expression) + ) ], - nil + source_map_operator( + operator: source_range_length(node.key.location.end_char, -1), + expression: source_range_node(node) + ) ) else s( @@ -411,6 +460,11 @@ def visit_binary(node) ) ) when :=~ + # When you use a regular expression on the left hand side of a =~ + # operator and it doesn't have interpolatoin, then its named capture + # groups introduce local variables into the scope. In this case the + # parser gem has a different node (match_with_lvasgn) instead of the + # regular send. if node.left.is_a?(RegexpLiteral) && node.left.parts.length == 1 && node.left.parts.first.is_a?(TStringContent) s( @@ -457,60 +511,124 @@ def visit_blockarg(node) # Visit a BlockVar node. def visit_block_var(node) shadowargs = - node.locals.map { |local| s(:shadowarg, [local.value.to_sym], nil) } - - # There is a special node type in the parser gem for when a single - # required parameter to a block would potentially be expanded - # automatically. We handle that case here. - if ::Parser::Builders::Default.emit_procarg0 - params = node.params - - if params.requireds.length == 1 && params.optionals.empty? && - params.rest.nil? && params.posts.empty? && - params.keywords.empty? && params.keyword_rest.nil? && - params.block.nil? - required = params.requireds.first + node.locals.map do |local| + s( + :shadowarg, + [local.value.to_sym], + source_map_variable( + name: source_range_node(local), + expression: source_range_node(local) + ) + ) + end + params = node.params + children = + if ::Parser::Builders::Default.emit_procarg0 && node.arg0? + # There is a special node type in the parser gem for when a single + # required parameter to a block would potentially be expanded + # automatically. We handle that case here. + required = params.requireds.first procarg0 = if ::Parser::Builders::Default.emit_arg_inside_procarg0 && required.is_a?(Ident) - s(:procarg0, [s(:arg, [required.value.to_sym], nil)], nil) + s( + :procarg0, + [ + s( + :arg, + [required.value.to_sym], + source_map_variable( + name: source_range_node(required), + expression: source_range_node(required) + ) + ) + ], + source_map_collection(expression: source_range_node(required)) + ) else - s(:procarg0, visit(required).children, nil) + child = visit(required) + s(:procarg0, child, child.location) end - return s(:args, [procarg0] + shadowargs, nil) + [procarg0] + else + visit(params).children end - end - s(:args, visit(node.params).children + shadowargs, nil) + s( + :args, + children + shadowargs, + source_map_collection( + begin_token: source_range_length(node.location.start_char, 1), + end_token: source_range_length(node.location.end_char, -1), + expression: source_range_node(node) + ) + ) end # Visit a BodyStmt node. def visit_bodystmt(node) - inner = visit(node.statements) + result = visit(node.statements) if node.rescue_clause - children = [inner] + visit(node.rescue_clause).children + rescue_node = visit(node.rescue_clause) + + children = [result] + rescue_node.children + location = rescue_node.location if node.else_clause children.pop children << visit(node.else_clause) + + location = + source_map_condition( + else_token: + source_range_length( + node.else_clause.location.start_char - 3, + -4 + ), + expression: + source_range( + location.expression.begin_pos, + node.else_clause.location.end_char + ) + ) end - inner = s(:rescue, children, nil) + result = s(rescue_node.type, children, location) end if node.ensure_clause - inner = s(:ensure, [inner] + visit(node.ensure_clause).children, nil) + ensure_node = visit(node.ensure_clause) + + expression = + ( + if result + result.location.expression.join(ensure_node.location.expression) + else + ensure_node.location.expression + end + ) + location = ensure_node.location.with_expression(expression) + + result = + s(ensure_node.type, [result] + ensure_node.children, location) end - inner + result end # Visit a Break node. def visit_break(node) - s(:break, visit_all(node.arguments.parts), nil) + s( + :break, + visit_all(node.arguments.parts), + source_map_keyword( + keyword: source_range_length(node.location.start_char, 5), + expression: source_range_node(node) + ) + ) end # Visit a CallNode node. @@ -606,6 +724,7 @@ def visit_command_call(node) visit(node.receiver), node.message == :call ? :call : node.message.value.to_sym ] + begin_token = nil end_token = nil @@ -649,13 +768,11 @@ def visit_command_call(node) if node.operator == :"::" source_range_find( node.receiver.location.end_char, - ( - if node.message == :call - dot_bound - else - node.message.location.start_char - end - ), + if node.message == :call + dot_bound + else + node.message.location.start_char + end, "::" ) elsif node.operator @@ -665,7 +782,18 @@ def visit_command_call(node) end_token: end_token, selector: node.message == :call ? nil : source_range_node(node.message), - expression: source_range_node(node) + expression: + if node.arguments.is_a?(ArgParen) || + (node.arguments.is_a?(Args) && node.arguments.parts.any?) + source_range( + node.location.start_char, + node.arguments.location.end_char + ) + elsif node.block + source_range_node(node.message) + else + source_range_node(node) + end ) ) @@ -798,31 +926,45 @@ def visit_def(node) s(:args, [], source_map_collection(expression: nil)) end - if node.target - target = node.target.is_a?(Paren) ? node.target.contents : node.target - - s( - :defs, - [visit(target), name, args, visit(node.bodystmt)], + location = + if node.endless? source_map_method_definition( keyword: source_range_length(node.location.start_char, 3), - operator: source_range_node(node.operator), + assignment: + source_range_find( + (node.params || node.name).location.end_char, + node.bodystmt.location.start_char, + "=" + ), name: source_range_node(node.name), - end_token: source_range_length(node.location.end_char, -3), expression: source_range_node(node) ) - ) - else - s( - :def, - [name, args, visit(node.bodystmt)], + else source_map_method_definition( keyword: source_range_length(node.location.start_char, 3), name: source_range_node(node.name), end_token: source_range_length(node.location.end_char, -3), expression: source_range_node(node) ) + end + + if node.target + target = node.target.is_a?(Paren) ? node.target.contents : node.target + + s( + :defs, + [visit(target), name, args, visit(node.bodystmt)], + source_map_method_definition( + keyword: location.keyword, + assignment: location.assignment, + operator: source_range_node(node.operator), + name: location.name, + end_token: location.end, + expression: location.expression + ) ) + else + s(:def, [name, args, visit(node.bodystmt)], location) end end @@ -934,7 +1076,22 @@ def visit_END(node) # Visit an Ensure node. def visit_ensure(node) - s(:ensure, [visit(node.statements)], nil) + start_char = node.location.start_char + end_char = + if node.statements.empty? + start_char + 6 + else + node.statements.body.last.location.end_char + end + + s( + :ensure, + [visit(node.statements)], + source_map_condition( + keyword: source_range_length(start_char, 6), + expression: source_range(start_char, end_char) + ) + ) end # Visit a Field node. @@ -1009,10 +1166,29 @@ def visit_fndptn(node) # Visit a For node. def visit_for(node) + begin_start = node.collection.location.end_char + begin_end = node.statements.location.start_char + + begin_token = + if buffer.source[begin_start...begin_end].include?("do") + source_range_find(begin_start, begin_end, "do") + end + s( :for, [visit(node.index), visit(node.collection), visit(node.statements)], - nil + source_map_for( + keyword: source_range_length(node.location.start_char, 3), + in_token: + source_range_find( + node.index.location.end_char, + node.collection.location.start_char, + "in" + ), + begin_token: begin_token, + end_token: source_range_length(node.location.end_char, -3), + expression: source_range_node(node) + ) ) end @@ -1223,6 +1399,19 @@ def visit_if(node) expression: source_range_node(node) ) else + begin_start = node.predicate.location.end_char + begin_end = + if node.statements.empty? + node.statements.location.end_char + else + node.statements.body.first.location.start_char + end + + begin_token = + if buffer.source[begin_start...begin_end].include?("then") + source_range_find(begin_start, begin_end, "then") + end + else_token = case node.consequent when Elsif @@ -1233,6 +1422,7 @@ def visit_if(node) source_map_condition( keyword: source_range_length(node.location.start_char, 2), + begin_token: begin_token, else_token: else_token, end_token: source_range_length(node.location.end_char, -3), expression: source_range_node(node) @@ -1288,10 +1478,20 @@ def visit_in(node) nil ) else + end_char = + if node.statements.empty? + node.statements.location.end_char - 1 + else + node.statements.body.first.location.start_char + end + s( :in_pattern, [visit(node.pattern), nil, visit(node.statements)], - nil + source_map_keyword( + keyword: source_range_length(node.location.start_char, 2), + expression: source_range(node.location.start_char, end_char) + ) ) end end @@ -1380,30 +1580,79 @@ def visit_label(node) # Visit a Lambda node. def visit_lambda(node) args = node.params.is_a?(LambdaVar) ? node.params : node.params.contents - - arguments = visit(args) - child = - if ::Parser::Builders::Default.emit_lambda - s(:lambda, [], nil) - else - s(:send, [nil, :lambda], nil) - end + args_node = visit(args) type = :block if args.empty? && (maximum = num_block_type(node.statements)) type = :numblock - arguments = maximum + args_node = maximum end - s(type, [child, arguments, visit(node.statements)], nil) + begin_start = node.params.location.end_char + begin_token, end_token = + if buffer.source[begin_start - 1] == "{" + [ + source_range_length(begin_start, -1), + source_range_length(node.location.end_char, -1) + ] + else + [ + source_range_length(begin_start, -2), + source_range_length(node.location.end_char, -3) + ] + end + + selector = source_range_length(node.location.start_char, 2) + + s( + type, + [ + if ::Parser::Builders::Default.emit_lambda + s(:lambda, [], source_map(expression: selector)) + else + s( + :send, + [nil, :lambda], + source_map_send(selector: selector, expression: selector) + ) + end, + args_node, + visit(node.statements) + ], + source_map_collection( + begin_token: begin_token, + end_token: end_token, + expression: source_range_node(node) + ) + ) end # Visit a LambdaVar node. def visit_lambda_var(node) shadowargs = - node.locals.map { |local| s(:shadowarg, [local.value.to_sym], nil) } + node.locals.map do |local| + s( + :shadowarg, + [local.value.to_sym], + source_map_variable( + name: source_range_node(local), + expression: source_range_node(local) + ) + ) + end + + location = + if node.location.start_char == node.location.end_char + source_map_collection(expression: nil) + else + source_map_collection( + begin_token: source_range_length(node.location.start_char, 1), + end_token: source_range_length(node.location.end_char, -1), + expression: source_range_node(node) + ) + end - s(:args, visit(node.params).children + shadowargs, nil) + s(:args, visit(node.params).children + shadowargs, location) end # Visit an MAssign node. @@ -1425,11 +1674,11 @@ def visit_massign(node) # Visit a MethodAddBlock node. def visit_method_add_block(node) - type, arguments = block_children(node.block) - case node.call when Break, Next, ReturnNode + type, arguments = block_children(node.block) call = visit(node.call) + s( call.type, [ @@ -1441,12 +1690,25 @@ def visit_method_add_block(node) ], nil ) - else + when ARef, Super, ZSuper + type, arguments = block_children(node.block) + s( type, [visit(node.call), arguments, visit(node.block.bodystmt)], nil ) + else + visit_command_call( + CommandCall.new( + receiver: node.call.receiver, + operator: node.call.operator, + message: node.call.message, + arguments: node.call.arguments, + block: node.block, + location: node.location + ) + ) end end @@ -1455,7 +1717,18 @@ def visit_mlhs(node) s( :mlhs, node.parts.map do |part| - part.is_a?(Ident) ? s(:arg, [part.value.to_sym], nil) : visit(part) + if part.is_a?(Ident) + s( + :arg, + [part.value.to_sym], + source_map_variable( + name: source_range_node(part), + expression: source_range_node(part) + ) + ) + else + visit(part) + end end, source_map_collection(expression: source_range_node(node)) ) @@ -1463,7 +1736,17 @@ def visit_mlhs(node) # Visit an MLHSParen node. def visit_mlhs_paren(node) - visit(node.contents) + child = visit(node.contents) + + s( + child.type, + child.children, + source_map_collection( + begin_token: source_range_length(node.location.start_char, 1), + end_token: source_range_length(node.location.end_char, -1), + expression: source_range_node(node) + ) + ) end # Visit a ModuleDeclaration node. @@ -1673,7 +1956,14 @@ def visit_params(node) when nil, ArgsForward # do nothing when :nil - children << s(:kwnilarg, [], nil) + children << s( + :kwnilarg, + [], + source_map_variable( + name: source_range_length(node.location.end_char, -3), + expression: source_range_node(node) + ) + ) else children << visit(node.keyword_rest) end @@ -1681,15 +1971,21 @@ def visit_params(node) children << visit(node.block) if node.block if node.keyword_rest.is_a?(ArgsForward) + location = + source_map(expression: source_range_node(node.keyword_rest)) + + # If there are no other arguments and we have the emit_forward_arg + # option enabled, then the entire argument list is represented by a + # single forward_args node. if children.empty? && !::Parser::Builders::Default.emit_forward_arg - return s(:forward_args, [], nil) + return s(:forward_args, [], location) end - children.insert( - node.requireds.length + node.optionals.length + - node.keywords.length, - s(:forward_arg, [], nil) - ) + # Otherwise, we need to insert a forward_arg node into the list of + # parameters before any keyword rest or block parameters. + index = + node.requireds.length + node.optionals.length + node.keywords.length + children.insert(index, s(:forward_arg, [], location)) end s(:args, children, nil) @@ -1697,31 +1993,19 @@ def visit_params(node) # Visit a Paren node. def visit_paren(node) + location = + source_map_collection( + begin_token: source_range_length(node.location.start_char, 1), + end_token: source_range_length(node.location.end_char, -1), + expression: source_range_node(node) + ) + if node.contents.nil? || - ( - node.contents.is_a?(Statements) && - node.contents.body.length == 1 && - node.contents.body.first.is_a?(VoidStmt) - ) - s(:begin, [], nil) - elsif stack[-2].is_a?(DefNode) && stack[-2].target.nil? && - stack[-2].target == node - visit(node.contents) + (node.contents.is_a?(Statements) && node.contents.empty?) + s(:begin, [], location) else child = visit(node.contents) - if child.type == :begin - child - else - s( - :begin, - [child], - source_map_collection( - begin_token: source_range_length(node.location.start_char, 1), - end_token: source_range_length(node.location.end_char, -1), - expression: source_range_node(node) - ) - ) - end + child.type == :begin ? child : s(:begin, [child], location) end end @@ -1847,23 +2131,86 @@ def visit_regexp_literal(node) # Visit a Rescue node. def visit_rescue(node) + # In the parser gem, there is a separation between the rescue node and + # the rescue body. They have different bounds, so we have to calculate + # those here. + start_char = node.location.start_char + + body_end_char = + if node.statements.empty? + start_char + 6 + else + node.statements.body.last.location.end_char + end + + end_char = + if node.consequent + end_node = node.consequent + end_node = end_node.consequent while end_node.consequent + + if end_node.statements.empty? + start_char + 6 + else + end_node.statements.body.last.location.end_char + end + else + body_end_char + end + + # These locations are reused for multiple children. + keyword = source_range_length(start_char, 6) + body_expression = source_range(start_char, body_end_char) + expression = source_range(start_char, end_char) + exceptions = case node.exception&.exceptions when nil nil - when VarRef - s(:array, [visit(node.exception.exceptions)], nil) when MRHS - s(:array, visit_all(node.exception.exceptions.parts), nil) + visit_array( + ArrayLiteral.new( + lbracket: nil, + contents: + Args.new( + parts: node.exception.exceptions.parts, + location: node.exception.exceptions.location + ), + location: node.exception.exceptions.location + ) + ) else - s(:array, [visit(node.exception.exceptions)], nil) + visit_array( + ArrayLiteral.new( + lbracket: nil, + contents: + Args.new( + parts: [node.exception.exceptions], + location: node.exception.exceptions.location + ), + location: node.exception.exceptions.location + ) + ) end resbody = if node.exception.nil? - s(:resbody, [nil, nil, visit(node.statements)], nil) + s( + :resbody, + [nil, nil, visit(node.statements)], + source_map_rescue_body( + keyword: keyword, + expression: body_expression + ) + ) elsif node.exception.variable.nil? - s(:resbody, [exceptions, nil, visit(node.statements)], nil) + s( + :resbody, + [exceptions, nil, visit(node.statements)], + source_map_rescue_body( + keyword: keyword, + expression: body_expression + ) + ) else s( :resbody, @@ -1872,7 +2219,16 @@ def visit_rescue(node) visit(node.exception.variable), visit(node.statements) ], - nil + source_map_rescue_body( + keyword: keyword, + assoc: + source_range_find( + node.location.start_char + 6, + node.exception.variable.location.start_char, + "=>" + ), + expression: body_expression + ) ) end @@ -1883,7 +2239,7 @@ def visit_rescue(node) children << nil end - s(:rescue, children, nil) + s(:rescue, children, source_map_condition(expression: expression)) end # Visit a RescueMod node. @@ -2314,59 +2670,58 @@ def visit_until(node) # Visit a VarField node. def visit_var_field(node) - is_match_var = ->(parent) do - case parent - when AryPtn, FndPtn, HshPtn, In, RAssign - true - when Binary - parent.operator == :"=>" - else - false + name = node.value.value.to_sym + match_var = + [stack[-3], stack[-2]].any? do |parent| + case parent + when AryPtn, FndPtn, HshPtn, In, RAssign + true + when Binary + parent.operator == :"=>" + else + false + end end - end - if [stack[-3], stack[-2]].any?(&is_match_var) - return( - s( - :match_var, - [node.value.value.to_sym], - source_map_variable( - name: source_range_node(node), - expression: source_range_node(node) - ) + if match_var + s( + :match_var, + [name], + source_map_variable( + name: source_range_node(node), + expression: source_range_node(node) ) ) - end - - case node.value - when Const + elsif node.value.is_a?(Const) s( :casgn, - [nil, node.value.value.to_sym], + [nil, name], source_map_constant( name: source_range_node(node.value), expression: source_range_node(node) ) ) - when CVar, GVar, Ident, IVar, VarRef - s( - { - CVar => :cvasgn, - GVar => :gvasgn, - Ident => :lvasgn, - IVar => :ivasgn, - VarRef => :lvasgn - }[ - node.value.class - ], - [node.value.value.to_sym], + else + location = source_map_variable( name: source_range_node(node), expression: source_range_node(node) ) - ) - else - s(:match_rest, [], nil) + + case node.value + when CVar + s(:cvasgn, [name], location) + when GVar + s(:gvasgn, [name], location) + when Ident + s(:lvasgn, [name], location) + when IVar + s(:ivasgn, [name], location) + when VarRef + s(:lvasgn, [name], location) + else + s(:match_rest, [], nil) + end end end @@ -2517,7 +2872,12 @@ def visit_zsuper(node) private def block_children(node) - arguments = (node.block_var ? visit(node.block_var) : s(:args, [], nil)) + arguments = + if node.block_var + visit(node.block_var) + else + s(:args, [], source_map_collection(expression: nil)) + end type = :block if !node.block_var && (maximum = num_block_type(node.bodystmt)) @@ -2688,6 +3048,23 @@ def source_map_definition( ) end + # Constructs a new source map for a for loop. + def source_map_for( + keyword: nil, + in_token: nil, + begin_token: nil, + end_token: nil, + expression: + ) + ::Parser::Source::Map::For.new( + keyword, + in_token, + begin_token, + end_token, + expression + ) + end + # Construct a source map for an index operation. def source_map_index(begin_token: nil, end_token: nil, expression:) ::Parser::Source::Map::Index.new(begin_token, end_token, expression) diff --git a/test/fixtures/break.rb b/test/fixtures/break.rb index a77c6b35..a608a6b2 100644 --- a/test/fixtures/break.rb +++ b/test/fixtures/break.rb @@ -27,3 +27,9 @@ ) % break foo.bar :baz do |qux| qux end +- +break( + foo.bar :baz do |qux| + qux + end +) diff --git a/test/node_test.rb b/test/node_test.rb index 7254c086..9660b341 100644 --- a/test/node_test.rb +++ b/test/node_test.rb @@ -131,7 +131,7 @@ def test_aryptn end SOURCE - at = location(lines: 2..2, chars: 18..47) + at = location(lines: 2..2, chars: 18..48) assert_node(AryPtn, source, at: at) { |node| node.consequent.pattern } end From 0f11b7e1d1afe7f3c9b284d5b140fed15ecf2a72 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 2 Feb 2023 13:31:56 -0500 Subject: [PATCH 345/536] Add query methods for instructions for branching logic --- lib/syntax_tree/yarv/instructions.rb | 774 ++++++--------------------- lib/syntax_tree/yarv/legacy.rb | 36 +- test/yarv_test.rb | 34 +- 3 files changed, 193 insertions(+), 651 deletions(-) diff --git a/lib/syntax_tree/yarv/instructions.rb b/lib/syntax_tree/yarv/instructions.rb index bba06f8d..c387e763 100644 --- a/lib/syntax_tree/yarv/instructions.rb +++ b/lib/syntax_tree/yarv/instructions.rb @@ -63,6 +63,50 @@ def self.calldata( CallData.new(method, argc, flags, kw_arg) end + # This is a base class for all YARV instructions. It provides a few + # convenience methods for working with instructions. + class Instruction + # This method creates an instruction that represents the canonical + # (non-specialized) form of this instruction. If this instruction is not + # a specialized instruction, then this method returns `self`. + def canonical + self + end + + # This returns the size of the instruction in terms of the number of slots + # it occupies in the instruction sequence. Effectively this is 1 plus the + # number of operands. + def length + 1 + end + + # This returns the number of values that are pushed onto the stack. + def pushes + 0 + end + + # This returns the number of values that are popped off the stack. + def pops + 0 + end + + # Whether or not this instruction is a branch instruction. + def branches? + false + end + + # Whether or not this instruction leaves the current frame. + def leaves? + false + end + + # Whether or not this instruction falls through to the next instruction if + # its branching fails. + def falls_through? + false + end + end + # ### Summary # # `adjuststack` accepts a single integer argument and removes that many @@ -76,7 +120,7 @@ def self.calldata( # x[0] # ~~~ # - class AdjustStack + class AdjustStack < Instruction attr_reader :number def initialize(number) @@ -107,14 +151,6 @@ def pops number end - def pushes - 0 - end - - def canonical - self - end - def call(vm) vm.pop(number) end @@ -138,7 +174,7 @@ def call(vm) # "#{5}" # ~~~ # - class AnyToString + class AnyToString < Instruction def disasm(fmt) fmt.instruction("anytostring") end @@ -155,10 +191,6 @@ def ==(other) other.is_a?(AnyToString) end - def length - 1 - end - def pops 2 end @@ -167,10 +199,6 @@ def pushes 1 end - def canonical - self - end - def call(vm) original, value = vm.pop(2) @@ -198,7 +226,7 @@ def call(vm) # puts x # ~~~ # - class BranchIf + class BranchIf < Instruction attr_reader :label def initialize(label) @@ -229,16 +257,16 @@ def pops 1 end - def pushes - 0 + def call(vm) + vm.jump(label) if vm.pop end - def canonical - self + def branches? + true end - def call(vm) - vm.jump(label) if vm.pop + def falls_through? + true end end @@ -259,7 +287,7 @@ def call(vm) # end # ~~~ # - class BranchNil + class BranchNil < Instruction attr_reader :label def initialize(label) @@ -290,16 +318,16 @@ def pops 1 end - def pushes - 0 + def call(vm) + vm.jump(label) if vm.pop.nil? end - def canonical - self + def branches? + true end - def call(vm) - vm.jump(label) if vm.pop.nil? + def falls_through? + true end end @@ -319,7 +347,7 @@ def call(vm) # end # ~~~ # - class BranchUnless + class BranchUnless < Instruction attr_reader :label def initialize(label) @@ -350,16 +378,16 @@ def pops 1 end - def pushes - 0 + def call(vm) + vm.jump(label) unless vm.pop end - def canonical - self + def branches? + true end - def call(vm) - vm.jump(label) unless vm.pop + def falls_through? + true end end @@ -382,7 +410,7 @@ def call(vm) # evaluate(value: 3) # ~~~ # - class CheckKeyword + class CheckKeyword < Instruction attr_reader :keyword_bits_index, :keyword_index def initialize(keyword_bits_index, keyword_index) @@ -419,18 +447,10 @@ def length 3 end - def pops - 0 - end - def pushes 1 end - def canonical - self - end - def call(vm) vm.push(vm.local_get(keyword_bits_index, 0)[keyword_index]) end @@ -448,7 +468,7 @@ def call(vm) # foo in Foo # ~~~ # - class CheckMatch + class CheckMatch < Instruction VM_CHECKMATCH_TYPE_WHEN = 1 VM_CHECKMATCH_TYPE_CASE = 2 VM_CHECKMATCH_TYPE_RESCUE = 3 @@ -489,10 +509,6 @@ def pushes 1 end - def canonical - self - end - def call(vm) target, pattern = vm.pop(2) @@ -536,7 +552,7 @@ def check?(pattern, target) # foo in [bar] # ~~~ # - class CheckType + class CheckType < Instruction TYPE_OBJECT = 0x01 TYPE_CLASS = 0x02 TYPE_MODULE = 0x03 @@ -643,10 +659,6 @@ def pushes 2 end - def canonical - self - end - def call(vm) object = vm.pop result = @@ -713,7 +725,7 @@ def call(vm) # [1, *2] # ~~~ # - class ConcatArray + class ConcatArray < Instruction def disasm(fmt) fmt.instruction("concatarray") end @@ -730,10 +742,6 @@ def ==(other) other.is_a?(ConcatArray) end - def length - 1 - end - def pops 2 end @@ -742,10 +750,6 @@ def pushes 1 end - def canonical - self - end - def call(vm) left, right = vm.pop(2) vm.push([*left, *right]) @@ -767,7 +771,7 @@ def call(vm) # "#{5}" # ~~~ # - class ConcatStrings + class ConcatStrings < Instruction attr_reader :number def initialize(number) @@ -802,10 +806,6 @@ def pushes 1 end - def canonical - self - end - def call(vm) vm.push(vm.pop(number).join) end @@ -826,7 +826,7 @@ def call(vm) # end # ~~~ # - class DefineClass + class DefineClass < Instruction TYPE_CLASS = 0 TYPE_SINGLETON_CLASS = 1 TYPE_MODULE = 2 @@ -874,10 +874,6 @@ def pushes 1 end - def canonical - self - end - def call(vm) object, superclass = vm.pop(2) @@ -914,7 +910,7 @@ def call(vm) # defined?(x) # ~~~ # - class Defined + class Defined < Instruction TYPE_NIL = 1 TYPE_IVAR = 2 TYPE_LVAR = 3 @@ -1011,10 +1007,6 @@ def pushes 1 end - def canonical - self - end - def call(vm) object = vm.pop @@ -1069,7 +1061,7 @@ def call(vm) # def value = "value" # ~~~ # - class DefineMethod + class DefineMethod < Instruction attr_reader :method_name, :method_iseq def initialize(method_name, method_iseq) @@ -1102,18 +1094,6 @@ def length 3 end - def pops - 0 - end - - def pushes - 0 - end - - def canonical - self - end - def call(vm) name = method_name nesting = vm.frame.nesting @@ -1150,7 +1130,7 @@ def call(vm) # def self.value = "value" # ~~~ # - class DefineSMethod + class DefineSMethod < Instruction attr_reader :method_name, :method_iseq def initialize(method_name, method_iseq) @@ -1187,14 +1167,6 @@ def pops 1 end - def pushes - 0 - end - - def canonical - self - end - def call(vm) name = method_name nesting = vm.frame.nesting @@ -1227,7 +1199,7 @@ def call(vm) # $global = 5 # ~~~ # - class Dup + class Dup < Instruction def disasm(fmt) fmt.instruction("dup") end @@ -1244,10 +1216,6 @@ def ==(other) other.is_a?(Dup) end - def length - 1 - end - def pops 1 end @@ -1256,10 +1224,6 @@ def pushes 2 end - def canonical - self - end - def call(vm) vm.push(vm.stack.last.dup) end @@ -1275,7 +1239,7 @@ def call(vm) # [true] # ~~~ # - class DupArray + class DupArray < Instruction attr_reader :object def initialize(object) @@ -1302,18 +1266,10 @@ def length 2 end - def pops - 0 - end - def pushes 1 end - def canonical - self - end - def call(vm) vm.push(object.dup) end @@ -1329,7 +1285,7 @@ def call(vm) # { a: 1 } # ~~~ # - class DupHash + class DupHash < Instruction attr_reader :object def initialize(object) @@ -1356,18 +1312,10 @@ def length 2 end - def pops - 0 - end - def pushes 1 end - def canonical - self - end - def call(vm) vm.push(object.dup) end @@ -1383,7 +1331,7 @@ def call(vm) # Object::X ||= true # ~~~ # - class DupN + class DupN < Instruction attr_reader :number def initialize(number) @@ -1410,18 +1358,10 @@ def length 2 end - def pops - 0 - end - def pushes number end - def canonical - self - end - def call(vm) values = vm.pop(number) vm.push(*values) @@ -1441,7 +1381,7 @@ def call(vm) # x, = [true, false, nil] # ~~~ # - class ExpandArray + class ExpandArray < Instruction attr_reader :number, :flags def initialize(number, flags) @@ -1478,10 +1418,6 @@ def pushes number end - def canonical - self - end - def call(vm) object = vm.pop object = @@ -1539,7 +1475,7 @@ def call(vm) # end # ~~~ # - class GetBlockParam + class GetBlockParam < Instruction attr_reader :index, :level def initialize(index, level) @@ -1570,18 +1506,10 @@ def length 3 end - def pops - 0 - end - def pushes 1 end - def canonical - self - end - def call(vm) vm.push(vm.local_get(index, level)) end @@ -1602,7 +1530,7 @@ def call(vm) # end # ~~~ # - class GetBlockParamProxy + class GetBlockParamProxy < Instruction attr_reader :index, :level def initialize(index, level) @@ -1636,18 +1564,10 @@ def length 3 end - def pops - 0 - end - def pushes 1 end - def canonical - self - end - def call(vm) vm.push(vm.local_get(index, level)) end @@ -1665,7 +1585,7 @@ def call(vm) # @@class_variable # ~~~ # - class GetClassVariable + class GetClassVariable < Instruction attr_reader :name, :cache def initialize(name, cache) @@ -1697,18 +1617,10 @@ def length 3 end - def pops - 0 - end - def pushes 1 end - def canonical - self - end - def call(vm) clazz = vm.frame._self clazz = clazz.class unless clazz.is_a?(Class) @@ -1728,7 +1640,7 @@ def call(vm) # Constant # ~~~ # - class GetConstant + class GetConstant < Instruction attr_reader :name def initialize(name) @@ -1763,10 +1675,6 @@ def pushes 1 end - def canonical - self - end - def call(vm) const_base, allow_nil = vm.pop(2) @@ -1798,7 +1706,7 @@ def call(vm) # $$ # ~~~ # - class GetGlobal + class GetGlobal < Instruction attr_reader :name def initialize(name) @@ -1825,18 +1733,10 @@ def length 2 end - def pops - 0 - end - def pushes 1 end - def canonical - self - end - def call(vm) # Evaluating the name of the global variable because there isn't a # reflection API for global variables. @@ -1861,7 +1761,7 @@ def call(vm) # @instance_variable # ~~~ # - class GetInstanceVariable + class GetInstanceVariable < Instruction attr_reader :name, :cache def initialize(name, cache) @@ -1893,18 +1793,10 @@ def length 3 end - def pops - 0 - end - def pushes 1 end - def canonical - self - end - def call(vm) method = Object.instance_method(:instance_variable_get) vm.push(method.bind(vm.frame._self).call(name)) @@ -1925,7 +1817,7 @@ def call(vm) # tap { tap { value } } # ~~~ # - class GetLocal + class GetLocal < Instruction attr_reader :index, :level def initialize(index, level) @@ -1955,18 +1847,10 @@ def length 3 end - def pops - 0 - end - def pushes 1 end - def canonical - self - end - def call(vm) vm.push(vm.local_get(index, level)) end @@ -1985,7 +1869,7 @@ def call(vm) # value # ~~~ # - class GetLocalWC0 + class GetLocalWC0 < Instruction attr_reader :index def initialize(index) @@ -2012,10 +1896,6 @@ def length 2 end - def pops - 0 - end - def pushes 1 end @@ -2042,7 +1922,7 @@ def call(vm) # self.then { value } # ~~~ # - class GetLocalWC1 + class GetLocalWC1 < Instruction attr_reader :index def initialize(index) @@ -2069,10 +1949,6 @@ def length 2 end - def pops - 0 - end - def pushes 1 end @@ -2096,7 +1972,7 @@ def call(vm) # 1 if (a == 1) .. (b == 2) # ~~~ # - class GetSpecial + class GetSpecial < Instruction SVAR_LASTLINE = 0 # $_ SVAR_BACKREF = 1 # $~ SVAR_FLIPFLOP_START = 2 # flipflop @@ -2128,18 +2004,10 @@ def length 3 end - def pops - 0 - end - def pushes 1 end - def canonical - self - end - def call(vm) case key when SVAR_LASTLINE @@ -2163,7 +2031,7 @@ def call(vm) # :"#{"foo"}" # ~~~ # - class Intern + class Intern < Instruction def disasm(fmt) fmt.instruction("intern") end @@ -2180,10 +2048,6 @@ def ==(other) other.is_a?(Intern) end - def length - 1 - end - def pops 1 end @@ -2192,10 +2056,6 @@ def pushes 1 end - def canonical - self - end - def call(vm) vm.push(vm.pop.to_sym) end @@ -2215,7 +2075,7 @@ def call(vm) # end # ~~~ # - class InvokeBlock + class InvokeBlock < Instruction attr_reader :calldata def initialize(calldata) @@ -2250,10 +2110,6 @@ def pushes 1 end - def canonical - self - end - def call(vm) vm.push(vm.frame_yield.block.call(*vm.pop(calldata.argc))) end @@ -2273,7 +2129,7 @@ def call(vm) # end # ~~~ # - class InvokeSuper + class InvokeSuper < Instruction attr_reader :calldata, :block_iseq def initialize(calldata, block_iseq) @@ -2302,10 +2158,6 @@ def ==(other) other.block_iseq == block_iseq end - def length - 1 - end - def pops argb = (calldata.flag?(CallData::CALL_ARGS_BLOCKARG) ? 1 : 0) argb + calldata.argc + 1 @@ -2315,10 +2167,6 @@ def pushes 1 end - def canonical - self - end - def call(vm) block = if (iseq = block_iseq) @@ -2358,7 +2206,7 @@ def call(vm) # end # ~~~ # - class Jump + class Jump < Instruction attr_reader :label def initialize(label) @@ -2385,21 +2233,13 @@ def length 2 end - def pops - 0 - end - - def pushes - 0 - end - - def canonical - self - end - def call(vm) vm.jump(label) end + + def branches? + true + end end # ### Summary @@ -2412,7 +2252,7 @@ def call(vm) # ;; # ~~~ # - class Leave + class Leave < Instruction def disasm(fmt) fmt.instruction("leave") end @@ -2429,10 +2269,6 @@ def ==(other) other.is_a?(Leave) end - def length - 1 - end - def pops 1 end @@ -2443,13 +2279,17 @@ def pushes 0 end - def canonical - self - end - def call(vm) vm.leave end + + def branches? + true + end + + def leaves? + true + end end # ### Summary @@ -2464,7 +2304,7 @@ def call(vm) # ["string"] # ~~~ # - class NewArray + class NewArray < Instruction attr_reader :number def initialize(number) @@ -2499,10 +2339,6 @@ def pushes 1 end - def canonical - self - end - def call(vm) vm.push(vm.pop(number)) end @@ -2520,7 +2356,7 @@ def call(vm) # ["string", **{ foo: "bar" }] # ~~~ # - class NewArrayKwSplat + class NewArrayKwSplat < Instruction attr_reader :number def initialize(number) @@ -2555,10 +2391,6 @@ def pushes 1 end - def canonical - self - end - def call(vm) vm.push(vm.pop(number)) end @@ -2578,7 +2410,7 @@ def call(vm) # end # ~~~ # - class NewHash + class NewHash < Instruction attr_reader :number def initialize(number) @@ -2613,10 +2445,6 @@ def pushes 1 end - def canonical - self - end - def call(vm) vm.push(vm.pop(number).each_slice(2).to_h) end @@ -2637,7 +2465,7 @@ def call(vm) # p (x..y), (x...y) # ~~~ # - class NewRange + class NewRange < Instruction attr_reader :exclude_end def initialize(exclude_end) @@ -2672,10 +2500,6 @@ def pushes 1 end - def canonical - self - end - def call(vm) vm.push(Range.new(*vm.pop(2), exclude_end == 1)) end @@ -2692,7 +2516,7 @@ def call(vm) # raise rescue true # ~~~ # - class Nop + class Nop < Instruction def disasm(fmt) fmt.instruction("nop") end @@ -2709,22 +2533,6 @@ def ==(other) other.is_a?(Nop) end - def length - 1 - end - - def pops - 0 - end - - def pushes - 0 - end - - def canonical - self - end - def call(vm) end end @@ -2743,7 +2551,7 @@ def call(vm) # "#{5}" # ~~~ # - class ObjToString + class ObjToString < Instruction attr_reader :calldata def initialize(calldata) @@ -2778,10 +2586,6 @@ def pushes 1 end - def canonical - self - end - def call(vm) vm.push(vm.pop.to_s) end @@ -2800,7 +2604,7 @@ def call(vm) # END { puts "END" } # ~~~ # - class Once + class Once < Instruction attr_reader :iseq, :cache def initialize(iseq, cache) @@ -2829,18 +2633,10 @@ def length 3 end - def pops - 0 - end - def pushes 1 end - def canonical - self - end - def call(vm) return if @executed vm.push(vm.run_block_frame(iseq, vm.frame)) @@ -2861,7 +2657,7 @@ def call(vm) # 2 & 3 # ~~~ # - class OptAnd + class OptAnd < Instruction attr_reader :calldata def initialize(calldata) @@ -2917,7 +2713,7 @@ def call(vm) # 7[2] # ~~~ # - class OptAref + class OptAref < Instruction attr_reader :calldata def initialize(calldata) @@ -2974,7 +2770,7 @@ def call(vm) # { 'test' => true }['test'] # ~~~ # - class OptArefWith + class OptArefWith < Instruction attr_reader :object, :calldata def initialize(object, calldata) @@ -3014,10 +2810,6 @@ def pushes 1 end - def canonical - self - end - def call(vm) vm.push(vm.pop[object]) end @@ -3036,7 +2828,7 @@ def call(vm) # {}[:key] = value # ~~~ # - class OptAset + class OptAset < Instruction attr_reader :calldata def initialize(calldata) @@ -3092,7 +2884,7 @@ def call(vm) # {}["key"] = value # ~~~ # - class OptAsetWith + class OptAsetWith < Instruction attr_reader :object, :calldata def initialize(object, calldata) @@ -3132,10 +2924,6 @@ def pushes 1 end - def canonical - self - end - def call(vm) hash, value = vm.pop(2) vm.push(hash[object] = value) @@ -3165,7 +2953,7 @@ def call(vm) # end # ~~~ # - class OptCaseDispatch + class OptCaseDispatch < Instruction attr_reader :case_dispatch_hash, :else_label def initialize(case_dispatch_hash, else_label) @@ -3206,16 +2994,16 @@ def pops 1 end - def pushes - 0 + def call(vm) + vm.jump(case_dispatch_hash.fetch(vm.pop, else_label)) end - def canonical - self + def branches? + true end - def call(vm) - vm.jump(case_dispatch_hash.fetch(vm.pop, else_label)) + def falls_through? + true end end @@ -3232,7 +3020,7 @@ def call(vm) # 2 / 3 # ~~~ # - class OptDiv + class OptDiv < Instruction attr_reader :calldata def initialize(calldata) @@ -3288,7 +3076,7 @@ def call(vm) # "".empty? # ~~~ # - class OptEmptyP + class OptEmptyP < Instruction attr_reader :calldata def initialize(calldata) @@ -3345,7 +3133,7 @@ def call(vm) # 2 == 2 # ~~~ # - class OptEq + class OptEq < Instruction attr_reader :calldata def initialize(calldata) @@ -3402,7 +3190,7 @@ def call(vm) # 4 >= 3 # ~~~ # - class OptGE + class OptGE < Instruction attr_reader :calldata def initialize(calldata) @@ -3458,7 +3246,7 @@ def call(vm) # ::Object # ~~~ # - class OptGetConstantPath + class OptGetConstantPath < Instruction attr_reader :names def initialize(names) @@ -3486,18 +3274,10 @@ def length 2 end - def pops - 0 - end - def pushes 1 end - def canonical - self - end - def call(vm) current = vm.frame._self current = current.class unless current.is_a?(Class) @@ -3523,7 +3303,7 @@ def call(vm) # 4 > 3 # ~~~ # - class OptGT + class OptGT < Instruction attr_reader :calldata def initialize(calldata) @@ -3580,7 +3360,7 @@ def call(vm) # 3 <= 4 # ~~~ # - class OptLE + class OptLE < Instruction attr_reader :calldata def initialize(calldata) @@ -3637,7 +3417,7 @@ def call(vm) # "".length # ~~~ # - class OptLength + class OptLength < Instruction attr_reader :calldata def initialize(calldata) @@ -3694,7 +3474,7 @@ def call(vm) # 3 < 4 # ~~~ # - class OptLT + class OptLT < Instruction attr_reader :calldata def initialize(calldata) @@ -3751,7 +3531,7 @@ def call(vm) # "" << 2 # ~~~ # - class OptLTLT + class OptLTLT < Instruction attr_reader :calldata def initialize(calldata) @@ -3809,7 +3589,7 @@ def call(vm) # 3 - 2 # ~~~ # - class OptMinus + class OptMinus < Instruction attr_reader :calldata def initialize(calldata) @@ -3866,7 +3646,7 @@ def call(vm) # 4 % 2 # ~~~ # - class OptMod + class OptMod < Instruction attr_reader :calldata def initialize(calldata) @@ -3923,7 +3703,7 @@ def call(vm) # 3 * 2 # ~~~ # - class OptMult + class OptMult < Instruction attr_reader :calldata def initialize(calldata) @@ -3982,7 +3762,7 @@ def call(vm) # 2 != 2 # ~~~ # - class OptNEq + class OptNEq < Instruction attr_reader :eq_calldata, :neq_calldata def initialize(eq_calldata, neq_calldata) @@ -4022,10 +3802,6 @@ def pushes 1 end - def canonical - self - end - def call(vm) receiver, argument = vm.pop(2) vm.push(receiver != argument) @@ -4044,7 +3820,7 @@ def call(vm) # [a, b, c].max # ~~~ # - class OptNewArrayMax + class OptNewArrayMax < Instruction attr_reader :number def initialize(number) @@ -4079,10 +3855,6 @@ def pushes 1 end - def canonical - self - end - def call(vm) vm.push(vm.pop(number).max) end @@ -4100,7 +3872,7 @@ def call(vm) # [a, b, c].min # ~~~ # - class OptNewArrayMin + class OptNewArrayMin < Instruction attr_reader :number def initialize(number) @@ -4135,10 +3907,6 @@ def pushes 1 end - def canonical - self - end - def call(vm) vm.push(vm.pop(number).min) end @@ -4157,7 +3925,7 @@ def call(vm) # "".nil? # ~~~ # - class OptNilP + class OptNilP < Instruction attr_reader :calldata def initialize(calldata) @@ -4212,7 +3980,7 @@ def call(vm) # !true # ~~~ # - class OptNot + class OptNot < Instruction attr_reader :calldata def initialize(calldata) @@ -4269,7 +4037,7 @@ def call(vm) # 2 | 3 # ~~~ # - class OptOr + class OptOr < Instruction attr_reader :calldata def initialize(calldata) @@ -4326,7 +4094,7 @@ def call(vm) # 2 + 3 # ~~~ # - class OptPlus + class OptPlus < Instruction attr_reader :calldata def initialize(calldata) @@ -4382,7 +4150,7 @@ def call(vm) # /a/ =~ "a" # ~~~ # - class OptRegExpMatch2 + class OptRegExpMatch2 < Instruction attr_reader :calldata def initialize(calldata) @@ -4438,7 +4206,7 @@ def call(vm) # puts "Hello, world!" # ~~~ # - class OptSendWithoutBlock + class OptSendWithoutBlock < Instruction attr_reader :calldata def initialize(calldata) @@ -4495,7 +4263,7 @@ def call(vm) # "".size # ~~~ # - class OptSize + class OptSize < Instruction attr_reader :calldata def initialize(calldata) @@ -4551,7 +4319,7 @@ def call(vm) # "hello".freeze # ~~~ # - class OptStrFreeze + class OptStrFreeze < Instruction attr_reader :object, :calldata def initialize(object, calldata) @@ -4583,18 +4351,10 @@ def length 3 end - def pops - 0 - end - def pushes 1 end - def canonical - self - end - def call(vm) vm.push(object.freeze) end @@ -4612,7 +4372,7 @@ def call(vm) # -"string" # ~~~ # - class OptStrUMinus + class OptStrUMinus < Instruction attr_reader :object, :calldata def initialize(object, calldata) @@ -4644,18 +4404,10 @@ def length 3 end - def pops - 0 - end - def pushes 1 end - def canonical - self - end - def call(vm) vm.push(-object) end @@ -4674,7 +4426,7 @@ def call(vm) # "".succ # ~~~ # - class OptSucc + class OptSucc < Instruction attr_reader :calldata def initialize(calldata) @@ -4728,7 +4480,7 @@ def call(vm) # a ||= 2 # ~~~ # - class Pop + class Pop < Instruction def disasm(fmt) fmt.instruction("pop") end @@ -4745,22 +4497,10 @@ def ==(other) other.is_a?(Pop) end - def length - 1 - end - def pops 1 end - def pushes - 0 - end - - def canonical - self - end - def call(vm) vm.pop end @@ -4776,7 +4516,7 @@ def call(vm) # nil # ~~~ # - class PutNil + class PutNil < Instruction def disasm(fmt) fmt.instruction("putnil") end @@ -4793,14 +4533,6 @@ def ==(other) other.is_a?(PutNil) end - def length - 1 - end - - def pops - 0 - end - def pushes 1 end @@ -4824,7 +4556,7 @@ def call(vm) # 5 # ~~~ # - class PutObject + class PutObject < Instruction attr_reader :object def initialize(object) @@ -4851,18 +4583,10 @@ def length 2 end - def pops - 0 - end - def pushes 1 end - def canonical - self - end - def call(vm) vm.push(object) end @@ -4880,7 +4604,7 @@ def call(vm) # 0 # ~~~ # - class PutObjectInt2Fix0 + class PutObjectInt2Fix0 < Instruction def disasm(fmt) fmt.instruction("putobject_INT2FIX_0_") end @@ -4897,14 +4621,6 @@ def ==(other) other.is_a?(PutObjectInt2Fix0) end - def length - 1 - end - - def pops - 0 - end - def pushes 1 end @@ -4930,7 +4646,7 @@ def call(vm) # 1 # ~~~ # - class PutObjectInt2Fix1 + class PutObjectInt2Fix1 < Instruction def disasm(fmt) fmt.instruction("putobject_INT2FIX_1_") end @@ -4947,14 +4663,6 @@ def ==(other) other.is_a?(PutObjectInt2Fix1) end - def length - 1 - end - - def pops - 0 - end - def pushes 1 end @@ -4978,7 +4686,7 @@ def call(vm) # puts "Hello, world!" # ~~~ # - class PutSelf + class PutSelf < Instruction def disasm(fmt) fmt.instruction("putself") end @@ -4995,22 +4703,10 @@ def ==(other) other.is_a?(PutSelf) end - def length - 1 - end - - def pops - 0 - end - def pushes 1 end - def canonical - self - end - def call(vm) vm.push(vm.frame._self) end @@ -5028,7 +4724,7 @@ def call(vm) # alias foo bar # ~~~ # - class PutSpecialObject + class PutSpecialObject < Instruction OBJECT_VMCORE = 1 OBJECT_CBASE = 2 OBJECT_CONST_BASE = 3 @@ -5059,18 +4755,10 @@ def length 2 end - def pops - 0 - end - def pushes 1 end - def canonical - self - end - def call(vm) case object when OBJECT_VMCORE @@ -5095,7 +4783,7 @@ def call(vm) # "foo" # ~~~ # - class PutString + class PutString < Instruction attr_reader :object def initialize(object) @@ -5122,18 +4810,10 @@ def length 2 end - def pops - 0 - end - def pushes 1 end - def canonical - self - end - def call(vm) vm.push(object.dup) end @@ -5152,7 +4832,7 @@ def call(vm) # "hello".tap { |i| p i } # ~~~ # - class Send + class Send < Instruction attr_reader :calldata, :block_iseq def initialize(calldata, block_iseq) @@ -5194,10 +4874,6 @@ def pushes 1 end - def canonical - self - end - def call(vm) block = if (iseq = block_iseq) @@ -5240,7 +4916,7 @@ def call(vm) # end # ~~~ # - class SetBlockParam + class SetBlockParam < Instruction attr_reader :index, :level def initialize(index, level) @@ -5275,14 +4951,6 @@ def pops 1 end - def pushes - 0 - end - - def canonical - self - end - def call(vm) vm.local_set(index, level, vm.pop) end @@ -5301,7 +4969,7 @@ def call(vm) # @@class_variable = 1 # ~~~ # - class SetClassVariable + class SetClassVariable < Instruction attr_reader :name, :cache def initialize(name, cache) @@ -5337,14 +5005,6 @@ def pops 1 end - def pushes - 0 - end - - def canonical - self - end - def call(vm) clazz = vm.frame._self clazz = clazz.class unless clazz.is_a?(Class) @@ -5363,7 +5023,7 @@ def call(vm) # Constant = 1 # ~~~ # - class SetConstant + class SetConstant < Instruction attr_reader :name def initialize(name) @@ -5394,14 +5054,6 @@ def pops 2 end - def pushes - 0 - end - - def canonical - self - end - def call(vm) value, parent = vm.pop(2) parent.const_set(name, value) @@ -5419,7 +5071,7 @@ def call(vm) # $global = 5 # ~~~ # - class SetGlobal + class SetGlobal < Instruction attr_reader :name def initialize(name) @@ -5450,14 +5102,6 @@ def pops 1 end - def pushes - 0 - end - - def canonical - self - end - def call(vm) # Evaluating the name of the global variable because there isn't a # reflection API for global variables. @@ -5481,7 +5125,7 @@ def call(vm) # @instance_variable = 1 # ~~~ # - class SetInstanceVariable + class SetInstanceVariable < Instruction attr_reader :name, :cache def initialize(name, cache) @@ -5517,14 +5161,6 @@ def pops 1 end - def pushes - 0 - end - - def canonical - self - end - def call(vm) method = Object.instance_method(:instance_variable_set) method.bind(vm.frame._self).call(name, vm.pop) @@ -5545,7 +5181,7 @@ def call(vm) # tap { tap { value = 10 } } # ~~~ # - class SetLocal + class SetLocal < Instruction attr_reader :index, :level def initialize(index, level) @@ -5579,14 +5215,6 @@ def pops 1 end - def pushes - 0 - end - - def canonical - self - end - def call(vm) vm.local_set(index, level, vm.pop) end @@ -5605,7 +5233,7 @@ def call(vm) # value = 5 # ~~~ # - class SetLocalWC0 + class SetLocalWC0 < Instruction attr_reader :index def initialize(index) @@ -5636,10 +5264,6 @@ def pops 1 end - def pushes - 0 - end - def canonical SetLocal.new(index, 0) end @@ -5662,7 +5286,7 @@ def call(vm) # self.then { value = 10 } # ~~~ # - class SetLocalWC1 + class SetLocalWC1 < Instruction attr_reader :index def initialize(index) @@ -5693,10 +5317,6 @@ def pops 1 end - def pushes - 0 - end - def canonical SetLocal.new(index, 1) end @@ -5717,7 +5337,7 @@ def call(vm) # {}[:key] = 'val' # ~~~ # - class SetN + class SetN < Instruction attr_reader :number def initialize(number) @@ -5752,10 +5372,6 @@ def pushes 1 end - def canonical - self - end - def call(vm) vm.stack[-number - 1] = vm.stack.last end @@ -5773,7 +5389,7 @@ def call(vm) # baz if (foo == 1) .. (bar == 1) # ~~~ # - class SetSpecial + class SetSpecial < Instruction attr_reader :key def initialize(key) @@ -5804,14 +5420,6 @@ def pops 1 end - def pushes - 0 - end - - def canonical - self - end - def call(vm) case key when GetSpecial::SVAR_LASTLINE @@ -5836,7 +5444,7 @@ def call(vm) # x = *(5) # ~~~ # - class SplatArray + class SplatArray < Instruction attr_reader :flag def initialize(flag) @@ -5871,10 +5479,6 @@ def pushes 1 end - def canonical - self - end - def call(vm) value = vm.pop @@ -5914,7 +5518,7 @@ def call(vm) # !!defined?([[]]) # ~~~ # - class Swap + class Swap < Instruction def disasm(fmt) fmt.instruction("swap") end @@ -5931,10 +5535,6 @@ def ==(other) other.is_a?(Swap) end - def length - 1 - end - def pops 2 end @@ -5943,10 +5543,6 @@ def pushes 2 end - def canonical - self - end - def call(vm) left, right = vm.pop(2) vm.push(right, left) @@ -5965,7 +5561,7 @@ def call(vm) # [1, 2, 3].map { break 2 } # ~~~ # - class Throw + class Throw < Instruction RUBY_TAG_NONE = 0x0 RUBY_TAG_RETURN = 0x1 RUBY_TAG_BREAK = 0x2 @@ -6013,10 +5609,6 @@ def pushes 1 end - def canonical - self - end - def call(vm) state = type & VM_THROW_STATE_MASK value = vm.pop @@ -6072,7 +5664,7 @@ def error_backtrace(vm) # end # ~~~ # - class TopN + class TopN < Instruction attr_reader :number def initialize(number) @@ -6099,18 +5691,10 @@ def length 2 end - def pops - 0 - end - def pushes 1 end - def canonical - self - end - def call(vm) vm.push(vm.stack[-number - 1]) end @@ -6127,7 +5711,7 @@ def call(vm) # /foo #{bar}/ # ~~~ # - class ToRegExp + class ToRegExp < Instruction attr_reader :options, :length def initialize(options, length) @@ -6160,10 +5744,6 @@ def pushes 1 end - def canonical - self - end - def call(vm) vm.push(Regexp.new(vm.pop(length).join, options)) end diff --git a/lib/syntax_tree/yarv/legacy.rb b/lib/syntax_tree/yarv/legacy.rb index ab9b00df..8e12ff16 100644 --- a/lib/syntax_tree/yarv/legacy.rb +++ b/lib/syntax_tree/yarv/legacy.rb @@ -19,7 +19,7 @@ module Legacy # @@class_variable # ~~~ # - class GetClassVariable + class GetClassVariable < Instruction attr_reader :name def initialize(name) @@ -46,10 +46,6 @@ def length 2 end - def pops - 0 - end - def pushes 1 end @@ -79,7 +75,7 @@ def call(vm) # Constant # ~~~ # - class OptGetInlineCache + class OptGetInlineCache < Instruction attr_reader :label, :cache def initialize(label, cache) @@ -111,21 +107,21 @@ def length 3 end - def pops - 0 - end - def pushes 1 end - def canonical - self - end - def call(vm) vm.push(nil) end + + def branches? + true + end + + def falls_through? + true + end end # ### Summary @@ -143,7 +139,7 @@ def call(vm) # Constant # ~~~ # - class OptSetInlineCache + class OptSetInlineCache < Instruction attr_reader :cache def initialize(cache) @@ -178,10 +174,6 @@ def pushes 1 end - def canonical - self - end - def call(vm) end end @@ -200,7 +192,7 @@ def call(vm) # @@class_variable = 1 # ~~~ # - class SetClassVariable + class SetClassVariable < Instruction attr_reader :name def initialize(name) @@ -231,10 +223,6 @@ def pops 1 end - def pushes - 0 - end - def canonical YARV::SetClassVariable.new(name, nil) end diff --git a/test/yarv_test.rb b/test/yarv_test.rb index e3995435..c4c4c3bd 100644 --- a/test/yarv_test.rb +++ b/test/yarv_test.rb @@ -288,38 +288,12 @@ def value end end - instructions = - YARV.constants.map { YARV.const_get(_1) } + - YARV::Legacy.constants.map { YARV::Legacy.const_get(_1) } - - [ - YARV::Assembler, - YARV::Bf, - YARV::CallData, - YARV::Compiler, - YARV::Decompiler, - YARV::Disassembler, - YARV::InstructionSequence, - YARV::Legacy, - YARV::LocalTable, - YARV::VM - ] + ObjectSpace.each_object(YARV::Instruction.singleton_class) do |instruction| + next if instruction == YARV::Instruction - interface = %i[ - disasm - to_a - deconstruct_keys - length - pops - pushes - canonical - call - == - ] - - instructions.each do |instruction| define_method("test_instruction_interface_#{instruction.name}") do - instance_methods = instruction.instance_methods(false) - assert_empty(interface - instance_methods) + methods = instruction.instance_methods(false) + assert_empty(%i[disasm to_a deconstruct_keys call ==] - methods) end end From 33d36ed2bbd61da601cfe6b7f5e248cd405d356f Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 2 Feb 2023 14:13:47 -0500 Subject: [PATCH 346/536] Add a control flow graph --- lib/syntax_tree.rb | 1 + lib/syntax_tree/yarv/control_flow_graph.rb | 162 +++++++++++++++++++++ 2 files changed, 163 insertions(+) create mode 100644 lib/syntax_tree/yarv/control_flow_graph.rb diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index 73add469..ea365172 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -31,6 +31,7 @@ require_relative "syntax_tree/yarv" require_relative "syntax_tree/yarv/bf" require_relative "syntax_tree/yarv/compiler" +require_relative "syntax_tree/yarv/control_flow_graph" require_relative "syntax_tree/yarv/decompiler" require_relative "syntax_tree/yarv/disassembler" require_relative "syntax_tree/yarv/instruction_sequence" diff --git a/lib/syntax_tree/yarv/control_flow_graph.rb b/lib/syntax_tree/yarv/control_flow_graph.rb new file mode 100644 index 00000000..15e0a767 --- /dev/null +++ b/lib/syntax_tree/yarv/control_flow_graph.rb @@ -0,0 +1,162 @@ +# frozen_string_literal: true + +module SyntaxTree + module YARV + # Constructs a control-flow-graph of a YARV instruction sequence. We use + # conventional basic-blocks. + class ControlFlowGraph + # This object represents a single basic block, wherein all contained + # instructions do not branch except for the last one. + class BasicBlock + # This is the index into the list of instructions where this block + # starts. + attr_reader :block_start + + # This is the set of instructions that this block contains. + attr_reader :insns + + # This is an array of basic blocks that are predecessors to this block. + attr_reader :preds + + # This is an array of basic blocks that are successors to this block. + attr_reader :succs + + def initialize(block_start, insns) + @block_start = block_start + @insns = insns + + @preds = [] + @succs = [] + end + + def id + "block_#{block_start}" + end + + def last + insns.last + end + end + + # This is the instruction sequence that this control flow graph + # corresponds to. + attr_reader :iseq + + # This is the list of instructions that this control flow graph contains. + # It is effectively the same as the list of instructions in the + # instruction sequence but with line numbers and events filtered out. + attr_reader :insns + + # This is the set of basic blocks that this control-flow graph contains. + attr_reader :blocks + + def initialize(iseq, insns, blocks) + @iseq = iseq + @insns = insns + @blocks = blocks + end + + def self.compile(iseq) + # First, we need to find all of the instructions that immediately follow + # labels so that when we are looking at instructions that branch we know + # where they branch to. + labels = {} + insns = [] + + iseq.insns.each do |insn| + case insn + when Instruction + insns << insn + when InstructionSequence::Label + labels[insn] = insns.length + end + end + + # Now we need to find the indices of the instructions that start a basic + # block because they're either: + # + # * the start of an instruction sequence + # * the target of a branch + # * fallen through to from a branch + # + block_starts = Set.new([0]) + + insns.each_with_index do |insn, index| + if insn.branches? + block_starts.add(labels[insn.label]) if insn.respond_to?(:label) + block_starts.add(index + 1) if insn.falls_through? + end + end + + block_starts = block_starts.to_a.sort + + # Now we can build up a set of basic blocks by iterating over the starts + # of each block. They are keyed by the index of their first instruction. + blocks = {} + block_starts.each_with_index do |block_start, block_index| + block_stop = (block_starts[(block_index + 1)..] + [insns.length]).min + + blocks[block_start] = + BasicBlock.new(block_start, insns[block_start...block_stop]) + end + + # Now we need to connect the blocks by letting them know which blocks + # precede them and which blocks follow them. + blocks.each do |block_start, block| + insn = block.last + + if insn.branches? && insn.respond_to?(:label) + block.succs << blocks.fetch(labels[insn.label]) + end + + if (!insn.branches? && !insn.leaves?) || insn.falls_through? + block.succs << blocks.fetch(block_start + block.insns.length) + end + + block.succs.each { |succ| succ.preds << block } + end + + # Here we're going to verify that we set up the control flow graph + # correctly. To do so we will assert that the only instruction in any + # given block that branches is the last instruction in the block. + blocks.each_value do |block| + block.insns[0...-1].each { |insn| raise if insn.branches? } + end + + # Finally we can return a new control flow graph with the given + # instruction sequence and our set of basic blocks. + new(iseq, insns, blocks.values) + end + + def disasm + fmt = Disassembler.new + + output = StringIO.new + output.puts "== cfg #{iseq.name}" + + blocks.each do |block| + output.print(block.id) + + unless block.preds.empty? + output.print(" # from: #{block.preds.map(&:id).join(", ")}") + end + + output.puts + + block.insns.each do |insn| + output.print(" ") + output.puts(insn.disasm(fmt)) + end + + succs = block.succs.map(&:id) + succs << "leaves" if block.last.leaves? + output.print(" # to: #{succs.join(", ")}") unless succs.empty? + + output.puts + end + + output.string + end + end + end +end From 7e6e4d139ccc83d8a3a9dec301fb955919ee98f9 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 2 Feb 2023 14:53:37 -0500 Subject: [PATCH 347/536] Build a data flow graph --- lib/syntax_tree.rb | 1 + lib/syntax_tree/yarv/control_flow_graph.rb | 1 - lib/syntax_tree/yarv/data_flow_graph.rb | 214 +++++++++++++++++++++ 3 files changed, 215 insertions(+), 1 deletion(-) create mode 100644 lib/syntax_tree/yarv/data_flow_graph.rb diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index ea365172..c6f1223b 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -32,6 +32,7 @@ require_relative "syntax_tree/yarv/bf" require_relative "syntax_tree/yarv/compiler" require_relative "syntax_tree/yarv/control_flow_graph" +require_relative "syntax_tree/yarv/data_flow_graph" require_relative "syntax_tree/yarv/decompiler" require_relative "syntax_tree/yarv/disassembler" require_relative "syntax_tree/yarv/instruction_sequence" diff --git a/lib/syntax_tree/yarv/control_flow_graph.rb b/lib/syntax_tree/yarv/control_flow_graph.rb index 15e0a767..26849b64 100644 --- a/lib/syntax_tree/yarv/control_flow_graph.rb +++ b/lib/syntax_tree/yarv/control_flow_graph.rb @@ -130,7 +130,6 @@ def self.compile(iseq) def disasm fmt = Disassembler.new - output = StringIO.new output.puts "== cfg #{iseq.name}" diff --git a/lib/syntax_tree/yarv/data_flow_graph.rb b/lib/syntax_tree/yarv/data_flow_graph.rb new file mode 100644 index 00000000..b028c521 --- /dev/null +++ b/lib/syntax_tree/yarv/data_flow_graph.rb @@ -0,0 +1,214 @@ +# frozen_string_literal: true + +module SyntaxTree + module YARV + # Constructs a data-flow-graph of a YARV instruction sequence, via a + # control-flow-graph. Data flow is discovered locally and then globally. The + # graph only considers data flow through the stack - local variables and + # objects are considered fully escaped in this analysis. + class DataFlowGraph + # This object represents the flow of data between instructions. + class DataFlow + attr_reader :in + attr_reader :out + + def initialize + @in = [] + @out = [] + end + end + + attr_reader :cfg, :insn_flows, :block_flows + + def initialize(cfg, insn_flows, block_flows) + @cfg = cfg + @insn_flows = insn_flows + @block_flows = block_flows + end + + def self.compile(cfg) + # First, create a data structure to encode data flow between + # instructions. + insn_flows = {} + cfg.insns.each_with_index do |insn, index| + insn_flows[index] = DataFlow.new + end + + # Next, create a data structure to encode data flow between basic + # blocks. + block_flows = {} + cfg.blocks.each do |block| + block_flows[block.block_start] = DataFlow.new + end + + # Now, discover the data flow within each basic block. Using an abstract + # stack, connect from consumers of data to the producers of that data. + cfg.blocks.each do |block| + block_flow = block_flows.fetch(block.block_start) + + stack = [] + stack_initial_depth = 0 + + # Go through each instruction in the block... + block.insns.each.with_index(block.block_start) do |insn, index| + insn_flow = insn_flows[index] + + # How many values will be missing from the local stack to run this + # instruction? + missing_stack_values = insn.pops - stack.size + + # For every value the instruction pops off the stack... + insn.pops.times do + # Was the value it pops off from another basic block? + if stack.empty? + # This is a basic block argument. + name = :"in_#{missing_stack_values - 1}" + + insn_flow.in.unshift(name) + block_flow.in.unshift(name) + + stack_initial_depth += 1 + missing_stack_values -= 1 + else + # Connect this consumer to the producer of the value. + insn_flow.in.unshift(stack.pop) + end + end + + # Record on our abstract stack that this instruction pushed + # this value onto the stack. + insn.pushes.times { stack << index } + end + + # Values that are left on the stack after going through all + # instructions are arguments to the basic block that we jump to. + stack.reverse_each.with_index do |producer, index| + block_flow.out << producer + insn_flows[producer].out << :"out_#{index}" + end + end + + # Go backwards and connect from producers to consumers. + cfg.insns.each_with_index do |insn, index| + # For every instruction that produced a value used in this + # instruction... + insn_flows[index].in.each do |producer| + # If it's actually another instruction and not a basic block + # argument... + if producer.is_a?(Integer) + # Record in the producing instruction that it produces a value + # used by this construction. + insn_flows[producer].out << index + end + end + end + + # Now, discover the data flow between basic blocks. + stack = [*cfg.blocks] + until stack.empty? + succ = stack.pop + succ_flow = block_flows.fetch(succ.block_start) + succ.preds.each do |pred| + pred_flow = block_flows.fetch(pred.block_start) + + # Does a predecessor block have fewer outputs than the successor + # has inputs? + if pred_flow.out.size < succ_flow.in.size + # If so then add arguments to pass data through from the + # predecessor's predecessors. + (succ_flow.in.size - pred_flow.out.size).times do |index| + name = :"pass_#{index}" + pred_flow.in.unshift(name) + pred_flow.out.unshift(name) + end + + # Since we modified the predecessor, add it back to the worklist + # so it'll be considered as a successor again, and propogate the + # global data flow back up the control flow graph. + stack << pred + end + end + end + + # Verify that we constructed the data flow graph correctly. Check that + # the first block has no arguments. + raise unless block_flows.fetch(cfg.blocks.first.block_start).in.empty? + + # Check all control flow edges between blocks pass the right number of + # arguments. + cfg.blocks.each do |pred| + pred_flow = block_flows.fetch(pred.block_start) + + if pred.succs.empty? + # With no successors, there should be no output arguments. + raise unless pred_flow.out.empty? + else + # Check with successor... + pred.succs.each do |succ| + succ_flow = block_flows.fetch(succ.block_start) + + # The predecessor should have as many output arguments as the + # success has input arguments. + raise unless pred_flow.out.size == succ_flow.in.size + end + end + end + + # Finally we can return the data flow graph. + new(cfg, insn_flows, block_flows) + end + + def disasm + fmt = Disassembler.new + output = StringIO.new + output.puts "== dfg #{cfg.iseq.name}" + + cfg.blocks.each do |block| + output.print(block.id) + unless block.preds.empty? + output.print(" # from: #{block.preds.map(&:id).join(", ")}") + end + output.puts + + block_flow = block_flows.fetch(block.block_start) + unless block_flow.in.empty? + output.puts " # in: #{block_flow.in.join(", ")}" + end + + block.insns.each.with_index(block.block_start) do |insn, index| + output.print(" ") + output.print(insn.disasm(fmt)) + + insn_flow = insn_flows[index] + if insn_flow.in.empty? && insn_flow.out.empty? + output.puts + next + end + + output.print(" # ") + unless insn_flow.in.empty? + output.print("in: #{insn_flow.in.join(", ")}") + output.print("; ") unless insn_flow.out.empty? + end + + unless insn_flow.out.empty? + output.print("out: #{insn_flow.out.join(", ")}") + end + + output.puts + end + + succs = block.succs.map(&:id) + succs << "leaves" if block.last.leaves? + output.puts(" # to: #{succs.join(", ")}") unless succs.empty? + + unless block_flow.out.empty? + output.puts " # out: #{block_flow.out.join(", ")}" + end + end + + output.string + end + end + end +end \ No newline at end of file From 907cf23b2e8245cd99b6839f06a2bae40b0ae393 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 2 Feb 2023 16:38:08 -0500 Subject: [PATCH 348/536] More documentation --- lib/syntax_tree/yarv/control_flow_graph.rb | 180 +++++++++++++-------- test/yarv_test.rb | 63 ++++++++ 2 files changed, 174 insertions(+), 69 deletions(-) diff --git a/lib/syntax_tree/yarv/control_flow_graph.rb b/lib/syntax_tree/yarv/control_flow_graph.rb index 26849b64..cd8a8324 100644 --- a/lib/syntax_tree/yarv/control_flow_graph.rb +++ b/lib/syntax_tree/yarv/control_flow_graph.rb @@ -2,12 +2,24 @@ module SyntaxTree module YARV - # Constructs a control-flow-graph of a YARV instruction sequence. We use - # conventional basic-blocks. + # This class represents a control flow graph of a YARV instruction sequence. + # It constructs a graph of basic blocks that hold subsets of the list of + # instructions from the instruction sequence. + # + # You can use this class by calling the ::compile method and passing it a + # YARV instruction sequence. It will return a control flow graph object. + # + # iseq = RubyVM::InstructionSequence.compile("1 + 2") + # iseq = SyntaxTree::YARV::InstructionSequence.from(iseq.to_a) + # cfg = SyntaxTree::YARV::ControlFlowGraph.compile(iseq) + # class ControlFlowGraph # This object represents a single basic block, wherein all contained # instructions do not branch except for the last one. class BasicBlock + # This is the unique identifier for this basic block. + attr_reader :id + # This is the index into the list of instructions where this block # starts. attr_reader :block_start @@ -22,6 +34,8 @@ class BasicBlock attr_reader :succs def initialize(block_start, insns) + @id = "block_#{block_start}" + @block_start = block_start @insns = insns @@ -29,8 +43,11 @@ def initialize(block_start, insns) @succs = [] end - def id - "block_#{block_start}" + # This method is used to verify that the basic block is well formed. It + # checks that the only instruction in this basic block that branches is + # the last instruction. + def verify + insns[0...-1].each { |insn| raise if insn.branches? } end def last @@ -38,94 +55,108 @@ def last end end - # This is the instruction sequence that this control flow graph - # corresponds to. - attr_reader :iseq - - # This is the list of instructions that this control flow graph contains. - # It is effectively the same as the list of instructions in the - # instruction sequence but with line numbers and events filtered out. - attr_reader :insns - - # This is the set of basic blocks that this control-flow graph contains. - attr_reader :blocks - - def initialize(iseq, insns, blocks) - @iseq = iseq - @insns = insns - @blocks = blocks - end - - def self.compile(iseq) - # First, we need to find all of the instructions that immediately follow - # labels so that when we are looking at instructions that branch we know - # where they branch to. - labels = {} - insns = [] - - iseq.insns.each do |insn| - case insn - when Instruction - insns << insn - when InstructionSequence::Label - labels[insn] = insns.length + # This class is responsible for creating a control flow graph from the + # given instruction sequence. + class Compiler + attr_reader :iseq, :labels, :insns + + def initialize(iseq) + @iseq = iseq + + # We need to find all of the instructions that immediately follow + # labels so that when we are looking at instructions that branch we + # know where they branch to. + @labels = {} + @insns = [] + + iseq.insns.each do |insn| + case insn + when Instruction + @insns << insn + when InstructionSequence::Label + @labels[insn] = @insns.length + end end end - # Now we need to find the indices of the instructions that start a basic - # block because they're either: + # This method is used to compile the instruction sequence into a control + # flow graph. It returns an instance of ControlFlowGraph. + def compile + blocks = connect_basic_blocks(build_basic_blocks) + ControlFlowGraph.new(iseq, insns, blocks.values).tap(&:verify) + end + + private + + # Finds the indices of the instructions that start a basic block because + # they're either: # # * the start of an instruction sequence # * the target of a branch # * fallen through to from a branch # - block_starts = Set.new([0]) - - insns.each_with_index do |insn, index| - if insn.branches? - block_starts.add(labels[insn.label]) if insn.respond_to?(:label) - block_starts.add(index + 1) if insn.falls_through? + def find_basic_block_starts + block_starts = Set.new([0]) + + insns.each_with_index do |insn, index| + if insn.branches? + block_starts.add(labels[insn.label]) if insn.respond_to?(:label) + block_starts.add(index + 1) if insn.falls_through? + end end + + block_starts.to_a.sort end - block_starts = block_starts.to_a.sort + # Builds up a set of basic blocks by iterating over the starts of each + # block. They are keyed by the index of their first instruction. + def build_basic_blocks + block_starts = find_basic_block_starts + blocks = {} - # Now we can build up a set of basic blocks by iterating over the starts - # of each block. They are keyed by the index of their first instruction. - blocks = {} - block_starts.each_with_index do |block_start, block_index| - block_stop = (block_starts[(block_index + 1)..] + [insns.length]).min + block_starts.each_with_index.to_h do |block_start, block_index| + block_end = (block_starts[(block_index + 1)..] + [insns.length]).min + block_insns = insns[block_start...block_end] - blocks[block_start] = - BasicBlock.new(block_start, insns[block_start...block_stop]) + [block_start, BasicBlock.new(block_start, block_insns)] + end end # Now we need to connect the blocks by letting them know which blocks # precede them and which blocks follow them. - blocks.each do |block_start, block| - insn = block.last + def connect_basic_blocks(blocks) + blocks.each do |block_start, block| + insn = block.last - if insn.branches? && insn.respond_to?(:label) - block.succs << blocks.fetch(labels[insn.label]) - end + if insn.branches? && insn.respond_to?(:label) + block.succs << blocks.fetch(labels[insn.label]) + end - if (!insn.branches? && !insn.leaves?) || insn.falls_through? - block.succs << blocks.fetch(block_start + block.insns.length) - end + if (!insn.branches? && !insn.leaves?) || insn.falls_through? + block.succs << blocks.fetch(block_start + block.insns.length) + end - block.succs.each { |succ| succ.preds << block } + block.succs.each { |succ| succ.preds << block } + end end + end - # Here we're going to verify that we set up the control flow graph - # correctly. To do so we will assert that the only instruction in any - # given block that branches is the last instruction in the block. - blocks.each_value do |block| - block.insns[0...-1].each { |insn| raise if insn.branches? } - end + # This is the instruction sequence that this control flow graph + # corresponds to. + attr_reader :iseq + + # This is the list of instructions that this control flow graph contains. + # It is effectively the same as the list of instructions in the + # instruction sequence but with line numbers and events filtered out. + attr_reader :insns + + # This is the set of basic blocks that this control-flow graph contains. + attr_reader :blocks - # Finally we can return a new control flow graph with the given - # instruction sequence and our set of basic blocks. - new(iseq, insns, blocks.values) + def initialize(iseq, insns, blocks) + @iseq = iseq + @insns = insns + @blocks = blocks end def disasm @@ -156,6 +187,17 @@ def disasm output.string end + + # This method is used to verify that the control flow graph is well + # formed. It does this by checking that each basic block is itself well + # formed. + def verify + blocks.each(&:verify) + end + + def self.compile(iseq) + Compiler.new(iseq).compile + end end end end diff --git a/test/yarv_test.rb b/test/yarv_test.rb index c4c4c3bd..e37afb63 100644 --- a/test/yarv_test.rb +++ b/test/yarv_test.rb @@ -297,6 +297,69 @@ def value end end + def test_cfg + iseq = RubyVM::InstructionSequence.compile("100 + (14 < 0 ? -1 : +1)") + iseq = SyntaxTree::YARV::InstructionSequence.from(iseq.to_a) + cfg = SyntaxTree::YARV::ControlFlowGraph.compile(iseq) + + assert_equal(<<~CFG, cfg.disasm) + == cfg + block_0 + putobject 100 + putobject 14 + putobject_INT2FIX_0_ + opt_lt + branchunless 13 + # to: block_7, block_5 + block_5 # from: block_0 + putobject -1 + jump 14 + # to: block_8 + block_7 # from: block_0 + putobject_INT2FIX_1_ + # to: block_8 + block_8 # from: block_5, block_7 + opt_plus + leave + # to: leaves + CFG + end + + def test_dfg + iseq = RubyVM::InstructionSequence.compile("100 + (14 < 0 ? -1 : +1)") + iseq = SyntaxTree::YARV::InstructionSequence.from(iseq.to_a) + cfg = SyntaxTree::YARV::ControlFlowGraph.compile(iseq) + dfg = SyntaxTree::YARV::DataFlowGraph.compile(cfg) + + assert_equal(<<~DFG, dfg.disasm) + == dfg + block_0 + putobject 100 # out: out_0 + putobject 14 # out: 3 + putobject_INT2FIX_0_ # out: 3 + opt_lt # in: 1, 2; out: 4 + branchunless 13 # in: 3 + # to: block_7, block_5 + # out: 0 + block_5 # from: block_0 + # in: pass_0 + putobject -1 # out: out_0 + jump 14 + # to: block_8 + # out: pass_0, 5 + block_7 # from: block_0 + # in: pass_0 + putobject_INT2FIX_1_ # out: out_0 + # to: block_8 + # out: pass_0, 7 + block_8 # from: block_5, block_7 + # in: in_0, in_1 + opt_plus # in: in_0, in_1; out: 9 + leave # in: 8 + # to: leaves + DFG + end + private def assert_decompiles(expected, source) From 7578736beb2f444a76f9ce60ca2181438922ef51 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 2 Feb 2023 16:43:46 -0500 Subject: [PATCH 349/536] More moving around and documentation --- lib/syntax_tree/yarv/control_flow_graph.rb | 136 +++++++++++---------- lib/syntax_tree/yarv/data_flow_graph.rb | 18 +-- 2 files changed, 78 insertions(+), 76 deletions(-) diff --git a/lib/syntax_tree/yarv/control_flow_graph.rb b/lib/syntax_tree/yarv/control_flow_graph.rb index cd8a8324..fa9823f1 100644 --- a/lib/syntax_tree/yarv/control_flow_graph.rb +++ b/lib/syntax_tree/yarv/control_flow_graph.rb @@ -14,6 +14,64 @@ module YARV # cfg = SyntaxTree::YARV::ControlFlowGraph.compile(iseq) # class ControlFlowGraph + # This is the instruction sequence that this control flow graph + # corresponds to. + attr_reader :iseq + + # This is the list of instructions that this control flow graph contains. + # It is effectively the same as the list of instructions in the + # instruction sequence but with line numbers and events filtered out. + attr_reader :insns + + # This is the set of basic blocks that this control-flow graph contains. + attr_reader :blocks + + def initialize(iseq, insns, blocks) + @iseq = iseq + @insns = insns + @blocks = blocks + end + + def disasm + fmt = Disassembler.new + output = StringIO.new + output.puts "== cfg #{iseq.name}" + + blocks.each do |block| + output.print(block.id) + + unless block.predecessors.empty? + output.print(" # from: #{block.predecessors.map(&:id).join(", ")}") + end + + output.puts + + block.insns.each do |insn| + output.print(" ") + output.puts(insn.disasm(fmt)) + end + + successors = block.successors.map(&:id) + successors << "leaves" if block.last.leaves? + output.print(" # to: #{successors.join(", ")}") unless successors.empty? + + output.puts + end + + output.string + end + + # This method is used to verify that the control flow graph is well + # formed. It does this by checking that each basic block is itself well + # formed. + def verify + blocks.each(&:verify) + end + + def self.compile(iseq) + Compiler.new(iseq).compile + end + # This object represents a single basic block, wherein all contained # instructions do not branch except for the last one. class BasicBlock @@ -28,10 +86,10 @@ class BasicBlock attr_reader :insns # This is an array of basic blocks that are predecessors to this block. - attr_reader :preds + attr_reader :predecessors # This is an array of basic blocks that are successors to this block. - attr_reader :succs + attr_reader :successors def initialize(block_start, insns) @id = "block_#{block_start}" @@ -39,8 +97,8 @@ def initialize(block_start, insns) @block_start = block_start @insns = insns - @preds = [] - @succs = [] + @predecessors = [] + @successors = [] end # This method is used to verify that the basic block is well formed. It @@ -122,81 +180,25 @@ def build_basic_blocks end end - # Now we need to connect the blocks by letting them know which blocks - # precede them and which blocks follow them. + # Connect the blocks by letting them know which blocks precede them and + # which blocks succeed them. def connect_basic_blocks(blocks) blocks.each do |block_start, block| insn = block.last if insn.branches? && insn.respond_to?(:label) - block.succs << blocks.fetch(labels[insn.label]) + block.successors << blocks.fetch(labels[insn.label]) end if (!insn.branches? && !insn.leaves?) || insn.falls_through? - block.succs << blocks.fetch(block_start + block.insns.length) + block.successors << blocks.fetch(block_start + block.insns.length) end - block.succs.each { |succ| succ.preds << block } - end - end - end - - # This is the instruction sequence that this control flow graph - # corresponds to. - attr_reader :iseq - - # This is the list of instructions that this control flow graph contains. - # It is effectively the same as the list of instructions in the - # instruction sequence but with line numbers and events filtered out. - attr_reader :insns - - # This is the set of basic blocks that this control-flow graph contains. - attr_reader :blocks - - def initialize(iseq, insns, blocks) - @iseq = iseq - @insns = insns - @blocks = blocks - end - - def disasm - fmt = Disassembler.new - output = StringIO.new - output.puts "== cfg #{iseq.name}" - - blocks.each do |block| - output.print(block.id) - - unless block.preds.empty? - output.print(" # from: #{block.preds.map(&:id).join(", ")}") - end - - output.puts - - block.insns.each do |insn| - output.print(" ") - output.puts(insn.disasm(fmt)) + block.successors.each do |successor| + successor.predecessors << block + end end - - succs = block.succs.map(&:id) - succs << "leaves" if block.last.leaves? - output.print(" # to: #{succs.join(", ")}") unless succs.empty? - - output.puts end - - output.string - end - - # This method is used to verify that the control flow graph is well - # formed. It does this by checking that each basic block is itself well - # formed. - def verify - blocks.each(&:verify) - end - - def self.compile(iseq) - Compiler.new(iseq).compile end end end diff --git a/lib/syntax_tree/yarv/data_flow_graph.rb b/lib/syntax_tree/yarv/data_flow_graph.rb index b028c521..13089dc7 100644 --- a/lib/syntax_tree/yarv/data_flow_graph.rb +++ b/lib/syntax_tree/yarv/data_flow_graph.rb @@ -108,7 +108,7 @@ def self.compile(cfg) until stack.empty? succ = stack.pop succ_flow = block_flows.fetch(succ.block_start) - succ.preds.each do |pred| + succ.predecessors.each do |pred| pred_flow = block_flows.fetch(pred.block_start) # Does a predecessor block have fewer outputs than the successor @@ -139,12 +139,12 @@ def self.compile(cfg) cfg.blocks.each do |pred| pred_flow = block_flows.fetch(pred.block_start) - if pred.succs.empty? + if pred.successors.empty? # With no successors, there should be no output arguments. raise unless pred_flow.out.empty? else # Check with successor... - pred.succs.each do |succ| + pred.successors.each do |succ| succ_flow = block_flows.fetch(succ.block_start) # The predecessor should have as many output arguments as the @@ -165,8 +165,8 @@ def disasm cfg.blocks.each do |block| output.print(block.id) - unless block.preds.empty? - output.print(" # from: #{block.preds.map(&:id).join(", ")}") + unless block.predecessors.empty? + output.print(" # from: #{block.predecessors.map(&:id).join(", ")}") end output.puts @@ -198,9 +198,9 @@ def disasm output.puts end - succs = block.succs.map(&:id) - succs << "leaves" if block.last.leaves? - output.puts(" # to: #{succs.join(", ")}") unless succs.empty? + successors = block.successors.map(&:id) + successors << "leaves" if block.last.leaves? + output.puts(" # to: #{successors.join(", ")}") unless successors.empty? unless block_flow.out.empty? output.puts " # out: #{block_flow.out.join(", ")}" @@ -211,4 +211,4 @@ def disasm end end end -end \ No newline at end of file +end From 7088c153057d92bbb03feb5120214fcfcdd553ea Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 2 Feb 2023 16:57:23 -0500 Subject: [PATCH 350/536] Support multiple branch targets per instruction --- lib/syntax_tree/yarv/control_flow_graph.rb | 17 +++++++----- lib/syntax_tree/yarv/instructions.rb | 30 ++++++++++------------ lib/syntax_tree/yarv/legacy.rb | 4 +-- 3 files changed, 26 insertions(+), 25 deletions(-) diff --git a/lib/syntax_tree/yarv/control_flow_graph.rb b/lib/syntax_tree/yarv/control_flow_graph.rb index fa9823f1..1d271768 100644 --- a/lib/syntax_tree/yarv/control_flow_graph.rb +++ b/lib/syntax_tree/yarv/control_flow_graph.rb @@ -105,7 +105,7 @@ def initialize(block_start, insns) # checks that the only instruction in this basic block that branches is # the last instruction. def verify - insns[0...-1].each { |insn| raise if insn.branches? } + insns[0...-1].each { |insn| raise unless insn.branch_targets.empty? } end def last @@ -157,8 +157,13 @@ def find_basic_block_starts block_starts = Set.new([0]) insns.each_with_index do |insn, index| - if insn.branches? - block_starts.add(labels[insn.label]) if insn.respond_to?(:label) + branch_targets = insn.branch_targets + + if branch_targets.any? + branch_targets.each do |branch_target| + block_starts.add(labels[branch_target]) + end + block_starts.add(index + 1) if insn.falls_through? end end @@ -186,11 +191,11 @@ def connect_basic_blocks(blocks) blocks.each do |block_start, block| insn = block.last - if insn.branches? && insn.respond_to?(:label) - block.successors << blocks.fetch(labels[insn.label]) + insn.branch_targets.each do |branch_target| + block.successors << blocks.fetch(labels[branch_target]) end - if (!insn.branches? && !insn.leaves?) || insn.falls_through? + if (insn.branch_targets.empty? && !insn.leaves?) || insn.falls_through? block.successors << blocks.fetch(block_start + block.insns.length) end diff --git a/lib/syntax_tree/yarv/instructions.rb b/lib/syntax_tree/yarv/instructions.rb index c387e763..97ccce15 100644 --- a/lib/syntax_tree/yarv/instructions.rb +++ b/lib/syntax_tree/yarv/instructions.rb @@ -90,9 +90,9 @@ def pops 0 end - # Whether or not this instruction is a branch instruction. - def branches? - false + # This returns an array of labels. + def branch_targets + [] end # Whether or not this instruction leaves the current frame. @@ -261,8 +261,8 @@ def call(vm) vm.jump(label) if vm.pop end - def branches? - true + def branch_targets + [label] end def falls_through? @@ -322,8 +322,8 @@ def call(vm) vm.jump(label) if vm.pop.nil? end - def branches? - true + def branch_targets + [label] end def falls_through? @@ -382,8 +382,8 @@ def call(vm) vm.jump(label) unless vm.pop end - def branches? - true + def branch_targets + [label] end def falls_through? @@ -2237,8 +2237,8 @@ def call(vm) vm.jump(label) end - def branches? - true + def branch_targets + [label] end end @@ -2283,10 +2283,6 @@ def call(vm) vm.leave end - def branches? - true - end - def leaves? true end @@ -2998,8 +2994,8 @@ def call(vm) vm.jump(case_dispatch_hash.fetch(vm.pop, else_label)) end - def branches? - true + def branch_targets + case_dispatch_hash.values.push(else_label) end def falls_through? diff --git a/lib/syntax_tree/yarv/legacy.rb b/lib/syntax_tree/yarv/legacy.rb index 8e12ff16..e20729d9 100644 --- a/lib/syntax_tree/yarv/legacy.rb +++ b/lib/syntax_tree/yarv/legacy.rb @@ -115,8 +115,8 @@ def call(vm) vm.push(nil) end - def branches? - true + def branch_targets + [label] end def falls_through? From b8dc90189aeb476913d8e12f2304b7223f5ccba9 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 2 Feb 2023 16:58:34 -0500 Subject: [PATCH 351/536] Remove BasicBlock.last --- lib/syntax_tree/yarv/control_flow_graph.rb | 8 ++------ lib/syntax_tree/yarv/data_flow_graph.rb | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/lib/syntax_tree/yarv/control_flow_graph.rb b/lib/syntax_tree/yarv/control_flow_graph.rb index 1d271768..1761127c 100644 --- a/lib/syntax_tree/yarv/control_flow_graph.rb +++ b/lib/syntax_tree/yarv/control_flow_graph.rb @@ -52,7 +52,7 @@ def disasm end successors = block.successors.map(&:id) - successors << "leaves" if block.last.leaves? + successors << "leaves" if block.insns.last.leaves? output.print(" # to: #{successors.join(", ")}") unless successors.empty? output.puts @@ -107,10 +107,6 @@ def initialize(block_start, insns) def verify insns[0...-1].each { |insn| raise unless insn.branch_targets.empty? } end - - def last - insns.last - end end # This class is responsible for creating a control flow graph from the @@ -189,7 +185,7 @@ def build_basic_blocks # which blocks succeed them. def connect_basic_blocks(blocks) blocks.each do |block_start, block| - insn = block.last + insn = block.insns.last insn.branch_targets.each do |branch_target| block.successors << blocks.fetch(labels[branch_target]) diff --git a/lib/syntax_tree/yarv/data_flow_graph.rb b/lib/syntax_tree/yarv/data_flow_graph.rb index 13089dc7..2af51883 100644 --- a/lib/syntax_tree/yarv/data_flow_graph.rb +++ b/lib/syntax_tree/yarv/data_flow_graph.rb @@ -199,7 +199,7 @@ def disasm end successors = block.successors.map(&:id) - successors << "leaves" if block.last.leaves? + successors << "leaves" if block.insns.last.leaves? output.puts(" # to: #{successors.join(", ")}") unless successors.empty? unless block_flow.out.empty? From 92cbfcae048c6867d0d5a6db5265591ed0b53076 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 2 Feb 2023 17:03:49 -0500 Subject: [PATCH 352/536] Provide BasicBlock.each_with_index --- lib/syntax_tree/yarv/control_flow_graph.rb | 7 ++++++- lib/syntax_tree/yarv/data_flow_graph.rb | 20 ++++++++++---------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/lib/syntax_tree/yarv/control_flow_graph.rb b/lib/syntax_tree/yarv/control_flow_graph.rb index 1761127c..5b4b5605 100644 --- a/lib/syntax_tree/yarv/control_flow_graph.rb +++ b/lib/syntax_tree/yarv/control_flow_graph.rb @@ -101,6 +101,12 @@ def initialize(block_start, insns) @successors = [] end + # Yield each instruction in this basic block along with its index from + # the original instruction sequence. + def each_with_index(&block) + insns.each.with_index(block_start, &block) + end + # This method is used to verify that the basic block is well formed. It # checks that the only instruction in this basic block that branches is # the last instruction. @@ -171,7 +177,6 @@ def find_basic_block_starts # block. They are keyed by the index of their first instruction. def build_basic_blocks block_starts = find_basic_block_starts - blocks = {} block_starts.each_with_index.to_h do |block_start, block_index| block_end = (block_starts[(block_index + 1)..] + [insns.length]).min diff --git a/lib/syntax_tree/yarv/data_flow_graph.rb b/lib/syntax_tree/yarv/data_flow_graph.rb index 2af51883..295308bd 100644 --- a/lib/syntax_tree/yarv/data_flow_graph.rb +++ b/lib/syntax_tree/yarv/data_flow_graph.rb @@ -38,19 +38,19 @@ def self.compile(cfg) # blocks. block_flows = {} cfg.blocks.each do |block| - block_flows[block.block_start] = DataFlow.new + block_flows[block.id] = DataFlow.new end # Now, discover the data flow within each basic block. Using an abstract # stack, connect from consumers of data to the producers of that data. cfg.blocks.each do |block| - block_flow = block_flows.fetch(block.block_start) + block_flow = block_flows.fetch(block.id) stack = [] stack_initial_depth = 0 # Go through each instruction in the block... - block.insns.each.with_index(block.block_start) do |insn, index| + block.each_with_index do |insn, index| insn_flow = insn_flows[index] # How many values will be missing from the local stack to run this @@ -107,9 +107,9 @@ def self.compile(cfg) stack = [*cfg.blocks] until stack.empty? succ = stack.pop - succ_flow = block_flows.fetch(succ.block_start) + succ_flow = block_flows.fetch(succ.id) succ.predecessors.each do |pred| - pred_flow = block_flows.fetch(pred.block_start) + pred_flow = block_flows.fetch(pred.id) # Does a predecessor block have fewer outputs than the successor # has inputs? @@ -132,12 +132,12 @@ def self.compile(cfg) # Verify that we constructed the data flow graph correctly. Check that # the first block has no arguments. - raise unless block_flows.fetch(cfg.blocks.first.block_start).in.empty? + raise unless block_flows.fetch(cfg.blocks.first.id).in.empty? # Check all control flow edges between blocks pass the right number of # arguments. cfg.blocks.each do |pred| - pred_flow = block_flows.fetch(pred.block_start) + pred_flow = block_flows.fetch(pred.id) if pred.successors.empty? # With no successors, there should be no output arguments. @@ -145,7 +145,7 @@ def self.compile(cfg) else # Check with successor... pred.successors.each do |succ| - succ_flow = block_flows.fetch(succ.block_start) + succ_flow = block_flows.fetch(succ.id) # The predecessor should have as many output arguments as the # success has input arguments. @@ -170,12 +170,12 @@ def disasm end output.puts - block_flow = block_flows.fetch(block.block_start) + block_flow = block_flows.fetch(block.id) unless block_flow.in.empty? output.puts " # in: #{block_flow.in.join(", ")}" end - block.insns.each.with_index(block.block_start) do |insn, index| + block.each_with_index do |insn, index| output.print(" ") output.print(insn.disasm(fmt)) From 439ffb6336f9af6c2386c291bb529488c6d79d03 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 2 Feb 2023 18:27:03 -0500 Subject: [PATCH 353/536] Refactor various graphs --- lib/syntax_tree.rb | 1 + lib/syntax_tree/yarv/basic_block.rb | 47 ++++ lib/syntax_tree/yarv/control_flow_graph.rb | 65 +---- lib/syntax_tree/yarv/data_flow_graph.rb | 296 +++++++++++---------- 4 files changed, 218 insertions(+), 191 deletions(-) create mode 100644 lib/syntax_tree/yarv/basic_block.rb diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index c6f1223b..e0e2a6be 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -29,6 +29,7 @@ require_relative "syntax_tree/index" require_relative "syntax_tree/yarv" +require_relative "syntax_tree/yarv/basic_block" require_relative "syntax_tree/yarv/bf" require_relative "syntax_tree/yarv/compiler" require_relative "syntax_tree/yarv/control_flow_graph" diff --git a/lib/syntax_tree/yarv/basic_block.rb b/lib/syntax_tree/yarv/basic_block.rb new file mode 100644 index 00000000..774a4c00 --- /dev/null +++ b/lib/syntax_tree/yarv/basic_block.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module SyntaxTree + module YARV + # This object represents a single basic block, wherein all contained + # instructions do not branch except for the last one. + class BasicBlock + # This is the unique identifier for this basic block. + attr_reader :id + + # This is the index into the list of instructions where this block starts. + attr_reader :block_start + + # This is the set of instructions that this block contains. + attr_reader :insns + + # This is an array of basic blocks that lead into this block. + attr_reader :incoming_blocks + + # This is an array of basic blocks that this block leads into. + attr_reader :outgoing_blocks + + def initialize(block_start, insns) + @id = "block_#{block_start}" + + @block_start = block_start + @insns = insns + + @incoming_blocks = [] + @outgoing_blocks = [] + end + + # Yield each instruction in this basic block along with its index from the + # original instruction sequence. + def each_with_index(&block) + insns.each.with_index(block_start, &block) + end + + # This method is used to verify that the basic block is well formed. It + # checks that the only instruction in this basic block that branches is + # the last instruction. + def verify + insns[0...-1].each { |insn| raise unless insn.branch_targets.empty? } + end + end + end +end diff --git a/lib/syntax_tree/yarv/control_flow_graph.rb b/lib/syntax_tree/yarv/control_flow_graph.rb index 5b4b5605..27df308e 100644 --- a/lib/syntax_tree/yarv/control_flow_graph.rb +++ b/lib/syntax_tree/yarv/control_flow_graph.rb @@ -40,8 +40,8 @@ def disasm blocks.each do |block| output.print(block.id) - unless block.predecessors.empty? - output.print(" # from: #{block.predecessors.map(&:id).join(", ")}") + unless block.incoming_blocks.empty? + output.print(" # from: #{block.incoming_blocks.map(&:id).join(", ")}") end output.puts @@ -51,9 +51,9 @@ def disasm output.puts(insn.disasm(fmt)) end - successors = block.successors.map(&:id) - successors << "leaves" if block.insns.last.leaves? - output.print(" # to: #{successors.join(", ")}") unless successors.empty? + dests = block.outgoing_blocks.map(&:id) + dests << "leaves" if block.insns.last.leaves? + output.print(" # to: #{dests.join(", ")}") unless dests.empty? output.puts end @@ -72,49 +72,6 @@ def self.compile(iseq) Compiler.new(iseq).compile end - # This object represents a single basic block, wherein all contained - # instructions do not branch except for the last one. - class BasicBlock - # This is the unique identifier for this basic block. - attr_reader :id - - # This is the index into the list of instructions where this block - # starts. - attr_reader :block_start - - # This is the set of instructions that this block contains. - attr_reader :insns - - # This is an array of basic blocks that are predecessors to this block. - attr_reader :predecessors - - # This is an array of basic blocks that are successors to this block. - attr_reader :successors - - def initialize(block_start, insns) - @id = "block_#{block_start}" - - @block_start = block_start - @insns = insns - - @predecessors = [] - @successors = [] - end - - # Yield each instruction in this basic block along with its index from - # the original instruction sequence. - def each_with_index(&block) - insns.each.with_index(block_start, &block) - end - - # This method is used to verify that the basic block is well formed. It - # checks that the only instruction in this basic block that branches is - # the last instruction. - def verify - insns[0...-1].each { |insn| raise unless insn.branch_targets.empty? } - end - end - # This class is responsible for creating a control flow graph from the # given instruction sequence. class Compiler @@ -186,22 +143,22 @@ def build_basic_blocks end end - # Connect the blocks by letting them know which blocks precede them and - # which blocks succeed them. + # Connect the blocks by letting them know which blocks are incoming and + # outgoing from each block. def connect_basic_blocks(blocks) blocks.each do |block_start, block| insn = block.insns.last insn.branch_targets.each do |branch_target| - block.successors << blocks.fetch(labels[branch_target]) + block.outgoing_blocks << blocks.fetch(labels[branch_target]) end if (insn.branch_targets.empty? && !insn.leaves?) || insn.falls_through? - block.successors << blocks.fetch(block_start + block.insns.length) + block.outgoing_blocks << blocks.fetch(block_start + block.insns.length) end - block.successors.each do |successor| - successor.predecessors << block + block.outgoing_blocks.each do |outgoing_block| + outgoing_block.incoming_blocks << block end end end diff --git a/lib/syntax_tree/yarv/data_flow_graph.rb b/lib/syntax_tree/yarv/data_flow_graph.rb index 295308bd..737518ce 100644 --- a/lib/syntax_tree/yarv/data_flow_graph.rb +++ b/lib/syntax_tree/yarv/data_flow_graph.rb @@ -26,138 +26,6 @@ def initialize(cfg, insn_flows, block_flows) @block_flows = block_flows end - def self.compile(cfg) - # First, create a data structure to encode data flow between - # instructions. - insn_flows = {} - cfg.insns.each_with_index do |insn, index| - insn_flows[index] = DataFlow.new - end - - # Next, create a data structure to encode data flow between basic - # blocks. - block_flows = {} - cfg.blocks.each do |block| - block_flows[block.id] = DataFlow.new - end - - # Now, discover the data flow within each basic block. Using an abstract - # stack, connect from consumers of data to the producers of that data. - cfg.blocks.each do |block| - block_flow = block_flows.fetch(block.id) - - stack = [] - stack_initial_depth = 0 - - # Go through each instruction in the block... - block.each_with_index do |insn, index| - insn_flow = insn_flows[index] - - # How many values will be missing from the local stack to run this - # instruction? - missing_stack_values = insn.pops - stack.size - - # For every value the instruction pops off the stack... - insn.pops.times do - # Was the value it pops off from another basic block? - if stack.empty? - # This is a basic block argument. - name = :"in_#{missing_stack_values - 1}" - - insn_flow.in.unshift(name) - block_flow.in.unshift(name) - - stack_initial_depth += 1 - missing_stack_values -= 1 - else - # Connect this consumer to the producer of the value. - insn_flow.in.unshift(stack.pop) - end - end - - # Record on our abstract stack that this instruction pushed - # this value onto the stack. - insn.pushes.times { stack << index } - end - - # Values that are left on the stack after going through all - # instructions are arguments to the basic block that we jump to. - stack.reverse_each.with_index do |producer, index| - block_flow.out << producer - insn_flows[producer].out << :"out_#{index}" - end - end - - # Go backwards and connect from producers to consumers. - cfg.insns.each_with_index do |insn, index| - # For every instruction that produced a value used in this - # instruction... - insn_flows[index].in.each do |producer| - # If it's actually another instruction and not a basic block - # argument... - if producer.is_a?(Integer) - # Record in the producing instruction that it produces a value - # used by this construction. - insn_flows[producer].out << index - end - end - end - - # Now, discover the data flow between basic blocks. - stack = [*cfg.blocks] - until stack.empty? - succ = stack.pop - succ_flow = block_flows.fetch(succ.id) - succ.predecessors.each do |pred| - pred_flow = block_flows.fetch(pred.id) - - # Does a predecessor block have fewer outputs than the successor - # has inputs? - if pred_flow.out.size < succ_flow.in.size - # If so then add arguments to pass data through from the - # predecessor's predecessors. - (succ_flow.in.size - pred_flow.out.size).times do |index| - name = :"pass_#{index}" - pred_flow.in.unshift(name) - pred_flow.out.unshift(name) - end - - # Since we modified the predecessor, add it back to the worklist - # so it'll be considered as a successor again, and propogate the - # global data flow back up the control flow graph. - stack << pred - end - end - end - - # Verify that we constructed the data flow graph correctly. Check that - # the first block has no arguments. - raise unless block_flows.fetch(cfg.blocks.first.id).in.empty? - - # Check all control flow edges between blocks pass the right number of - # arguments. - cfg.blocks.each do |pred| - pred_flow = block_flows.fetch(pred.id) - - if pred.successors.empty? - # With no successors, there should be no output arguments. - raise unless pred_flow.out.empty? - else - # Check with successor... - pred.successors.each do |succ| - succ_flow = block_flows.fetch(succ.id) - - # The predecessor should have as many output arguments as the - # success has input arguments. - raise unless pred_flow.out.size == succ_flow.in.size - end - end - end - - # Finally we can return the data flow graph. - new(cfg, insn_flows, block_flows) - end - def disasm fmt = Disassembler.new output = StringIO.new @@ -165,8 +33,9 @@ def disasm cfg.blocks.each do |block| output.print(block.id) - unless block.predecessors.empty? - output.print(" # from: #{block.predecessors.map(&:id).join(", ")}") + unless block.incoming_blocks.empty? + srcs = block.incoming_blocks.map(&:id) + output.print(" # from: #{srcs.join(", ")}") end output.puts @@ -198,9 +67,9 @@ def disasm output.puts end - successors = block.successors.map(&:id) - successors << "leaves" if block.insns.last.leaves? - output.puts(" # to: #{successors.join(", ")}") unless successors.empty? + dests = block.outgoing_blocks.map(&:id) + dests << "leaves" if block.insns.last.leaves? + output.puts(" # to: #{dests.join(", ")}") unless dests.empty? unless block_flow.out.empty? output.puts " # out: #{block_flow.out.join(", ")}" @@ -209,6 +78,159 @@ def disasm output.string end + + # Verify that we constructed the data flow graph correctly. + def verify + # Check that the first block has no arguments. + raise unless block_flows.fetch(cfg.blocks.first.id).in.empty? + + # Check all control flow edges between blocks pass the right number of + # arguments. + cfg.blocks.each do |block| + block_flow = block_flows.fetch(block.id) + + if block.outgoing_blocks.empty? + # With no outgoing blocks, there should be no output arguments. + raise unless block_flow.out.empty? + else + # Check with outgoing blocks... + block.outgoing_blocks.each do |outgoing_block| + outgoing_flow = block_flows.fetch(outgoing_block.id) + + # The block should have as many output arguments as the + # outgoing block has input arguments. + raise unless block_flow.out.size == outgoing_flow.in.size + end + end + end + end + + def self.compile(cfg) + Compiler.new(cfg).compile + end + + # This class is responsible for creating a data flow graph from the given + # control flow graph. + class Compiler + attr_reader :cfg, :insn_flows, :block_flows + + def initialize(cfg) + @cfg = cfg + + # This data structure will hold the data flow between instructions + # within individual basic blocks. + @insn_flows = {} + cfg.insns.each_with_index do |insn, index| + @insn_flows[index] = DataFlow.new + end + + # This data structure will hold the data flow between basic blocks. + @block_flows = {} + cfg.blocks.each do |block| + @block_flows[block.id] = DataFlow.new + end + end + + def compile + find_local_flow + find_global_flow + DataFlowGraph.new(cfg, insn_flows, block_flows).tap(&:verify) + end + + private + + # Find the data flow within each basic block. Using an abstract stack, + # connect from consumers of data to the producers of that data. + def find_local_flow + cfg.blocks.each do |block| + block_flow = block_flows.fetch(block.id) + stack = [] + + # Go through each instruction in the block... + block.each_with_index do |insn, index| + insn_flow = insn_flows[index] + + # How many values will be missing from the local stack to run this + # instruction? + missing = insn.pops - stack.size + + # For every value the instruction pops off the stack... + insn.pops.times do + # Was the value it pops off from another basic block? + if stack.empty? + # This is a basic block argument. + missing -= 1 + name = :"in_#{missing}" + + insn_flow.in.unshift(name) + block_flow.in.unshift(name) + else + # Connect this consumer to the producer of the value. + insn_flow.in.unshift(stack.pop) + end + end + + # Record on our abstract stack that this instruction pushed + # this value onto the stack. + insn.pushes.times { stack << index } + end + + # Values that are left on the stack after going through all + # instructions are arguments to the basic block that we jump to. + stack.reverse_each.with_index do |producer, index| + block_flow.out << producer + insn_flows[producer].out << :"out_#{index}" + end + end + + # Go backwards and connect from producers to consumers. + cfg.insns.each_with_index do |insn, index| + # For every instruction that produced a value used in this + # instruction... + insn_flows[index].in.each do |producer| + # If it's actually another instruction and not a basic block + # argument... + if producer.is_a?(Integer) + # Record in the producing instruction that it produces a value + # used by this construction. + insn_flows[producer].out << index + end + end + end + end + + # Find the data that flows between basic blocks. + def find_global_flow + stack = [*cfg.blocks] + + until stack.empty? + block = stack.pop + block_flow = block_flows.fetch(block.id) + + block.incoming_blocks.each do |incoming_block| + incoming_flow = block_flows.fetch(incoming_block.id) + + # Does a predecessor block have fewer outputs than the successor + # has inputs? + if incoming_flow.out.size < block_flow.in.size + # If so then add arguments to pass data through from the + # incoming block's incoming blocks. + (block_flow.in.size - incoming_flow.out.size).times do |index| + name = :"pass_#{index}" + + incoming_flow.in.unshift(name) + incoming_flow.out.unshift(name) + end + + # Since we modified the incoming block, add it back to the stack + # so it'll be considered as an outgoing block again, and + # propogate the global data flow back up the control flow graph. + stack << incoming_block + end + end + end + end + end end end end From f600b0694e2c64bb4c6ce7d0d29d60533fdc1ab6 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 2 Feb 2023 20:07:06 -0500 Subject: [PATCH 354/536] Properly use the disassembler for the cfg --- lib/syntax_tree/yarv/control_flow_graph.rb | 22 ++-- lib/syntax_tree/yarv/disassembler.rb | 112 ++++++++++--------- lib/syntax_tree/yarv/instruction_sequence.rb | 7 +- test/yarv_test.rb | 20 ++-- 4 files changed, 82 insertions(+), 79 deletions(-) diff --git a/lib/syntax_tree/yarv/control_flow_graph.rb b/lib/syntax_tree/yarv/control_flow_graph.rb index 27df308e..3b3f9b82 100644 --- a/lib/syntax_tree/yarv/control_flow_graph.rb +++ b/lib/syntax_tree/yarv/control_flow_graph.rb @@ -33,32 +33,28 @@ def initialize(iseq, insns, blocks) end def disasm - fmt = Disassembler.new - output = StringIO.new - output.puts "== cfg #{iseq.name}" + fmt = Disassembler.new(iseq) + fmt.output.puts "== cfg #{iseq.name}" blocks.each do |block| - output.print(block.id) + fmt.output.print(block.id) unless block.incoming_blocks.empty? - output.print(" # from: #{block.incoming_blocks.map(&:id).join(", ")}") + fmt.output.print(" # from: #{block.incoming_blocks.map(&:id).join(", ")}") end - output.puts + fmt.output.puts - block.insns.each do |insn| - output.print(" ") - output.puts(insn.disasm(fmt)) - end + fmt.with_prefix(" ") { fmt.format_insns!(block.insns) } dests = block.outgoing_blocks.map(&:id) dests << "leaves" if block.insns.last.leaves? - output.print(" # to: #{dests.join(", ")}") unless dests.empty? + fmt.output.print(" # to: #{dests.join(", ")}") unless dests.empty? - output.puts + fmt.output.puts end - output.string + fmt.string end # This method is used to verify that the control flow graph is well diff --git a/lib/syntax_tree/yarv/disassembler.rb b/lib/syntax_tree/yarv/disassembler.rb index d303bcb7..0b445e02 100644 --- a/lib/syntax_tree/yarv/disassembler.rb +++ b/lib/syntax_tree/yarv/disassembler.rb @@ -4,15 +4,16 @@ module SyntaxTree module YARV class Disassembler attr_reader :output, :queue + attr_reader :current_prefix attr_accessor :current_iseq - def initialize + def initialize(current_iseq = nil) @output = StringIO.new @queue = [] @current_prefix = "" - @current_iseq = nil + @current_iseq = current_iseq end ######################################################################## @@ -97,16 +98,69 @@ def object(value) end ######################################################################## - # Main entrypoint + # Entrypoints ######################################################################## + def string + output.string + end + def format! while (@current_iseq = queue.shift) output << "\n" if output.pos > 0 format_iseq(@current_iseq) end + end - output.string + def format_insns!(insns, length = 0) + events = [] + lines = [] + + insns.each do |insn| + case insn + when Integer + lines << insn + when Symbol + events << event(insn) + when InstructionSequence::Label + # skip + else + output << "#{current_prefix}%04d " % length + + disasm = insn.disasm(self) + output << disasm + + if lines.any? + output << " " * (65 - disasm.length) if disasm.length < 65 + elsif events.any? + output << " " * (39 - disasm.length) if disasm.length < 39 + end + + if lines.any? + output << "(%4d)" % lines.last + lines.clear + end + + if events.any? + output << "[#{events.join}]" + events.clear + end + + output << "\n" + length += insn.length + end + end + end + + def with_prefix(value) + previous = @current_prefix + + begin + @current_prefix = value + yield + ensure + @current_prefix = previous + end end private @@ -157,55 +211,7 @@ def format_iseq(iseq) output << "#{current_prefix}#{locals.join(" ")}\n" end - length = 0 - events = [] - lines = [] - - iseq.insns.each do |insn| - case insn - when Integer - lines << insn - when Symbol - events << event(insn) - when InstructionSequence::Label - # skip - else - output << "#{current_prefix}%04d " % length - - disasm = insn.disasm(self) - output << disasm - - if lines.any? - output << " " * (65 - disasm.length) if disasm.length < 65 - elsif events.any? - output << " " * (39 - disasm.length) if disasm.length < 39 - end - - if lines.any? - output << "(%4d)" % lines.last - lines.clear - end - - if events.any? - output << "[#{events.join}]" - events.clear - end - - output << "\n" - length += insn.length - end - end - end - - def with_prefix(value) - previous = @current_prefix - - begin - @current_prefix = value - yield - ensure - @current_prefix = previous - end + format_insns!(iseq.insns) end end end diff --git a/lib/syntax_tree/yarv/instruction_sequence.rb b/lib/syntax_tree/yarv/instruction_sequence.rb index 6aa7279e..1281eba4 100644 --- a/lib/syntax_tree/yarv/instruction_sequence.rb +++ b/lib/syntax_tree/yarv/instruction_sequence.rb @@ -270,9 +270,10 @@ def to_a end def disasm - disassembler = Disassembler.new - disassembler.enqueue(self) - disassembler.format! + fmt = Disassembler.new + fmt.enqueue(self) + fmt.format! + fmt.string end # This method converts our linked list of instructions into a final array diff --git a/test/yarv_test.rb b/test/yarv_test.rb index e37afb63..91147dc3 100644 --- a/test/yarv_test.rb +++ b/test/yarv_test.rb @@ -305,22 +305,22 @@ def test_cfg assert_equal(<<~CFG, cfg.disasm) == cfg block_0 - putobject 100 - putobject 14 - putobject_INT2FIX_0_ - opt_lt - branchunless 13 + 0000 putobject 100 + 0002 putobject 14 + 0004 putobject_INT2FIX_0_ + 0005 opt_lt + 0007 branchunless 13 # to: block_7, block_5 block_5 # from: block_0 - putobject -1 - jump 14 + 0000 putobject -1 + 0002 jump 14 # to: block_8 block_7 # from: block_0 - putobject_INT2FIX_1_ + 0000 putobject_INT2FIX_1_ # to: block_8 block_8 # from: block_5, block_7 - opt_plus - leave + 0000 opt_plus + 0002 leave # to: leaves CFG end From d66c977eb37d7f01f3221fdc0bcde086e56e1b8e Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 2 Feb 2023 20:43:29 -0500 Subject: [PATCH 355/536] Use length for offsets to make it more readable --- lib/syntax_tree/yarv/basic_block.rb | 10 +++- lib/syntax_tree/yarv/control_flow_graph.rb | 69 +++++++++++++--------- lib/syntax_tree/yarv/data_flow_graph.rb | 20 +++---- test/yarv_test.rb | 57 +++++++++--------- 4 files changed, 90 insertions(+), 66 deletions(-) diff --git a/lib/syntax_tree/yarv/basic_block.rb b/lib/syntax_tree/yarv/basic_block.rb index 774a4c00..6798a092 100644 --- a/lib/syntax_tree/yarv/basic_block.rb +++ b/lib/syntax_tree/yarv/basic_block.rb @@ -32,8 +32,14 @@ def initialize(block_start, insns) # Yield each instruction in this basic block along with its index from the # original instruction sequence. - def each_with_index(&block) - insns.each.with_index(block_start, &block) + def each_with_length + return enum_for(:each_with_length) unless block_given? + + length = block_start + insns.each do |insn| + yield insn, length + length += insn.length + end end # This method is used to verify that the basic block is well formed. It diff --git a/lib/syntax_tree/yarv/control_flow_graph.rb b/lib/syntax_tree/yarv/control_flow_graph.rb index 3b3f9b82..bcf9f26e 100644 --- a/lib/syntax_tree/yarv/control_flow_graph.rb +++ b/lib/syntax_tree/yarv/control_flow_graph.rb @@ -34,24 +34,23 @@ def initialize(iseq, insns, blocks) def disasm fmt = Disassembler.new(iseq) - fmt.output.puts "== cfg #{iseq.name}" + fmt.output.print("== cfg: #:1 ") + fmt.output.puts("(#{iseq.line},0)-(#{iseq.line},0)>") blocks.each do |block| - fmt.output.print(block.id) - - unless block.incoming_blocks.empty? - fmt.output.print(" # from: #{block.incoming_blocks.map(&:id).join(", ")}") - end - - fmt.output.puts - - fmt.with_prefix(" ") { fmt.format_insns!(block.insns) } + fmt.output.puts(block.id) + fmt.with_prefix(" ") do + unless block.incoming_blocks.empty? + from = block.incoming_blocks.map(&:id).join(", ") + fmt.output.puts("#{fmt.current_prefix}== from: #{from}") + end - dests = block.outgoing_blocks.map(&:id) - dests << "leaves" if block.insns.last.leaves? - fmt.output.print(" # to: #{dests.join(", ")}") unless dests.empty? + fmt.format_insns!(block.insns, block.block_start) - fmt.output.puts + to = block.outgoing_blocks.map(&:id) + to << "leaves" if block.insns.last.leaves? + fmt.output.puts("#{fmt.current_prefix}== to: #{to.join(", ")}") + end end fmt.string @@ -71,23 +70,34 @@ def self.compile(iseq) # This class is responsible for creating a control flow graph from the # given instruction sequence. class Compiler - attr_reader :iseq, :labels, :insns + # This is the instruction sequence that is being compiled. + attr_reader :iseq + + # This is a hash of indices in the YARV instruction sequence that point + # to their corresponding instruction. + attr_reader :insns + + # This is a hash of labels that point to their corresponding index into + # the YARV instruction sequence. Note that this is not the same as the + # index into the list of instructions on the instruction sequence + # object. Instead, this is the index into the C array, so it includes + # operands. + attr_reader :labels def initialize(iseq) @iseq = iseq - # We need to find all of the instructions that immediately follow - # labels so that when we are looking at instructions that branch we - # know where they branch to. + @insns = {} @labels = {} - @insns = [] + length = 0 iseq.insns.each do |insn| case insn when Instruction - @insns << insn + @insns[length] = insn + length += insn.length when InstructionSequence::Label - @labels[insn] = @insns.length + @labels[insn] = length end end end @@ -111,7 +121,7 @@ def compile def find_basic_block_starts block_starts = Set.new([0]) - insns.each_with_index do |insn, index| + insns.each do |index, insn| branch_targets = insn.branch_targets if branch_targets.any? @@ -119,7 +129,7 @@ def find_basic_block_starts block_starts.add(labels[branch_target]) end - block_starts.add(index + 1) if insn.falls_through? + block_starts.add(index + insn.length) if insn.falls_through? end end @@ -131,10 +141,14 @@ def find_basic_block_starts def build_basic_blocks block_starts = find_basic_block_starts - block_starts.each_with_index.to_h do |block_start, block_index| - block_end = (block_starts[(block_index + 1)..] + [insns.length]).min - block_insns = insns[block_start...block_end] + length = 0 + blocks = + iseq.insns.grep(Instruction).slice_after do |insn| + length += insn.length + block_starts.include?(length) + end + block_starts.zip(blocks).to_h do |block_start, block_insns| [block_start, BasicBlock.new(block_start, block_insns)] end end @@ -150,7 +164,8 @@ def connect_basic_blocks(blocks) end if (insn.branch_targets.empty? && !insn.leaves?) || insn.falls_through? - block.outgoing_blocks << blocks.fetch(block_start + block.insns.length) + fall_through_start = block_start + block.insns.sum(&:length) + block.outgoing_blocks << blocks.fetch(fall_through_start) end block.outgoing_blocks.each do |outgoing_block| diff --git a/lib/syntax_tree/yarv/data_flow_graph.rb b/lib/syntax_tree/yarv/data_flow_graph.rb index 737518ce..670e0daf 100644 --- a/lib/syntax_tree/yarv/data_flow_graph.rb +++ b/lib/syntax_tree/yarv/data_flow_graph.rb @@ -44,11 +44,11 @@ def disasm output.puts " # in: #{block_flow.in.join(", ")}" end - block.each_with_index do |insn, index| + block.each_with_length do |insn, length| output.print(" ") output.print(insn.disasm(fmt)) - insn_flow = insn_flows[index] + insn_flow = insn_flows[length] if insn_flow.in.empty? && insn_flow.out.empty? output.puts next @@ -120,8 +120,8 @@ def initialize(cfg) # This data structure will hold the data flow between instructions # within individual basic blocks. @insn_flows = {} - cfg.insns.each_with_index do |insn, index| - @insn_flows[index] = DataFlow.new + cfg.insns.each_key do |length| + @insn_flows[length] = DataFlow.new end # This data structure will hold the data flow between basic blocks. @@ -147,8 +147,8 @@ def find_local_flow stack = [] # Go through each instruction in the block... - block.each_with_index do |insn, index| - insn_flow = insn_flows[index] + block.each_with_length do |insn, length| + insn_flow = insn_flows[length] # How many values will be missing from the local stack to run this # instruction? @@ -172,7 +172,7 @@ def find_local_flow # Record on our abstract stack that this instruction pushed # this value onto the stack. - insn.pushes.times { stack << index } + insn.pushes.times { stack << length } end # Values that are left on the stack after going through all @@ -184,16 +184,16 @@ def find_local_flow end # Go backwards and connect from producers to consumers. - cfg.insns.each_with_index do |insn, index| + cfg.insns.each_key do |length| # For every instruction that produced a value used in this # instruction... - insn_flows[index].in.each do |producer| + insn_flows[length].in.each do |producer| # If it's actually another instruction and not a basic block # argument... if producer.is_a?(Integer) # Record in the producing instruction that it produces a value # used by this construction. - insn_flows[producer].out << index + insn_flows[producer].out << length end end end diff --git a/test/yarv_test.rb b/test/yarv_test.rb index 91147dc3..7a998fa4 100644 --- a/test/yarv_test.rb +++ b/test/yarv_test.rb @@ -303,25 +303,28 @@ def test_cfg cfg = SyntaxTree::YARV::ControlFlowGraph.compile(iseq) assert_equal(<<~CFG, cfg.disasm) - == cfg + == cfg: #@:1 (1,0)-(1,0)> block_0 0000 putobject 100 0002 putobject 14 0004 putobject_INT2FIX_0_ 0005 opt_lt 0007 branchunless 13 - # to: block_7, block_5 - block_5 # from: block_0 - 0000 putobject -1 - 0002 jump 14 - # to: block_8 - block_7 # from: block_0 - 0000 putobject_INT2FIX_1_ - # to: block_8 - block_8 # from: block_5, block_7 - 0000 opt_plus - 0002 leave - # to: leaves + == to: block_13, block_9 + block_9 + == from: block_0 + 0009 putobject -1 + 0011 jump 14 + == to: block_14 + block_13 + == from: block_0 + 0013 putobject_INT2FIX_1_ + == to: block_14 + block_14 + == from: block_9, block_13 + 0014 opt_plus + 0016 leave + == to: leaves CFG end @@ -335,27 +338,27 @@ def test_dfg == dfg block_0 putobject 100 # out: out_0 - putobject 14 # out: 3 - putobject_INT2FIX_0_ # out: 3 - opt_lt # in: 1, 2; out: 4 - branchunless 13 # in: 3 - # to: block_7, block_5 + putobject 14 # out: 5 + putobject_INT2FIX_0_ # out: 5 + opt_lt # in: 2, 4; out: 7 + branchunless 13 # in: 5 + # to: block_13, block_9 # out: 0 - block_5 # from: block_0 + block_9 # from: block_0 # in: pass_0 putobject -1 # out: out_0 jump 14 - # to: block_8 - # out: pass_0, 5 - block_7 # from: block_0 + # to: block_14 + # out: pass_0, 9 + block_13 # from: block_0 # in: pass_0 putobject_INT2FIX_1_ # out: out_0 - # to: block_8 - # out: pass_0, 7 - block_8 # from: block_5, block_7 + # to: block_14 + # out: pass_0, 13 + block_14 # from: block_9, block_13 # in: in_0, in_1 - opt_plus # in: in_0, in_1; out: 9 - leave # in: 8 + opt_plus # in: in_0, in_1; out: 16 + leave # in: 14 # to: leaves DFG end From 7d1cf1ce3aba3bc1a1251637304f298cb9f84fae Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 2 Feb 2023 20:55:33 -0500 Subject: [PATCH 356/536] Properly use disassembler for DFG --- lib/syntax_tree/yarv/control_flow_graph.rb | 3 +- lib/syntax_tree/yarv/data_flow_graph.rb | 68 +++++++++----------- lib/syntax_tree/yarv/disassembler.rb | 12 ++-- lib/syntax_tree/yarv/instruction_sequence.rb | 4 ++ test/yarv_test.rb | 51 ++++++++------- 5 files changed, 67 insertions(+), 71 deletions(-) diff --git a/lib/syntax_tree/yarv/control_flow_graph.rb b/lib/syntax_tree/yarv/control_flow_graph.rb index bcf9f26e..ef779c54 100644 --- a/lib/syntax_tree/yarv/control_flow_graph.rb +++ b/lib/syntax_tree/yarv/control_flow_graph.rb @@ -34,8 +34,7 @@ def initialize(iseq, insns, blocks) def disasm fmt = Disassembler.new(iseq) - fmt.output.print("== cfg: #:1 ") - fmt.output.puts("(#{iseq.line},0)-(#{iseq.line},0)>") + fmt.output.puts("== cfg: #{iseq.inspect}") blocks.each do |block| fmt.output.puts(block.id) diff --git a/lib/syntax_tree/yarv/data_flow_graph.rb b/lib/syntax_tree/yarv/data_flow_graph.rb index 670e0daf..09ba84a4 100644 --- a/lib/syntax_tree/yarv/data_flow_graph.rb +++ b/lib/syntax_tree/yarv/data_flow_graph.rb @@ -27,56 +27,48 @@ def initialize(cfg, insn_flows, block_flows) end def disasm - fmt = Disassembler.new - output = StringIO.new - output.puts "== dfg #{cfg.iseq.name}" + fmt = Disassembler.new(cfg.iseq) + fmt.output.puts("== dfg: #{cfg.iseq.inspect}") cfg.blocks.each do |block| - output.print(block.id) - unless block.incoming_blocks.empty? - srcs = block.incoming_blocks.map(&:id) - output.print(" # from: #{srcs.join(", ")}") - end - output.puts - - block_flow = block_flows.fetch(block.id) - unless block_flow.in.empty? - output.puts " # in: #{block_flow.in.join(", ")}" - end - - block.each_with_length do |insn, length| - output.print(" ") - output.print(insn.disasm(fmt)) - - insn_flow = insn_flows[length] - if insn_flow.in.empty? && insn_flow.out.empty? - output.puts - next + fmt.output.puts(block.id) + fmt.with_prefix(" ") do + unless block.incoming_blocks.empty? + from = block.incoming_blocks.map(&:id).join(", ") + fmt.output.puts("#{fmt.current_prefix}== from: #{from}") end - output.print(" # ") - unless insn_flow.in.empty? - output.print("in: #{insn_flow.in.join(", ")}") - output.print("; ") unless insn_flow.out.empty? + block_flow = block_flows.fetch(block.id) + unless block_flow.in.empty? + fmt.output.puts("#{fmt.current_prefix}== in: #{block_flow.in.join(", ")}") end - unless insn_flow.out.empty? - output.print("out: #{insn_flow.out.join(", ")}") + fmt.format_insns!(block.insns, block.block_start) do |insn, length| + insn_flow = insn_flows[length] + next if insn_flow.in.empty? && insn_flow.out.empty? + + fmt.output.print(" # ") + unless insn_flow.in.empty? + fmt.output.print("in: #{insn_flow.in.join(", ")}") + fmt.output.print("; ") unless insn_flow.out.empty? + end + + unless insn_flow.out.empty? + fmt.output.print("out: #{insn_flow.out.join(", ")}") + end end - output.puts - end - - dests = block.outgoing_blocks.map(&:id) - dests << "leaves" if block.insns.last.leaves? - output.puts(" # to: #{dests.join(", ")}") unless dests.empty? + to = block.outgoing_blocks.map(&:id) + to << "leaves" if block.insns.last.leaves? + fmt.output.puts("#{fmt.current_prefix}== to: #{to.join(", ")}") - unless block_flow.out.empty? - output.puts " # out: #{block_flow.out.join(", ")}" + unless block_flow.out.empty? + fmt.output.puts("#{fmt.current_prefix}== out: #{block_flow.out.join(", ")}") + end end end - output.string + fmt.string end # Verify that we constructed the data flow graph correctly. diff --git a/lib/syntax_tree/yarv/disassembler.rb b/lib/syntax_tree/yarv/disassembler.rb index 0b445e02..8b86851e 100644 --- a/lib/syntax_tree/yarv/disassembler.rb +++ b/lib/syntax_tree/yarv/disassembler.rb @@ -146,6 +146,10 @@ def format_insns!(insns, length = 0) events.clear end + # A hook here to allow for custom formatting of instructions after + # the main body has been processed. + yield insn, length if block_given? + output << "\n" length += insn.length end @@ -166,13 +170,7 @@ def with_prefix(value) private def format_iseq(iseq) - output << "#{current_prefix}== disasm: " - output << "#:1 " - - location = Location.fixed(line: iseq.line, char: 0, column: 0) - output << "(#{location.start_line},#{location.start_column})-" - output << "(#{location.end_line},#{location.end_column})" - output << "> " + output << "#{current_prefix}== disasm: #{iseq.inspect} " if iseq.catch_table.any? output << "(catch: TRUE)\n" diff --git a/lib/syntax_tree/yarv/instruction_sequence.rb b/lib/syntax_tree/yarv/instruction_sequence.rb index 1281eba4..83453837 100644 --- a/lib/syntax_tree/yarv/instruction_sequence.rb +++ b/lib/syntax_tree/yarv/instruction_sequence.rb @@ -276,6 +276,10 @@ def disasm fmt.string end + def inspect + "#:1 (#{line},#{0})-(#{line},#{0})>" + end + # This method converts our linked list of instructions into a final array # and performs any other compilation steps necessary. def compile! diff --git a/test/yarv_test.rb b/test/yarv_test.rb index 7a998fa4..5ac37504 100644 --- a/test/yarv_test.rb +++ b/test/yarv_test.rb @@ -335,31 +335,34 @@ def test_dfg dfg = SyntaxTree::YARV::DataFlowGraph.compile(cfg) assert_equal(<<~DFG, dfg.disasm) - == dfg + == dfg: #@:1 (1,0)-(1,0)> block_0 - putobject 100 # out: out_0 - putobject 14 # out: 5 - putobject_INT2FIX_0_ # out: 5 - opt_lt # in: 2, 4; out: 7 - branchunless 13 # in: 5 - # to: block_13, block_9 - # out: 0 - block_9 # from: block_0 - # in: pass_0 - putobject -1 # out: out_0 - jump 14 - # to: block_14 - # out: pass_0, 9 - block_13 # from: block_0 - # in: pass_0 - putobject_INT2FIX_1_ # out: out_0 - # to: block_14 - # out: pass_0, 13 - block_14 # from: block_9, block_13 - # in: in_0, in_1 - opt_plus # in: in_0, in_1; out: 16 - leave # in: 14 - # to: leaves + 0000 putobject 100 # out: out_0 + 0002 putobject 14 # out: 5 + 0004 putobject_INT2FIX_0_ # out: 5 + 0005 opt_lt # in: 2, 4; out: 7 + 0007 branchunless 13 # in: 5 + == to: block_13, block_9 + == out: 0 + block_9 + == from: block_0 + == in: pass_0 + 0009 putobject -1 # out: out_0 + 0011 jump 14 + == to: block_14 + == out: pass_0, 9 + block_13 + == from: block_0 + == in: pass_0 + 0013 putobject_INT2FIX_1_ # out: out_0 + == to: block_14 + == out: pass_0, 13 + block_14 + == from: block_9, block_13 + == in: in_0, in_1 + 0014 opt_plus # in: in_0, in_1; out: 16 + 0016 leave # in: 14 + == to: leaves DFG end From 28c5a4ac92745c26590794366f014742bc02eebd Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 2 Feb 2023 21:01:49 -0500 Subject: [PATCH 357/536] Various formatting for CFG and DFG --- lib/syntax_tree/yarv/control_flow_graph.rb | 30 +++++++------ lib/syntax_tree/yarv/data_flow_graph.rb | 45 +++++++++----------- lib/syntax_tree/yarv/disassembler.rb | 2 +- lib/syntax_tree/yarv/instruction_sequence.rb | 2 +- 4 files changed, 41 insertions(+), 38 deletions(-) diff --git a/lib/syntax_tree/yarv/control_flow_graph.rb b/lib/syntax_tree/yarv/control_flow_graph.rb index ef779c54..fb8f97f3 100644 --- a/lib/syntax_tree/yarv/control_flow_graph.rb +++ b/lib/syntax_tree/yarv/control_flow_graph.rb @@ -38,17 +38,17 @@ def disasm blocks.each do |block| fmt.output.puts(block.id) - fmt.with_prefix(" ") do + fmt.with_prefix(" ") do |prefix| unless block.incoming_blocks.empty? - from = block.incoming_blocks.map(&:id).join(", ") - fmt.output.puts("#{fmt.current_prefix}== from: #{from}") + from = block.incoming_blocks.map(&:id) + fmt.output.puts("#{prefix}== from: #{from.join(", ")}") end fmt.format_insns!(block.insns, block.block_start) to = block.outgoing_blocks.map(&:id) to << "leaves" if block.insns.last.leaves? - fmt.output.puts("#{fmt.current_prefix}== to: #{to.join(", ")}") + fmt.output.puts("#{prefix}== to: #{to.join(", ")}") end end @@ -142,14 +142,19 @@ def build_basic_blocks length = 0 blocks = - iseq.insns.grep(Instruction).slice_after do |insn| - length += insn.length - block_starts.include?(length) - end + iseq + .insns + .grep(Instruction) + .slice_after do |insn| + length += insn.length + block_starts.include?(length) + end - block_starts.zip(blocks).to_h do |block_start, block_insns| - [block_start, BasicBlock.new(block_start, block_insns)] - end + block_starts + .zip(blocks) + .to_h do |block_start, block_insns| + [block_start, BasicBlock.new(block_start, block_insns)] + end end # Connect the blocks by letting them know which blocks are incoming and @@ -162,7 +167,8 @@ def connect_basic_blocks(blocks) block.outgoing_blocks << blocks.fetch(labels[branch_target]) end - if (insn.branch_targets.empty? && !insn.leaves?) || insn.falls_through? + if (insn.branch_targets.empty? && !insn.leaves?) || + insn.falls_through? fall_through_start = block_start + block.insns.sum(&:length) block.outgoing_blocks << blocks.fetch(fall_through_start) end diff --git a/lib/syntax_tree/yarv/data_flow_graph.rb b/lib/syntax_tree/yarv/data_flow_graph.rb index 09ba84a4..614d1233 100644 --- a/lib/syntax_tree/yarv/data_flow_graph.rb +++ b/lib/syntax_tree/yarv/data_flow_graph.rb @@ -32,27 +32,27 @@ def disasm cfg.blocks.each do |block| fmt.output.puts(block.id) - fmt.with_prefix(" ") do + fmt.with_prefix(" ") do |prefix| unless block.incoming_blocks.empty? - from = block.incoming_blocks.map(&:id).join(", ") - fmt.output.puts("#{fmt.current_prefix}== from: #{from}") + from = block.incoming_blocks.map(&:id) + fmt.output.puts("#{prefix}== from: #{from.join(", ")}") end block_flow = block_flows.fetch(block.id) unless block_flow.in.empty? - fmt.output.puts("#{fmt.current_prefix}== in: #{block_flow.in.join(", ")}") + fmt.output.puts("#{prefix}== in: #{block_flow.in.join(", ")}") end - fmt.format_insns!(block.insns, block.block_start) do |insn, length| + fmt.format_insns!(block.insns, block.block_start) do |_, length| insn_flow = insn_flows[length] next if insn_flow.in.empty? && insn_flow.out.empty? - + fmt.output.print(" # ") unless insn_flow.in.empty? fmt.output.print("in: #{insn_flow.in.join(", ")}") fmt.output.print("; ") unless insn_flow.out.empty? end - + unless insn_flow.out.empty? fmt.output.print("out: #{insn_flow.out.join(", ")}") end @@ -60,11 +60,11 @@ def disasm to = block.outgoing_blocks.map(&:id) to << "leaves" if block.insns.last.leaves? - fmt.output.puts("#{fmt.current_prefix}== to: #{to.join(", ")}") + fmt.output.puts("#{prefix}== to: #{to.join(", ")}") unless block_flow.out.empty? - fmt.output.puts("#{fmt.current_prefix}== out: #{block_flow.out.join(", ")}") - end + fmt.output.puts("#{prefix}== out: #{block_flow.out.join(", ")}") + end end end @@ -104,23 +104,20 @@ def self.compile(cfg) # This class is responsible for creating a data flow graph from the given # control flow graph. class Compiler - attr_reader :cfg, :insn_flows, :block_flows + # This is the control flow graph that is being compiled. + attr_reader :cfg - def initialize(cfg) - @cfg = cfg + # This data structure will hold the data flow between instructions + # within individual basic blocks. + attr_reader :insn_flows - # This data structure will hold the data flow between instructions - # within individual basic blocks. - @insn_flows = {} - cfg.insns.each_key do |length| - @insn_flows[length] = DataFlow.new - end + # This data structure will hold the data flow between basic blocks. + attr_reader :block_flows - # This data structure will hold the data flow between basic blocks. - @block_flows = {} - cfg.blocks.each do |block| - @block_flows[block.id] = DataFlow.new - end + def initialize(cfg) + @cfg = cfg + @insn_flows = cfg.insns.to_h { |length, _| [length, DataFlow.new] } + @block_flows = cfg.blocks.to_h { |block| [block.id, DataFlow.new] } end def compile diff --git a/lib/syntax_tree/yarv/disassembler.rb b/lib/syntax_tree/yarv/disassembler.rb index 8b86851e..7756d125 100644 --- a/lib/syntax_tree/yarv/disassembler.rb +++ b/lib/syntax_tree/yarv/disassembler.rb @@ -161,7 +161,7 @@ def with_prefix(value) begin @current_prefix = value - yield + yield value ensure @current_prefix = previous end diff --git a/lib/syntax_tree/yarv/instruction_sequence.rb b/lib/syntax_tree/yarv/instruction_sequence.rb index 83453837..45fc6121 100644 --- a/lib/syntax_tree/yarv/instruction_sequence.rb +++ b/lib/syntax_tree/yarv/instruction_sequence.rb @@ -277,7 +277,7 @@ def disasm end def inspect - "#:1 (#{line},#{0})-(#{line},#{0})>" + "#:1 (#{line},0)-(#{line},0)>" end # This method converts our linked list of instructions into a final array From 5526f399e81e7ec418a4a667e7c86d0082de9b1f Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 2 Feb 2023 21:13:00 -0500 Subject: [PATCH 358/536] Split out calldata into its own file --- lib/syntax_tree.rb | 1 + lib/syntax_tree/yarv/calldata.rb | 91 ++++++++++++++++++++++++++++ lib/syntax_tree/yarv/disassembler.rb | 25 +------- lib/syntax_tree/yarv/instructions.rb | 61 ------------------- 4 files changed, 93 insertions(+), 85 deletions(-) create mode 100644 lib/syntax_tree/yarv/calldata.rb diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index e0e2a6be..ade9ff5e 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -31,6 +31,7 @@ require_relative "syntax_tree/yarv" require_relative "syntax_tree/yarv/basic_block" require_relative "syntax_tree/yarv/bf" +require_relative "syntax_tree/yarv/calldata" require_relative "syntax_tree/yarv/compiler" require_relative "syntax_tree/yarv/control_flow_graph" require_relative "syntax_tree/yarv/data_flow_graph" diff --git a/lib/syntax_tree/yarv/calldata.rb b/lib/syntax_tree/yarv/calldata.rb new file mode 100644 index 00000000..fadea61b --- /dev/null +++ b/lib/syntax_tree/yarv/calldata.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +module SyntaxTree + module YARV + # This is an operand to various YARV instructions that represents the + # information about a specific call site. + class CallData + CALL_ARGS_SPLAT = 1 << 0 + CALL_ARGS_BLOCKARG = 1 << 1 + CALL_FCALL = 1 << 2 + CALL_VCALL = 1 << 3 + CALL_ARGS_SIMPLE = 1 << 4 + CALL_BLOCKISEQ = 1 << 5 + CALL_KWARG = 1 << 6 + CALL_KW_SPLAT = 1 << 7 + CALL_TAILCALL = 1 << 8 + CALL_SUPER = 1 << 9 + CALL_ZSUPER = 1 << 10 + CALL_OPT_SEND = 1 << 11 + CALL_KW_SPLAT_MUT = 1 << 12 + + attr_reader :method, :argc, :flags, :kw_arg + + def initialize( + method, + argc = 0, + flags = CallData::CALL_ARGS_SIMPLE, + kw_arg = nil + ) + @method = method + @argc = argc + @flags = flags + @kw_arg = kw_arg + end + + def flag?(mask) + (flags & mask) > 0 + end + + def to_h + result = { mid: method, flag: flags, orig_argc: argc } + result[:kw_arg] = kw_arg if kw_arg + result + end + + def inspect + names = [] + names << :ARGS_SPLAT if flag?(CALL_ARGS_SPLAT) + names << :ARGS_BLOCKARG if flag?(CALL_ARGS_BLOCKARG) + names << :FCALL if flag?(CALL_FCALL) + names << :VCALL if flag?(CALL_VCALL) + names << :ARGS_SIMPLE if flag?(CALL_ARGS_SIMPLE) + names << :BLOCKISEQ if flag?(CALL_BLOCKISEQ) + names << :KWARG if flag?(CALL_KWARG) + names << :KW_SPLAT if flag?(CALL_KW_SPLAT) + names << :TAILCALL if flag?(CALL_TAILCALL) + names << :SUPER if flag?(CALL_SUPER) + names << :ZSUPER if flag?(CALL_ZSUPER) + names << :OPT_SEND if flag?(CALL_OPT_SEND) + names << :KW_SPLAT_MUT if flag?(CALL_KW_SPLAT_MUT) + + parts = [] + parts << "mid:#{method}" if method + parts << "argc:#{argc}" + parts << "kw:[#{kw_arg.join(", ")}]" if kw_arg + parts << names.join("|") if names.any? + + "" + end + + def self.from(serialized) + new( + serialized[:mid], + serialized[:orig_argc], + serialized[:flag], + serialized[:kw_arg] + ) + end + end + + # A convenience method for creating a CallData object. + def self.calldata( + method, + argc = 0, + flags = CallData::CALL_ARGS_SIMPLE, + kw_arg = nil + ) + CallData.new(method, argc, flags, kw_arg) + end + end +end diff --git a/lib/syntax_tree/yarv/disassembler.rb b/lib/syntax_tree/yarv/disassembler.rb index 7756d125..ad66d0bf 100644 --- a/lib/syntax_tree/yarv/disassembler.rb +++ b/lib/syntax_tree/yarv/disassembler.rb @@ -21,30 +21,7 @@ def initialize(current_iseq = nil) ######################################################################## def calldata(value) - flag_names = [] - flag_names << :ARGS_SPLAT if value.flag?(CallData::CALL_ARGS_SPLAT) - if value.flag?(CallData::CALL_ARGS_BLOCKARG) - flag_names << :ARGS_BLOCKARG - end - flag_names << :FCALL if value.flag?(CallData::CALL_FCALL) - flag_names << :VCALL if value.flag?(CallData::CALL_VCALL) - flag_names << :ARGS_SIMPLE if value.flag?(CallData::CALL_ARGS_SIMPLE) - flag_names << :BLOCKISEQ if value.flag?(CallData::CALL_BLOCKISEQ) - flag_names << :KWARG if value.flag?(CallData::CALL_KWARG) - flag_names << :KW_SPLAT if value.flag?(CallData::CALL_KW_SPLAT) - flag_names << :TAILCALL if value.flag?(CallData::CALL_TAILCALL) - flag_names << :SUPER if value.flag?(CallData::CALL_SUPER) - flag_names << :ZSUPER if value.flag?(CallData::CALL_ZSUPER) - flag_names << :OPT_SEND if value.flag?(CallData::CALL_OPT_SEND) - flag_names << :KW_SPLAT_MUT if value.flag?(CallData::CALL_KW_SPLAT_MUT) - - parts = [] - parts << "mid:#{value.method}" if value.method - parts << "argc:#{value.argc}" - parts << "kw:[#{value.kw_arg.join(", ")}]" if value.kw_arg - parts << flag_names.join("|") if flag_names.any? - - "" + value.inspect end def enqueue(iseq) diff --git a/lib/syntax_tree/yarv/instructions.rb b/lib/syntax_tree/yarv/instructions.rb index 97ccce15..9bd8f0cd 100644 --- a/lib/syntax_tree/yarv/instructions.rb +++ b/lib/syntax_tree/yarv/instructions.rb @@ -2,67 +2,6 @@ module SyntaxTree module YARV - # This is an operand to various YARV instructions that represents the - # information about a specific call site. - class CallData - CALL_ARGS_SPLAT = 1 << 0 - CALL_ARGS_BLOCKARG = 1 << 1 - CALL_FCALL = 1 << 2 - CALL_VCALL = 1 << 3 - CALL_ARGS_SIMPLE = 1 << 4 - CALL_BLOCKISEQ = 1 << 5 - CALL_KWARG = 1 << 6 - CALL_KW_SPLAT = 1 << 7 - CALL_TAILCALL = 1 << 8 - CALL_SUPER = 1 << 9 - CALL_ZSUPER = 1 << 10 - CALL_OPT_SEND = 1 << 11 - CALL_KW_SPLAT_MUT = 1 << 12 - - attr_reader :method, :argc, :flags, :kw_arg - - def initialize( - method, - argc = 0, - flags = CallData::CALL_ARGS_SIMPLE, - kw_arg = nil - ) - @method = method - @argc = argc - @flags = flags - @kw_arg = kw_arg - end - - def flag?(mask) - (flags & mask) > 0 - end - - def to_h - result = { mid: method, flag: flags, orig_argc: argc } - result[:kw_arg] = kw_arg if kw_arg - result - end - - def self.from(serialized) - new( - serialized[:mid], - serialized[:orig_argc], - serialized[:flag], - serialized[:kw_arg] - ) - end - end - - # A convenience method for creating a CallData object. - def self.calldata( - method, - argc = 0, - flags = CallData::CALL_ARGS_SIMPLE, - kw_arg = nil - ) - CallData.new(method, argc, flags, kw_arg) - end - # This is a base class for all YARV instructions. It provides a few # convenience methods for working with instructions. class Instruction From 02ec2ad5441b797382d026ecd31b5cc4eeeed35b Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 2 Feb 2023 21:32:23 -0500 Subject: [PATCH 359/536] Simplify disassembler API --- lib/syntax_tree/yarv/control_flow_graph.rb | 8 +++--- lib/syntax_tree/yarv/data_flow_graph.rb | 29 ++++++++++++++-------- lib/syntax_tree/yarv/disassembler.rb | 16 +++++++++--- 3 files changed, 35 insertions(+), 18 deletions(-) diff --git a/lib/syntax_tree/yarv/control_flow_graph.rb b/lib/syntax_tree/yarv/control_flow_graph.rb index fb8f97f3..dc900e50 100644 --- a/lib/syntax_tree/yarv/control_flow_graph.rb +++ b/lib/syntax_tree/yarv/control_flow_graph.rb @@ -34,21 +34,21 @@ def initialize(iseq, insns, blocks) def disasm fmt = Disassembler.new(iseq) - fmt.output.puts("== cfg: #{iseq.inspect}") + fmt.puts("== cfg: #{iseq.inspect}") blocks.each do |block| - fmt.output.puts(block.id) + fmt.puts(block.id) fmt.with_prefix(" ") do |prefix| unless block.incoming_blocks.empty? from = block.incoming_blocks.map(&:id) - fmt.output.puts("#{prefix}== from: #{from.join(", ")}") + fmt.puts("#{prefix}== from: #{from.join(", ")}") end fmt.format_insns!(block.insns, block.block_start) to = block.outgoing_blocks.map(&:id) to << "leaves" if block.insns.last.leaves? - fmt.output.puts("#{prefix}== to: #{to.join(", ")}") + fmt.puts("#{prefix}== to: #{to.join(", ")}") end end diff --git a/lib/syntax_tree/yarv/data_flow_graph.rb b/lib/syntax_tree/yarv/data_flow_graph.rb index 614d1233..f98eedda 100644 --- a/lib/syntax_tree/yarv/data_flow_graph.rb +++ b/lib/syntax_tree/yarv/data_flow_graph.rb @@ -6,6 +6,15 @@ module YARV # control-flow-graph. Data flow is discovered locally and then globally. The # graph only considers data flow through the stack - local variables and # objects are considered fully escaped in this analysis. + # + # You can use this class by calling the ::compile method and passing it a + # control flow graph. It will return a data flow graph object. + # + # iseq = RubyVM::InstructionSequence.compile("1 + 2") + # iseq = SyntaxTree::YARV::InstructionSequence.from(iseq.to_a) + # cfg = SyntaxTree::YARV::ControlFlowGraph.compile(iseq) + # dfg = SyntaxTree::YARV::DataFlowGraph.compile(cfg) + # class DataFlowGraph # This object represents the flow of data between instructions. class DataFlow @@ -28,42 +37,42 @@ def initialize(cfg, insn_flows, block_flows) def disasm fmt = Disassembler.new(cfg.iseq) - fmt.output.puts("== dfg: #{cfg.iseq.inspect}") + fmt.puts("== dfg: #{cfg.iseq.inspect}") cfg.blocks.each do |block| - fmt.output.puts(block.id) + fmt.puts(block.id) fmt.with_prefix(" ") do |prefix| unless block.incoming_blocks.empty? from = block.incoming_blocks.map(&:id) - fmt.output.puts("#{prefix}== from: #{from.join(", ")}") + fmt.puts("#{prefix}== from: #{from.join(", ")}") end block_flow = block_flows.fetch(block.id) unless block_flow.in.empty? - fmt.output.puts("#{prefix}== in: #{block_flow.in.join(", ")}") + fmt.puts("#{prefix}== in: #{block_flow.in.join(", ")}") end fmt.format_insns!(block.insns, block.block_start) do |_, length| insn_flow = insn_flows[length] next if insn_flow.in.empty? && insn_flow.out.empty? - fmt.output.print(" # ") + fmt.print(" # ") unless insn_flow.in.empty? - fmt.output.print("in: #{insn_flow.in.join(", ")}") - fmt.output.print("; ") unless insn_flow.out.empty? + fmt.print("in: #{insn_flow.in.join(", ")}") + fmt.print("; ") unless insn_flow.out.empty? end unless insn_flow.out.empty? - fmt.output.print("out: #{insn_flow.out.join(", ")}") + fmt.print("out: #{insn_flow.out.join(", ")}") end end to = block.outgoing_blocks.map(&:id) to << "leaves" if block.insns.last.leaves? - fmt.output.puts("#{prefix}== to: #{to.join(", ")}") + fmt.puts("#{prefix}== to: #{to.join(", ")}") unless block_flow.out.empty? - fmt.output.puts("#{prefix}== out: #{block_flow.out.join(", ")}") + fmt.puts("#{prefix}== out: #{block_flow.out.join(", ")}") end end end diff --git a/lib/syntax_tree/yarv/disassembler.rb b/lib/syntax_tree/yarv/disassembler.rb index ad66d0bf..a758bce3 100644 --- a/lib/syntax_tree/yarv/disassembler.rb +++ b/lib/syntax_tree/yarv/disassembler.rb @@ -78,10 +78,6 @@ def object(value) # Entrypoints ######################################################################## - def string - output.string - end - def format! while (@current_iseq = queue.shift) output << "\n" if output.pos > 0 @@ -133,6 +129,18 @@ def format_insns!(insns, length = 0) end end + def print(string) + output.print(string) + end + + def puts(string) + output.puts(string) + end + + def string + output.string + end + def with_prefix(value) previous = @current_prefix From b34e5d4f0e75bd44e9ce34faeddca3616c546d92 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 3 Feb 2023 10:26:22 -0500 Subject: [PATCH 360/536] Speed up ractor tests --- test/ractor_test.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/ractor_test.rb b/test/ractor_test.rb index bcdb2a51..7e0201ca 100644 --- a/test/ractor_test.rb +++ b/test/ractor_test.rb @@ -33,7 +33,7 @@ def test_formatting private def filepaths - Dir.glob(File.expand_path("../lib/syntax_tree/{node,parser}.rb", __dir__)) + Dir.glob(File.expand_path("../lib/syntax_tree/plugin/*.rb", __dir__)) end # Ractors still warn about usage, so I'm disabling that warning here just to From da08570e9b46e0d29085e185fc76a82b04e0ae6e Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Sat, 4 Feb 2023 16:40:23 -0500 Subject: [PATCH 361/536] EmbDoc fixes --- lib/syntax_tree/node.rb | 31 +++++++++++++++++++++++++++++-- test/fixtures/call.rb | 5 +++++ test/fixtures/def.rb | 16 ++++++++++++++++ test/fixtures/symbols.rb | 5 +++++ 4 files changed, 55 insertions(+), 2 deletions(-) diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index fc5517cf..55b381c3 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -4090,7 +4090,8 @@ def deconstruct_keys(_keys) def format(q) q.group do q.group do - q.text("def ") + q.text("def") + q.text(" ") if target || name.comments.empty? if target q.format(target) @@ -4872,6 +4873,25 @@ class EmbDoc < Node def initialize(value:, location:) @value = value @location = location + + @leading = false + @trailing = false + end + + def leading! + @leading = true + end + + def leading? + @leading + end + + def trailing! + @trailing = true + end + + def trailing? + @trailing end def inline? @@ -4908,7 +4928,13 @@ def deconstruct_keys(_keys) end def format(q) - q.trim + if (q.parent.is_a?(DefNode) && q.parent.endless?) || + q.parent.is_a?(Statements) + q.trim + else + q.breakable_return + end + q.text(value) end @@ -10465,6 +10491,7 @@ def deconstruct_keys(_keys) def format(q) q.text(":") + q.text("\\") if value.comments.any? q.format(value) end diff --git a/test/fixtures/call.rb b/test/fixtures/call.rb index c41ee4ac..d35c6036 100644 --- a/test/fixtures/call.rb +++ b/test/fixtures/call.rb @@ -60,3 +60,8 @@ % a b do end.c d +% +self. +=begin +=end + to_s diff --git a/test/fixtures/def.rb b/test/fixtures/def.rb index a827adfe..1441bf04 100644 --- a/test/fixtures/def.rb +++ b/test/fixtures/def.rb @@ -23,3 +23,19 @@ def foo() # comment def foo( # comment ) end +% +def +=begin +=end +a +end +% +def a() +=begin +=end +=1 +- +def a() = +=begin +=end + 1 diff --git a/test/fixtures/symbols.rb b/test/fixtures/symbols.rb index 5e2673f3..12f0a22f 100644 --- a/test/fixtures/symbols.rb +++ b/test/fixtures/symbols.rb @@ -19,3 +19,8 @@ %I[foo] # comment % %I{foo[]} +% +:\ +=begin +=end +symbol From a5ad966a44c70f2861ed3ad2a26804d58a3515e0 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Sun, 5 Feb 2023 09:32:36 -0500 Subject: [PATCH 362/536] Fix up Ruby 2.7.0 build --- test/fixtures/def.rb | 10 ---------- test/fixtures/def_endless.rb | 10 ++++++++++ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/test/fixtures/def.rb b/test/fixtures/def.rb index 1441bf04..0cc49e0a 100644 --- a/test/fixtures/def.rb +++ b/test/fixtures/def.rb @@ -29,13 +29,3 @@ def foo( # comment =end a end -% -def a() -=begin -=end -=1 -- -def a() = -=begin -=end - 1 diff --git a/test/fixtures/def_endless.rb b/test/fixtures/def_endless.rb index 4595fba9..8d1f9d33 100644 --- a/test/fixtures/def_endless.rb +++ b/test/fixtures/def_endless.rb @@ -22,3 +22,13 @@ def self.foo = bar baz end def foo? = true +% +def a() +=begin +=end +=1 +- +def a() = +=begin +=end + 1 From 4ec195bef0f61cbd098119eab56bc16190dd925b Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 6 Feb 2023 13:55:59 -0500 Subject: [PATCH 363/536] Mermaid visitor --- lib/syntax_tree.rb | 2 + lib/syntax_tree/node.rb | 12 ++-- lib/syntax_tree/visitor/mermaid_visitor.rb | 81 ++++++++++++++++++++++ 3 files changed, 90 insertions(+), 5 deletions(-) create mode 100644 lib/syntax_tree/visitor/mermaid_visitor.rb diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index ade9ff5e..1af1b476 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +require "cgi" require "etc" require "fiddle" require "json" @@ -18,6 +19,7 @@ require_relative "syntax_tree/visitor/field_visitor" require_relative "syntax_tree/visitor/json_visitor" require_relative "syntax_tree/visitor/match_visitor" +require_relative "syntax_tree/visitor/mermaid_visitor" require_relative "syntax_tree/visitor/mutation_visitor" require_relative "syntax_tree/visitor/pretty_print_visitor" require_relative "syntax_tree/visitor/environment" diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 1a814aaf..8ffbcd2d 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -127,17 +127,19 @@ def format(q) end def pretty_print(q) - visitor = Visitor::PrettyPrintVisitor.new(q) - visitor.visit(self) + accept(Visitor::PrettyPrintVisitor.new(q)) end def to_json(*opts) - visitor = Visitor::JSONVisitor.new - visitor.visit(self).to_json(*opts) + accept(Visitor::JSONVisitor.new).to_json(*opts) end def construct_keys - PrettierPrint.format(+"") { |q| Visitor::MatchVisitor.new(q).visit(self) } + PrettierPrint.format(+"") { |q| accept(Visitor::MatchVisitor.new(q)) } + end + + def mermaid + accept(Visitor::MermaidVisitor.new) end end diff --git a/lib/syntax_tree/visitor/mermaid_visitor.rb b/lib/syntax_tree/visitor/mermaid_visitor.rb new file mode 100644 index 00000000..2b06049a --- /dev/null +++ b/lib/syntax_tree/visitor/mermaid_visitor.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +module SyntaxTree + class Visitor + # This visitor transforms the AST into a mermaid flow chart. + class MermaidVisitor < FieldVisitor + attr_reader :output, :target + + def initialize + @output = StringIO.new + @output.puts("flowchart TD") + + @target = nil + end + + def visit_program(node) + super + output.string + end + + private + + def comments(node) + # Ignore + end + + def field(name, value) + case value + when Node + node_id = visit(value) + output.puts(" #{target} -- \"#{name}\" --> #{node_id}") + when String + node_id = "#{target}_#{name}" + output.puts(" #{node_id}([#{CGI.escapeHTML(value.inspect)}])") + output.puts(" #{target} -- \"#{name}\" --> #{node_id}") + when nil + # skip + else + node_id = "#{target}_#{name}" + output.puts(" #{node_id}([\"#{CGI.escapeHTML(value.inspect)}\"])") + output.puts(" #{target} -- \"#{name}\" --> #{node_id}") + end + end + + def list(name, values) + values.each_with_index do |value, index| + field("#{name}[#{index}]", value) + end + end + + def node(node, type) + previous_target = target + + begin + @target = "node_#{node.object_id}" + + yield + + output.puts(" #{@target}[\"#{type}\"]") + @target + ensure + @target = previous_target + end + end + + def pairs(name, values) + values.each_with_index do |(key, value), index| + node_id = "#{target}_#{name}_#{index}" + output.puts(" #{node_id}((\" \"))") + output.puts(" #{target} -- \"#{name}[#{index}]\" --> #{node_id}") + output.puts(" #{node_id} -- \"[0]\" --> #{visit(key)}") + output.puts(" #{node_id} -- \"[1]\" --> #{visit(value)}") if value + end + end + + def text(name, value) + field(name, value) + end + end + end +end From e7c5adf1de9fcac198fdbbdc1350515c3bf02210 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 6 Feb 2023 14:12:36 -0500 Subject: [PATCH 364/536] Control flow graphs to mermaid --- .rubocop.yml | 3 ++ lib/syntax_tree/node.rb | 8 ++--- lib/syntax_tree/yarv/control_flow_graph.rb | 34 +++++++++++++++++++++ lib/syntax_tree/yarv/disassembler.rb | 35 ++++++++++++++++++++++ 4 files changed, 76 insertions(+), 4 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index 381d7a27..62e78453 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -90,6 +90,9 @@ Style/CaseLikeIf: Style/ClassVars: Enabled: false +Style/CombinableLoops: + Enabled: false + Style/DocumentDynamicEvalDefinition: Enabled: false diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 8ffbcd2d..b1ecfdc7 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -134,12 +134,12 @@ def to_json(*opts) accept(Visitor::JSONVisitor.new).to_json(*opts) end - def construct_keys - PrettierPrint.format(+"") { |q| accept(Visitor::MatchVisitor.new(q)) } + def to_mermaid + accept(Visitor::MermaidVisitor.new) end - def mermaid - accept(Visitor::MermaidVisitor.new) + def construct_keys + PrettierPrint.format(+"") { |q| accept(Visitor::MatchVisitor.new(q)) } end end diff --git a/lib/syntax_tree/yarv/control_flow_graph.rb b/lib/syntax_tree/yarv/control_flow_graph.rb index dc900e50..a9f3e093 100644 --- a/lib/syntax_tree/yarv/control_flow_graph.rb +++ b/lib/syntax_tree/yarv/control_flow_graph.rb @@ -55,6 +55,40 @@ def disasm fmt.string end + def to_mermaid + output = StringIO.new + output.puts("flowchart TD") + + fmt = Disassembler::Mermaid.new + blocks.each do |block| + output.puts(" subgraph #{block.id}") + previous = nil + + block.each_with_length do |insn, length| + node_id = "node_#{length}" + label = "%04d %s" % [length, insn.disasm(fmt)] + + output.puts(" #{node_id}(\"#{CGI.escapeHTML(label)}\")") + output.puts(" #{previous} --> #{node_id}") if previous + + previous = node_id + end + + output.puts(" end") + end + + blocks.each do |block| + block.outgoing_blocks.each do |outgoing| + offset = + block.block_start + block.insns.sum(&:length) - + block.insns.last.length + output.puts(" node_#{offset} --> node_#{outgoing.block_start}") + end + end + + output.string + end + # This method is used to verify that the control flow graph is well # formed. It does this by checking that each basic block is itself well # formed. diff --git a/lib/syntax_tree/yarv/disassembler.rb b/lib/syntax_tree/yarv/disassembler.rb index a758bce3..f60af0fd 100644 --- a/lib/syntax_tree/yarv/disassembler.rb +++ b/lib/syntax_tree/yarv/disassembler.rb @@ -3,6 +3,41 @@ module SyntaxTree module YARV class Disassembler + # This class is another object that handles disassembling a YARV + # instruction sequence but it does so in order to provide a label for a + # mermaid diagram. + class Mermaid + def calldata(value) + value.inspect + end + + def enqueue(iseq) + end + + def event(name) + end + + def inline_storage(cache) + "" + end + + def instruction(name, operands = []) + operands.empty? ? name : "#{name} #{operands.join(", ")}" + end + + def label(value) + "%04d" % value.name["label_".length..] + end + + def local(index, **) + index.inspect + end + + def object(value) + value.inspect + end + end + attr_reader :output, :queue attr_reader :current_prefix From e642348dc2da3e2a8299ebc9e56b0fe6e965446f Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 6 Feb 2023 14:32:44 -0500 Subject: [PATCH 365/536] DFG to mermaid --- lib/syntax_tree/yarv/control_flow_graph.rb | 1 + lib/syntax_tree/yarv/data_flow_graph.rb | 61 ++++++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/lib/syntax_tree/yarv/control_flow_graph.rb b/lib/syntax_tree/yarv/control_flow_graph.rb index a9f3e093..328ffc4c 100644 --- a/lib/syntax_tree/yarv/control_flow_graph.rb +++ b/lib/syntax_tree/yarv/control_flow_graph.rb @@ -82,6 +82,7 @@ def to_mermaid offset = block.block_start + block.insns.sum(&:length) - block.insns.last.length + output.puts(" node_#{offset} --> node_#{outgoing.block_start}") end end diff --git a/lib/syntax_tree/yarv/data_flow_graph.rb b/lib/syntax_tree/yarv/data_flow_graph.rb index f98eedda..7423d022 100644 --- a/lib/syntax_tree/yarv/data_flow_graph.rb +++ b/lib/syntax_tree/yarv/data_flow_graph.rb @@ -80,6 +80,67 @@ def disasm fmt.string end + def to_mermaid + output = StringIO.new + output.puts("flowchart TD") + + fmt = Disassembler::Mermaid.new + links = [] + + cfg.blocks.each do |block| + block_flow = block_flows.fetch(block.id) + graph_name = + if block_flow.in.any? + "#{block.id} #{block_flows[block.id].in.join(", ")}" + else + block.id + end + + output.puts(" subgraph \"#{CGI.escapeHTML(graph_name)}\"") + previous = nil + + block.each_with_length do |insn, length| + node_id = "node_#{length}" + label = "%04d %s" % [length, insn.disasm(fmt)] + + output.puts(" #{node_id}(\"#{CGI.escapeHTML(label)}\")") + + if previous + output.puts(" #{previous} --> #{node_id}") + links << "red" + end + + insn_flows[length].in.each do |input| + if input.is_a?(Integer) + output.puts(" node_#{input} --> #{node_id}") + links << "green" + end + end + + previous = node_id + end + + output.puts(" end") + end + + cfg.blocks.each do |block| + block.outgoing_blocks.each do |outgoing| + offset = + block.block_start + block.insns.sum(&:length) - + block.insns.last.length + + output.puts(" node_#{offset} --> node_#{outgoing.block_start}") + links << "red" + end + end + + links.each_with_index do |color, index| + output.puts(" linkStyle #{index} stroke:#{color}") + end + + output.string + end + # Verify that we constructed the data flow graph correctly. def verify # Check that the first block has no arguments. From 4796d1cae3c22431e1256703a7cb194023696064 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Sat, 4 Feb 2023 16:25:06 -0500 Subject: [PATCH 366/536] Sea of nodes --- lib/syntax_tree.rb | 1 + lib/syntax_tree/yarv/control_flow_graph.rb | 212 +++++---- lib/syntax_tree/yarv/data_flow_graph.rb | 99 +++- lib/syntax_tree/yarv/instruction_sequence.rb | 4 + lib/syntax_tree/yarv/instructions.rb | 39 ++ lib/syntax_tree/yarv/sea_of_nodes.rb | 464 +++++++++++++++++++ test/yarv_test.rb | 110 +++++ 7 files changed, 817 insertions(+), 112 deletions(-) create mode 100644 lib/syntax_tree/yarv/sea_of_nodes.rb diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index 1af1b476..cd1f1ce4 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -43,6 +43,7 @@ require_relative "syntax_tree/yarv/instructions" require_relative "syntax_tree/yarv/legacy" require_relative "syntax_tree/yarv/local_table" +require_relative "syntax_tree/yarv/sea_of_nodes" require_relative "syntax_tree/yarv/assembler" require_relative "syntax_tree/yarv/vm" diff --git a/lib/syntax_tree/yarv/control_flow_graph.rb b/lib/syntax_tree/yarv/control_flow_graph.rb index 328ffc4c..1a361e5e 100644 --- a/lib/syntax_tree/yarv/control_flow_graph.rb +++ b/lib/syntax_tree/yarv/control_flow_graph.rb @@ -14,93 +14,6 @@ module YARV # cfg = SyntaxTree::YARV::ControlFlowGraph.compile(iseq) # class ControlFlowGraph - # This is the instruction sequence that this control flow graph - # corresponds to. - attr_reader :iseq - - # This is the list of instructions that this control flow graph contains. - # It is effectively the same as the list of instructions in the - # instruction sequence but with line numbers and events filtered out. - attr_reader :insns - - # This is the set of basic blocks that this control-flow graph contains. - attr_reader :blocks - - def initialize(iseq, insns, blocks) - @iseq = iseq - @insns = insns - @blocks = blocks - end - - def disasm - fmt = Disassembler.new(iseq) - fmt.puts("== cfg: #{iseq.inspect}") - - blocks.each do |block| - fmt.puts(block.id) - fmt.with_prefix(" ") do |prefix| - unless block.incoming_blocks.empty? - from = block.incoming_blocks.map(&:id) - fmt.puts("#{prefix}== from: #{from.join(", ")}") - end - - fmt.format_insns!(block.insns, block.block_start) - - to = block.outgoing_blocks.map(&:id) - to << "leaves" if block.insns.last.leaves? - fmt.puts("#{prefix}== to: #{to.join(", ")}") - end - end - - fmt.string - end - - def to_mermaid - output = StringIO.new - output.puts("flowchart TD") - - fmt = Disassembler::Mermaid.new - blocks.each do |block| - output.puts(" subgraph #{block.id}") - previous = nil - - block.each_with_length do |insn, length| - node_id = "node_#{length}" - label = "%04d %s" % [length, insn.disasm(fmt)] - - output.puts(" #{node_id}(\"#{CGI.escapeHTML(label)}\")") - output.puts(" #{previous} --> #{node_id}") if previous - - previous = node_id - end - - output.puts(" end") - end - - blocks.each do |block| - block.outgoing_blocks.each do |outgoing| - offset = - block.block_start + block.insns.sum(&:length) - - block.insns.last.length - - output.puts(" node_#{offset} --> node_#{outgoing.block_start}") - end - end - - output.string - end - - # This method is used to verify that the control flow graph is well - # formed. It does this by checking that each basic block is itself well - # formed. - def verify - blocks.each(&:verify) - end - - def self.compile(iseq) - Compiler.new(iseq).compile - end - # This class is responsible for creating a control flow graph from the # given instruction sequence. class Compiler @@ -139,7 +52,11 @@ def initialize(iseq) # This method is used to compile the instruction sequence into a control # flow graph. It returns an instance of ControlFlowGraph. def compile - blocks = connect_basic_blocks(build_basic_blocks) + blocks = build_basic_blocks + + connect_basic_blocks(blocks) + prune_basic_blocks(blocks) + ControlFlowGraph.new(iseq, insns, blocks.values).tap(&:verify) end @@ -187,7 +104,16 @@ def build_basic_blocks block_starts .zip(blocks) - .to_h do |block_start, block_insns| + .to_h do |block_start, insns| + # It's possible that we have not detected a block start but still + # have branching instructions inside of a basic block. This can + # happen if you have an unconditional jump which is followed by + # instructions that are unreachable. As of Ruby 3.2, this is + # possible with something as simple as "1 => a". In this case we + # can discard all instructions that follow branching instructions. + block_insns = + insns.slice_after { |insn| insn.branch_targets.any? }.first + [block_start, BasicBlock.new(block_start, block_insns)] end end @@ -213,6 +139,114 @@ def connect_basic_blocks(blocks) end end end + + # If there are blocks that are unreachable, we can remove them from the + # graph entirely at this point. + def prune_basic_blocks(blocks) + visited = Set.new + queue = [blocks.fetch(0)] + + until queue.empty? + current_block = queue.shift + next if visited.include?(current_block) + + visited << current_block + queue.concat(current_block.outgoing_blocks) + end + + blocks.select! { |_, block| visited.include?(block) } + end + end + + # This is the instruction sequence that this control flow graph + # corresponds to. + attr_reader :iseq + + # This is the list of instructions that this control flow graph contains. + # It is effectively the same as the list of instructions in the + # instruction sequence but with line numbers and events filtered out. + attr_reader :insns + + # This is the set of basic blocks that this control-flow graph contains. + attr_reader :blocks + + def initialize(iseq, insns, blocks) + @iseq = iseq + @insns = insns + @blocks = blocks + end + + def disasm + fmt = Disassembler.new(iseq) + fmt.puts("== cfg: #{iseq.inspect}") + + blocks.each do |block| + fmt.puts(block.id) + fmt.with_prefix(" ") do |prefix| + unless block.incoming_blocks.empty? + from = block.incoming_blocks.map(&:id) + fmt.puts("#{prefix}== from: #{from.join(", ")}") + end + + fmt.format_insns!(block.insns, block.block_start) + + to = block.outgoing_blocks.map(&:id) + to << "leaves" if block.insns.last.leaves? + fmt.puts("#{prefix}== to: #{to.join(", ")}") + end + end + + fmt.string + end + + def to_dfg + DataFlowGraph.compile(self) + end + + def to_mermaid + output = StringIO.new + output.puts("flowchart TD") + + fmt = Disassembler::Mermaid.new + blocks.each do |block| + output.puts(" subgraph #{block.id}") + previous = nil + + block.each_with_length do |insn, length| + node_id = "node_#{length}" + label = "%04d %s" % [length, insn.disasm(fmt)] + + output.puts(" #{node_id}(\"#{CGI.escapeHTML(label)}\")") + output.puts(" #{previous} --> #{node_id}") if previous + + previous = node_id + end + + output.puts(" end") + end + + blocks.each do |block| + block.outgoing_blocks.each do |outgoing| + offset = + block.block_start + block.insns.sum(&:length) - + block.insns.last.length + + output.puts(" node_#{offset} --> node_#{outgoing.block_start}") + end + end + + output.string + end + + # This method is used to verify that the control flow graph is well + # formed. It does this by checking that each basic block is itself well + # formed. + def verify + blocks.each(&:verify) + end + + def self.compile(iseq) + Compiler.new(iseq).compile end end end diff --git a/lib/syntax_tree/yarv/data_flow_graph.rb b/lib/syntax_tree/yarv/data_flow_graph.rb index 7423d022..ace40296 100644 --- a/lib/syntax_tree/yarv/data_flow_graph.rb +++ b/lib/syntax_tree/yarv/data_flow_graph.rb @@ -27,6 +27,42 @@ def initialize end end + # This represents an object that goes on the stack that is passed between + # basic blocks. + class BlockArgument + attr_reader :name + + def initialize(name) + @name = name + end + + def local? + false + end + + def to_str + name.to_s + end + end + + # This represents an object that goes on the stack that is passed between + # instructions within a basic block. + class LocalArgument + attr_reader :name, :length + + def initialize(length) + @length = length + end + + def local? + true + end + + def to_str + length.to_s + end + end + attr_reader :cfg, :insn_flows, :block_flows def initialize(cfg, insn_flows, block_flows) @@ -35,11 +71,15 @@ def initialize(cfg, insn_flows, block_flows) @block_flows = block_flows end + def blocks + cfg.blocks + end + def disasm fmt = Disassembler.new(cfg.iseq) fmt.puts("== dfg: #{cfg.iseq.inspect}") - cfg.blocks.each do |block| + blocks.each do |block| fmt.puts(block.id) fmt.with_prefix(" ") do |prefix| unless block.incoming_blocks.empty? @@ -80,6 +120,10 @@ def disasm fmt.string end + def to_son + SeaOfNodes.compile(self) + end + def to_mermaid output = StringIO.new output.puts("flowchart TD") @@ -87,7 +131,7 @@ def to_mermaid fmt = Disassembler::Mermaid.new links = [] - cfg.blocks.each do |block| + blocks.each do |block| block_flow = block_flows.fetch(block.id) graph_name = if block_flow.in.any? @@ -123,7 +167,7 @@ def to_mermaid output.puts(" end") end - cfg.blocks.each do |block| + blocks.each do |block| block.outgoing_blocks.each do |outgoing| offset = block.block_start + block.insns.sum(&:length) - @@ -144,11 +188,11 @@ def to_mermaid # Verify that we constructed the data flow graph correctly. def verify # Check that the first block has no arguments. - raise unless block_flows.fetch(cfg.blocks.first.id).in.empty? + raise unless block_flows.fetch(blocks.first.id).in.empty? # Check all control flow edges between blocks pass the right number of # arguments. - cfg.blocks.each do |block| + blocks.each do |block| block_flow = block_flows.fetch(block.id) if block.outgoing_blocks.empty? @@ -191,8 +235,8 @@ def initialize(cfg) end def compile - find_local_flow - find_global_flow + find_internal_flow + find_external_flow DataFlowGraph.new(cfg, insn_flows, block_flows).tap(&:verify) end @@ -200,45 +244,53 @@ def compile # Find the data flow within each basic block. Using an abstract stack, # connect from consumers of data to the producers of that data. - def find_local_flow + def find_internal_flow cfg.blocks.each do |block| block_flow = block_flows.fetch(block.id) stack = [] - # Go through each instruction in the block... + # Go through each instruction in the block. block.each_with_length do |insn, length| insn_flow = insn_flows[length] # How many values will be missing from the local stack to run this - # instruction? + # instruction? This will be used to determine if the values that + # are being used by this instruction are coming from previous + # instructions or from previous basic blocks. missing = insn.pops - stack.size - # For every value the instruction pops off the stack... + # For every value the instruction pops off the stack. insn.pops.times do # Was the value it pops off from another basic block? if stack.empty? - # This is a basic block argument. + # If the stack is empty, then there aren't enough values being + # pushed from previous instructions to fulfill the needs of + # this instruction. In that case the values must be coming + # from previous basic blocks. missing -= 1 - name = :"in_#{missing}" + argument = BlockArgument.new(:"in_#{missing}") - insn_flow.in.unshift(name) - block_flow.in.unshift(name) + insn_flow.in.unshift(argument) + block_flow.in.unshift(argument) else - # Connect this consumer to the producer of the value. + # Since there are values in the stack, we can connect this + # consumer to the producer of the value. insn_flow.in.unshift(stack.pop) end end # Record on our abstract stack that this instruction pushed # this value onto the stack. - insn.pushes.times { stack << length } + insn.pushes.times { stack << LocalArgument.new(length) } end # Values that are left on the stack after going through all # instructions are arguments to the basic block that we jump to. stack.reverse_each.with_index do |producer, index| block_flow.out << producer - insn_flows[producer].out << :"out_#{index}" + + argument = BlockArgument.new(:"out_#{index}") + insn_flows[producer.length].out << argument end end @@ -249,17 +301,17 @@ def find_local_flow insn_flows[length].in.each do |producer| # If it's actually another instruction and not a basic block # argument... - if producer.is_a?(Integer) + if producer.is_a?(LocalArgument) # Record in the producing instruction that it produces a value # used by this construction. - insn_flows[producer].out << length + insn_flows[producer.length].out << LocalArgument.new(length) end end end end # Find the data that flows between basic blocks. - def find_global_flow + def find_external_flow stack = [*cfg.blocks] until stack.empty? @@ -275,7 +327,7 @@ def find_global_flow # If so then add arguments to pass data through from the # incoming block's incoming blocks. (block_flow.in.size - incoming_flow.out.size).times do |index| - name = :"pass_#{index}" + name = BlockArgument.new(:"pass_#{index}") incoming_flow.in.unshift(name) incoming_flow.out.unshift(name) @@ -283,7 +335,8 @@ def find_global_flow # Since we modified the incoming block, add it back to the stack # so it'll be considered as an outgoing block again, and - # propogate the global data flow back up the control flow graph. + # propogate the external data flow back up the control flow + # graph. stack << incoming_block end end diff --git a/lib/syntax_tree/yarv/instruction_sequence.rb b/lib/syntax_tree/yarv/instruction_sequence.rb index 45fc6121..918a3c86 100644 --- a/lib/syntax_tree/yarv/instruction_sequence.rb +++ b/lib/syntax_tree/yarv/instruction_sequence.rb @@ -269,6 +269,10 @@ def to_a ] end + def to_cfg + ControlFlowGraph.compile(self) + end + def disasm fmt = Disassembler.new fmt.enqueue(self) diff --git a/lib/syntax_tree/yarv/instructions.rb b/lib/syntax_tree/yarv/instructions.rb index 9bd8f0cd..38c80fde 100644 --- a/lib/syntax_tree/yarv/instructions.rb +++ b/lib/syntax_tree/yarv/instructions.rb @@ -44,6 +44,13 @@ def leaves? def falls_through? false end + + # Does the instruction have side effects? Control-flow counts as a + # side-effect, as do some special-case instructions like Leave. By default + # every instruction is marked as having side effects. + def side_effects? + true + end end # ### Summary @@ -1166,6 +1173,10 @@ def pushes def call(vm) vm.push(vm.stack.last.dup) end + + def side_effects? + false + end end # ### Summary @@ -2470,6 +2481,10 @@ def ==(other) def call(vm) end + + def side_effects? + false + end end # ### Summary @@ -4439,6 +4454,10 @@ def pops def call(vm) vm.pop end + + def side_effects? + false + end end # ### Summary @@ -4479,6 +4498,10 @@ def canonical def call(vm) canonical.call(vm) end + + def side_effects? + false + end end # ### Summary @@ -4525,6 +4548,10 @@ def pushes def call(vm) vm.push(object) end + + def side_effects? + false + end end # ### Summary @@ -4567,6 +4594,10 @@ def canonical def call(vm) canonical.call(vm) end + + def side_effects? + false + end end # ### Summary @@ -4609,6 +4640,10 @@ def canonical def call(vm) canonical.call(vm) end + + def side_effects? + false + end end # ### Summary @@ -4645,6 +4680,10 @@ def pushes def call(vm) vm.push(vm.frame._self) end + + def side_effects? + false + end end # ### Summary diff --git a/lib/syntax_tree/yarv/sea_of_nodes.rb b/lib/syntax_tree/yarv/sea_of_nodes.rb new file mode 100644 index 00000000..be027f39 --- /dev/null +++ b/lib/syntax_tree/yarv/sea_of_nodes.rb @@ -0,0 +1,464 @@ +# frozen_string_literal: true + +module SyntaxTree + module YARV + # A sea of nodes is an intermediate representation used by a compiler to + # represent both control and data flow in the same graph. The way we use it + # allows us to have the vertices of the graph represent either an + # instruction in the instruction sequence or a synthesized node that we add + # to the graph. The edges of the graph represent either control flow or data + # flow. + class SeaOfNodes + # This object represents a node in the graph that holds a YARV + # instruction. + class InsnNode + attr_reader :inputs, :outputs, :insn, :offset + + def initialize(insn, offset) + @inputs = [] + @outputs = [] + + @insn = insn + @offset = offset + end + + def id + offset + end + + def label + "%04d %s" % [offset, insn.disasm(Disassembler::Mermaid.new)] + end + end + + # Phi nodes are used to represent the merging of data flow from multiple + # incoming blocks. + class PhiNode + attr_reader :inputs, :outputs, :id + + def initialize(id) + @inputs = [] + @outputs = [] + @id = id + end + + def label + "#{id} φ" + end + end + + # Merge nodes are present in any block that has multiple incoming blocks. + # It provides a place for Phi nodes to attach their results. + class MergeNode + attr_reader :inputs, :outputs, :id + + def initialize(id) + @inputs = [] + @outputs = [] + @id = id + end + + def label + "#{id} ψ" + end + end + + # The edge of a graph represents either control flow or data flow. + class Edge + TYPES = %i[data control info].freeze + + attr_reader :from + attr_reader :to + attr_reader :type + attr_reader :label + + def initialize(from, to, type, label) + raise unless TYPES.include?(type) + + @from = from + @to = to + @type = type + @label = label + end + end + + # A subgraph represents the local data and control flow of a single basic + # block. + class SubGraph + attr_reader :first_fixed, :last_fixed, :inputs, :outputs + + def initialize(first_fixed, last_fixed, inputs, outputs) + @first_fixed = first_fixed + @last_fixed = last_fixed + @inputs = inputs + @outputs = outputs + end + end + + # The compiler is responsible for taking a data flow graph and turning it + # into a sea of nodes. + class Compiler + attr_reader :dfg, :nodes + + def initialize(dfg) + @dfg = dfg + @nodes = [] + + # We need to put a unique ID on the synthetic nodes in the graph, so + # we keep a counter that we increment any time we create a new + # synthetic node. + @id_counter = 999 + end + + def compile + local_graphs = {} + dfg.blocks.each do |block| + local_graphs[block.id] = create_local_graph(block) + end + + connect_local_graphs_control(local_graphs) + connect_local_graphs_data(local_graphs) + cleanup + + SeaOfNodes.new(dfg, nodes, local_graphs).tap(&:verify) + end + + private + + # Counter for synthetic nodes. + def id_counter + @id_counter += 1 + end + + # Create a sub-graph for a single basic block - block block argument + # inputs and outputs will be left dangling, to be connected later. + def create_local_graph(block) + block_flow = dfg.block_flows.fetch(block.id) + + # A map of instructions to nodes. + insn_nodes = {} + + # Create a node for each instruction in the block. + block.each_with_length do |insn, offset| + node = InsnNode.new(insn, offset) + insn_nodes[offset] = node + nodes << node + end + + # The first and last node in the sub-graph, and the last fixed node. + previous_fixed = nil + first_fixed = nil + last_fixed = nil + + # The merge node for the phi nodes to attach to. + merge_node = nil + + # If there is more than one predecessor and we have basic block + # arguments coming in, then we need a merge node for the phi nodes to + # attach to. + if block.incoming_blocks.size > 1 && !block_flow.in.empty? + merge_node = MergeNode.new(id_counter) + nodes << merge_node + + previous_fixed = merge_node + first_fixed = merge_node + last_fixed = merge_node + end + + # Connect local control flow (only nodes with side effects.) + block.each_with_length do |insn, length| + if insn.side_effects? + insn_node = insn_nodes[length] + connect previous_fixed, insn_node, :control if previous_fixed + previous_fixed = insn_node + first_fixed ||= insn_node + last_fixed = insn_node + end + end + + # Connect basic block arguments. + inputs = {} + outputs = {} + block_flow.in.each do |arg| + # Each basic block argument gets a phi node. Even if there's only + # one predecessor! We'll tidy this up later. + phi = PhiNode.new(id_counter) + connect(phi, merge_node, :info) if merge_node + nodes << phi + inputs[arg] = phi + + block.each_with_length do |_, consumer_offset| + consumer_flow = dfg.insn_flows[consumer_offset] + consumer_flow.in.each_with_index do |producer, input_index| + if producer == arg + connect(phi, insn_nodes[consumer_offset], :data, input_index) + end + end + end + + block_flow.out.each { |out| outputs[out] = phi if out == arg } + end + + # Connect local dataflow from consumers back to producers. + block.each_with_length do |_, consumer_offset| + consumer_flow = dfg.insn_flows.fetch(consumer_offset) + consumer_flow.in.each_with_index do |producer, input_index| + if producer.local? + connect( + insn_nodes[producer.length], + insn_nodes[consumer_offset], + :data, + input_index + ) + end + end + end + + # Connect dataflow from producers that leaves the block. + block.each_with_length do |_, producer_pc| + dfg + .insn_flows + .fetch(producer_pc) + .out + .each do |consumer| + unless consumer.local? + # This is an argument to the successor block - not to an + # instruction here. + outputs[consumer.name] = insn_nodes[producer_pc] + end + end + end + + # A graph with only side-effect free instructions will currently have + # no fixed nodes! In that case just use the first instruction's node + # for both first and last. But it's a bug that it'll appear in the + # control flow path! + SubGraph.new( + first_fixed || insn_nodes[block.block_start], + last_fixed || insn_nodes[block.block_start], + inputs, + outputs + ) + end + + # Connect control flow that flows between basic blocks. + def connect_local_graphs_control(local_graphs) + dfg.blocks.each do |predecessor| + predecessor_last = local_graphs[predecessor.id].last_fixed + predecessor.outgoing_blocks.each_with_index do |successor, index| + label = + if index > 0 && + index == (predecessor.outgoing_blocks.length - 1) + # If there are multiple outgoing blocks from this block, then + # the last one is a fallthrough. Otherwise it's a branch. + :fallthrough + else + :"branch#{index}" + end + + connect( + predecessor_last, + local_graphs[successor.id].first_fixed, + :control, + label + ) + end + end + end + + # Connect data flow that flows between basic blocks. + def connect_local_graphs_data(local_graphs) + dfg.blocks.each do |predecessor| + arg_outs = local_graphs[predecessor.id].outputs.values + arg_outs.each_with_index do |arg_out, arg_n| + predecessor.outgoing_blocks.each do |successor| + successor_graph = local_graphs[successor.id] + arg_in = successor_graph.inputs.values[arg_n] + + # We're connecting to a phi node, so we may need a special + # label. + raise unless arg_in.is_a?(PhiNode) + + label = + case arg_out + when InsnNode + # Instructions that go into a phi node are labelled by the + # offset of last instruction in the block that executed + # them. This way you know which value to use for the phi, + # based on the last instruction you executed. + dfg.blocks.find do |block| + block_start = block.block_start + block_end = + block_start + block.insns.sum(&:length) - + block.insns.last.length + + if (block_start..block_end).cover?(arg_out.offset) + break block_end + end + end + when PhiNode + # Phi nodes to phi nodes are not labelled. + else + raise + end + + connect(arg_out, arg_in, :data, label) + end + end + end + end + + # We don't always build things in an optimal way. Go back and fix up + # some mess we left. Ideally we wouldn't create these problems in the + # first place. + def cleanup + nodes.dup.each do |node| # dup because we're mutating + next unless node.is_a?(PhiNode) + + if node.inputs.size == 1 + # Remove phi nodes with a single input. + node.inputs.each do |producer_edge| + node.outputs.each do |consumer_edge| + connect( + producer_edge.from, + consumer_edge.to, + producer_edge.type, + consumer_edge.label + ) + end + end + + remove(node) + elsif node.inputs.map(&:from).uniq.size == 1 + # Remove phi nodes where all inputs are the same. + producer_edge = node.inputs.first + consumer_edge = node.outputs.find { |e| !e.to.is_a?(MergeNode) } + connect( + producer_edge.from, + consumer_edge.to, + :data, + consumer_edge.label + ) + remove(node) + end + end + end + + # Connect one node to another. + def connect(from, to, type, label = nil) + raise if from == to + raise if !to.is_a?(PhiNode) && type == :data && label.nil? + + edge = Edge.new(from, to, type, label) + from.outputs << edge + to.inputs << edge + end + + # Remove a node from the graph. + def remove(node) + node.inputs.each do |producer_edge| + producer_edge.from.outputs.reject! { |edge| edge.to == node } + end + + node.outputs.each do |consumer_edge| + consumer_edge.to.inputs.reject! { |edge| edge.from == node } + end + + nodes.delete(node) + end + end + + attr_reader :dfg, :nodes, :local_graphs + + def initialize(dfg, nodes, local_graphs) + @dfg = dfg + @nodes = nodes + @local_graphs = local_graphs + end + + def to_mermaid + output = StringIO.new + output.puts("flowchart TD") + + nodes.each do |node| + escaped = "\"#{CGI.escapeHTML(node.label)}\"" + output.puts(" node_#{node.id}(#{escaped})") + end + + link_counter = 0 + nodes.each do |producer| + producer.outputs.each do |consumer_edge| + case consumer_edge.type + when :data + edge = "-->" + edge_style = "stroke:green;" + when :control + edge = "-->" + edge_style = "stroke:red;" + when :info + edge = "-.->" + else + raise + end + + label = + if !consumer_edge.label + "" + elsif consumer_edge.to.is_a?(PhiNode) + # Edges into phi nodes are labelled by the offset of the + # instruction going into the merge. + "|%04d| " % consumer_edge.label + else + "|#{consumer_edge.label}| " + end + + to_id = "node_#{consumer_edge.to.id}" + output.puts(" node_#{producer.id} #{edge} #{label}#{to_id}") + + if edge_style + output.puts(" linkStyle #{link_counter} #{edge_style}") + end + + link_counter += 1 + end + end + + output.string + end + + def verify + # Verify edge labels. + nodes.each do |node| + # Not talking about phi nodes right now. + next if node.is_a?(PhiNode) + + if node.is_a?(InsnNode) && node.insn.branch_targets.any? && + !node.insn.is_a?(Leave) + # A branching node must have at least one branch edge and + # potentially a fallthrough edge coming out. + + labels = node.outputs.map(&:label).sort + raise if labels[0] != :branch0 + raise if labels[1] != :fallthrough && labels.size > 2 + else + labels = node.inputs.filter { |e| e.type == :data }.map(&:label) + next if labels.empty? + + # No nil labels + raise if labels.any?(&:nil?) + + # Labels should start at zero. + raise unless labels.min.zero? + + # Labels should be contiguous. + raise unless labels.sort == (labels.min..labels.max).to_a + end + end + end + + def self.compile(dfg) + Compiler.new(dfg).compile + end + end + end +end diff --git a/test/yarv_test.rb b/test/yarv_test.rb index 5ac37504..e6a3adda 100644 --- a/test/yarv_test.rb +++ b/test/yarv_test.rb @@ -366,6 +366,116 @@ def test_dfg DFG end + def test_son + iseq = RubyVM::InstructionSequence.compile("(14 < 0 ? -1 : +1) + 100") + iseq = SyntaxTree::YARV::InstructionSequence.from(iseq.to_a) + cfg = SyntaxTree::YARV::ControlFlowGraph.compile(iseq) + dfg = SyntaxTree::YARV::DataFlowGraph.compile(cfg) + son = SyntaxTree::YARV::SeaOfNodes.compile(dfg) + + assert_equal(<<~SON, son.to_mermaid) + flowchart TD + node_0("0000 putobject 14") + node_2("0002 putobject_INT2FIX_0_") + node_3("0003 opt_lt <calldata!mid:<, argc:1, ARGS_SIMPLE>") + node_5("0005 branchunless 0011") + node_7("0007 putobject -1") + node_9("0009 jump 0012") + node_11("0011 putobject_INT2FIX_1_") + node_12("0012 putobject 100") + node_14("0014 opt_plus <calldata!mid:+, argc:1, ARGS_SIMPLE>") + node_16("0016 leave") + node_1000("1000 ψ") + node_1001("1001 φ") + node_0 --> |0| node_3 + linkStyle 0 stroke:green; + node_2 --> |1| node_3 + linkStyle 1 stroke:green; + node_3 --> node_5 + linkStyle 2 stroke:red; + node_3 --> |0| node_5 + linkStyle 3 stroke:green; + node_5 --> |branch0| node_11 + linkStyle 4 stroke:red; + node_5 --> |fallthrough| node_9 + linkStyle 5 stroke:red; + node_7 --> |0009| node_1001 + linkStyle 6 stroke:green; + node_9 --> |branch0| node_1000 + linkStyle 7 stroke:red; + node_11 --> |branch0| node_1000 + linkStyle 8 stroke:red; + node_11 --> |0011| node_1001 + linkStyle 9 stroke:green; + node_12 --> |1| node_14 + linkStyle 10 stroke:green; + node_14 --> node_16 + linkStyle 11 stroke:red; + node_14 --> |0| node_16 + linkStyle 12 stroke:green; + node_1000 --> node_14 + linkStyle 13 stroke:red; + node_1001 -.-> node_1000 + node_1001 --> |0| node_14 + linkStyle 15 stroke:green; + SON + end + + def test_son_indirect_basic_block_argument + iseq = RubyVM::InstructionSequence.compile("100 + (14 < 0 ? -1 : +1)") + iseq = SyntaxTree::YARV::InstructionSequence.from(iseq.to_a) + cfg = SyntaxTree::YARV::ControlFlowGraph.compile(iseq) + dfg = SyntaxTree::YARV::DataFlowGraph.compile(cfg) + son = SyntaxTree::YARV::SeaOfNodes.compile(dfg) + + assert_equal(<<~SON, son.to_mermaid) + flowchart TD + node_0("0000 putobject 100") + node_2("0002 putobject 14") + node_4("0004 putobject_INT2FIX_0_") + node_5("0005 opt_lt <calldata!mid:<, argc:1, ARGS_SIMPLE>") + node_7("0007 branchunless 0013") + node_9("0009 putobject -1") + node_11("0011 jump 0014") + node_13("0013 putobject_INT2FIX_1_") + node_14("0014 opt_plus <calldata!mid:+, argc:1, ARGS_SIMPLE>") + node_16("0016 leave") + node_1002("1002 ψ") + node_1004("1004 φ") + node_0 --> |0| node_14 + linkStyle 0 stroke:green; + node_2 --> |0| node_5 + linkStyle 1 stroke:green; + node_4 --> |1| node_5 + linkStyle 2 stroke:green; + node_5 --> node_7 + linkStyle 3 stroke:red; + node_5 --> |0| node_7 + linkStyle 4 stroke:green; + node_7 --> |branch0| node_13 + linkStyle 5 stroke:red; + node_7 --> |fallthrough| node_11 + linkStyle 6 stroke:red; + node_9 --> |0011| node_1004 + linkStyle 7 stroke:green; + node_11 --> |branch0| node_1002 + linkStyle 8 stroke:red; + node_13 --> |branch0| node_1002 + linkStyle 9 stroke:red; + node_13 --> |0013| node_1004 + linkStyle 10 stroke:green; + node_14 --> node_16 + linkStyle 11 stroke:red; + node_14 --> |0| node_16 + linkStyle 12 stroke:green; + node_1002 --> node_14 + linkStyle 13 stroke:red; + node_1004 -.-> node_1002 + node_1004 --> |1| node_14 + linkStyle 15 stroke:green; + SON + end + private def assert_decompiles(expected, source) From 9e09fd005663d6539c2b5570a3cb8c11bf23e311 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 7 Feb 2023 08:33:30 -0500 Subject: [PATCH 367/536] Sea of nodes optimizations and convenience functions --- lib/syntax_tree/yarv/control_flow_graph.rb | 4 + lib/syntax_tree/yarv/instruction_sequence.rb | 8 ++ lib/syntax_tree/yarv/sea_of_nodes.rb | 91 +++++++++++++++++--- test/yarv_test.rb | 52 +++++------ 4 files changed, 113 insertions(+), 42 deletions(-) diff --git a/lib/syntax_tree/yarv/control_flow_graph.rb b/lib/syntax_tree/yarv/control_flow_graph.rb index 1a361e5e..73d30208 100644 --- a/lib/syntax_tree/yarv/control_flow_graph.rb +++ b/lib/syntax_tree/yarv/control_flow_graph.rb @@ -203,6 +203,10 @@ def to_dfg DataFlowGraph.compile(self) end + def to_son + to_dfg.to_son + end + def to_mermaid output = StringIO.new output.puts("flowchart TD") diff --git a/lib/syntax_tree/yarv/instruction_sequence.rb b/lib/syntax_tree/yarv/instruction_sequence.rb index 918a3c86..821738c9 100644 --- a/lib/syntax_tree/yarv/instruction_sequence.rb +++ b/lib/syntax_tree/yarv/instruction_sequence.rb @@ -273,6 +273,14 @@ def to_cfg ControlFlowGraph.compile(self) end + def to_dfg + to_cfg.to_dfg + end + + def to_son + to_dfg.to_son + end + def disasm fmt = Disassembler.new fmt.enqueue(self) diff --git a/lib/syntax_tree/yarv/sea_of_nodes.rb b/lib/syntax_tree/yarv/sea_of_nodes.rb index be027f39..fdf905a7 100644 --- a/lib/syntax_tree/yarv/sea_of_nodes.rb +++ b/lib/syntax_tree/yarv/sea_of_nodes.rb @@ -118,7 +118,8 @@ def compile connect_local_graphs_control(local_graphs) connect_local_graphs_data(local_graphs) - cleanup + cleanup_phi_nodes + cleanup_insn_nodes SeaOfNodes.new(dfg, nodes, local_graphs).tap(&:verify) end @@ -311,23 +312,13 @@ def connect_local_graphs_data(local_graphs) # We don't always build things in an optimal way. Go back and fix up # some mess we left. Ideally we wouldn't create these problems in the # first place. - def cleanup + def cleanup_phi_nodes nodes.dup.each do |node| # dup because we're mutating next unless node.is_a?(PhiNode) if node.inputs.size == 1 # Remove phi nodes with a single input. - node.inputs.each do |producer_edge| - node.outputs.each do |consumer_edge| - connect( - producer_edge.from, - consumer_edge.to, - producer_edge.type, - consumer_edge.label - ) - end - end - + connect_over(node) remove(node) elsif node.inputs.map(&:from).uniq.size == 1 # Remove phi nodes where all inputs are the same. @@ -344,6 +335,66 @@ def cleanup end end + # Eliminate as many unnecessary nodes as we can. + def cleanup_insn_nodes + nodes.dup.each do |node| + next unless node.is_a?(InsnNode) + + case node.insn + when AdjustStack + # If there are any inputs to the adjust stack that are immediately + # discarded, we can remove them from the input list. + number = node.insn.number + + node.inputs.dup.each do |input_edge| + next if input_edge.type != :data + + from = input_edge.from + next unless from.is_a?(InsnNode) + + if from.inputs.empty? && from.outputs.size == 1 + number -= 1 + remove(input_edge.from) + elsif from.insn.is_a?(Dup) + number -= 1 + connect_over(from) + remove(from) + + new_edge = node.inputs.last + new_edge.from.outputs.delete(new_edge) + node.inputs.delete(new_edge) + end + end + + if number == 0 + connect_over(node) + remove(node) + else + next_node = + if number == 1 + InsnNode.new(Pop.new, node.offset) + else + InsnNode.new(AdjustStack.new(number), node.offset) + end + + next_node.inputs.concat(node.inputs) + next_node.outputs.concat(node.outputs) + + # Dynamically finding the index of the node in the nodes array + # because we're mutating the array as we go. + nodes[nodes.index(node)] = next_node + end + when Jump + # When you have a jump instruction that only has one input and one + # output, you can just connect over top of it and remove it. + if node.inputs.size == 1 && node.outputs.size == 1 + connect_over(node) + remove(node) + end + end + end + end + # Connect one node to another. def connect(from, to, type, label = nil) raise if from == to @@ -354,6 +405,20 @@ def connect(from, to, type, label = nil) to.inputs << edge end + # Connect all of the inputs to all of the outputs of a node. + def connect_over(node) + node.inputs.each do |producer_edge| + node.outputs.each do |consumer_edge| + connect( + producer_edge.from, + consumer_edge.to, + producer_edge.type, + producer_edge.label + ) + end + end + end + # Remove a node from the graph. def remove(node) node.inputs.each do |producer_edge| diff --git a/test/yarv_test.rb b/test/yarv_test.rb index e6a3adda..a1e89568 100644 --- a/test/yarv_test.rb +++ b/test/yarv_test.rb @@ -302,7 +302,7 @@ def test_cfg iseq = SyntaxTree::YARV::InstructionSequence.from(iseq.to_a) cfg = SyntaxTree::YARV::ControlFlowGraph.compile(iseq) - assert_equal(<<~CFG, cfg.disasm) + assert_equal(<<~DISASM, cfg.disasm) == cfg: #@:1 (1,0)-(1,0)> block_0 0000 putobject 100 @@ -325,7 +325,7 @@ def test_cfg 0014 opt_plus 0016 leave == to: leaves - CFG + DISASM end def test_dfg @@ -334,7 +334,7 @@ def test_dfg cfg = SyntaxTree::YARV::ControlFlowGraph.compile(iseq) dfg = SyntaxTree::YARV::DataFlowGraph.compile(cfg) - assert_equal(<<~DFG, dfg.disasm) + assert_equal(<<~DISASM, dfg.disasm) == dfg: #@:1 (1,0)-(1,0)> block_0 0000 putobject 100 # out: out_0 @@ -363,7 +363,7 @@ def test_dfg 0014 opt_plus # in: in_0, in_1; out: 16 0016 leave # in: 14 == to: leaves - DFG + DISASM end def test_son @@ -373,14 +373,13 @@ def test_son dfg = SyntaxTree::YARV::DataFlowGraph.compile(cfg) son = SyntaxTree::YARV::SeaOfNodes.compile(dfg) - assert_equal(<<~SON, son.to_mermaid) + assert_equal(<<~MERMAID, son.to_mermaid) flowchart TD node_0("0000 putobject 14") node_2("0002 putobject_INT2FIX_0_") node_3("0003 opt_lt <calldata!mid:<, argc:1, ARGS_SIMPLE>") node_5("0005 branchunless 0011") node_7("0007 putobject -1") - node_9("0009 jump 0012") node_11("0011 putobject_INT2FIX_1_") node_12("0012 putobject 100") node_14("0014 opt_plus <calldata!mid:+, argc:1, ARGS_SIMPLE>") @@ -397,28 +396,26 @@ def test_son linkStyle 3 stroke:green; node_5 --> |branch0| node_11 linkStyle 4 stroke:red; - node_5 --> |fallthrough| node_9 + node_5 --> |fallthrough| node_1000 linkStyle 5 stroke:red; node_7 --> |0009| node_1001 linkStyle 6 stroke:green; - node_9 --> |branch0| node_1000 - linkStyle 7 stroke:red; node_11 --> |branch0| node_1000 - linkStyle 8 stroke:red; + linkStyle 7 stroke:red; node_11 --> |0011| node_1001 - linkStyle 9 stroke:green; + linkStyle 8 stroke:green; node_12 --> |1| node_14 - linkStyle 10 stroke:green; + linkStyle 9 stroke:green; node_14 --> node_16 - linkStyle 11 stroke:red; + linkStyle 10 stroke:red; node_14 --> |0| node_16 - linkStyle 12 stroke:green; + linkStyle 11 stroke:green; node_1000 --> node_14 - linkStyle 13 stroke:red; + linkStyle 12 stroke:red; node_1001 -.-> node_1000 node_1001 --> |0| node_14 - linkStyle 15 stroke:green; - SON + linkStyle 14 stroke:green; + MERMAID end def test_son_indirect_basic_block_argument @@ -428,7 +425,7 @@ def test_son_indirect_basic_block_argument dfg = SyntaxTree::YARV::DataFlowGraph.compile(cfg) son = SyntaxTree::YARV::SeaOfNodes.compile(dfg) - assert_equal(<<~SON, son.to_mermaid) + assert_equal(<<~MERMAID, son.to_mermaid) flowchart TD node_0("0000 putobject 100") node_2("0002 putobject 14") @@ -436,7 +433,6 @@ def test_son_indirect_basic_block_argument node_5("0005 opt_lt <calldata!mid:<, argc:1, ARGS_SIMPLE>") node_7("0007 branchunless 0013") node_9("0009 putobject -1") - node_11("0011 jump 0014") node_13("0013 putobject_INT2FIX_1_") node_14("0014 opt_plus <calldata!mid:+, argc:1, ARGS_SIMPLE>") node_16("0016 leave") @@ -454,26 +450,24 @@ def test_son_indirect_basic_block_argument linkStyle 4 stroke:green; node_7 --> |branch0| node_13 linkStyle 5 stroke:red; - node_7 --> |fallthrough| node_11 + node_7 --> |fallthrough| node_1002 linkStyle 6 stroke:red; node_9 --> |0011| node_1004 linkStyle 7 stroke:green; - node_11 --> |branch0| node_1002 - linkStyle 8 stroke:red; node_13 --> |branch0| node_1002 - linkStyle 9 stroke:red; + linkStyle 8 stroke:red; node_13 --> |0013| node_1004 - linkStyle 10 stroke:green; + linkStyle 9 stroke:green; node_14 --> node_16 - linkStyle 11 stroke:red; + linkStyle 10 stroke:red; node_14 --> |0| node_16 - linkStyle 12 stroke:green; + linkStyle 11 stroke:green; node_1002 --> node_14 - linkStyle 13 stroke:red; + linkStyle 12 stroke:red; node_1004 -.-> node_1002 node_1004 --> |1| node_14 - linkStyle 15 stroke:green; - SON + linkStyle 14 stroke:green; + MERMAID end private From 93ec53b1a042ff5d5575a0f6a5dba728884572fb Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 7 Feb 2023 11:12:01 -0500 Subject: [PATCH 368/536] Optimize pop nodes --- lib/syntax_tree/yarv/sea_of_nodes.rb | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/lib/syntax_tree/yarv/sea_of_nodes.rb b/lib/syntax_tree/yarv/sea_of_nodes.rb index fdf905a7..181d729c 100644 --- a/lib/syntax_tree/yarv/sea_of_nodes.rb +++ b/lib/syntax_tree/yarv/sea_of_nodes.rb @@ -391,6 +391,30 @@ def cleanup_insn_nodes connect_over(node) remove(node) end + when Pop + from = node.inputs.find { |edge| edge.type == :data }.from + next unless from.is_a?(InsnNode) + + removed = + if from.inputs.empty? && from.outputs.size == 1 + remove(from) + true + elsif from.insn.is_a?(Dup) + connect_over(from) + remove(from) + + new_edge = node.inputs.last + new_edge.from.outputs.delete(new_edge) + node.inputs.delete(new_edge) + true + else + false + end + + if removed + connect_over(node) + remove(node) + end end end end From 0411bdda92897879390b7541b133d553ef0707f5 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 7 Feb 2023 12:26:09 -0500 Subject: [PATCH 369/536] Documentation on changing the structure of the AST --- doc/changing_structure.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 doc/changing_structure.md diff --git a/doc/changing_structure.md b/doc/changing_structure.md new file mode 100644 index 00000000..74012f26 --- /dev/null +++ b/doc/changing_structure.md @@ -0,0 +1,16 @@ +# Changing structure + +First and foremost, changing the structure of the tree in any way is a major breaking change. It forces the consumers to update their visitors, pattern matches, and method calls. It should not be taking lightly, and can only happen on a major version change. So keep that in mind. + +That said, if you do want to change the structure of the tree, there are a few steps that you have to take. They are enumerated below. + +1. Change the structure in the required node classes. This could mean adding/removing classes or adding/removing fields. Be sure to also update the `copy` and `===` methods to be sure that they are correct. +2. Update the parser to correctly create the new structure. +3. Update any visitor methods that are affected by the change. For example, if adding a new node make sure to create the new visit method alias in the `Visitor` class. +4. Update the `FieldVisitor` class to be sure that the various serializers, pretty printers, and matchers all get updated accordingly. +5. Update the `DSL` module to be sure that folks can correctly create nodes with the new structure. +6. Ensure the formatting of the code hasn't changed. This can mostly be done by running the tests, but if there's a corner case that we don't cover that is now exposed by your change be sure to add test cases. +7. Update the translation visitors to ensure we're still translating into other ASTs correctly. +8. Update the YARV compiler visitor to ensure we're still compiling correctly. +9. Make sure we aren't referencing the previous structure in any documentation or tests. +10. Be sure to update `CHANGELOG.md` with a description of the change that you made. From c13bfda6d167908437f0518d0dfe1cfe14d439c5 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 7 Feb 2023 12:10:06 -0500 Subject: [PATCH 370/536] More locations for the parser translation --- lib/syntax_tree/node.rb | 5 +- lib/syntax_tree/parser.rb | 33 +- lib/syntax_tree/translation/parser.rb | 819 ++++++++++++++------------ test/fixtures/next.rb | 7 + test/node_test.rb | 12 +- 5 files changed, 474 insertions(+), 402 deletions(-) diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index b1ecfdc7..ff8ee95a 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -11527,8 +11527,9 @@ def ===(other) # # To be clear, this method should just not exist. It's not good. It's a # place of shame. But it's necessary for now, so I'm keeping it. - def pin(parent) - replace = PinnedVarRef.new(value: value, location: location) + def pin(parent, pin) + replace = + PinnedVarRef.new(value: value, location: pin.location.to(location)) parent .deconstruct_keys([]) diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index 75af65bf..59128875 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -641,8 +641,7 @@ def visit(node) end def visit_var_ref(node) - pins.shift - node.pin(stack[-2]) + node.pin(stack[-2], pins.shift) end def self.visit(node, tokens) @@ -1683,6 +1682,22 @@ def on_float(value) # VarField right # ) -> FndPtn def on_fndptn(constant, left, values, right) + # The left and right of a find pattern are always going to be splats, so + # we're going to consume the * operators and use their location + # information to extend the location of the splats. + right, left = + [right, left].map do |node| + operator = consume_operator(:*) + location = + if node.value + operator.location.to(node.location) + else + operator.location + end + + node.copy(location: location) + end + # The opening of this find pattern is either going to be a left bracket, a # right left parenthesis, or the left splat. We're going to use this to # determine how to find the closing of the pattern, as well as determining @@ -1791,7 +1806,7 @@ def on_heredoc_beg(value) line: lineno, char: char_pos, column: current_column, - size: value.size + 1 + size: value.size ) # Here we're going to artificially create an extra node type so that if @@ -1826,7 +1841,7 @@ def on_heredoc_end(value) line: lineno, char: char_pos, column: current_column, - size: value.size + 1 + size: value.size ) heredoc_end = HeredocEnd.new(value: value.chomp, location: location) @@ -1841,9 +1856,9 @@ def on_heredoc_end(value) start_line: heredoc.location.start_line, start_char: heredoc.location.start_char, start_column: heredoc.location.start_column, - end_line: lineno, - end_char: char_pos, - end_column: current_column + end_line: location.end_line, + end_char: location.end_char, + end_column: location.end_column ) ) end @@ -2357,14 +2372,14 @@ def on_method_add_arg(call, arguments) # :call-seq: # on_method_add_block: ( - # (Break | Call | Command | CommandCall) call, + # (Break | Call | Command | CommandCall, Next) call, # Block block # ) -> Break | MethodAddBlock def on_method_add_block(call, block) location = call.location.to(block.location) case call - when Break + when Break, Next parts = call.arguments.parts node = parts.pop diff --git a/lib/syntax_tree/translation/parser.rb b/lib/syntax_tree/translation/parser.rb index 1e47b4e7..4a4b6ade 100644 --- a/lib/syntax_tree/translation/parser.rb +++ b/lib/syntax_tree/translation/parser.rb @@ -27,9 +27,9 @@ def visit_alias(node) s( :alias, [visit(node.left), visit(node.right)], - source_map_keyword( - keyword: source_range_length(node.location.start_char, 5), - expression: source_range_node(node) + source_map_keyword_bare( + source_range_length(node.location.start_char, 5), + source_range_node(node) ) ) end @@ -58,11 +58,7 @@ def visit_aref(node) [visit(node.collection)].concat(visit_all(node.index.parts)), source_map_index( begin_token: - source_range_find( - node.collection.location.end_char, - node.index.location.start_char, - "[" - ), + source_range_find_between(node.collection, node.index, "["), end_token: source_range_length(node.location.end_char, -1), expression: source_range_node(node) ) @@ -90,9 +86,9 @@ def visit_aref(node) source_map_send( selector: source_range( - source_range_find( - node.collection.location.end_char, - node.index.location.start_char, + source_range_find_between( + node.collection, + node.index, "[" ).begin_pos, node.location.end_char @@ -128,11 +124,7 @@ def visit_aref_field(node) [visit(node.collection)].concat(visit_all(node.index.parts)), source_map_index( begin_token: - source_range_find( - node.collection.location.end_char, - node.index.location.start_char, - "[" - ), + source_range_find_between(node.collection, node.index, "["), end_token: source_range_length(node.location.end_char, -1), expression: source_range_node(node) ) @@ -162,9 +154,9 @@ def visit_aref_field(node) source_map_send( selector: source_range( - source_range_find( - node.collection.location.end_char, - node.index.location.start_char, + source_range_find_between( + node.collection, + node.index, "[" ).begin_pos, node.location.end_char @@ -182,8 +174,8 @@ def visit_arg_block(node) :block_pass, [visit(node.value)], source_map_operator( - operator: source_range_length(node.location.start_char, 1), - expression: source_range_node(node) + source_range_length(node.location.start_char, 1), + source_range_node(node) ) ) end @@ -192,18 +184,14 @@ def visit_arg_block(node) def visit_arg_star(node) if stack[-3].is_a?(MLHSParen) && stack[-3].contents.is_a?(MLHS) if node.value.nil? - s( - :restarg, - [], - source_map_variable(expression: source_range_node(node)) - ) + s(:restarg, [], source_map_variable(nil, source_range_node(node))) else s( :restarg, [node.value.value.to_sym], source_map_variable( - name: source_range_node(node.value), - expression: source_range_node(node) + source_range_node(node.value), + source_range_node(node) ) ) end @@ -212,8 +200,8 @@ def visit_arg_star(node) :splat, node.value.nil? ? [] : [visit(node.value)], source_map_operator( - operator: source_range_length(node.location.start_char, 1), - expression: source_range_node(node) + source_range_length(node.location.start_char, 1), + source_range_node(node) ) ) end @@ -307,11 +295,7 @@ def visit_assign(node) target .location .with_operator( - source_range_find( - node.target.location.end_char, - node.value.location.start_char, - "=" - ) + source_range_find_between(node.target, node.value, "=") ) .with_expression(source_range_node(node)) @@ -324,19 +308,25 @@ def visit_assoc(node) expression = source_range(node.location.start_char, node.location.end_char - 1) + type, location = + if node.key.value.start_with?(/[A-Z]/) + [:const, source_map_constant(nil, expression, expression)] + else + [ + :send, + source_map_send(selector: expression, expression: expression) + ] + end + s( :pair, [ visit(node.key), - s( - node.key.value.start_with?(/[A-Z]/) ? :const : :send, - [nil, node.key.value.chomp(":").to_sym], - source_map_send(selector: expression, expression: expression) - ) + s(type, [nil, node.key.value.chomp(":").to_sym], location) ], source_map_operator( - operator: source_range_length(node.key.location.end_char, -1), - expression: source_range_node(node) + source_range_length(node.key.location.end_char, -1), + source_range_node(node) ) ) else @@ -344,8 +334,9 @@ def visit_assoc(node) :pair, [visit(node.key), visit(node.value)], source_map_operator( - operator: source_range_length(node.key.location.end_char, -1), - expression: source_range_node(node) + source_range_search_between(node.key, node.value, "=>") || + source_range_length(node.key.location.end_char, -1), + source_range_node(node) ) ) end @@ -357,8 +348,8 @@ def visit_assoc_splat(node) :kwsplat, [visit(node.value)], source_map_operator( - operator: source_range_length(node.location.start_char, 2), - expression: source_range_node(node) + source_range_length(node.location.start_char, 2), + source_range_node(node) ) ) end @@ -394,15 +385,14 @@ def visit_BEGIN(node) :preexe, [visit(node.statements)], source_map_keyword( - keyword: source_range_length(node.location.start_char, 5), - begin_token: - source_range_find( - node.location.start_char + 5, - node.statements.location.start_char, - "{" - ), - end_token: source_range_length(node.location.end_char, -1), - expression: source_range_node(node) + source_range_length(node.location.start_char, 5), + source_range_find( + node.location.start_char + 5, + node.statements.location.start_char, + "{" + ), + source_range_length(node.location.end_char, -1), + source_range_node(node) ) ) end @@ -450,13 +440,12 @@ def visit_binary(node) ), [visit(node.left), visit(node.right)], source_map_operator( - operator: - source_range_find( - node.left.location.end_char, - node.right.location.start_char, - node.operator.to_s - ), - expression: source_range_node(node) + source_range_find_between( + node.left, + node.right, + node.operator.to_s + ), + source_range_node(node) ) ) when :=~ @@ -471,13 +460,12 @@ def visit_binary(node) :match_with_lvasgn, [visit(node.left), visit(node.right)], source_map_operator( - operator: - source_range_find( - node.left.location.end_char, - node.right.location.start_char, - node.operator.to_s - ), - expression: source_range_node(node) + source_range_find_between( + node.left, + node.right, + node.operator.to_s + ), + source_range_node(node) ) ) else @@ -491,18 +479,14 @@ def visit_binary(node) # Visit a BlockArg node. def visit_blockarg(node) if node.name.nil? - s( - :blockarg, - [nil], - source_map_variable(expression: source_range_node(node)) - ) + s(:blockarg, [nil], source_map_variable(nil, source_range_node(node))) else s( :blockarg, [node.name.value.to_sym], source_map_variable( - name: source_range_node(node.name), - expression: source_range_node(node) + source_range_node(node.name), + source_range_node(node) ) ) end @@ -516,8 +500,8 @@ def visit_block_var(node) :shadowarg, [local.value.to_sym], source_map_variable( - name: source_range_node(local), - expression: source_range_node(local) + source_range_node(local), + source_range_node(local) ) ) end @@ -539,8 +523,8 @@ def visit_block_var(node) :arg, [required.value.to_sym], source_map_variable( - name: source_range_node(required), - expression: source_range_node(required) + source_range_node(required), + source_range_node(required) ) ) ], @@ -624,9 +608,9 @@ def visit_break(node) s( :break, visit_all(node.arguments.parts), - source_map_keyword( - keyword: source_range_length(node.location.start_char, 5), - expression: source_range_node(node) + source_map_keyword_bare( + source_range_length(node.location.start_char, 5), + source_range_node(node) ) ) end @@ -685,11 +669,7 @@ def visit_CHAR(node) def visit_class(node) operator = if node.superclass - source_range_find( - node.constant.location.end_char, - node.superclass.location.start_char, - "<" - ) + source_range_find_between(node.constant, node.superclass, "<") end s( @@ -824,8 +804,9 @@ def visit_const(node) :const, [nil, node.value.to_sym], source_map_constant( - name: source_range_node(node), - expression: source_range_node(node) + nil, + source_range_node(node), + source_range_node(node) ) ) end @@ -840,14 +821,9 @@ def visit_const_path_field(node) :casgn, [visit(node.parent), node.constant.value.to_sym], source_map_constant( - double_colon: - source_range_find( - node.parent.location.end_char, - node.constant.location.start_char, - "::" - ), - name: source_range_node(node.constant), - expression: source_range_node(node) + source_range_find_between(node.parent, node.constant, "::"), + source_range_node(node.constant), + source_range_node(node) ) ) end @@ -859,14 +835,9 @@ def visit_const_path_ref(node) :const, [visit(node.parent), node.constant.value.to_sym], source_map_constant( - double_colon: - source_range_find( - node.parent.location.end_char, - node.constant.location.start_char, - "::" - ), - name: source_range_node(node.constant), - expression: source_range_node(node) + source_range_find_between(node.parent, node.constant, "::"), + source_range_node(node.constant), + source_range_node(node) ) ) end @@ -877,8 +848,9 @@ def visit_const_ref(node) :const, [nil, node.constant.value.to_sym], source_map_constant( - name: source_range_node(node.constant), - expression: source_range_node(node) + nil, + source_range_node(node.constant), + source_range_node(node) ) ) end @@ -888,10 +860,7 @@ def visit_cvar(node) s( :cvar, [node.value.to_sym], - source_map_variable( - name: source_range_node(node), - expression: source_range_node(node) - ) + source_map_variable(source_range_node(node), source_range_node(node)) ) end @@ -931,9 +900,9 @@ def visit_def(node) source_map_method_definition( keyword: source_range_length(node.location.start_char, 3), assignment: - source_range_find( - (node.params || node.name).location.end_char, - node.bodystmt.location.start_char, + source_range_find_between( + (node.params || node.name), + node.bodystmt, "=" ), name: source_range_node(node.name), @@ -983,10 +952,10 @@ def visit_defined(node) :defined?, [visit(node.value)], source_map_keyword( - keyword: source_range_length(node.location.start_char, 8), - begin_token: begin_token, - end_token: end_token, - expression: source_range_node(node) + source_range_length(node.location.start_char, 8), + begin_token, + end_token, + source_range_node(node) ) ) end @@ -1061,15 +1030,14 @@ def visit_END(node) :postexe, [visit(node.statements)], source_map_keyword( - keyword: source_range_length(node.location.start_char, 3), - begin_token: - source_range_find( - node.location.start_char + 3, - node.statements.location.start_char, - "{" - ), - end_token: source_range_length(node.location.end_char, -1), - expression: source_range_node(node) + source_range_length(node.location.start_char, 3), + source_range_find( + node.location.start_char + 3, + node.statements.location.start_char, + "{" + ), + source_range_length(node.location.end_char, -1), + source_range_node(node) ) ) end @@ -1129,32 +1097,36 @@ def visit_float(node) s( :float, [node.value.to_f], - source_map_operator( - operator: operator, - expression: source_range_node(node) - ) + source_map_operator(operator, source_range_node(node)) ) end # Visit a FndPtn node. def visit_fndptn(node) - make_match_rest = ->(child) do - if child.is_a?(VarField) && child.value.nil? - s(:match_rest, [], nil) - else - s(:match_rest, [visit(child)], nil) + left, right = + [node.left, node.right].map do |child| + location = + source_map_operator( + source_range_length(child.location.start_char, 1), + source_range_node(child) + ) + + if child.is_a?(VarField) && child.value.nil? + s(:match_rest, [], location) + else + s(:match_rest, [visit(child)], location) + end end - end inner = s( :find_pattern, - [ - make_match_rest[node.left], - *visit_all(node.values), - make_match_rest[node.right] - ], - nil + [left, *visit_all(node.values), right], + source_map_collection( + begin_token: source_range_length(node.location.start_char, 1), + end_token: source_range_length(node.location.end_char, -1), + expression: source_range_node(node) + ) ) if node.constant @@ -1166,28 +1138,15 @@ def visit_fndptn(node) # Visit a For node. def visit_for(node) - begin_start = node.collection.location.end_char - begin_end = node.statements.location.start_char - - begin_token = - if buffer.source[begin_start...begin_end].include?("do") - source_range_find(begin_start, begin_end, "do") - end - s( :for, [visit(node.index), visit(node.collection), visit(node.statements)], source_map_for( - keyword: source_range_length(node.location.start_char, 3), - in_token: - source_range_find( - node.index.location.end_char, - node.collection.location.start_char, - "in" - ), - begin_token: begin_token, - end_token: source_range_length(node.location.end_char, -3), - expression: source_range_node(node) + source_range_length(node.location.start_char, 3), + source_range_find_between(node.index, node.collection, "in"), + source_range_search_between(node.collection, node.statements, "do"), + source_range_length(node.location.end_char, -3), + source_range_node(node) ) ) end @@ -1197,10 +1156,7 @@ def visit_gvar(node) s( :gvar, [node.value.to_sym], - source_map_variable( - name: source_range_node(node), - expression: source_range_node(node) - ) + source_map_variable(source_range_node(node), source_range_node(node)) ) end @@ -1303,15 +1259,32 @@ def visit_heredoc(node) end heredoc_segments.trim! + location = + source_map_heredoc( + source_range_node(node.beginning), + source_range( + if node.parts.empty? + node.beginning.location.end_char + else + node.parts.first.location.start_char + end, + node.ending.location.start_char + ), + source_range( + node.ending.location.start_char, + node.ending.location.end_char - 1 + ) + ) if node.beginning.value.match?(/`\w+`\z/) - s(:xstr, heredoc_segments.segments, nil) + s(:xstr, heredoc_segments.segments, location) elsif heredoc_segments.segments.length > 1 - s(:dstr, heredoc_segments.segments, nil) + s(:dstr, heredoc_segments.segments, location) elsif heredoc_segments.segments.empty? - s(:dstr, [], nil) + s(:dstr, [], location) else - heredoc_segments.segments.first + segment = heredoc_segments.segments.first + s(segment.type, segment.children, location) end end @@ -1353,10 +1326,7 @@ def visit_ident(node) s( :lvar, [node.value.to_sym], - source_map_variable( - name: source_range_node(node), - expression: source_range_node(node) - ) + source_map_variable(source_range_node(node), source_range_node(node)) ) end @@ -1389,14 +1359,9 @@ def visit_if(node) :if, [predicate, visit(node.statements), visit(node.consequent)], if node.modifier? - source_map_keyword( - keyword: - source_range_find( - node.statements.location.end_char, - node.predicate.location.start_char, - "if" - ), - expression: source_range_node(node) + source_map_keyword_bare( + source_range_find_between(node.statements, node.predicate, "if"), + source_range_node(node) ) else begin_start = node.predicate.location.end_char @@ -1410,6 +1375,8 @@ def visit_if(node) begin_token = if buffer.source[begin_start...begin_end].include?("then") source_range_find(begin_start, begin_end, "then") + elsif buffer.source[begin_start...begin_end].include?(";") + source_range_find(begin_start, begin_end, ";") end else_token = @@ -1450,7 +1417,7 @@ def visit_imaginary(node) # case. Maybe there's an API for this but I can't find it. eval(node.value) ], - source_map_operator(expression: source_range_node(node)) + source_map_operator(nil, source_range_node(node)) ) end @@ -1478,19 +1445,24 @@ def visit_in(node) nil ) else + begin_token = + source_range_search_between(node.pattern, node.statements, "then") + end_char = - if node.statements.empty? + if begin_token || node.statements.empty? node.statements.location.end_char - 1 else - node.statements.body.first.location.start_char + node.statements.body.last.location.start_char end s( :in_pattern, [visit(node.pattern), nil, visit(node.statements)], source_map_keyword( - keyword: source_range_length(node.location.start_char, 2), - expression: source_range(node.location.start_char, end_char) + source_range_length(node.location.start_char, 2), + begin_token, + nil, + source_range(node.location.start_char, end_char) ) ) end @@ -1506,10 +1478,7 @@ def visit_int(node) s( :int, [node.value.to_i], - source_map_operator( - operator: operator, - expression: source_range_node(node) - ) + source_map_operator(operator, source_range_node(node)) ) end @@ -1518,10 +1487,7 @@ def visit_ivar(node) s( :ivar, [node.value.to_sym], - source_map_variable( - name: source_range_node(node), - expression: source_range_node(node) - ) + source_map_variable(source_range_node(node), source_range_node(node)) ) end @@ -1548,18 +1514,14 @@ def visit_kw(node) # Visit a KwRestParam node. def visit_kwrest_param(node) if node.name.nil? - s( - :kwrestarg, - [], - source_map_variable(expression: source_range_node(node)) - ) + s(:kwrestarg, [], source_map_variable(nil, source_range_node(node))) else s( :kwrestarg, [node.name.value.to_sym], source_map_variable( - name: source_range_node(node.name), - expression: source_range_node(node) + source_range_node(node.name), + source_range_node(node) ) ) end @@ -1635,8 +1597,8 @@ def visit_lambda_var(node) :shadowarg, [local.value.to_sym], source_map_variable( - name: source_range_node(local), - expression: source_range_node(local) + source_range_node(local), + source_range_node(local) ) ) end @@ -1661,13 +1623,8 @@ def visit_massign(node) :masgn, [visit(node.target), visit(node.value)], source_map_operator( - operator: - source_range_find( - node.target.location.end_char, - node.value.location.start_char, - "=" - ), - expression: source_range_node(node) + source_range_find_between(node.target, node.value, "="), + source_range_node(node) ) ) end @@ -1722,8 +1679,8 @@ def visit_mlhs(node) :arg, [part.value.to_sym], source_map_variable( - name: source_range_node(part), - expression: source_range_node(part) + source_range_node(part), + source_range_node(part) ) ) else @@ -1778,9 +1735,9 @@ def visit_next(node) s( :next, visit_all(node.arguments.parts), - source_map_keyword( - keyword: source_range_length(node.location.start_char, 4), - expression: source_range_node(node) + source_map_keyword_bare( + source_range_length(node.location.start_char, 4), + source_range_node(node) ) ) end @@ -1839,10 +1796,45 @@ def visit_not(node) # Visit an OpAssign node. def visit_opassign(node) location = - source_map_variable( - name: source_range_node(node.target), - expression: source_range_node(node) - ).with_operator(source_range_node(node.operator)) + case node.target + when ARefField + source_map_index( + begin_token: + source_range_find( + node.target.collection.location.end_char, + if node.target.index + node.target.index.location.start_char + else + node.target.location.end_char + end, + "[" + ), + end_token: source_range_length(node.target.location.end_char, -1), + expression: source_range_node(node) + ) + when Field + source_map_send( + dot: + if node.target.operator == :"::" + source_range_find_between( + node.target.parent, + node.target.name, + "::" + ) + else + source_range_node(node.target.operator) + end, + selector: source_range_node(node.target.name), + expression: source_range_node(node) + ) + else + source_map_variable( + source_range_node(node.target), + source_range_node(node) + ) + end + + location = location.with_operator(source_range_node(node.operator)) case node.operator.value when "||=" @@ -1876,8 +1868,8 @@ def visit_params(node) :arg, [required.value.to_sym], source_map_variable( - name: source_range_node(required), - expression: source_range_node(required) + source_range_node(required), + source_range_node(required) ) ) end @@ -1889,16 +1881,9 @@ def visit_params(node) :optarg, [name.value.to_sym, visit(value)], source_map_variable( - name: source_range_node(name), - expression: - source_range_node(name).join(source_range_node(value)) - ).with_operator( - source_range_find( - name.location.end_char, - value.location.start_char, - "=" - ) - ) + source_range_node(name), + source_range_node(name).join(source_range_node(value)) + ).with_operator(source_range_find_between(name, value, "=")) ) end @@ -1912,8 +1897,8 @@ def visit_params(node) :arg, [post.value.to_sym], source_map_variable( - name: source_range_node(post), - expression: source_range_node(post) + source_range_node(post), + source_range_node(post) ) ) end @@ -1927,13 +1912,11 @@ def visit_params(node) :kwoptarg, [key, visit(value)], source_map_variable( - name: - source_range( - name.location.start_char, - name.location.end_char - 1 - ), - expression: - source_range_node(name).join(source_range_node(value)) + source_range( + name.location.start_char, + name.location.end_char - 1 + ), + source_range_node(name).join(source_range_node(value)) ) ) else @@ -1941,12 +1924,11 @@ def visit_params(node) :kwarg, [key], source_map_variable( - name: - source_range( - name.location.start_char, - name.location.end_char - 1 - ), - expression: source_range_node(name) + source_range( + name.location.start_char, + name.location.end_char - 1 + ), + source_range_node(name) ) ) end @@ -1960,8 +1942,8 @@ def visit_params(node) :kwnilarg, [], source_map_variable( - name: source_range_length(node.location.end_char, -3), - expression: source_range_node(node) + source_range_length(node.location.end_char, -3), + source_range_node(node) ) ) else @@ -2011,12 +1993,41 @@ def visit_paren(node) # Visit a PinnedBegin node. def visit_pinned_begin(node) - s(:pin, [s(:begin, [visit(node.statement)], nil)], nil) + s( + :pin, + [ + s( + :begin, + [visit(node.statement)], + source_map_collection( + begin_token: + source_range_length(node.location.start_char + 1, 1), + end_token: source_range_length(node.location.end_char, -1), + expression: + source_range( + node.location.start_char + 1, + node.location.end_char + ) + ) + ) + ], + source_map_send( + selector: source_range_length(node.location.start_char, 1), + expression: source_range_node(node) + ) + ) end # Visit a PinnedVarRef node. def visit_pinned_var_ref(node) - s(:pin, [visit(node.value)], nil) + s( + :pin, + [visit(node.value)], + source_map_send( + selector: source_range_length(node.location.start_char, 1), + expression: source_range_node(node) + ) + ) end # Visit a Program node. @@ -2057,8 +2068,8 @@ def visit_range(node) node.operator.value == ".." ? :irange : :erange, [visit(node.left), visit(node.right)], source_map_operator( - operator: source_range_node(node.operator), - expression: source_range_node(node) + source_range_node(node.operator), + source_range_node(node) ) ) end @@ -2069,8 +2080,8 @@ def visit_rassign(node) node.operator.value == "=>" ? :match_pattern : :match_pattern_p, [visit(node.value), visit(node.pattern)], source_map_operator( - operator: source_range_node(node.operator), - expression: source_range_node(node) + source_range_node(node.operator), + source_range_node(node) ) ) end @@ -2080,7 +2091,7 @@ def visit_rational(node) s( :rational, [node.value.to_r], - source_map_operator(expression: source_range_node(node)) + source_map_operator(nil, source_range_node(node)) ) end @@ -2089,9 +2100,9 @@ def visit_redo(node) s( :redo, [], - source_map_keyword( - keyword: source_range_node(node), - expression: source_range_node(node) + source_map_keyword_bare( + source_range_node(node), + source_range_node(node) ) ) end @@ -2245,11 +2256,7 @@ def visit_rescue(node) # Visit a RescueMod node. def visit_rescue_mod(node) keyword = - source_range_find( - node.statement.location.end_char, - node.value.location.start_char, - "rescue" - ) + source_range_find_between(node.statement, node.value, "rescue") s( :rescue, @@ -2276,16 +2283,12 @@ def visit_rest_param(node) :restarg, [node.name.value.to_sym], source_map_variable( - name: source_range_node(node.name), - expression: source_range_node(node) + source_range_node(node.name), + source_range_node(node) ) ) else - s( - :restarg, - [], - source_map_variable(expression: source_range_node(node)) - ) + s(:restarg, [], source_map_variable(nil, source_range_node(node))) end end @@ -2294,9 +2297,9 @@ def visit_retry(node) s( :retry, [], - source_map_keyword( - keyword: source_range_node(node), - expression: source_range_node(node) + source_map_keyword_bare( + source_range_node(node), + source_range_node(node) ) ) end @@ -2306,9 +2309,9 @@ def visit_return(node) s( :return, node.arguments ? visit_all(node.arguments.parts) : [], - source_map_keyword( - keyword: source_range_length(node.location.start_char, 6), - expression: source_range_node(node) + source_map_keyword_bare( + source_range_length(node.location.start_char, 6), + source_range_node(node) ) ) end @@ -2399,7 +2402,11 @@ def visit_string_literal(node) location = if node.quote source_map_collection( - begin_token: source_range_length(node.location.start_char, 1), + begin_token: + source_range_length( + node.location.start_char, + node.quote.length + ), end_token: source_range_length(node.location.end_char, -1), expression: source_range_node(node) ) @@ -2423,9 +2430,9 @@ def visit_super(node) s( :super, visit_all(node.arguments.parts), - source_map_keyword( - keyword: source_range_node(node), - expression: source_range_node(node) + source_map_keyword_bare( + source_range_length(node.location.start_char, 5), + source_range_node(node) ) ) else @@ -2435,15 +2442,14 @@ def visit_super(node) :super, [], source_map_keyword( - keyword: source_range_length(node.location.start_char, 5), - begin_token: - source_range_find( - node.location.start_char + 5, - node.location.end_char, - "(" - ), - end_token: source_range_length(node.location.end_char, -1), - expression: source_range_node(node) + source_range_length(node.location.start_char, 5), + source_range_find( + node.location.start_char + 5, + node.location.end_char, + "(" + ), + source_range_length(node.location.end_char, -1), + source_range_node(node) ) ) when ArgsForward @@ -2453,15 +2459,14 @@ def visit_super(node) :super, visit_all(node.arguments.arguments.parts), source_map_keyword( - keyword: source_range_length(node.location.start_char, 5), - begin_token: - source_range_find( - node.location.start_char + 5, - node.location.end_char, - "(" - ), - end_token: source_range_length(node.location.end_char, -1), - expression: source_range_node(node) + source_range_length(node.location.start_char, 5), + source_range_find( + node.location.start_char + 5, + node.location.end_char, + "(" + ), + source_range_length(node.location.end_char, -1), + source_range_node(node) ) ) end @@ -2526,9 +2531,9 @@ def visit_top_const_field(node) node.constant.value.to_sym ], source_map_constant( - double_colon: source_range_length(node.location.start_char, 2), - name: source_range_node(node.constant), - expression: source_range_node(node) + source_range_length(node.location.start_char, 2), + source_range_node(node.constant), + source_range_node(node) ) ) end @@ -2548,9 +2553,9 @@ def visit_top_const_ref(node) node.constant.value.to_sym ], source_map_constant( - double_colon: source_range_length(node.location.start_char, 2), - name: source_range_node(node.constant), - expression: source_range_node(node) + source_range_length(node.location.start_char, 2), + source_range_node(node.constant), + source_range_node(node) ) ) end @@ -2592,9 +2597,9 @@ def visit_undef(node) s( :undef, visit_all(node.symbols), - source_map_keyword( - keyword: source_range_length(node.location.start_char, 5), - expression: source_range_node(node) + source_map_keyword_bare( + source_range_length(node.location.start_char, 5), + source_range_node(node) ) ) end @@ -2624,14 +2629,13 @@ def visit_unless(node) :if, [predicate, visit(node.consequent), visit(node.statements)], if node.modifier? - source_map_keyword( - keyword: - source_range_find( - node.statements.location.end_char, - node.predicate.location.start_char, - "unless" - ), - expression: source_range_node(node) + source_map_keyword_bare( + source_range_find_between( + node.statements, + node.predicate, + "unless" + ), + source_range_node(node) ) else source_map_condition( @@ -2649,20 +2653,20 @@ def visit_until(node) loop_post?(node) ? :until_post : :until, [visit(node.predicate), visit(node.statements)], if node.modifier? - source_map_keyword( - keyword: - source_range_find( - node.statements.location.end_char, - node.predicate.location.start_char, - "until" - ), - expression: source_range_node(node) + source_map_keyword_bare( + source_range_find_between( + node.statements, + node.predicate, + "until" + ), + source_range_node(node) ) else source_map_keyword( - keyword: source_range_length(node.location.start_char, 5), - end_token: source_range_length(node.location.end_char, -3), - expression: source_range_node(node) + source_range_length(node.location.start_char, 5), + nil, + source_range_length(node.location.end_char, -3), + source_range_node(node) ) end ) @@ -2688,8 +2692,8 @@ def visit_var_field(node) :match_var, [name], source_map_variable( - name: source_range_node(node), - expression: source_range_node(node) + source_range_node(node.value), + source_range_node(node.value) ) ) elsif node.value.is_a?(Const) @@ -2697,15 +2701,16 @@ def visit_var_field(node) :casgn, [nil, name], source_map_constant( - name: source_range_node(node.value), - expression: source_range_node(node) + nil, + source_range_node(node.value), + source_range_node(node) ) ) else location = source_map_variable( - name: source_range_node(node), - expression: source_range_node(node) + source_range_node(node), + source_range_node(node) ) case node.value @@ -2747,17 +2752,26 @@ def visit_vcall(node) # Visit a When node. def visit_when(node) keyword = source_range_length(node.location.start_char, 4) + begin_token = + if buffer.source[node.statements.location.start_char] == ";" + source_range_length(node.statements.location.start_char, 1) + end + + end_char = + if node.statements.body.empty? + node.statements.location.end_char + else + node.statements.body.last.location.end_char + end s( :when, visit_all(node.arguments.parts) + [visit(node.statements)], source_map_keyword( - keyword: keyword, - expression: - source_range( - keyword.begin_pos, - node.statements.location.end_char - 1 - ) + keyword, + begin_token, + nil, + source_range(keyword.begin_pos, end_char) ) ) end @@ -2768,20 +2782,20 @@ def visit_while(node) loop_post?(node) ? :while_post : :while, [visit(node.predicate), visit(node.statements)], if node.modifier? - source_map_keyword( - keyword: - source_range_find( - node.statements.location.end_char, - node.predicate.location.start_char, - "while" - ), - expression: source_range_node(node) + source_map_keyword_bare( + source_range_find_between( + node.statements, + node.predicate, + "while" + ), + source_range_node(node) ) else source_map_keyword( - keyword: source_range_length(node.location.start_char, 5), - end_token: source_range_length(node.location.end_char, -3), - expression: source_range_node(node) + source_range_length(node.location.start_char, 5), + nil, + source_range_length(node.location.end_char, -3), + source_range_node(node) ) end ) @@ -2828,18 +2842,18 @@ def visit_yield(node) s( :yield, [], - source_map_keyword( - keyword: source_range_length(node.location.start_char, 5), - expression: source_range_node(node) + source_map_keyword_bare( + source_range_length(node.location.start_char, 5), + source_range_node(node) ) ) when Args s( :yield, visit_all(node.arguments.parts), - source_map_keyword( - keyword: source_range_length(node.location.start_char, 5), - expression: source_range_node(node) + source_map_keyword_bare( + source_range_length(node.location.start_char, 5), + source_range_node(node) ) ) else @@ -2847,11 +2861,10 @@ def visit_yield(node) :yield, visit_all(node.arguments.contents.parts), source_map_keyword( - keyword: source_range_length(node.location.start_char, 5), - begin_token: - source_range_length(node.arguments.location.start_char, 1), - end_token: source_range_length(node.location.end_char, -1), - expression: source_range_node(node) + source_range_length(node.location.start_char, 5), + source_range_length(node.arguments.location.start_char, 1), + source_range_length(node.location.end_char, -1), + source_range_node(node) ) ) end @@ -2862,9 +2875,9 @@ def visit_zsuper(node) s( :zsuper, [], - source_map_keyword( - keyword: source_range_length(node.location.start_char, 5), - expression: source_range_node(node) + source_map_keyword_bare( + source_range_length(node.location.start_char, 5), + source_range_node(node) ) ) end @@ -3029,7 +3042,7 @@ def source_map_condition( end # Constructs a new source map for a constant reference. - def source_map_constant(double_colon: nil, name: nil, expression:) + def source_map_constant(double_colon, name, expression) ::Parser::Source::Map::Constant.new(double_colon, name, expression) end @@ -3049,13 +3062,7 @@ def source_map_definition( end # Constructs a new source map for a for loop. - def source_map_for( - keyword: nil, - in_token: nil, - begin_token: nil, - end_token: nil, - expression: - ) + def source_map_for(keyword, in_token, begin_token, end_token, expression) ::Parser::Source::Map::For.new( keyword, in_token, @@ -3065,18 +3072,22 @@ def source_map_for( ) end + # Constructs a new source map for a heredoc. + def source_map_heredoc(expression, heredoc_body, heredoc_end) + ::Parser::Source::Map::Heredoc.new( + expression, + heredoc_body, + heredoc_end + ) + end + # Construct a source map for an index operation. def source_map_index(begin_token: nil, end_token: nil, expression:) ::Parser::Source::Map::Index.new(begin_token, end_token, expression) end # Constructs a new source map for the use of a keyword. - def source_map_keyword( - keyword: nil, - begin_token: nil, - end_token: nil, - expression: - ) + def source_map_keyword(keyword, begin_token, end_token, expression) ::Parser::Source::Map::Keyword.new( keyword, begin_token, @@ -3085,6 +3096,12 @@ def source_map_keyword( ) end + # Constructs a new source map for the use of a keyword without a begin or + # end token. + def source_map_keyword_bare(keyword, expression) + source_map_keyword(keyword, nil, nil, expression) + end + # Constructs a new source map for a method definition. def source_map_method_definition( keyword: nil, @@ -3105,7 +3122,7 @@ def source_map_method_definition( end # Constructs a new source map for an operator. - def source_map_operator(operator: nil, expression:) + def source_map_operator(operator, expression) ::Parser::Source::Map::Operator.new(operator, expression) end @@ -3142,7 +3159,7 @@ def source_map_send( end # Constructs a new source map for a variable. - def source_map_variable(name: nil, expression:) + def source_map_variable(name, expression) ::Parser::Source::Map::Variable.new(name, expression) end @@ -3152,16 +3169,48 @@ def source_range(start_char, end_char) end # Constructs a new source range by finding the given needle in the given - # range of the source. - def source_range_find(start_char, end_char, needle) + # range of the source. If the needle is not found, returns nil. + def source_range_search(start_char, end_char, needle) index = buffer.source[start_char...end_char].index(needle) - unless index + return unless index + + offset = start_char + index + source_range(offset, offset + needle.length) + end + + # Constructs a new source range by searching for the given needle between + # the end location of the start node and the start location of the end + # node. If the needle is not found, returns nil. + def source_range_search_between(start_node, end_node, needle) + source_range_search( + start_node.location.end_char, + end_node.location.start_char, + needle + ) + end + + # Constructs a new source range by finding the given needle in the given + # range of the source. If it needle is not found, raises an error. + def source_range_find(start_char, end_char, needle) + source_range = source_range_search(start_char, end_char, needle) + + unless source_range slice = buffer.source[start_char...end_char].inspect raise "Could not find #{needle.inspect} in #{slice}" end - offset = start_char + index - source_range(offset, offset + needle.length) + source_range + end + + # Constructs a new source range by finding the given needle between the + # end location of the start node and the start location of the end node. + # If the needle is not found, returns raises an error. + def source_range_find_between(start_node, end_node, needle) + source_range_find( + start_node.location.end_char, + end_node.location.start_char, + needle + ) end # Constructs a new source range from the given start offset and length. diff --git a/test/fixtures/next.rb b/test/fixtures/next.rb index be667951..79a8c62e 100644 --- a/test/fixtures/next.rb +++ b/test/fixtures/next.rb @@ -65,3 +65,10 @@ next([1, 2]) - next 1, 2 +% +next fun foo do end +- +next( + fun foo do + end +) diff --git a/test/node_test.rb b/test/node_test.rb index 9660b341..19fbeed2 100644 --- a/test/node_test.rb +++ b/test/node_test.rb @@ -60,7 +60,7 @@ def test_arg_paren_heredoc ARGUMENT SOURCE - at = location(lines: 1..3, chars: 6..28) + at = location(lines: 1..3, chars: 6..37) assert_node(ArgParen, source, at: at, &:arguments) end @@ -533,7 +533,7 @@ def test_heredoc HEREDOC SOURCE - at = location(lines: 1..3, chars: 0..22) + at = location(lines: 1..3, chars: 0..30) assert_node(Heredoc, source, at: at) end @@ -544,7 +544,7 @@ def test_heredoc_beg HEREDOC SOURCE - at = location(chars: 0..11) + at = location(chars: 0..10) assert_node(HeredocBeg, source, at: at, &:beginning) end @@ -555,7 +555,7 @@ def test_heredoc_end HEREDOC SOURCE - at = location(lines: 3..3, chars: 22..31, columns: 0..9) + at = location(lines: 3..3, chars: 22..30, columns: 0..8) assert_node(HeredocEnd, source, at: at, &:ending) end @@ -950,7 +950,7 @@ def test_var_field guard_version("3.1.0") do def test_pinned_var_ref source = "foo in ^bar" - at = location(chars: 8..11) + at = location(chars: 7..11) assert_node(PinnedVarRef, source, at: at, &:pattern) end @@ -1008,7 +1008,7 @@ def test_xstring_heredoc HEREDOC SOURCE - at = location(lines: 1..3, chars: 0..18) + at = location(lines: 1..3, chars: 0..26) assert_node(Heredoc, source, at: at) end From 3f308340c97c56eedb580263c66b0d5c65a23bf8 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 7 Feb 2023 16:25:29 -0500 Subject: [PATCH 371/536] Strip out whitequark/parser submodule --- .gitmodules | 6 - Rakefile | 18 +- tasks/spec.rake | 10 + tasks/whitequark.rake | 87 ++ test/ruby-syntax-fixtures | 1 - test/ruby_syntax_fixtures_test.rb | 19 - test/suites/helper.rb | 3 - test/suites/parse_helper.rb | 175 --- test/suites/parser | 1 - test/translation/parser.txt | 1824 +++++++++++++++++++++++++++++ test/translation/parser_test.rb | 168 +++ 11 files changed, 2092 insertions(+), 220 deletions(-) create mode 100644 tasks/spec.rake create mode 100644 tasks/whitequark.rake delete mode 160000 test/ruby-syntax-fixtures delete mode 100644 test/ruby_syntax_fixtures_test.rb delete mode 100644 test/suites/helper.rb delete mode 100644 test/suites/parse_helper.rb delete mode 160000 test/suites/parser create mode 100644 test/translation/parser.txt create mode 100644 test/translation/parser_test.rb diff --git a/.gitmodules b/.gitmodules index 8287c5e3..f5477ea3 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,9 +4,3 @@ [submodule "spec"] path = spec/ruby url = git@github.com:ruby/spec.git -[submodule "test/ruby-syntax-fixtures"] - path = test/ruby-syntax-fixtures - url = https://github.com/ruby-syntax-tree/ruby-syntax-fixtures -[submodule "test/suites/parser"] - path = test/suites/parser - url = https://github.com/whitequark/parser diff --git a/Rakefile b/Rakefile index cb96e7bf..aa8d29f6 100644 --- a/Rakefile +++ b/Rakefile @@ -4,18 +4,13 @@ require "bundler/gem_tasks" require "rake/testtask" require "syntax_tree/rake_tasks" +Rake.add_rakelib "tasks" + Rake::TestTask.new(:test) do |t| t.libs << "test" t.libs << "test/suites" t.libs << "lib" - - # These are our own tests. - test_files = FileList["test/**/*_test.rb"] - - # This is a big test file from the parser gem that tests its functionality. - test_files << "test/suites/parser/test/test_parser.rb" - - t.test_files = test_files + t.test_files = FileList["test/**/*_test.rb"] end task default: :test @@ -34,10 +29,3 @@ end SyntaxTree::Rake::CheckTask.new(&configure) SyntaxTree::Rake::WriteTask.new(&configure) - -desc "Run mspec tests using YARV emulation" -task :spec do - Dir["./spec/ruby/language/**/*_spec.rb"].each do |filepath| - sh "exe/yarv ./spec/mspec/bin/mspec-tag #{filepath}" - end -end diff --git a/tasks/spec.rake b/tasks/spec.rake new file mode 100644 index 00000000..c361fe8e --- /dev/null +++ b/tasks/spec.rake @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +desc "Run mspec tests using YARV emulation" +task :spec do + specs = File.expand_path("../spec/ruby/language/**/*_spec.rb", __dir__) + + Dir[specs].each do |filepath| + sh "exe/yarv ./spec/mspec/bin/mspec-tag #{filepath}" + end +end diff --git a/tasks/whitequark.rake b/tasks/whitequark.rake new file mode 100644 index 00000000..4f7ee650 --- /dev/null +++ b/tasks/whitequark.rake @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +# This file's purpose is to extract the examples from the whitequark/parser +# gem and generate a test file that we can use to ensure that our parser +# generates equivalent syntax trees when translating. To do this, it runs the +# parser's test suite but overrides the `assert_parses` method to collect the +# examples into a hash. Then, it writes out the hash to a file that we can use +# to generate our own tests. +# +# To run the test suite, it's important to note that we have to mirror both any +# APIs provided to the test suite (for example the ParseHelper module below). +# This is obviously relatively brittle, but it's effective for now. + +require "ast" + +module ParseHelper + # This object is going to collect all of the examples from the parser gem into + # a hash that we can use to generate our own tests. + COLLECTED = Hash.new { |hash, key| hash[key] = [] } + + include AST::Sexp + ALL_VERSIONS = %w[3.1 3.2] + + private + + def assert_context(*) + end + + def assert_diagnoses(*) + end + + def assert_diagnoses_many(*) + end + + def refute_diagnoses(*) + end + + def with_versions(*) + end + + def assert_parses(_ast, code, _source_maps = "", versions = ALL_VERSIONS) + # We're going to skip any examples that are for older Ruby versions + # that we do not support. + return if (versions & %w[3.1 3.2]).empty? + + entry = caller.find { _1.include?("test_parser.rb") } + _, lineno, name = *entry.match(/(\d+):in `(.+)'/) + + COLLECTED["#{name}:#{lineno}"] << code + end +end + +namespace :extract do + desc "Extract the whitequark/parser tests" + task :whitequark do + directory = File.expand_path("../tmp/parser", __dir__) + unless File.directory?(directory) + sh "git clone --depth 1 https://github.com/whitequark/parser #{directory}" + end + + mkdir_p "#{directory}/extract" + touch "#{directory}/extract/helper.rb" + touch "#{directory}/extract/parse_helper.rb" + touch "#{directory}/extract/extracted.txt" + $:.unshift "#{directory}/extract" + + require "parser/current" + require "minitest/autorun" + require_relative "#{directory}/test/test_parser" + + Minitest.after_run do + filepath = File.expand_path("../test/translation/parser.txt", __dir__) + + File.open(filepath, "w") do |file| + ParseHelper::COLLECTED.sort.each do |(key, codes)| + if codes.length == 1 + file.puts("!!! #{key}\n#{codes.first}") + else + codes.each_with_index do |code, index| + file.puts("!!! #{key}:#{index}\n#{code}") + end + end + end + end + end + end +end diff --git a/test/ruby-syntax-fixtures b/test/ruby-syntax-fixtures deleted file mode 160000 index 5b333f5a..00000000 --- a/test/ruby-syntax-fixtures +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 5b333f5a34d6fb08f88acc93b69c7d19b3fee8e7 diff --git a/test/ruby_syntax_fixtures_test.rb b/test/ruby_syntax_fixtures_test.rb deleted file mode 100644 index c5c13b27..00000000 --- a/test/ruby_syntax_fixtures_test.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true - -# The ruby-syntax-fixtures repository tests against the current Ruby syntax, so -# we don't execute this test unless we're running 3.2 or above. -return unless RUBY_VERSION >= "3.2" - -require_relative "test_helper" - -module SyntaxTree - class RubySyntaxFixturesTest < Minitest::Test - Dir[ - File.expand_path("ruby-syntax-fixtures/**/*.rb", __dir__) - ].each do |file| - define_method "test_ruby_syntax_fixtures_#{file}" do - refute_nil(SyntaxTree.parse(SyntaxTree.read(file))) - end - end - end -end diff --git a/test/suites/helper.rb b/test/suites/helper.rb deleted file mode 100644 index b0f8c427..00000000 --- a/test/suites/helper.rb +++ /dev/null @@ -1,3 +0,0 @@ -# frozen_string_literal: true - -require "parser/current" diff --git a/test/suites/parse_helper.rb b/test/suites/parse_helper.rb deleted file mode 100644 index 04fe8123..00000000 --- a/test/suites/parse_helper.rb +++ /dev/null @@ -1,175 +0,0 @@ -# frozen_string_literal: true - -module ParseHelper - include AST::Sexp - - CURRENT_VERSION = RUBY_VERSION.split(".")[0..1].join(".").freeze - ALL_VERSIONS = %w[1.8 1.9 2.0 2.1 2.2 2.3 2.4 2.5 2.6 2.7 3.0 3.1 3.2 mac ios] - - known_failures = [ - # I think this may be a bug in the parser gem's precedence calculation. - # Unary plus appears to be parsed as part of the number literal in CRuby, - # but parser is parsing it as a separate operator. - "test_unary_num_pow_precedence:3505", - - # Not much to be done about this. Basically, regular expressions with named - # capture groups that use the =~ operator inject local variables into the - # current scope. In the parser gem, it detects this and changes future - # references to that name to be a local variable instead of a potential - # method call. CRuby does not do this. - "test_lvar_injecting_match:3778", - - # This is failing because CRuby is not marking values captured in hash - # patterns as local variables, while the parser gem is. - "test_pattern_matching_hash:8971", - - # This is not actually allowed in the CRuby parser but the parser gem thinks - # it is allowed. - "test_pattern_matching_hash_with_string_keys:9016", - "test_pattern_matching_hash_with_string_keys:9027", - "test_pattern_matching_hash_with_string_keys:9038", - "test_pattern_matching_hash_with_string_keys:9060", - "test_pattern_matching_hash_with_string_keys:9071", - "test_pattern_matching_hash_with_string_keys:9082", - - # This happens with pattern matching where you're matching a literal value - # inside parentheses, which doesn't really do anything. Ripper doesn't - # capture that this value is inside a parentheses, so it's hard to translate - # properly. - "test_pattern_matching_expr_in_paren:9206", - - # These are also failing because of CRuby not marking values captured in - # hash patterns as local variables. - "test_pattern_matching_single_line_allowed_omission_of_parentheses:9205", - "test_pattern_matching_single_line_allowed_omission_of_parentheses:9581", - "test_pattern_matching_single_line_allowed_omission_of_parentheses:9611", - - # I'm not even sure what this is testing, because the code is invalid in - # CRuby. - "test_control_meta_escape_chars_in_regexp__since_31:*", - ] - - # These are failures that we need to take care of (or determine the reason - # that we're not going to handle them). - todo_failures = [ - "test_dedenting_heredoc:334", - "test_dedenting_heredoc:390", - "test_dedenting_heredoc:399", - "test_slash_newline_in_heredocs:7194", - "test_parser_slash_slash_n_escaping_in_literals:*", - "test_cond_match_current_line:4801", - "test_forwarded_restarg:*", - "test_forwarded_kwrestarg:*", - "test_forwarded_argument_with_restarg:*", - "test_forwarded_argument_with_kwrestarg:*" - ] - - if CURRENT_VERSION <= "2.7" - # I'm not sure why this is failing on 2.7.0, but we'll turn it off for now - # until we have more time to investigate. - todo_failures.push("test_pattern_matching_hash:*") - end - - if CURRENT_VERSION <= "3.0" - # In < 3.0, there are some changes to the way the parser gem handles - # forwarded args. We should eventually support this, but for now we're going - # to mark them as todo. - todo_failures.push( - "test_forward_arg:*", - "test_forward_args_legacy:*", - "test_endless_method_forwarded_args_legacy:*", - "test_trailing_forward_arg:*" - ) - end - - if CURRENT_VERSION == "3.1" - # This test actually fails on 3.1.0, even though it's marked as being since - # 3.1. So we're going to skip this test on 3.1, but leave it in for other - # versions. - known_failures.push( - "test_multiple_pattern_matches:11086", - "test_multiple_pattern_matches:11102" - ) - end - - # This is the list of all failures. - FAILURES = (known_failures + todo_failures).freeze - - private - - def assert_context(*) - end - - def assert_diagnoses(*) - end - - def assert_diagnoses_many(*) - end - - def refute_diagnoses(*) - end - - def with_versions(*) - end - - def assert_parses(_ast, code, _source_maps = "", versions = ALL_VERSIONS) - # We're going to skip any examples that aren't for the current version of - # Ruby. - return unless versions.include?(CURRENT_VERSION) - - # We're going to skip any examples that are for older Ruby versions that we - # do not support. - return if (versions & %w[3.1 3.2]).empty? - - caller(1, 3).each do |line| - _, lineno, name = *line.match(/(\d+):in `(.+)'/) - - # Return directly and don't do anything if it's a known failure. - return if FAILURES.include?("#{name}:#{lineno}") - return if FAILURES.include?("#{name}:*") - end - - expected = parse(code) - return if expected.nil? - - buffer = expected.location.expression.source_buffer - actual = SyntaxTree::Translation.to_parser(SyntaxTree.parse(code), buffer) - assert_equal(expected, actual) - end - - def parse(code) - parser = Parser::CurrentRuby.default_parser - parser.diagnostics.consumer = ->(*) {} - - buffer = Parser::Source::Buffer.new("(string)", 1) - buffer.source = code - - parser.parse(buffer) - rescue Parser::SyntaxError - end -end - -if ENV["PARSER_LOCATION"] - # Modify the source map == check so that it doesn't check against the node - # itself so we don't get into a recursive loop. - Parser::Source::Map.prepend( - Module.new do - def ==(other) - self.class == other.class && - (instance_variables - %i[@node]).map do |ivar| - instance_variable_get(ivar) == other.instance_variable_get(ivar) - end.reduce(:&) - end - end - ) - - # Next, ensure that we're comparing the nodes and also comparing the source - # ranges so that we're getting all of the necessary information. - Parser::AST::Node.prepend( - Module.new do - def ==(other) - super && (location == other.location) - end - end - ) -end diff --git a/test/suites/parser b/test/suites/parser deleted file mode 160000 index 8de8b7fa..00000000 --- a/test/suites/parser +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8de8b7fa7af471a2159860d6a0a5b615eac9c83c diff --git a/test/translation/parser.txt b/test/translation/parser.txt new file mode 100644 index 00000000..5e9e8d31 --- /dev/null +++ b/test/translation/parser.txt @@ -0,0 +1,1824 @@ +!!! assert_parses_args:2249:0 +def f (foo: 1, bar: 2, **baz, &b); end +!!! assert_parses_args:2249:1 +def f (foo: 1, &b); end +!!! assert_parses_args:2249:2 +def f **baz, &b; end +!!! assert_parses_args:2249:3 +def f *, **; end +!!! assert_parses_args:2249:4 +def f a, o=1, *r, &b; end +!!! assert_parses_args:2249:5 +def f a, o=1, *r, p, &b; end +!!! assert_parses_args:2249:6 +def f a, o=1, &b; end +!!! assert_parses_args:2249:7 +def f a, o=1, p, &b; end +!!! assert_parses_args:2249:8 +def f a, *r, &b; end +!!! assert_parses_args:2249:9 +def f a, *r, p, &b; end +!!! assert_parses_args:2249:10 +def f a, &b; end +!!! assert_parses_args:2249:11 +def f o=1, *r, &b; end +!!! assert_parses_args:2249:12 +def f o=1, *r, p, &b; end +!!! assert_parses_args:2249:13 +def f o=1, &b; end +!!! assert_parses_args:2249:14 +def f o=1, p, &b; end +!!! assert_parses_args:2249:15 +def f *r, &b; end +!!! assert_parses_args:2249:16 +def f *r, p, &b; end +!!! assert_parses_args:2249:17 +def f &b; end +!!! assert_parses_args:2249:18 +def f ; end +!!! assert_parses_args:2249:19 +def f (((a))); end +!!! assert_parses_args:2249:20 +def f ((a, a1)); end +!!! assert_parses_args:2249:21 +def f ((a, *r)); end +!!! assert_parses_args:2249:22 +def f ((a, *r, p)); end +!!! assert_parses_args:2249:23 +def f ((a, *)); end +!!! assert_parses_args:2249:24 +def f ((a, *, p)); end +!!! assert_parses_args:2249:25 +def f ((*r)); end +!!! assert_parses_args:2249:26 +def f ((*r, p)); end +!!! assert_parses_args:2249:27 +def f ((*)); end +!!! assert_parses_args:2249:28 +def f ((*, p)); end +!!! assert_parses_args:2249:29 +def f foo: +; end +!!! assert_parses_args:2249:30 +def f foo: -1 +; end +!!! assert_parses_blockargs:2506:0 +f{ |a| } +!!! assert_parses_blockargs:2506:1 +f{ |a, b,| } +!!! assert_parses_blockargs:2506:2 +f{ |a| } +!!! assert_parses_blockargs:2506:3 +f{ |foo:| } +!!! assert_parses_blockargs:2506:4 +f{ } +!!! assert_parses_blockargs:2506:5 +f{ | | } +!!! assert_parses_blockargs:2506:6 +f{ |;a| } +!!! assert_parses_blockargs:2506:7 +f{ |; +a +| } +!!! assert_parses_blockargs:2506:8 +f{ || } +!!! assert_parses_blockargs:2506:9 +f{ |a| } +!!! assert_parses_blockargs:2506:10 +f{ |a, c| } +!!! assert_parses_blockargs:2506:11 +f{ |a,| } +!!! assert_parses_blockargs:2506:12 +f{ |a, &b| } +!!! assert_parses_blockargs:2506:13 +f{ |a, *s, &b| } +!!! assert_parses_blockargs:2506:14 +f{ |a, *, &b| } +!!! assert_parses_blockargs:2506:15 +f{ |a, *s| } +!!! assert_parses_blockargs:2506:16 +f{ |a, *| } +!!! assert_parses_blockargs:2506:17 +f{ |*s, &b| } +!!! assert_parses_blockargs:2506:18 +f{ |*, &b| } +!!! assert_parses_blockargs:2506:19 +f{ |*s| } +!!! assert_parses_blockargs:2506:20 +f{ |*| } +!!! assert_parses_blockargs:2506:21 +f{ |&b| } +!!! assert_parses_blockargs:2506:22 +f{ |a, o=1, o1=2, *r, &b| } +!!! assert_parses_blockargs:2506:23 +f{ |a, o=1, *r, p, &b| } +!!! assert_parses_blockargs:2506:24 +f{ |a, o=1, &b| } +!!! assert_parses_blockargs:2506:25 +f{ |a, o=1, p, &b| } +!!! assert_parses_blockargs:2506:26 +f{ |a, *r, p, &b| } +!!! assert_parses_blockargs:2506:27 +f{ |o=1, *r, &b| } +!!! assert_parses_blockargs:2506:28 +f{ |o=1, *r, p, &b| } +!!! assert_parses_blockargs:2506:29 +f{ |o=1, &b| } +!!! assert_parses_blockargs:2506:30 +f{ |o=1, p, &b| } +!!! assert_parses_blockargs:2506:31 +f{ |*r, p, &b| } +!!! assert_parses_blockargs:2506:32 +f{ |foo: 1, bar: 2, **baz, &b| } +!!! assert_parses_blockargs:2506:33 +f{ |foo: 1, &b| } +!!! assert_parses_blockargs:2506:34 +f{ |**baz, &b| } +!!! assert_parses_pattern_match:8503:0 +case foo; in self then true; end +!!! assert_parses_pattern_match:8503:1 +case foo; in 1..2 then true; end +!!! assert_parses_pattern_match:8503:2 +case foo; in 1.. then true; end +!!! assert_parses_pattern_match:8503:3 +case foo; in ..2 then true; end +!!! assert_parses_pattern_match:8503:4 +case foo; in 1...2 then true; end +!!! assert_parses_pattern_match:8503:5 +case foo; in 1... then true; end +!!! assert_parses_pattern_match:8503:6 +case foo; in ...2 then true; end +!!! assert_parses_pattern_match:8503:7 +case foo; in [*x, 1 => a, *y] then true; end +!!! assert_parses_pattern_match:8503:8 +case foo; in String(*, 1, *) then true; end +!!! assert_parses_pattern_match:8503:9 +case foo; in Array[*, 1, *] then true; end +!!! assert_parses_pattern_match:8503:10 +case foo; in *, 42, * then true; end +!!! assert_parses_pattern_match:8503:11 +case foo; in x, then nil; end +!!! assert_parses_pattern_match:8503:12 +case foo; in *x then nil; end +!!! assert_parses_pattern_match:8503:13 +case foo; in * then nil; end +!!! assert_parses_pattern_match:8503:14 +case foo; in x, y then nil; end +!!! assert_parses_pattern_match:8503:15 +case foo; in x, y, then nil; end +!!! assert_parses_pattern_match:8503:16 +case foo; in x, *y, z then nil; end +!!! assert_parses_pattern_match:8503:17 +case foo; in *x, y, z then nil; end +!!! assert_parses_pattern_match:8503:18 +case foo; in 1, "a", [], {} then nil; end +!!! assert_parses_pattern_match:8503:19 +case foo; in ->{ 42 } then true; end +!!! assert_parses_pattern_match:8503:20 +case foo; in A(1, 2) then true; end +!!! assert_parses_pattern_match:8503:21 +case foo; in A(x:) then true; end +!!! assert_parses_pattern_match:8503:22 +case foo; in A() then true; end +!!! assert_parses_pattern_match:8503:23 +case foo; in A[1, 2] then true; end +!!! assert_parses_pattern_match:8503:24 +case foo; in A[x:] then true; end +!!! assert_parses_pattern_match:8503:25 +case foo; in A[] then true; end +!!! assert_parses_pattern_match:8503:26 +case foo; in x then x; end +!!! assert_parses_pattern_match:8503:27 +case foo; in {} then true; end +!!! assert_parses_pattern_match:8503:28 +case foo; in a: 1 then true; end +!!! assert_parses_pattern_match:8503:29 +case foo; in { a: 1 } then true; end +!!! assert_parses_pattern_match:8503:30 +case foo; in { a: 1, } then true; end +!!! assert_parses_pattern_match:8503:31 +case foo; in a: then true; end +!!! assert_parses_pattern_match:8503:32 +case foo; in **a then true; end +!!! assert_parses_pattern_match:8503:33 +case foo; in ** then true; end +!!! assert_parses_pattern_match:8503:34 +case foo; in a: 1, b: 2 then true; end +!!! assert_parses_pattern_match:8503:35 +case foo; in a:, b: then true; end +!!! assert_parses_pattern_match:8503:36 +case foo; in a: 1, _a:, ** then true; end +!!! assert_parses_pattern_match:8503:37 +case foo; + in {a: 1 + } + false + ; end +!!! assert_parses_pattern_match:8503:38 +case foo; + in {a: + 2} + false + ; end +!!! assert_parses_pattern_match:8503:39 +case foo; + in {Foo: 42 + } + false + ; end +!!! assert_parses_pattern_match:8503:40 +case foo; + in a: {b:}, c: + p c + ; end +!!! assert_parses_pattern_match:8503:41 +case foo; + in {a: + } + true + ; end +!!! assert_parses_pattern_match:8503:42 +case foo; in A then true; end +!!! assert_parses_pattern_match:8503:43 +case foo; in A::B then true; end +!!! assert_parses_pattern_match:8503:44 +case foo; in ::A then true; end +!!! assert_parses_pattern_match:8503:45 +case foo; in [x] then nil; end +!!! assert_parses_pattern_match:8503:46 +case foo; in [x,] then nil; end +!!! assert_parses_pattern_match:8503:47 +case foo; in [x, y] then true; end +!!! assert_parses_pattern_match:8503:48 +case foo; in [x, y,] then true; end +!!! assert_parses_pattern_match:8503:49 +case foo; in [x, y, *] then true; end +!!! assert_parses_pattern_match:8503:50 +case foo; in [x, y, *z] then true; end +!!! assert_parses_pattern_match:8503:51 +case foo; in [x, *y, z] then true; end +!!! assert_parses_pattern_match:8503:52 +case foo; in [x, *, y] then true; end +!!! assert_parses_pattern_match:8503:53 +case foo; in [*x, y] then true; end +!!! assert_parses_pattern_match:8503:54 +case foo; in [*, x] then true; end +!!! assert_parses_pattern_match:8503:55 +case foo; in (1) then true; end +!!! assert_parses_pattern_match:8503:56 +case foo; in x if true; nil; end +!!! assert_parses_pattern_match:8503:57 +case foo; in x unless true; nil; end +!!! assert_parses_pattern_match:8503:58 +case foo; in 1; end +!!! assert_parses_pattern_match:8503:59 +case foo; in ^foo then nil; end +!!! assert_parses_pattern_match:8503:60 +case foo; in "a": then true; end +!!! assert_parses_pattern_match:8503:61 +case foo; in "#{ 'a' }": then true; end +!!! assert_parses_pattern_match:8503:62 +case foo; in "#{ %q{a} }": then true; end +!!! assert_parses_pattern_match:8503:63 +case foo; in "#{ %Q{a} }": then true; end +!!! assert_parses_pattern_match:8503:64 +case foo; in "a": 1 then true; end +!!! assert_parses_pattern_match:8503:65 +case foo; in "#{ 'a' }": 1 then true; end +!!! assert_parses_pattern_match:8503:66 +case foo; in "#{ %q{a} }": 1 then true; end +!!! assert_parses_pattern_match:8503:67 +case foo; in "#{ %Q{a} }": 1 then true; end +!!! assert_parses_pattern_match:8503:68 +case foo; in ^(42) then nil; end +!!! assert_parses_pattern_match:8503:69 +case foo; in { foo: ^(42) } then nil; end +!!! assert_parses_pattern_match:8503:70 +case foo; in ^(0+0) then nil; end +!!! assert_parses_pattern_match:8503:71 +case foo; in ^@a; end +!!! assert_parses_pattern_match:8503:72 +case foo; in ^@@TestPatternMatching; end +!!! assert_parses_pattern_match:8503:73 +case foo; in ^$TestPatternMatching; end +!!! assert_parses_pattern_match:8503:74 +case foo; in ^(1 +); end +!!! assert_parses_pattern_match:8503:75 +case foo; in 1 | 2 then true; end +!!! assert_parses_pattern_match:8503:76 +case foo; in 1 => a then true; end +!!! assert_parses_pattern_match:8503:77 +case foo; in **nil then true; end +!!! block in test_endless_comparison_method:10392:0 +def ===(other) = do_something +!!! block in test_endless_comparison_method:10392:1 +def ==(other) = do_something +!!! block in test_endless_comparison_method:10392:2 +def !=(other) = do_something +!!! block in test_endless_comparison_method:10392:3 +def <=(other) = do_something +!!! block in test_endless_comparison_method:10392:4 +def >=(other) = do_something +!!! block in test_endless_comparison_method:10392:5 +def !=(other) = do_something +!!! block in test_parser_slash_slash_n_escaping_in_literals:7327:0 +'a\ +b' +!!! block in test_parser_slash_slash_n_escaping_in_literals:7327:1 +<<-'HERE' +a\ +b +HERE +!!! block in test_parser_slash_slash_n_escaping_in_literals:7327:2 +%q{a\ +b} +!!! block in test_parser_slash_slash_n_escaping_in_literals:7327:3 +"a\ +b" +!!! block in test_parser_slash_slash_n_escaping_in_literals:7327:4 +<<-"HERE" +a\ +b +HERE +!!! block in test_parser_slash_slash_n_escaping_in_literals:7327:5 +%{a\ +b} +!!! block in test_parser_slash_slash_n_escaping_in_literals:7327:6 +%Q{a\ +b} +!!! block in test_parser_slash_slash_n_escaping_in_literals:7327:7 +%w{a\ +b} +!!! block in test_parser_slash_slash_n_escaping_in_literals:7327:8 +%W{a\ +b} +!!! block in test_parser_slash_slash_n_escaping_in_literals:7327:9 +%i{a\ +b} +!!! block in test_parser_slash_slash_n_escaping_in_literals:7327:10 +%I{a\ +b} +!!! block in test_parser_slash_slash_n_escaping_in_literals:7327:11 +:'a\ +b' +!!! block in test_parser_slash_slash_n_escaping_in_literals:7327:12 +%s{a\ +b} +!!! block in test_parser_slash_slash_n_escaping_in_literals:7327:13 +:"a\ +b" +!!! block in test_parser_slash_slash_n_escaping_in_literals:7327:14 +/a\ +b/ +!!! block in test_parser_slash_slash_n_escaping_in_literals:7327:15 +%r{a\ +b} +!!! block in test_parser_slash_slash_n_escaping_in_literals:7327:16 +%x{a\ +b} +!!! block in test_parser_slash_slash_n_escaping_in_literals:7327:17 +`a\ +b` +!!! block in test_parser_slash_slash_n_escaping_in_literals:7327:18 +<<-`HERE` +a\ +b +HERE +!!! block in test_ruby_bug_11873_a:6017:0 +a b{c d}, :e do end +!!! block in test_ruby_bug_11873_a:6017:1 +a b{c d}, 1 do end +!!! block in test_ruby_bug_11873_a:6017:2 +a b{c d}, 1.0 do end +!!! block in test_ruby_bug_11873_a:6017:3 +a b{c d}, 1.0r do end +!!! block in test_ruby_bug_11873_a:6017:4 +a b{c d}, 1.0i do end +!!! block in test_ruby_bug_11873_a:6022:0 +a b{c(d)}, :e do end +!!! block in test_ruby_bug_11873_a:6022:1 +a b{c(d)}, 1 do end +!!! block in test_ruby_bug_11873_a:6022:2 +a b{c(d)}, 1.0 do end +!!! block in test_ruby_bug_11873_a:6022:3 +a b{c(d)}, 1.0r do end +!!! block in test_ruby_bug_11873_a:6022:4 +a b{c(d)}, 1.0i do end +!!! block in test_ruby_bug_11873_a:6036:0 +a b(c d), :e do end +!!! block in test_ruby_bug_11873_a:6036:1 +a b(c d), 1 do end +!!! block in test_ruby_bug_11873_a:6036:2 +a b(c d), 1.0 do end +!!! block in test_ruby_bug_11873_a:6036:3 +a b(c d), 1.0r do end +!!! block in test_ruby_bug_11873_a:6036:4 +a b(c d), 1.0i do end +!!! block in test_ruby_bug_11873_a:6041:0 +a b(c(d)), :e do end +!!! block in test_ruby_bug_11873_a:6041:1 +a b(c(d)), 1 do end +!!! block in test_ruby_bug_11873_a:6041:2 +a b(c(d)), 1.0 do end +!!! block in test_ruby_bug_11873_a:6041:3 +a b(c(d)), 1.0r do end +!!! block in test_ruby_bug_11873_a:6041:4 +a b(c(d)), 1.0i do end +!!! test___ENCODING__:1037 +__ENCODING__ +!!! test___ENCODING___legacy_:1046 +__ENCODING__ +!!! test_alias:2020 +alias :foo bar +!!! test_alias_gvar:2032 +alias $a $b +!!! test_alias_gvar:2037 +alias $a $+ +!!! test_ambiuous_quoted_label_in_ternary_operator:7204 +a ? b & '': nil +!!! test_and:4447 +foo and bar +!!! test_and:4453 +foo && bar +!!! test_and_asgn:1748 +foo.a &&= 1 +!!! test_and_asgn:1758 +foo[0, 1] &&= 2 +!!! test_and_or_masgn:4475 +foo && (a, b = bar) +!!! test_and_or_masgn:4484 +foo || (a, b = bar) +!!! test_anonymous_blockarg:10861 +def foo(&); bar(&); end +!!! test_arg:2055 +def f(foo); end +!!! test_arg:2066 +def f(foo, bar); end +!!! test_arg_duplicate_ignored:2958 +def foo(_, _); end +!!! test_arg_duplicate_ignored:2972 +def foo(_a, _a); end +!!! test_arg_label:3012 +def foo() a:b end +!!! test_arg_label:3019 +def foo + a:b end +!!! test_arg_label:3026 +f { || a:b } +!!! test_arg_scope:2238 +lambda{|;a|a} +!!! test_args_args_assocs:4077 +fun(foo, :foo => 1) +!!! test_args_args_assocs:4083 +fun(foo, :foo => 1, &baz) +!!! test_args_args_assocs_comma:4092 +foo[bar, :baz => 1,] +!!! test_args_args_comma:3941 +foo[bar,] +!!! test_args_args_star:3908 +fun(foo, *bar) +!!! test_args_args_star:3913 +fun(foo, *bar, &baz) +!!! test_args_assocs:4001 +fun(:foo => 1) +!!! test_args_assocs:4006 +fun(:foo => 1, &baz) +!!! test_args_assocs:4012 +self[:bar => 1] +!!! test_args_assocs:4021 +self.[]= foo, :a => 1 +!!! test_args_assocs:4031 +yield(:foo => 42) +!!! test_args_assocs:4039 +super(:foo => 42) +!!! test_args_assocs_comma:4068 +foo[:baz => 1,] +!!! test_args_assocs_legacy:3951 +fun(:foo => 1) +!!! test_args_assocs_legacy:3956 +fun(:foo => 1, &baz) +!!! test_args_assocs_legacy:3962 +self[:bar => 1] +!!! test_args_assocs_legacy:3971 +self.[]= foo, :a => 1 +!!! test_args_assocs_legacy:3981 +yield(:foo => 42) +!!! test_args_assocs_legacy:3989 +super(:foo => 42) +!!! test_args_block_pass:3934 +fun(&bar) +!!! test_args_cmd:3901 +fun(f bar) +!!! test_args_star:3921 +fun(*bar) +!!! test_args_star:3926 +fun(*bar, &baz) +!!! test_array_assocs:629 +[ 1 => 2 ] +!!! test_array_assocs:637 +[ 1, 2 => 3 ] +!!! test_array_plain:589 +[1, 2] +!!! test_array_splat:598 +[1, *foo, 2] +!!! test_array_splat:611 +[1, *foo] +!!! test_array_splat:622 +[*foo] +!!! test_array_symbols:695 +%i[foo bar] +!!! test_array_symbols_empty:732 +%i[] +!!! test_array_symbols_empty:740 +%I() +!!! test_array_symbols_interp:706 +%I[foo #{bar}] +!!! test_array_symbols_interp:721 +%I[foo#{bar}] +!!! test_array_words:647 +%w[foo bar] +!!! test_array_words_empty:682 +%w[] +!!! test_array_words_empty:689 +%W() +!!! test_array_words_interp:657 +%W[foo #{bar}] +!!! test_array_words_interp:671 +%W[foo #{bar}foo#@baz] +!!! test_asgn_cmd:1126 +foo = m foo +!!! test_asgn_cmd:1130 +foo = bar = m foo +!!! test_asgn_mrhs:1449 +foo = bar, 1 +!!! test_asgn_mrhs:1456 +foo = *bar +!!! test_asgn_mrhs:1461 +foo = baz, *bar +!!! test_back_ref:995 +$+ +!!! test_bang:3434 +!foo +!!! test_bang_cmd:3448 +!m foo +!!! test_begin_cmdarg:5526 +p begin 1.times do 1 end end +!!! test_beginless_erange_after_newline:935 +foo +...100 +!!! test_beginless_irange_after_newline:923 +foo +..100 +!!! test_beginless_range:903 +..100 +!!! test_beginless_range:912 +...100 +!!! test_blockarg:2187 +def f(&block); end +!!! test_break:5037 +break(foo) +!!! test_break:5051 +break foo +!!! test_break:5057 +break() +!!! test_break:5064 +break +!!! test_break_block:5072 +break fun foo do end +!!! test_bug_435:7067 +"#{-> foo {}}" +!!! test_bug_447:7046 +m [] do end +!!! test_bug_447:7055 +m [], 1 do end +!!! test_bug_452:7080 +td (1_500).toString(); td.num do; end +!!! test_bug_466:7096 +foo "#{(1+1).to_i}" do; end +!!! test_bug_473:7113 +m "#{[]}" +!!! test_bug_480:7124 +m "#{}#{()}" +!!! test_bug_481:7136 +m def x(); end; 1.tap do end +!!! test_bug_ascii_8bit_in_literal:5880 +# coding:utf-8 + "\xD0\xBF\xD1\x80\xD0\xBE\xD0\xB2\xD0\xB5\xD1\x80\xD0\xBA\xD0\xB0" +!!! test_bug_cmd_string_lookahead:5752 +desc "foo" do end +!!! test_bug_cmdarg:5549 +assert dogs +!!! test_bug_cmdarg:5554 +assert do: true +!!! test_bug_cmdarg:5562 +f x: -> do meth do end end +!!! test_bug_def_no_paren_eql_begin:5799 +def foo +=begin +=end +end +!!! test_bug_do_block_in_call_args:5762 +bar def foo; self.each do end end +!!! test_bug_do_block_in_cmdarg:5777 +tap (proc do end) +!!! test_bug_do_block_in_hash_brace:6569 +p :foo, {a: proc do end, b: proc do end} +!!! test_bug_do_block_in_hash_brace:6587 +p :foo, {:a => proc do end, b: proc do end} +!!! test_bug_do_block_in_hash_brace:6605 +p :foo, {"a": proc do end, b: proc do end} +!!! test_bug_do_block_in_hash_brace:6623 +p :foo, {proc do end => proc do end, b: proc do end} +!!! test_bug_do_block_in_hash_brace:6643 +p :foo, {** proc do end, b: proc do end} +!!! test_bug_heredoc_do:5835 +f <<-TABLE do +TABLE +end +!!! test_bug_interp_single:5789 +"#{1}" +!!! test_bug_interp_single:5793 +%W"#{1}" +!!! test_bug_lambda_leakage:6550 +->(scope) {}; scope +!!! test_bug_regex_verification:6563 +/#)/x +!!! test_bug_rescue_empty_else:5813 +begin; rescue LoadError; else; end +!!! test_bug_while_not_parens_do:5805 +while not (true) do end +!!! test_case_cond:4844 +case; when foo; 'foo'; end +!!! test_case_cond_else:4857 +case; when foo; 'foo'; else 'bar'; end +!!! test_case_expr:4816 +case foo; when 'bar'; bar; end +!!! test_case_expr_else:4830 +case foo; when 'bar'; bar; else baz; end +!!! test_casgn_scoped:1192 +Bar::Foo = 10 +!!! test_casgn_toplevel:1181 +::Foo = 10 +!!! test_casgn_unscoped:1203 +Foo = 10 +!!! test_character:248 +?a +!!! test_class:1827 +class Foo; end +!!! test_class:1837 +class Foo end +!!! test_class_definition_in_while_cond:6870 +while class Foo; tap do end; end; break; end +!!! test_class_definition_in_while_cond:6882 +while class Foo a = tap do end; end; break; end +!!! test_class_definition_in_while_cond:6895 +while class << self; tap do end; end; break; end +!!! test_class_definition_in_while_cond:6907 +while class << self; a = tap do end; end; break; end +!!! test_class_super:1848 +class Foo < Bar; end +!!! test_class_super_label:1860 +class Foo < a:b; end +!!! test_comments_before_leading_dot__27:7750 +a # +# +.foo +!!! test_comments_before_leading_dot__27:7757 +a # + # +.foo +!!! test_comments_before_leading_dot__27:7764 +a # +# +&.foo +!!! test_comments_before_leading_dot__27:7771 +a # + # +&.foo +!!! test_complex:156 +42i +!!! test_complex:162 +42ri +!!! test_complex:168 +42.1i +!!! test_complex:174 +42.1ri +!!! test_cond_begin:4686 +if (bar); foo; end +!!! test_cond_begin_masgn:4695 +if (bar; a, b = foo); end +!!! test_cond_eflipflop:4758 +if foo...bar; end +!!! test_cond_eflipflop:4772 +!(foo...bar) +!!! test_cond_iflipflop:4735 +if foo..bar; end +!!! test_cond_iflipflop:4749 +!(foo..bar) +!!! test_cond_match_current_line:4781 +if /wat/; end +!!! test_cond_match_current_line:4801 +!/wat/ +!!! test_const_op_asgn:1536 +A += 1 +!!! test_const_op_asgn:1542 +::A += 1 +!!! test_const_op_asgn:1550 +B::A += 1 +!!! test_const_op_asgn:1558 +def x; self::A ||= 1; end +!!! test_const_op_asgn:1567 +def x; ::A ||= 1; end +!!! test_const_scoped:1020 +Bar::Foo +!!! test_const_toplevel:1011 +::Foo +!!! test_const_unscoped:1029 +Foo +!!! test_control_meta_escape_chars_in_regexp__since_31:10686 +/\c\xFF/ +!!! test_control_meta_escape_chars_in_regexp__since_31:10692 +/\c\M-\xFF/ +!!! test_control_meta_escape_chars_in_regexp__since_31:10698 +/\C-\xFF/ +!!! test_control_meta_escape_chars_in_regexp__since_31:10704 +/\C-\M-\xFF/ +!!! test_control_meta_escape_chars_in_regexp__since_31:10710 +/\M-\xFF/ +!!! test_control_meta_escape_chars_in_regexp__since_31:10716 +/\M-\C-\xFF/ +!!! test_control_meta_escape_chars_in_regexp__since_31:10722 +/\M-\c\xFF/ +!!! test_cpath:1807 +module ::Foo; end +!!! test_cpath:1813 +module Bar::Foo; end +!!! test_cvar:973 +@@foo +!!! test_cvasgn:1106 +@@var = 10 +!!! test_dedenting_heredoc:297 +p <<~E +E +!!! test_dedenting_heredoc:304 +p <<~E + E +!!! test_dedenting_heredoc:311 +p <<~E + x +E +!!! test_dedenting_heredoc:318 +p <<~E + ð +E +!!! test_dedenting_heredoc:325 +p <<~E + x + y +E +!!! test_dedenting_heredoc:334 +p <<~E + x + y +E +!!! test_dedenting_heredoc:343 +p <<~E + x + y +E +!!! test_dedenting_heredoc:352 +p <<~E + x + y +E +!!! test_dedenting_heredoc:361 +p <<~E + x + y +E +!!! test_dedenting_heredoc:370 +p <<~E + x + +y +E +!!! test_dedenting_heredoc:380 +p <<~E + x + + y +E +!!! test_dedenting_heredoc:390 +p <<~E + x + \ y +E +!!! test_dedenting_heredoc:399 +p <<~E + x + \ y +E +!!! test_dedenting_heredoc:408 +p <<~"E" + x + #{foo} +E +!!! test_dedenting_heredoc:419 +p <<~`E` + x + #{foo} +E +!!! test_dedenting_heredoc:430 +p <<~"E" + x + #{" y"} +E +!!! test_dedenting_interpolating_heredoc_fake_line_continuation:459 +<<~'FOO' + baz\\ + qux +FOO +!!! test_dedenting_non_interpolating_heredoc_line_continuation:451 +<<~'FOO' + baz\ + qux +FOO +!!! test_def:1899 +def foo; end +!!! test_def:1907 +def String; end +!!! test_def:1911 +def String=; end +!!! test_def:1915 +def until; end +!!! test_def:1919 +def BEGIN; end +!!! test_def:1923 +def END; end +!!! test_defined:1058 +defined? foo +!!! test_defined:1064 +defined?(foo) +!!! test_defined:1072 +defined? @foo +!!! test_defs:1929 +def self.foo; end +!!! test_defs:1937 +def self::foo; end +!!! test_defs:1945 +def (foo).foo; end +!!! test_defs:1949 +def String.foo; end +!!! test_defs:1954 +def String::foo; end +!!! test_empty_stmt:60 +!!! test_endless_method:9786 +def foo() = 42 +!!! test_endless_method:9798 +def inc(x) = x + 1 +!!! test_endless_method:9811 +def obj.foo() = 42 +!!! test_endless_method:9823 +def obj.inc(x) = x + 1 +!!! test_endless_method_command_syntax:9880 +def foo = puts "Hello" +!!! test_endless_method_command_syntax:9892 +def foo() = puts "Hello" +!!! test_endless_method_command_syntax:9904 +def foo(x) = puts x +!!! test_endless_method_command_syntax:9917 +def obj.foo = puts "Hello" +!!! test_endless_method_command_syntax:9931 +def obj.foo() = puts "Hello" +!!! test_endless_method_command_syntax:9945 +def rescued(x) = raise "to be caught" rescue "instance #{x}" +!!! test_endless_method_command_syntax:9964 +def self.rescued(x) = raise "to be caught" rescue "class #{x}" +!!! test_endless_method_command_syntax:9985 +def obj.foo(x) = puts x +!!! test_endless_method_forwarded_args_legacy:9840 +def foo(...) = bar(...) +!!! test_endless_method_with_rescue_mod:9855 +def m() = 1 rescue 2 +!!! test_endless_method_with_rescue_mod:9866 +def self.m() = 1 rescue 2 +!!! test_endless_method_without_args:10404 +def foo = 42 +!!! test_endless_method_without_args:10412 +def foo = 42 rescue nil +!!! test_endless_method_without_args:10423 +def self.foo = 42 +!!! test_endless_method_without_args:10432 +def self.foo = 42 rescue nil +!!! test_ensure:5261 +begin; meth; ensure; bar; end +!!! test_ensure_empty:5274 +begin ensure end +!!! test_false:96 +false +!!! test_float:129 +1.33 +!!! test_float:134 +-1.33 +!!! test_for:5002 +for a in foo do p a; end +!!! test_for:5014 +for a in foo; p a; end +!!! test_for_mlhs:5023 +for a, b in foo; p a, b; end +!!! test_forward_arg:7899 +def foo(...); bar(...); end +!!! test_forward_arg_with_open_args:10745 +def foo ... +end +!!! test_forward_arg_with_open_args:10752 +def foo a, b = 1, ... +end +!!! test_forward_arg_with_open_args:10770 +def foo(a, ...) bar(...) end +!!! test_forward_arg_with_open_args:10781 +def foo a, ... + bar(...) +end +!!! test_forward_arg_with_open_args:10792 +def foo b = 1, ... + bar(...) +end +!!! test_forward_arg_with_open_args:10804 +def foo ...; bar(...); end +!!! test_forward_arg_with_open_args:10814 +def foo a, ...; bar(...); end +!!! test_forward_arg_with_open_args:10825 +def foo b = 1, ...; bar(...); end +!!! test_forward_arg_with_open_args:10837 +(def foo ... + bar(...) +end) +!!! test_forward_arg_with_open_args:10848 +(def foo ...; bar(...); end) +!!! test_forward_args_legacy:7863 +def foo(...); bar(...); end +!!! test_forward_args_legacy:7875 +def foo(...); super(...); end +!!! test_forward_args_legacy:7887 +def foo(...); end +!!! test_forwarded_argument_with_kwrestarg:10962 +def foo(argument, **); bar(argument, **); end +!!! test_forwarded_argument_with_restarg:10923 +def foo(argument, *); bar(argument, *); end +!!! test_forwarded_kwrestarg:10943 +def foo(**); bar(**); end +!!! test_forwarded_restarg:10905 +def foo(*); bar(*); end +!!! test_gvar:980 +$foo +!!! test_gvasgn:1116 +$var = 10 +!!! test_hash_empty:750 +{ } +!!! test_hash_hashrocket:759 +{ 1 => 2 } +!!! test_hash_hashrocket:768 +{ 1 => 2, :foo => "bar" } +!!! test_hash_kwsplat:821 +{ foo: 2, **bar } +!!! test_hash_label:776 +{ foo: 2 } +!!! test_hash_label_end:789 +{ 'foo': 2 } +!!! test_hash_label_end:802 +{ 'foo': 2, 'bar': {}} +!!! test_hash_label_end:810 +f(a ? "a":1) +!!! test_hash_pair_value_omission:10040 +{a:, b:} +!!! test_hash_pair_value_omission:10054 +{puts:} +!!! test_hash_pair_value_omission:10065 +{BAR:} +!!! test_heredoc:263 +<(**nil) {} +!!! test_kwoptarg:2124 +def f(foo: 1); end +!!! test_kwrestarg_named:2135 +def f(**foo); end +!!! test_kwrestarg_unnamed:2146 +def f(**); end +!!! test_lbrace_arg_after_command_args:7235 +let (:a) { m do; end } +!!! test_lparenarg_after_lvar__since_25:6679 +meth (-1.3).abs +!!! test_lparenarg_after_lvar__since_25:6688 +foo (-1.3).abs +!!! test_lvar:959 +foo +!!! test_lvar_injecting_match:3778 +/(?bar)/ =~ 'bar'; match +!!! test_lvasgn:1084 +var = 10; var +!!! test_masgn:1247 +foo, bar = 1, 2 +!!! test_masgn:1258 +(foo, bar) = 1, 2 +!!! test_masgn:1268 +foo, bar, baz = 1, 2 +!!! test_masgn_attr:1390 +self.a, self[1, 2] = foo +!!! test_masgn_attr:1403 +self::a, foo = foo +!!! test_masgn_attr:1411 +self.A, foo = foo +!!! test_masgn_cmd:1439 +foo, bar = m foo +!!! test_masgn_const:1421 +self::A, foo = foo +!!! test_masgn_const:1429 +::A, foo = foo +!!! test_masgn_nested:1365 +a, (b, c) = foo +!!! test_masgn_nested:1379 +((b, )) = foo +!!! test_masgn_splat:1279 +@foo, @@bar = *foo +!!! test_masgn_splat:1288 +a, b = *foo, bar +!!! test_masgn_splat:1296 +a, *b = bar +!!! test_masgn_splat:1302 +a, *b, c = bar +!!! test_masgn_splat:1313 +a, * = bar +!!! test_masgn_splat:1319 +a, *, c = bar +!!! test_masgn_splat:1330 +*b = bar +!!! test_masgn_splat:1336 +*b, c = bar +!!! test_masgn_splat:1346 +* = bar +!!! test_masgn_splat:1352 +*, c, d = bar +!!! test_method_definition_in_while_cond:6816 +while def foo; tap do end; end; break; end +!!! test_method_definition_in_while_cond:6828 +while def self.foo; tap do end; end; break; end +!!! test_method_definition_in_while_cond:6841 +while def foo a = tap do end; end; break; end +!!! test_method_definition_in_while_cond:6854 +while def self.foo a = tap do end; end; break; end +!!! test_module:1789 +module Foo; end +!!! test_multiple_pattern_matches:11086 +{a: 0} => a: +{a: 0} => a: +!!! test_multiple_pattern_matches:11102 +{a: 0} in a: +{a: 0} in a: +!!! test_newline_in_hash_argument:11035 +obj.set foo: +1 +!!! test_newline_in_hash_argument:11046 +obj.set "foo": +1 +!!! test_newline_in_hash_argument:11057 +case foo +in a: +0 +true +in "b": +0 +true +end +!!! test_next:5131 +next(foo) +!!! test_next:5145 +next foo +!!! test_next:5151 +next() +!!! test_next:5158 +next +!!! test_next_block:5166 +next fun foo do end +!!! test_nil:66 +nil +!!! test_nil_expression:73 +() +!!! test_nil_expression:80 +begin end +!!! test_non_lvar_injecting_match:3793 +/#{1}(?bar)/ =~ 'bar' +!!! test_not:3462 +not foo +!!! test_not:3468 +not(foo) +!!! test_not:3474 +not() +!!! test_not_cmd:3488 +not m foo +!!! test_not_masgn__24:4672 +!(a, b = foo) +!!! test_nth_ref:1002 +$10 +!!! test_numbered_args_after_27:7358 +m { _1 + _9 } +!!! test_numbered_args_after_27:7373 +m do _1 + _9 end +!!! test_numbered_args_after_27:7390 +-> { _1 + _9} +!!! test_numbered_args_after_27:7405 +-> do _1 + _9 end +!!! test_numparam_outside_block:7512 +class A; _1; end +!!! test_numparam_outside_block:7520 +module A; _1; end +!!! test_numparam_outside_block:7528 +class << foo; _1; end +!!! test_numparam_outside_block:7536 +def self.m; _1; end +!!! test_numparam_outside_block:7545 +_1 +!!! test_op_asgn:1606 +foo.a += 1 +!!! test_op_asgn:1616 +foo::a += 1 +!!! test_op_asgn:1622 +foo.A += 1 +!!! test_op_asgn_cmd:1630 +foo.a += m foo +!!! test_op_asgn_cmd:1636 +foo::a += m foo +!!! test_op_asgn_cmd:1642 +foo.A += m foo +!!! test_op_asgn_cmd:1654 +foo::A += m foo +!!! test_op_asgn_index:1664 +foo[0, 1] += 2 +!!! test_op_asgn_index_cmd:1678 +foo[0, 1] += m foo +!!! test_optarg:2074 +def f foo = 1; end +!!! test_optarg:2084 +def f(foo=1, bar=2); end +!!! test_or:4461 +foo or bar +!!! test_or:4467 +foo || bar +!!! test_or_asgn:1724 +foo.a ||= 1 +!!! test_or_asgn:1734 +foo[0, 1] ||= 2 +!!! test_parser_bug_272:6528 +a @b do |c|;end +!!! test_parser_bug_490:7151 +def m; class << self; class C; end; end; end +!!! test_parser_bug_490:7162 +def m; class << self; module M; end; end; end +!!! test_parser_bug_490:7173 +def m; class << self; A = nil; end; end +!!! test_parser_bug_507:7265 +m = -> *args do end +!!! test_parser_bug_518:7277 +class A < B +end +!!! test_parser_bug_525:7287 +m1 :k => m2 do; m3() do end; end +!!! test_parser_bug_604:7737 +m a + b do end +!!! test_parser_bug_640:443 +<<~FOO + baz\ + qux +FOO +!!! test_parser_bug_645:9774 +-> (arg={}) {} +!!! test_parser_bug_830:10630 +/\(/ +!!! test_parser_drops_truncated_parts_of_squiggly_heredoc:10446 +<<~HERE + #{} +HERE +!!! test_pattern_matching__FILE__LINE_literals:9473 + case [__FILE__, __LINE__ + 1, __ENCODING__] + in [__FILE__, __LINE__, __ENCODING__] + end +!!! test_pattern_matching_blank_else:9390 +case 1; in 2; 3; else; end +!!! test_pattern_matching_else:9376 +case 1; in 2; 3; else; 4; end +!!! test_pattern_matching_single_line:9540 +1 => [a]; a +!!! test_pattern_matching_single_line:9552 +1 in [a]; a +!!! test_pattern_matching_single_line_allowed_omission_of_parentheses:9566 +[1, 2] => a, b; a +!!! test_pattern_matching_single_line_allowed_omission_of_parentheses:9581 +{a: 1} => a:; a +!!! test_pattern_matching_single_line_allowed_omission_of_parentheses:9596 +[1, 2] in a, b; a +!!! test_pattern_matching_single_line_allowed_omission_of_parentheses:9611 +{a: 1} in a:; a +!!! test_pattern_matching_single_line_allowed_omission_of_parentheses:9626 +{key: :value} in key: value; value +!!! test_pattern_matching_single_line_allowed_omission_of_parentheses:9643 +{key: :value} => key: value; value +!!! test_postexe:5486 +END { 1 } +!!! test_preexe:5467 +BEGIN { 1 } +!!! test_procarg0:2803 +m { |foo| } +!!! test_procarg0:2812 +m { |(foo, bar)| } +!!! test_range_endless:869 +1.. +!!! test_range_endless:877 +1... +!!! test_range_exclusive:861 +1...2 +!!! test_range_inclusive:853 +1..2 +!!! test_rational:142 +42r +!!! test_rational:148 +42.1r +!!! test_redo:5178 +redo +!!! test_regex_interp:551 +/foo#{bar}baz/ +!!! test_regex_plain:541 +/source/im +!!! test_resbody_list:5398 +begin; meth; rescue Exception; bar; end +!!! test_resbody_list_mrhs:5411 +begin; meth; rescue Exception, foo; bar; end +!!! test_resbody_list_var:5444 +begin; meth; rescue foo => ex; bar; end +!!! test_resbody_var:5426 +begin; meth; rescue => ex; bar; end +!!! test_resbody_var:5434 +begin; meth; rescue => @ex; bar; end +!!! test_rescue:5188 +begin; meth; rescue; foo; end +!!! test_rescue_else:5203 +begin; meth; rescue; foo; else; bar; end +!!! test_rescue_else_ensure:5302 +begin; meth; rescue; baz; else foo; ensure; bar end +!!! test_rescue_ensure:5286 +begin; meth; rescue; baz; ensure; bar; end +!!! test_rescue_in_lambda_block:6928 +-> do rescue; end +!!! test_rescue_mod:5319 +meth rescue bar +!!! test_rescue_mod_asgn:5331 +foo = meth rescue bar +!!! test_rescue_mod_masgn:5345 +foo, bar = meth rescue [1, 2] +!!! test_rescue_mod_op_assign:5365 +foo += meth rescue bar +!!! test_rescue_without_begin_end:5381 +meth do; foo; rescue; bar; end +!!! test_restarg_named:2094 +def f(*foo); end +!!! test_restarg_unnamed:2104 +def f(*); end +!!! test_retry:5457 +retry +!!! test_return:5084 +return(foo) +!!! test_return:5098 +return foo +!!! test_return:5104 +return() +!!! test_return:5111 +return +!!! test_return_block:5119 +return fun foo do end +!!! test_ruby_bug_10279:5905 +{a: if true then 42 end} +!!! test_ruby_bug_10653:5915 +true ? 1.tap do |n| p n end : 0 +!!! test_ruby_bug_10653:5945 +false ? raise {} : tap {} +!!! test_ruby_bug_10653:5958 +false ? raise do end : tap do end +!!! test_ruby_bug_11107:5973 +p ->() do a() do end end +!!! test_ruby_bug_11380:5985 +p -> { :hello }, a: 1 do end +!!! test_ruby_bug_11873:6353 +a b{c d}, "x" do end +!!! test_ruby_bug_11873:6367 +a b(c d), "x" do end +!!! test_ruby_bug_11873:6380 +a b{c(d)}, "x" do end +!!! test_ruby_bug_11873:6394 +a b(c(d)), "x" do end +!!! test_ruby_bug_11873:6407 +a b{c d}, /x/ do end +!!! test_ruby_bug_11873:6421 +a b(c d), /x/ do end +!!! test_ruby_bug_11873:6434 +a b{c(d)}, /x/ do end +!!! test_ruby_bug_11873:6448 +a b(c(d)), /x/ do end +!!! test_ruby_bug_11873:6461 +a b{c d}, /x/m do end +!!! test_ruby_bug_11873:6475 +a b(c d), /x/m do end +!!! test_ruby_bug_11873:6488 +a b{c(d)}, /x/m do end +!!! test_ruby_bug_11873:6502 +a b(c(d)), /x/m do end +!!! test_ruby_bug_11873_b:6050 +p p{p(p);p p}, tap do end +!!! test_ruby_bug_11989:6069 +p <<~"E" + x\n y +E +!!! test_ruby_bug_11990:6078 +p <<~E " y" + x +E +!!! test_ruby_bug_12073:6089 +a = 1; a b: 1 +!!! test_ruby_bug_12073:6102 +def foo raise; raise A::B, ''; end +!!! test_ruby_bug_12402:6116 +foo = raise(bar) rescue nil +!!! test_ruby_bug_12402:6127 +foo += raise(bar) rescue nil +!!! test_ruby_bug_12402:6139 +foo[0] += raise(bar) rescue nil +!!! test_ruby_bug_12402:6153 +foo.m += raise(bar) rescue nil +!!! test_ruby_bug_12402:6166 +foo::m += raise(bar) rescue nil +!!! test_ruby_bug_12402:6179 +foo.C += raise(bar) rescue nil +!!! test_ruby_bug_12402:6192 +foo::C ||= raise(bar) rescue nil +!!! test_ruby_bug_12402:6205 +foo = raise bar rescue nil +!!! test_ruby_bug_12402:6216 +foo += raise bar rescue nil +!!! test_ruby_bug_12402:6228 +foo[0] += raise bar rescue nil +!!! test_ruby_bug_12402:6242 +foo.m += raise bar rescue nil +!!! test_ruby_bug_12402:6255 +foo::m += raise bar rescue nil +!!! test_ruby_bug_12402:6268 +foo.C += raise bar rescue nil +!!! test_ruby_bug_12402:6281 +foo::C ||= raise bar rescue nil +!!! test_ruby_bug_12669:6296 +a = b = raise :x +!!! test_ruby_bug_12669:6305 +a += b = raise :x +!!! test_ruby_bug_12669:6314 +a = b += raise :x +!!! test_ruby_bug_12669:6323 +a += b += raise :x +!!! test_ruby_bug_12686:6334 +f (g rescue nil) +!!! test_ruby_bug_13547:7018 +meth[] {} +!!! test_ruby_bug_14690:7250 +let () { m(a) do; end } +!!! test_ruby_bug_15789:7622 +m ->(a = ->{_1}) {a} +!!! test_ruby_bug_15789:7636 +m ->(a: ->{_1}) {a} +!!! test_ruby_bug_9669:5889 +def a b: +return +end +!!! test_ruby_bug_9669:5895 +o = { +a: +1 +} +!!! test_sclass:1884 +class << foo; nil; end +!!! test_self:952 +self +!!! test_send_attr_asgn:3528 +foo.a = 1 +!!! test_send_attr_asgn:3536 +foo::a = 1 +!!! test_send_attr_asgn:3544 +foo.A = 1 +!!! test_send_attr_asgn:3552 +foo::A = 1 +!!! test_send_attr_asgn_conditional:3751 +a&.b = 1 +!!! test_send_binary_op:3308 +foo + 1 +!!! test_send_binary_op:3314 +foo - 1 +!!! test_send_binary_op:3318 +foo * 1 +!!! test_send_binary_op:3322 +foo / 1 +!!! test_send_binary_op:3326 +foo % 1 +!!! test_send_binary_op:3330 +foo ** 1 +!!! test_send_binary_op:3334 +foo | 1 +!!! test_send_binary_op:3338 +foo ^ 1 +!!! test_send_binary_op:3342 +foo & 1 +!!! test_send_binary_op:3346 +foo <=> 1 +!!! test_send_binary_op:3350 +foo < 1 +!!! test_send_binary_op:3354 +foo <= 1 +!!! test_send_binary_op:3358 +foo > 1 +!!! test_send_binary_op:3362 +foo >= 1 +!!! test_send_binary_op:3366 +foo == 1 +!!! test_send_binary_op:3376 +foo != 1 +!!! test_send_binary_op:3382 +foo === 1 +!!! test_send_binary_op:3386 +foo =~ 1 +!!! test_send_binary_op:3396 +foo !~ 1 +!!! test_send_binary_op:3402 +foo << 1 +!!! test_send_binary_op:3406 +foo >> 1 +!!! test_send_block_chain_cmd:3201 +meth 1 do end.fun bar +!!! test_send_block_chain_cmd:3212 +meth 1 do end.fun(bar) +!!! test_send_block_chain_cmd:3225 +meth 1 do end::fun bar +!!! test_send_block_chain_cmd:3236 +meth 1 do end::fun(bar) +!!! test_send_block_chain_cmd:3249 +meth 1 do end.fun bar do end +!!! test_send_block_chain_cmd:3261 +meth 1 do end.fun(bar) {} +!!! test_send_block_chain_cmd:3273 +meth 1 do end.fun {} +!!! test_send_block_conditional:3759 +foo&.bar {} +!!! test_send_call:3721 +foo.(1) +!!! test_send_call:3731 +foo::(1) +!!! test_send_conditional:3743 +a&.b +!!! test_send_index:3562 +foo[1, 2] +!!! test_send_index_asgn:3591 +foo[1, 2] = 3 +!!! test_send_index_asgn_legacy:3603 +foo[1, 2] = 3 +!!! test_send_index_cmd:3584 +foo[m bar] +!!! test_send_index_legacy:3573 +foo[1, 2] +!!! test_send_lambda:3615 +->{ } +!!! test_send_lambda:3625 +-> * { } +!!! test_send_lambda:3636 +-> do end +!!! test_send_lambda_args:3648 +->(a) { } +!!! test_send_lambda_args:3662 +-> (a) { } +!!! test_send_lambda_args_noparen:3686 +-> a: 1 { } +!!! test_send_lambda_args_noparen:3695 +-> a: { } +!!! test_send_lambda_args_shadow:3673 +->(a; foo, bar) { } +!!! test_send_lambda_legacy:3707 +->{ } +!!! test_send_op_asgn_conditional:3770 +a&.b &&= 1 +!!! test_send_plain:3105 +foo.fun +!!! test_send_plain:3112 +foo::fun +!!! test_send_plain:3119 +foo::Fun() +!!! test_send_plain_cmd:3128 +foo.fun bar +!!! test_send_plain_cmd:3135 +foo::fun bar +!!! test_send_plain_cmd:3142 +foo::Fun bar +!!! test_send_self:3044 +fun +!!! test_send_self:3050 +fun! +!!! test_send_self:3056 +fun(1) +!!! test_send_self_block:3066 +fun { } +!!! test_send_self_block:3070 +fun() { } +!!! test_send_self_block:3074 +fun(1) { } +!!! test_send_self_block:3078 +fun do end +!!! test_send_unary_op:3412 +-foo +!!! test_send_unary_op:3418 ++foo +!!! test_send_unary_op:3422 +~foo +!!! test_slash_newline_in_heredocs:7186 +<<~E + 1 \ + 2 + 3 +E +!!! test_slash_newline_in_heredocs:7194 +<<-E + 1 \ + 2 + 3 +E +!!! test_space_args_arg:4132 +fun (1) +!!! test_space_args_arg_block:4146 +fun (1) {} +!!! test_space_args_arg_block:4160 +foo.fun (1) {} +!!! test_space_args_arg_block:4176 +foo::fun (1) {} +!!! test_space_args_arg_call:4198 +fun (1).to_i +!!! test_space_args_arg_newline:4138 +fun (1 +) +!!! test_space_args_block:4430 +fun () {} +!!! test_space_args_cmd:4125 +fun (f bar) +!!! test_string___FILE__:241 +__FILE__ +!!! test_string_concat:226 +"foo#@a" "bar" +!!! test_string_dvar:215 +"#@a #@@a #$a" +!!! test_string_interp:200 +"foo#{bar}baz" +!!! test_string_plain:184 +'foobar' +!!! test_string_plain:191 +%q(foobar) +!!! test_super:3807 +super(foo) +!!! test_super:3815 +super foo +!!! test_super:3821 +super() +!!! test_super_block:3839 +super foo, bar do end +!!! test_super_block:3845 +super do end +!!! test_symbol_interp:484 +:"foo#{bar}baz" +!!! test_symbol_plain:469 +:foo +!!! test_symbol_plain:475 +:'foo' +!!! test_ternary:4605 +foo ? 1 : 2 +!!! test_ternary_ambiguous_symbol:4614 +t=1;(foo)?t:T +!!! test_trailing_forward_arg:8022 +def foo(a, b, ...); bar(a, 42, ...); end +!!! test_true:89 +true +!!! test_unary_num_pow_precedence:3505 ++2.0 ** 10 +!!! test_unary_num_pow_precedence:3512 +-2 ** 10 +!!! test_unary_num_pow_precedence:3519 +-2.0 ** 10 +!!! test_undef:2003 +undef foo, :bar, :"foo#{1}" +!!! test_unless:4529 +unless foo then bar; end +!!! test_unless:4537 +unless foo; bar; end +!!! test_unless_else:4573 +unless foo then bar; else baz; end +!!! test_unless_else:4582 +unless foo; bar; else baz; end +!!! test_unless_mod:4546 +bar unless foo +!!! test_until:4948 +until foo do meth end +!!! test_until:4955 +until foo; meth end +!!! test_until_mod:4963 +meth until foo +!!! test_until_post:4978 +begin meth end until foo +!!! test_var_and_asgn:1714 +a &&= 1 +!!! test_var_op_asgn:1498 +a += 1 +!!! test_var_op_asgn:1504 +@a |= 1 +!!! test_var_op_asgn:1510 +@@var |= 10 +!!! test_var_op_asgn:1514 +def a; @@var |= 10; end +!!! test_var_op_asgn_cmd:1521 +foo += m foo +!!! test_var_or_asgn:1706 +a ||= 1 +!!! test_when_multi:4895 +case foo; when 'bar', 'baz'; bar; end +!!! test_when_splat:4904 +case foo; when 1, *baz; bar; when *foo; end +!!! test_when_then:4883 +case foo; when 'bar' then bar; end +!!! test_while:4924 +while foo do meth end +!!! test_while:4932 +while foo; meth end +!!! test_while_mod:4941 +meth while foo +!!! test_while_post:4970 +begin meth end while foo +!!! test_xstring_interp:524 +`foo#{bar}baz` +!!! test_xstring_plain:515 +`foobar` +!!! test_yield:3855 +yield(foo) +!!! test_yield:3863 +yield foo +!!! test_yield:3869 +yield() +!!! test_yield:3877 +yield +!!! test_zsuper:3831 +super diff --git a/test/translation/parser_test.rb b/test/translation/parser_test.rb new file mode 100644 index 00000000..576d4ac1 --- /dev/null +++ b/test/translation/parser_test.rb @@ -0,0 +1,168 @@ +# frozen_string_literal: true + +require_relative "../test_helper" +require "parser/current" + +Parser::Builders::Default.modernize + +module SyntaxTree + module Translation + class ParserTest < Minitest::Test + known_failures = [ + # I think this may be a bug in the parser gem's precedence calculation. + # Unary plus appears to be parsed as part of the number literal in + # CRuby, but parser is parsing it as a separate operator. + "test_unary_num_pow_precedence:3505", + + # Not much to be done about this. Basically, regular expressions with + # named capture groups that use the =~ operator inject local variables + # into the current scope. In the parser gem, it detects this and changes + # future references to that name to be a local variable instead of a + # potential method call. CRuby does not do this. + "test_lvar_injecting_match:3778", + + # This is failing because CRuby is not marking values captured in hash + # patterns as local variables, while the parser gem is. + "test_pattern_matching_hash:8971", + + # This is not actually allowed in the CRuby parser but the parser gem + # thinks it is allowed. + "test_pattern_matching_hash_with_string_keys:9016", + "test_pattern_matching_hash_with_string_keys:9027", + "test_pattern_matching_hash_with_string_keys:9038", + "test_pattern_matching_hash_with_string_keys:9060", + "test_pattern_matching_hash_with_string_keys:9071", + "test_pattern_matching_hash_with_string_keys:9082", + + # This happens with pattern matching where you're matching a literal + # value inside parentheses, which doesn't really do anything. Ripper + # doesn't capture that this value is inside a parentheses, so it's hard + # to translate properly. + "test_pattern_matching_expr_in_paren:9206", + + # These are also failing because of CRuby not marking values captured in + # hash patterns as local variables. + "test_pattern_matching_single_line_allowed_omission_of_parentheses:*", + + # I'm not even sure what this is testing, because the code is invalid in + # CRuby. + "test_control_meta_escape_chars_in_regexp__since_31:*", + ] + + todo_failures = [ + "test_dedenting_heredoc:334", + "test_dedenting_heredoc:390", + "test_dedenting_heredoc:399", + "test_slash_newline_in_heredocs:7194", + "test_parser_slash_slash_n_escaping_in_literals:*", + "test_cond_match_current_line:4801", + "test_forwarded_restarg:*", + "test_forwarded_kwrestarg:*", + "test_forwarded_argument_with_restarg:*", + "test_forwarded_argument_with_kwrestarg:*" + ] + + current_version = RUBY_VERSION.split(".")[0..1].join(".") + + if current_version <= "2.7" + # I'm not sure why this is failing on 2.7.0, but we'll turn it off for + # now until we have more time to investigate. + todo_failures.push( + "test_pattern_matching_hash:*", + "test_pattern_matching_single_line:9552" + ) + end + + if current_version <= "3.0" + # In < 3.0, there are some changes to the way the parser gem handles + # forwarded args. We should eventually support this, but for now we're + # going to mark them as todo. + todo_failures.push( + "test_forward_arg:*", + "test_forward_args_legacy:*", + "test_endless_method_forwarded_args_legacy:*", + "test_trailing_forward_arg:*", + "test_forward_arg_with_open_args:10770", + ) + end + + if current_version == "3.1" + # This test actually fails on 3.1.0, even though it's marked as being + # since 3.1. So we're going to skip this test on 3.1, but leave it in + # for other versions. + known_failures.push( + "test_multiple_pattern_matches:11086", + "test_multiple_pattern_matches:11102" + ) + end + + if current_version < "3.2" || RUBY_ENGINE == "truffleruby" + known_failures.push( + "test_if_while_after_class__since_32:11004", + "test_if_while_after_class__since_32:11014", + "test_newline_in_hash_argument:11057" + ) + end + + all_failures = known_failures + todo_failures + + File + .foreach(File.expand_path("parser.txt", __dir__), chomp: true) + .slice_before { |line| line.start_with?("!!!") } + .each do |(prefix, *lines)| + name = prefix[4..] + next if all_failures.any? { |pattern| File.fnmatch?(pattern, name) } + + define_method(name) { assert_parses(lines.join("\n")) } + end + + private + + def assert_parses(source) + parser = ::Parser::CurrentRuby.default_parser + parser.diagnostics.consumer = ->(*) {} + + buffer = ::Parser::Source::Buffer.new("(string)", 1) + buffer.source = source + + expected = + begin + parser.parse(buffer) + rescue ::Parser::SyntaxError + # We can get a syntax error if we're parsing a fixture that was + # designed for a later Ruby version but we're running an earlier + # Ruby version. In this case we can just return early from the test. + end + + return if expected.nil? + node = SyntaxTree.parse(source) + assert_equal expected, SyntaxTree::Translation.to_parser(node, buffer) + end + end + end +end + +if ENV["PARSER_LOCATION"] + # Modify the source map == check so that it doesn't check against the node + # itself so we don't get into a recursive loop. + Parser::Source::Map.prepend( + Module.new do + def ==(other) + self.class == other.class && + (instance_variables - %i[@node]).map do |ivar| + instance_variable_get(ivar) == other.instance_variable_get(ivar) + end.reduce(:&) + end + end + ) + + # Next, ensure that we're comparing the nodes and also comparing the source + # ranges so that we're getting all of the necessary information. + Parser::AST::Node.prepend( + Module.new do + def ==(other) + super && (location == other.location) + end + end + ) +end From 946bc61c485c9fa325a7df60821c1815e76e995c Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 8 Feb 2023 14:20:06 -0500 Subject: [PATCH 372/536] Don't rely on parent being present --- lib/syntax_tree/node.rb | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index ff8ee95a..70fbdf4c 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -1539,7 +1539,7 @@ def ===(other) private def format_contents(q) - q.parent.format_key(q, key) + (q.parent || HashKeyFormatter::Identity.new).format_key(q, key) return unless value if key.comments.empty? && AssignFormatting.skip_indent?(value) @@ -1756,6 +1756,20 @@ def format_key(q, key) end end + # When formatting a single assoc node without the context of the parent + # hash, this formatter is used. It uses whatever is present in the node, + # because there is nothing to be consistent with. + class Identity + def format_key(q, key) + if key.is_a?(Label) + q.format(key) + else + q.format(key) + q.text(" =>") + end + end + end + def self.for(container) labels = container.assocs.all? do |assoc| @@ -4328,7 +4342,7 @@ def format(q) # are no parentheses around the arguments to that command, so we need to # break the block. case q.parent - when Command, CommandCall + when nil, Command, CommandCall q.break_parent format_break(q, break_opening, break_closing) return @@ -4382,7 +4396,7 @@ def unchangeable_bounds?(q) # If we're a sibling of a control-flow keyword, then we're going to have to # use the do..end bounds. def forced_do_end_bounds?(q) - case q.parent.call + case q.parent&.call when Break, Next, ReturnNode, Super true else From 2119110732d4dcd426a4caf183c142b75d96eb27 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 8 Feb 2023 15:41:19 -0500 Subject: [PATCH 373/536] Do not rely on fiddle being present --- lib/syntax_tree.rb | 1 - lib/syntax_tree/yarv/instruction_sequence.rb | 38 ++++++++++++-------- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index cd1f1ce4..e5bc5ab5 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -2,7 +2,6 @@ require "cgi" require "etc" -require "fiddle" require "json" require "pp" require "prettier_print" diff --git a/lib/syntax_tree/yarv/instruction_sequence.rb b/lib/syntax_tree/yarv/instruction_sequence.rb index 821738c9..45b543e6 100644 --- a/lib/syntax_tree/yarv/instruction_sequence.rb +++ b/lib/syntax_tree/yarv/instruction_sequence.rb @@ -7,6 +7,28 @@ module YARV # list of instructions along with the metadata pertaining to them. It also # functions as a builder for the instruction sequence. class InstructionSequence + # This provides a handle to the rb_iseq_load function, which allows you + # to pass a serialized iseq to Ruby and have it return a + # RubyVM::InstructionSequence object. + def self.iseq_load(iseq) + require "fiddle" + + @iseq_load_function ||= + Fiddle::Function.new( + Fiddle::Handle::DEFAULT["rb_iseq_load"], + [Fiddle::TYPE_VOIDP] * 3, + Fiddle::TYPE_VOIDP + ) + + Fiddle.dlunwrap(@iseq_load_function.call(Fiddle.dlwrap(iseq), 0, nil)) + rescue LoadError + raise "Could not load the Fiddle library" + rescue NameError + raise "Unable to find rb_iseq_load" + rescue Fiddle::DLError + raise "Unable to perform a dynamic load" + end + # When the list of instructions is first being created, it's stored as a # linked list. This is to make it easier to perform peephole optimizations # and other transformations like instruction specialization. @@ -60,19 +82,6 @@ def push(instruction) MAGIC = "YARVInstructionSequence/SimpleDataFormat" - # This provides a handle to the rb_iseq_load function, which allows you to - # pass a serialized iseq to Ruby and have it return a - # RubyVM::InstructionSequence object. - ISEQ_LOAD = - begin - Fiddle::Function.new( - Fiddle::Handle::DEFAULT["rb_iseq_load"], - [Fiddle::TYPE_VOIDP] * 3, - Fiddle::TYPE_VOIDP - ) - rescue NameError, Fiddle::DLError - end - # This object is used to track the size of the stack at any given time. It # is effectively a mini symbolic interpreter. It's necessary because when # instruction sequences get serialized they include a :stack_max field on @@ -221,8 +230,7 @@ def length end def eval - raise "Unsupported platform" if ISEQ_LOAD.nil? - Fiddle.dlunwrap(ISEQ_LOAD.call(Fiddle.dlwrap(to_a), 0, nil)).eval + InstructionSequence.iseq_load(to_a).eval end def to_a From 4f76ffab5d742c42ab982a77b696256c8ffb9090 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 8 Feb 2023 15:47:35 -0500 Subject: [PATCH 374/536] Strip out mspec for now --- .gitmodules | 6 ------ Rakefile | 1 - spec/mspec | 1 - spec/ruby | 1 - tasks/spec.rake | 10 ---------- 5 files changed, 19 deletions(-) delete mode 100644 .gitmodules delete mode 160000 spec/mspec delete mode 160000 spec/ruby delete mode 100644 tasks/spec.rake diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index f5477ea3..00000000 --- a/.gitmodules +++ /dev/null @@ -1,6 +0,0 @@ -[submodule "mspec"] - path = spec/mspec - url = git@github.com:ruby/mspec.git -[submodule "spec"] - path = spec/ruby - url = git@github.com:ruby/spec.git diff --git a/Rakefile b/Rakefile index aa8d29f6..22d7d1fe 100644 --- a/Rakefile +++ b/Rakefile @@ -8,7 +8,6 @@ Rake.add_rakelib "tasks" Rake::TestTask.new(:test) do |t| t.libs << "test" - t.libs << "test/suites" t.libs << "lib" t.test_files = FileList["test/**/*_test.rb"] end diff --git a/spec/mspec b/spec/mspec deleted file mode 160000 index 4877d58d..00000000 --- a/spec/mspec +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 4877d58dff577641bc1ecd1bf3d3c3daa93b423f diff --git a/spec/ruby b/spec/ruby deleted file mode 160000 index 71873ae4..00000000 --- a/spec/ruby +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 71873ae4421f5b551a5af0f3427e901414736835 diff --git a/tasks/spec.rake b/tasks/spec.rake deleted file mode 100644 index c361fe8e..00000000 --- a/tasks/spec.rake +++ /dev/null @@ -1,10 +0,0 @@ -# frozen_string_literal: true - -desc "Run mspec tests using YARV emulation" -task :spec do - specs = File.expand_path("../spec/ruby/language/**/*_spec.rb", __dir__) - - Dir[specs].each do |filepath| - sh "exe/yarv ./spec/mspec/bin/mspec-tag #{filepath}" - end -end From f44046d115b04b1c442634cad84be8a8c6e01afd Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 8 Feb 2023 15:49:39 -0500 Subject: [PATCH 375/536] Update rubocop version --- .rubocop.yml | 3 +++ Gemfile.lock | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.rubocop.yml b/.rubocop.yml index 62e78453..33636c44 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -30,6 +30,9 @@ Lint/AmbiguousRange: Lint/BooleanSymbol: Enabled: false +Lint/Debugger: + Enabled: false + Lint/DuplicateBranch: Enabled: false diff --git a/Gemfile.lock b/Gemfile.lock index 799bd891..46111ea4 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -19,7 +19,7 @@ GEM rake (13.0.6) regexp_parser (2.6.2) rexml (3.2.5) - rubocop (1.44.1) + rubocop (1.45.1) json (~> 2.3) parallel (~> 1.10) parser (>= 3.2.0.0) From 72c4f5c9c25d9e34b2e09b66c439c018bcf9a571 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 9 Feb 2023 11:14:30 -0500 Subject: [PATCH 376/536] Provide a reflection API --- bin/console | 1 + lib/syntax_tree/node.rb | 133 ++++++++++---------- lib/syntax_tree/reflection.rb | 224 ++++++++++++++++++++++++++++++++++ 3 files changed, 290 insertions(+), 68 deletions(-) create mode 100644 lib/syntax_tree/reflection.rb diff --git a/bin/console b/bin/console index 1c18bd62..6f35f1ec 100755 --- a/bin/console +++ b/bin/console @@ -3,6 +3,7 @@ require "bundler/setup" require "syntax_tree" +require "syntax_tree/reflection" require "irb" IRB.start(__FILE__) diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 70fbdf4c..4ac5aa24 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -557,7 +557,7 @@ def var_alias? # collection[] # class ARef < Node - # [untyped] the value being indexed + # [Node] the value being indexed attr_reader :collection # [nil | Args] the value being passed within the brackets @@ -635,7 +635,7 @@ def ===(other) # collection[index] = value # class ARefField < Node - # [untyped] the value being indexed + # [Node] the value being indexed attr_reader :collection # [nil | Args] the value being passed within the brackets @@ -810,7 +810,7 @@ def trailing_comma? # method(first, second, third) # class Args < Node - # [Array[ untyped ]] the arguments that this node wraps + # [Array[ Node ]] the arguments that this node wraps attr_reader :parts # [Array[ Comment | EmbDoc ]] the comments attached to this node @@ -876,7 +876,7 @@ def arity # method(&expression) # class ArgBlock < Node - # [nil | untyped] the expression being turned into a block + # [nil | Node] the expression being turned into a block attr_reader :value # [Array[ Comment | EmbDoc ]] the comments attached to this node @@ -928,7 +928,7 @@ def ===(other) # method(*arguments) # class ArgStar < Node - # [nil | untyped] the expression being splatted + # [nil | Node] the expression being splatted attr_reader :value # [Array[ Comment | EmbDoc ]] the comments attached to this node @@ -1289,7 +1289,7 @@ def format(q) # [nil | VarRef] the optional constant wrapper attr_reader :constant - # [Array[ untyped ]] the regular positional arguments that this array + # [Array[ Node ]] the regular positional arguments that this array # pattern is matching against attr_reader :requireds @@ -1297,7 +1297,7 @@ def format(q) # positional arguments attr_reader :rest - # [Array[ untyped ]] the list of positional arguments occurring after the + # [Array[ Node ]] the list of positional arguments occurring after the # optional star if there is one attr_reader :posts @@ -1407,7 +1407,7 @@ class Assign < Node # to assign the result of the expression to attr_reader :target - # [untyped] the expression to be assigned + # [Node] the expression to be assigned attr_reader :value # [Array[ Comment | EmbDoc ]] the comments attached to this node @@ -1482,10 +1482,10 @@ def skip_indent? # # In the above example, the would be two Assoc nodes. class Assoc < Node - # [untyped] the key of this pair + # [Node] the key of this pair attr_reader :key - # [untyped] the value of this pair + # [Node] the value of this pair attr_reader :value # [Array[ Comment | EmbDoc ]] the comments attached to this node @@ -1560,7 +1560,7 @@ def format_contents(q) # { **pairs } # class AssocSplat < Node - # [nil | untyped] the expression that is being splatted + # [nil | Node] the expression that is being splatted attr_reader :value # [Array[ Comment | EmbDoc ]] the comments attached to this node @@ -1924,7 +1924,7 @@ def ===(other) # end # class PinnedBegin < Node - # [untyped] the expression being pinned + # [Node] the expression being pinned attr_reader :statement # [Array[ Comment | EmbDoc ]] the comments attached to this node @@ -2005,13 +2005,13 @@ def name } end - # [untyped] the left-hand side of the expression + # [Node] the left-hand side of the expression attr_reader :left # [Symbol] the operator used between the two expressions attr_reader :operator - # [untyped] the right-hand side of the expression + # [Node] the right-hand side of the expression attr_reader :right # [Array[ Comment | EmbDoc ]] the comments attached to this node @@ -2670,7 +2670,7 @@ def format(q) # Of course there are a lot of caveats to that, including trailing operators # when necessary, where comments are places, how blocks are aligned, etc. class CallChainFormatter - # [Call | MethodAddBlock] the top of the call chain + # [CallNode | MethodAddBlock] the top of the call chain attr_reader :node def initialize(node) @@ -2891,7 +2891,7 @@ def format_child( # receiver.message # class CallNode < Node - # [nil | untyped] the receiver of the method call + # [nil | Node] the receiver of the method call attr_reader :receiver # [nil | :"::" | Op | Period] the operator being used to send the message @@ -3067,7 +3067,7 @@ class Case < Node # [Kw] the keyword that opens this expression attr_reader :keyword - # [nil | untyped] optional value being switched on + # [nil | Node] optional value being switched on attr_reader :value # [In | When] the next clause in the chain @@ -3146,14 +3146,14 @@ def ===(other) # value => pattern # class RAssign < Node - # [untyped] the left-hand expression + # [Node] the left-hand expression attr_reader :value # [Kw | Op] the operator being used to match against the pattern, which is # either => or in attr_reader :operator - # [untyped] the pattern on the right-hand side of the expression + # [Node] the pattern on the right-hand side of the expression attr_reader :pattern # [Array[ Comment | EmbDoc ]] the comments attached to this node @@ -3264,7 +3264,7 @@ class ClassDeclaration < Node # defined attr_reader :constant - # [nil | untyped] the optional superclass declaration + # [nil | Node] the optional superclass declaration attr_reader :superclass # [BodyStmt] the expressions to execute within the context of the class @@ -3402,7 +3402,7 @@ class Command < Node # [Args] the arguments being sent with the message attr_reader :arguments - # [nil | Block] the optional block being passed to the method + # [nil | BlockNode] the optional block being passed to the method attr_reader :block # [Array[ Comment | EmbDoc ]] the comments attached to this node @@ -3508,7 +3508,7 @@ def align(q, node, &block) # object.method argument # class CommandCall < Node - # [untyped] the receiver of the message + # [Node] the receiver of the message attr_reader :receiver # [:"::" | Op | Period] the operator used to send the message @@ -3520,7 +3520,7 @@ class CommandCall < Node # [nil | Args] the arguments going along with the message attr_reader :arguments - # [nil | Block] the block associated with this method call + # [nil | BlockNode] the block associated with this method call attr_reader :block # [Array[ Comment | EmbDoc ]] the comments attached to this node @@ -3806,7 +3806,7 @@ def ===(other) # object::Const = value # class ConstPathField < Node - # [untyped] the source of the constant + # [Node] the source of the constant attr_reader :parent # [Const] the constant itself @@ -3870,7 +3870,7 @@ def ===(other) # object::Const # class ConstPathRef < Node - # [untyped] the source of the constant + # [Node] the source of the constant attr_reader :parent # [Const] the constant itself @@ -4039,7 +4039,7 @@ def ===(other) # def object.method(param) result end # class DefNode < Node - # [nil | untyped] the target where the method is being defined + # [nil | Node] the target where the method is being defined attr_reader :target # [nil | Op | Period] the operator being used to declare the method @@ -4051,7 +4051,7 @@ class DefNode < Node # [nil | Params | Paren] the parameter declaration for the method attr_reader :params - # [BodyStmt | untyped] the expressions to be executed by the method + # [BodyStmt | Node] the expressions to be executed by the method attr_reader :bodystmt # [Array[ Comment | EmbDoc ]] the comments attached to this node @@ -4185,7 +4185,7 @@ def arity # defined?(variable) # class Defined < Node - # [untyped] the value being sent to the keyword + # [Node] the value being sent to the keyword attr_reader :value # [Array[ Comment | EmbDoc ]] the comments attached to this node @@ -4476,13 +4476,13 @@ def format_flat(q, flat_opening, flat_closing) # # One of the sides of the expression may be nil, but not both. class RangeNode < Node - # [nil | untyped] the left side of the expression + # [nil | Node] the left side of the expression attr_reader :left # [Op] the operator used for this range attr_reader :operator - # [nil | untyped] the right side of the expression + # [nil | Node] the right side of the expression attr_reader :right # [Array[ Comment | EmbDoc ]] the comments attached to this node @@ -4801,7 +4801,7 @@ def ===(other) # end # class Elsif < Node - # [untyped] the expression to be checked + # [Node] the expression to be checked attr_reader :predicate # [Statements] the expressions to be executed @@ -5227,7 +5227,7 @@ def ===(other) # object.variable = value # class Field < Node - # [untyped] the parent object that owns the field being assigned + # [Node] the parent object that owns the field being assigned attr_reader :parent # [:"::" | Op | Period] the operator being used for the assignment @@ -5353,13 +5353,13 @@ def ===(other) # end # class FndPtn < Node - # [nil | untyped] the optional constant wrapper + # [nil | Node] the optional constant wrapper attr_reader :constant # [VarField] the splat on the left-hand side attr_reader :left - # [Array[ untyped ]] the list of positional expressions in the pattern that + # [Array[ Node ]] the list of positional expressions in the pattern that # are being matched attr_reader :values @@ -5455,7 +5455,7 @@ class For < Node # pull values out of the object being enumerated attr_reader :index - # [untyped] the object being enumerated in the loop + # [Node] the object being enumerated in the loop attr_reader :collection # [Statements] the statements to be executed @@ -5934,7 +5934,7 @@ class KeywordFormatter # [Label] the keyword being used attr_reader :key - # [untyped] the optional value for the keyword + # [Node] the optional value for the keyword attr_reader :value def initialize(key, value) @@ -5975,10 +5975,10 @@ def format(q) end end - # [nil | untyped] the optional constant wrapper + # [nil | Node] the optional constant wrapper attr_reader :constant - # [Array[ [Label, untyped] ]] the set of tuples representing the keywords + # [Array[ [Label, Node] ]] the set of tuples representing the keywords # that should be matched against in the pattern attr_reader :keywords @@ -6404,7 +6404,7 @@ def contains_conditional? # end # class IfNode < Node - # [untyped] the expression to be checked + # [Node] the expression to be checked attr_reader :predicate # [Statements] the expressions to be executed @@ -6477,13 +6477,13 @@ def modifier? # predicate ? truthy : falsy # class IfOp < Node - # [untyped] the expression to be checked + # [Node] the expression to be checked attr_reader :predicate - # [untyped] the expression to be executed if the predicate is truthy + # [Node] the expression to be executed if the predicate is truthy attr_reader :truthy - # [untyped] the expression to be executed if the predicate is falsy + # [Node] the expression to be executed if the predicate is falsy attr_reader :falsy # [Array[ Comment | EmbDoc ]] the comments attached to this node @@ -6667,7 +6667,7 @@ def ===(other) # end # class In < Node - # [untyped] the pattern to check against + # [Node] the pattern to check against attr_reader :pattern # [Statements] the expressions to execute if the pattern matched @@ -7450,7 +7450,7 @@ class MAssign < Node # [MLHS | MLHSParen] the target of the multiple assignment attr_reader :target - # [untyped] the value being assigned + # [Node] the value being assigned attr_reader :value # [Array[ Comment | EmbDoc ]] the comments attached to this node @@ -7510,10 +7510,10 @@ def ===(other) # method {} # class MethodAddBlock < Node - # [Call | Command | CommandCall] the method call + # [CallNode | Command | CommandCall] the method call attr_reader :call - # [Block] the block being sent with the method call + # [BlockNode] the block being sent with the method call attr_reader :block # [Array[ Comment | EmbDoc ]] the comments attached to this node @@ -7585,7 +7585,7 @@ def format_contents(q) # first, second, third = value # class MLHS < Node - # Array[ARefField | ArgStar | Field | Ident | MLHSParen | VarField] the + # [Array[ARefField | ArgStar | Field | Ident | MLHSParen | VarField]] the # parts of the left-hand side of a multiple assignment attr_reader :parts @@ -7812,7 +7812,7 @@ def format_declaration(q) # values = first, second, third # class MRHS < Node - # Array[untyped] the parts that are being assigned + # [Array[Node]] the parts that are being assigned attr_reader :parts # [Array[ Comment | EmbDoc ]] the comments attached to this node @@ -7988,7 +7988,7 @@ class OpAssign < Node # [Op] the operator being used for the assignment attr_reader :operator - # [untyped] the expression to be assigned + # [Node] the expression to be assigned attr_reader :value # [Array[ Comment | EmbDoc ]] the comments attached to this node @@ -8145,7 +8145,7 @@ class OptionalFormatter # [Ident] the name of the parameter attr_reader :name - # [untyped] the value of the parameter + # [Node] the value of the parameter attr_reader :value def initialize(name, value) @@ -8170,7 +8170,7 @@ class KeywordFormatter # [Ident] the name of the parameter attr_reader :name - # [nil | untyped] the value of the parameter + # [nil | Node] the value of the parameter attr_reader :value def initialize(name, value) @@ -8214,7 +8214,7 @@ def format(q) # [Array[ Ident ]] any required parameters attr_reader :requireds - # [Array[ [ Ident, untyped ] ]] any optional parameters and their default + # [Array[ [ Ident, Node ] ]] any optional parameters and their default # values attr_reader :optionals @@ -8226,7 +8226,7 @@ def format(q) # parameter attr_reader :posts - # [Array[ [ Ident, nil | untyped ] ]] any keyword parameters and their + # [Array[ [ Ident, nil | Node ] ]] any keyword parameters and their # optional default values attr_reader :keywords @@ -8419,7 +8419,7 @@ class Paren < Node # [LParen] the left parenthesis that opened this statement attr_reader :lparen - # [nil | untyped] the expression inside the parentheses + # [nil | Node] the expression inside the parentheses attr_reader :contents # [Array[ Comment | EmbDoc ]] the comments attached to this node @@ -9268,7 +9268,7 @@ def ambiguous?(q) # end # class RescueEx < Node - # [untyped] the list of exceptions being rescued + # [Node] the list of exceptions being rescued attr_reader :exceptions # [nil | Field | VarField] the expression being used to capture the raised @@ -9466,10 +9466,10 @@ def ===(other) # expression rescue value # class RescueMod < Node - # [untyped] the expression to execute + # [Node] the expression to execute attr_reader :statement - # [untyped] the value to use if the executed expression raises an error + # [Node] the value to use if the executed expression raises an error attr_reader :value # [Array[ Comment | EmbDoc ]] the comments attached to this node @@ -9728,7 +9728,7 @@ def ===(other) # end # class SClass < Node - # [untyped] the target of the singleton class to enter + # [Node] the target of the singleton class to enter attr_reader :target # [BodyStmt] the expressions to be executed @@ -9802,10 +9802,10 @@ def ===(other) # propagate that onto void_stmt nodes inside the stmts in order to make sure # all comments get printed appropriately. class Statements < Node - # [SyntaxTree] the parser that is generating this node + # [Parser] the parser that is generating this node attr_reader :parser - # [Array[ untyped ]] the list of expressions contained within this node + # [Array[ Node ]] the list of expressions contained within this node attr_reader :body # [Array[ Comment | EmbDoc ]] the comments attached to this node @@ -10985,7 +10985,7 @@ def ===(other) # not value # class Not < Node - # [nil | untyped] the statement on which to operate + # [nil | Node] the statement on which to operate attr_reader :statement # [boolean] whether or not parentheses were used @@ -11072,7 +11072,7 @@ class Unary < Node # [String] the operator being used attr_reader :operator - # [untyped] the statement on which to operate + # [Node] the statement on which to operate attr_reader :statement # [Array[ Comment | EmbDoc ]] the comments attached to this node @@ -11216,7 +11216,7 @@ def ===(other) # end # class UnlessNode < Node - # [untyped] the expression to be checked + # [Node] the expression to be checked attr_reader :predicate # [Statements] the expressions to be executed @@ -11362,7 +11362,7 @@ def format_break(q) # end # class UntilNode < Node - # [untyped] the expression to be checked + # [Node] the expression to be checked attr_reader :predicate # [Statements] the expressions to be executed @@ -11683,9 +11683,6 @@ def arity # ;; # class VoidStmt < Node - # [Location] the location of this node - attr_reader :location - # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments @@ -11846,7 +11843,7 @@ def ===(other) # end # class WhileNode < Node - # [untyped] the expression to be checked + # [Node] the expression to be checked attr_reader :predicate # [Statements] the expressions to be executed diff --git a/lib/syntax_tree/reflection.rb b/lib/syntax_tree/reflection.rb new file mode 100644 index 00000000..2457fe49 --- /dev/null +++ b/lib/syntax_tree/reflection.rb @@ -0,0 +1,224 @@ +# frozen_string_literal: true + +module SyntaxTree + # This module is used to provide some reflection on the various types of nodes + # and their attributes. As soon as it is required it collects all of its + # information. + module Reflection + # This module represents the type of the values being passed to attributes + # of nodes. It is used as part of the documentation of the attributes. + module Type + CONSTANTS = SyntaxTree.constants.to_h { [_1, SyntaxTree.const_get(_1)] } + + # Represents an array type that holds another type. + class ArrayType + attr_reader :type + + def initialize(type) + @type = type + end + + def ===(value) + value.is_a?(Array) && value.all? { type === _1 } + end + end + + # Represents a tuple type that holds a number of types in order. + class TupleType + attr_reader :types + + def initialize(types) + @types = types + end + + def ===(value) + value.is_a?(Array) && value.length == types.length && + value.zip(types).all? { _2 === _1 } + end + end + + # Represents a union type that can be one of a number of types. + class UnionType + attr_reader :types + + def initialize(types) + @types = types + end + + def ===(value) + types.any? { _1 === value } + end + end + + class << self + def parse(comment) + unless comment.start_with?("[") + raise "Comment does not start with a bracket: #{comment.inspect}" + end + + count = 1 + found = + comment.chars[1..].find.with_index(1) do |char, index| + count += { "[" => 1, "]" => -1 }.fetch(char, 0) + break index if count == 0 + end + + # If we weren't able to find the end of the balanced brackets, then + # the comment is malformed. + if found.nil? + raise "Comment does not have balanced brackets: #{comment.inspect}" + end + + parse_type(comment[1...found].strip) + end + + private + + def parse_type(value) + case value + when "Integer" + Integer + when "String" + String + when "Symbol" + Symbol + when "boolean" + UnionType.new([TrueClass, FalseClass]) + when "nil" + NilClass + when ":\"::\"" + :"::" + when ":call" + :call + when ":nil" + :nil + when /\AArray\[(.+)\]\z/ + ArrayType.new(parse_type($1.strip)) + when /\A\[(.+)\]\z/ + TupleType.new($1.strip.split(/\s*,\s*/).map { parse_type(_1) }) + else + if value.include?("|") + UnionType.new(value.split(/\s*\|\s*/).map { parse_type(_1) }) + else + CONSTANTS.fetch(value.to_sym) + end + end + end + end + end + + # This class represents one of the attributes on a node in the tree. + class Attribute + attr_reader :name, :comment, :type + + def initialize(name, comment) + @name = name + @comment = comment + @type = Type.parse(comment) + end + end + + # This class represents one of our nodes in the tree. We're going to use it + # as a placeholder for collecting all of the various places that nodes are + # used. + class Node + attr_reader :name, :comment, :attributes + + def initialize(name, comment, attributes) + @name = name + @comment = comment + @attributes = attributes + end + end + + class << self + # This is going to hold a hash of all of the nodes in the tree. The keys + # are the names of the nodes as symbols. + attr_reader :nodes + + # This expects a node name as a symbol and returns the node object for + # that node. + def node(name) + nodes.fetch(name) + end + + private + + def parse_comments(statements, index) + statements[0...index] + .reverse_each + .take_while { _1.is_a?(SyntaxTree::Comment) } + .reverse_each + .map { _1.value[2..] } + end + end + + @nodes = {} + + # For each node, we're going to parse out its attributes and other metadata. + # We'll use this as the basis for our report. + program = + SyntaxTree.parse(SyntaxTree.read(File.expand_path("node.rb", __dir__))) + + main_statements = program.statements.body.last.bodystmt.statements.body + main_statements.each_with_index do |main_statement, main_statement_index| + # Ensure we are only looking at class declarations. + next unless main_statement.is_a?(SyntaxTree::ClassDeclaration) + + # Ensure we're looking at class declarations with superclasses. + next unless main_statement.superclass.is_a?(SyntaxTree::VarRef) + + # Ensure we're looking at class declarations that inherit from Node. + next unless main_statement.superclass.value.value == "Node" + + # All child nodes inherit the location attr_reader from Node, so we'll add + # that to the list of attributes first. + attributes = { + location: + Attribute.new(:location, "[Location] the location of this node") + } + + statements = main_statement.bodystmt.statements.body + statements.each_with_index do |statement, statement_index| + case statement + when SyntaxTree::Command + # We only use commands in node classes to define attributes. So, we + # can safely assume that we're looking at an attribute definition. + unless %w[attr_reader attr_accessor].include?(statement.message.value) + raise "Unexpected command: #{statement.message.value.inspect}" + end + + # The arguments to the command are the attributes that we're defining. + # We want to ensure that we're only defining one at a time. + if statement.arguments.parts.length != 1 + raise "Declaring more than one attribute at a time is not permitted" + end + + attribute = + Attribute.new( + statement.arguments.parts.first.value.value.to_sym, + parse_comments(statements, statement_index).join(" ") + ) + + # Ensure that we don't already have an attribute named the same as + # this one, and then add it to the list of attributes. + if attributes.key?(attribute.name) + raise "Duplicate attribute: #{attribute.name}" + end + + attributes[attribute.name] = attribute + end + end + + # Finally, set it up in the hash of nodes so that we can use it later. + node = + Node.new( + main_statement.constant.constant.value.to_sym, + parse_comments(main_statements, main_statement_index).join("\n"), + attributes + ) + + @nodes[node.name] = node + end + end +end From 45d8c4c3dfb544b8ef5644bc3bab54377607f9b9 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 9 Feb 2023 12:40:53 -0500 Subject: [PATCH 377/536] Enforce types in the test suite --- lib/syntax_tree/node.rb | 86 +++++++++++++++++++-------- lib/syntax_tree/parser.rb | 67 +++++++++++++++++---- lib/syntax_tree/reflection.rb | 12 ++++ lib/syntax_tree/translation/parser.rb | 19 +++--- lib/syntax_tree/yarv/compiler.rb | 4 +- lib/syntax_tree/yarv/decompiler.rb | 2 +- test/formatting_test.rb | 1 + test/interface_test.rb | 72 ---------------------- test/test_helper.rb | 30 ++++++++++ 9 files changed, 173 insertions(+), 120 deletions(-) delete mode 100644 test/interface_test.rb diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 4ac5aa24..4a98dae4 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -1131,7 +1131,8 @@ def format(q) end end - # [LBracket] the bracket that opens this array + # [nil | LBracket | QSymbolsBeg | QWordsBeg | SymbolsBeg | WordsBeg] the + # bracket that opens this array attr_reader :lbracket # [nil | Args] the contents of the array @@ -1485,7 +1486,7 @@ class Assoc < Node # [Node] the key of this pair attr_reader :key - # [Node] the value of this pair + # [nil | Node] the value of this pair attr_reader :value # [Array[ Comment | EmbDoc ]] the comments attached to this node @@ -3508,16 +3509,16 @@ def align(q, node, &block) # object.method argument # class CommandCall < Node - # [Node] the receiver of the message + # [nil | Node] the receiver of the message attr_reader :receiver - # [:"::" | Op | Period] the operator used to send the message + # [nil | :"::" | Op | Period] the operator used to send the message attr_reader :operator - # [Const | Ident | Op] the message being send + # [:call | Const | Ident | Op] the message being send attr_reader :message - # [nil | Args] the arguments going along with the message + # [nil | Args | ArgParen] the arguments going along with the message attr_reader :arguments # [nil | BlockNode] the block associated with this method call @@ -4603,7 +4604,7 @@ class DynaSymbol < Node # dynamic symbol attr_reader :parts - # [String] the quote used to delimit the dynamic symbol + # [nil | String] the quote used to delimit the dynamic symbol attr_reader :quote # [Array[ Comment | EmbDoc ]] the comments attached to this node @@ -5947,7 +5948,7 @@ def comments end def format(q) - q.format(key) + HashKeyFormatter::Labels.new.format_key(q, key) if value q.text(" ") @@ -5978,8 +5979,8 @@ def format(q) # [nil | Node] the optional constant wrapper attr_reader :constant - # [Array[ [Label, Node] ]] the set of tuples representing the keywords - # that should be matched against in the pattern + # [Array[ [DynaSymbol | Label, nil | Node] ]] the set of tuples + # representing the keywords that should be matched against in the pattern attr_reader :keywords # [nil | VarField] an optional parameter to gather up all remaining keywords @@ -7510,7 +7511,7 @@ def ===(other) # method {} # class MethodAddBlock < Node - # [CallNode | Command | CommandCall] the method call + # [ARef | CallNode | Command | CommandCall | Super | ZSuper] the method call attr_reader :call # [BlockNode] the block being sent with the method call @@ -7585,8 +7586,12 @@ def format_contents(q) # first, second, third = value # class MLHS < Node - # [Array[ARefField | ArgStar | Field | Ident | MLHSParen | VarField]] the - # parts of the left-hand side of a multiple assignment + # [ + # Array[ + # ARefField | ArgStar | ConstPathField | Field | Ident | MLHSParen | + # TopConstField | VarField + # ] + # ] the parts of the left-hand side of a multiple assignment attr_reader :parts # [boolean] whether or not there is a trailing comma at the end of this @@ -8211,7 +8216,7 @@ def format(q) end end - # [Array[ Ident ]] any required parameters + # [Array[ Ident | MLHSParen ]] any required parameters attr_reader :requireds # [Array[ [ Ident, Node ] ]] any optional parameters and their default @@ -8226,11 +8231,12 @@ def format(q) # parameter attr_reader :posts - # [Array[ [ Ident, nil | Node ] ]] any keyword parameters and their + # [Array[ [ Label, nil | Node ] ]] any keyword parameters and their # optional default values attr_reader :keywords - # [nil | :nil | KwRestParam] the optional keyword rest parameter + # [nil | :nil | ArgsForward | KwRestParam] the optional keyword rest + # parameter attr_reader :keyword_rest # [nil | BlockArg] the optional block parameter @@ -9268,7 +9274,7 @@ def ambiguous?(q) # end # class RescueEx < Node - # [Node] the list of exceptions being rescued + # [nil | Node] the list of exceptions being rescued attr_reader :exceptions # [nil | Field | VarField] the expression being used to capture the raised @@ -9346,7 +9352,7 @@ class Rescue < Node # [Kw] the rescue keyword attr_reader :keyword - # [RescueEx] the exceptions being rescued + # [nil | RescueEx] the exceptions being rescued attr_reader :exception # [Statements] the expressions to evaluate when an error is rescued @@ -9995,9 +10001,13 @@ class StringContent < Node # string attr_reader :parts + # [Array[ Comment | EmbDoc ]] the comments attached to this node + attr_reader :comments + def initialize(parts:, location:) @parts = parts @location = location + @comments = [] end def accept(visitor) @@ -10024,6 +10034,33 @@ def deconstruct_keys(_keys) def ===(other) other.is_a?(StringContent) && ArrayMatch.call(parts, other.parts) end + + def format(q) + q.text(q.quote) + q.group do + parts.each do |part| + if part.is_a?(TStringContent) + value = Quotes.normalize(part.value, q.quote) + first = true + + value.each_line(chomp: true) do |line| + if first + first = false + else + q.breakable_return + end + + q.text(line) + end + + q.breakable_return if value.end_with?("\n") + else + q.format(part) + end + end + end + q.text(q.quote) + end end # StringConcat represents concatenating two strings together using a backward @@ -10033,7 +10070,8 @@ def ===(other) # "second" # class StringConcat < Node - # [StringConcat | StringLiteral] the left side of the concatenation + # [Heredoc | StringConcat | StringLiteral] the left side of the + # concatenation attr_reader :left # [StringLiteral] the right side of the concatenation @@ -10230,7 +10268,7 @@ class StringLiteral < Node # string literal attr_reader :parts - # [String] which quote was used by the string literal + # [nil | String] which quote was used by the string literal attr_reader :quote # [Array[ Comment | EmbDoc ]] the comments attached to this node @@ -10475,8 +10513,8 @@ def ===(other) # :symbol # class SymbolLiteral < Node - # [Backtick | Const | CVar | GVar | Ident | IVar | Kw | Op] the value of the - # symbol + # [Backtick | Const | CVar | GVar | Ident | IVar | Kw | Op | TStringContent] + # the value of the symbol attr_reader :value # [Array[ Comment | EmbDoc ]] the comments attached to this node @@ -11430,7 +11468,7 @@ def modifier? # # In the example above, the VarField node represents the +variable+ token. class VarField < Node - # [nil | Const | CVar | GVar | Ident | IVar] the target of this node + # [nil | :nil | Const | CVar | GVar | Ident | IVar] the target of this node attr_reader :value # [Array[ Comment | EmbDoc ]] the comments attached to this node @@ -11569,7 +11607,7 @@ def pin(parent, pin) # This can be a plain local variable like the example above. It can also be a # a class variable, a global variable, or an instance variable. class PinnedVarRef < Node - # [VarRef] the value of this node + # [Const | CVar | GVar | Ident | IVar] the value of this node attr_reader :value # [Array[ Comment | EmbDoc ]] the comments attached to this node diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index 59128875..ca006c31 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -908,6 +908,13 @@ def on_blockarg(name) # (nil | Ensure) ensure_clause # ) -> BodyStmt def on_bodystmt(statements, rescue_clause, else_clause, ensure_clause) + # In certain versions of Ruby, the `statements` argument can be any node + # in the case that we're inside of an endless method definition. In this + # case we'll wrap it in a Statements node to be consistent. + unless statements.is_a?(Statements) + statements = Statements.new(self, body: [statements], location: statements.location) + end + parts = [statements, rescue_clause, else_clause, ensure_clause].compact BodyStmt.new( @@ -1157,13 +1164,23 @@ def on_const(value) end # :call-seq: - # on_const_path_field: (untyped parent, Const constant) -> ConstPathField + # on_const_path_field: (untyped parent, Const constant) -> + # ConstPathField | Field def on_const_path_field(parent, constant) - ConstPathField.new( - parent: parent, - constant: constant, - location: parent.location.to(constant.location) - ) + if constant.is_a?(Const) + ConstPathField.new( + parent: parent, + constant: constant, + location: parent.location.to(constant.location) + ) + else + Field.new( + parent: parent, + operator: consume_operator(:"::"), + name: constant, + location: parent.location.to(constant.location) + ) + end end # :call-seq: @@ -1866,10 +1883,40 @@ def on_heredoc_end(value) # :call-seq: # on_hshptn: ( # (nil | untyped) constant, - # Array[[Label, untyped]] keywords, + # Array[[Label | StringContent, untyped]] keywords, # (nil | VarField) keyword_rest # ) -> HshPtn def on_hshptn(constant, keywords, keyword_rest) + keywords = + (keywords || []).map do |(label, value)| + if label.is_a?(Label) + [label, value] + else + tstring_beg_index = + tokens.rindex do |token| + token.is_a?(TStringBeg) && token.location.start_char < label.location.start_char + end + + tstring_beg = tokens.delete_at(tstring_beg_index) + + label_end_index = + tokens.rindex do |token| + token.is_a?(LabelEnd) && token.location.start_char == label.location.end_char + end + + label_end = tokens.delete_at(label_end_index) + + [ + DynaSymbol.new( + parts: label.parts, + quote: label_end.value[0], + location: tstring_beg.location.to(label_end.location) + ), + value + ] + end + end + if keyword_rest # We're doing this to delete the token from the list so that it doesn't # confuse future patterns by thinking they have an extra ** on the end. @@ -1882,7 +1929,7 @@ def on_hshptn(constant, keywords, keyword_rest) keyword_rest = VarField.new(value: nil, location: token.location) end - parts = [constant, *keywords&.flatten(1), keyword_rest].compact + parts = [constant, *keywords.flatten(1), keyword_rest].compact # If there's no constant, there may be braces, so we're going to look for # those to get our bounds. @@ -1899,7 +1946,7 @@ def on_hshptn(constant, keywords, keyword_rest) HshPtn.new( constant: constant, - keywords: keywords || [], + keywords: keywords, keyword_rest: keyword_rest, location: parts[0].location.to(parts[-1].location) ) @@ -2379,7 +2426,7 @@ def on_method_add_block(call, block) location = call.location.to(block.location) case call - when Break, Next + when Break, Next, ReturnNode parts = call.arguments.parts node = parts.pop diff --git a/lib/syntax_tree/reflection.rb b/lib/syntax_tree/reflection.rb index 2457fe49..ec4345e1 100644 --- a/lib/syntax_tree/reflection.rb +++ b/lib/syntax_tree/reflection.rb @@ -21,6 +21,10 @@ def initialize(type) def ===(value) value.is_a?(Array) && value.all? { type === _1 } end + + def inspect + "Array<#{type.inspect}>" + end end # Represents a tuple type that holds a number of types in order. @@ -35,6 +39,10 @@ def ===(value) value.is_a?(Array) && value.length == types.length && value.zip(types).all? { _2 === _1 } end + + def inspect + "[#{types.map(&:inspect).join(", ")}]" + end end # Represents a union type that can be one of a number of types. @@ -48,6 +56,10 @@ def initialize(types) def ===(value) types.any? { _1 === value } end + + def inspect + types.map(&:inspect).join(" | ") + end end class << self diff --git a/lib/syntax_tree/translation/parser.rb b/lib/syntax_tree/translation/parser.rb index 4a4b6ade..65bf918d 100644 --- a/lib/syntax_tree/translation/parser.rb +++ b/lib/syntax_tree/translation/parser.rb @@ -1068,7 +1068,7 @@ def visit_field(node) case stack[-2] when Assign, MLHS Ident.new( - value: :"#{node.name.value}=", + value: "#{node.name.value}=", location: node.name.location ) else @@ -1295,11 +1295,11 @@ def visit_hshptn(node) next s(:pair, [visit(keyword), visit(value)], nil) if value case keyword - when Label - s(:match_var, [keyword.value.chomp(":").to_sym], nil) - when StringContent + when DynaSymbol raise if keyword.parts.length > 1 s(:match_var, [keyword.parts.first.value.to_sym], nil) + when Label + s(:match_var, [keyword.value.chomp(":").to_sym], nil) end end @@ -2364,13 +2364,10 @@ def visit_statements(node) # Visit a StringConcat node. def visit_string_concat(node) - visit_string_literal( - StringLiteral.new( - parts: [node.left, node.right], - quote: nil, - location: node.location - ) - ) + location = + source_map_collection(expression: source_range_node(node)) + + s(:dstr, [visit(node.left), visit(node.right)], location) end # Visit a StringContent node. diff --git a/lib/syntax_tree/yarv/compiler.rb b/lib/syntax_tree/yarv/compiler.rb index c1b4d6dd..1899140a 100644 --- a/lib/syntax_tree/yarv/compiler.rb +++ b/lib/syntax_tree/yarv/compiler.rb @@ -1050,11 +1050,11 @@ def visit_if_op(node) visit_if( IfNode.new( predicate: node.predicate, - statements: node.truthy, + statements: Statements.new(nil, body: [node.truthy], location: Location.default), consequent: Else.new( keyword: Kw.new(value: "else", location: Location.default), - statements: node.falsy, + statements: Statements.new(nil, body: [node.falsy], location: Location.default), location: Location.default ), location: Location.default diff --git a/lib/syntax_tree/yarv/decompiler.rb b/lib/syntax_tree/yarv/decompiler.rb index 753ba80a..4ea99e3a 100644 --- a/lib/syntax_tree/yarv/decompiler.rb +++ b/lib/syntax_tree/yarv/decompiler.rb @@ -151,7 +151,7 @@ def decompile(iseq) elsif argc == 1 && method.end_with?("=") receiver, argument = clause.pop(2) clause << Assign( - CallNode(receiver, Period("."), Ident(method[0..-2]), nil), + Field(receiver, Period("."), Ident(method[0..-2])), argument ) else diff --git a/test/formatting_test.rb b/test/formatting_test.rb index 37ca29e1..5e5f9e9f 100644 --- a/test/formatting_test.rb +++ b/test/formatting_test.rb @@ -7,6 +7,7 @@ class FormattingTest < Minitest::Test Fixtures.each_fixture do |fixture| define_method(:"test_formatted_#{fixture.name}") do assert_equal(fixture.formatted, SyntaxTree.format(fixture.source)) + assert_syntax_tree(SyntaxTree.parse(fixture.source)) end end diff --git a/test/interface_test.rb b/test/interface_test.rb deleted file mode 100644 index 5086680e..00000000 --- a/test/interface_test.rb +++ /dev/null @@ -1,72 +0,0 @@ -# frozen_string_literal: true - -require_relative "test_helper" - -module SyntaxTree - class InterfaceTest < Minitest::Test - ObjectSpace.each_object(Node.singleton_class) do |klass| - next if klass == Node - - define_method(:"test_instantiate_#{klass.name}") do - assert_syntax_tree(instantiate(klass)) - end - end - - Fixtures.each_fixture do |fixture| - define_method(:"test_#{fixture.name}") do - assert_syntax_tree(SyntaxTree.parse(fixture.source)) - end - end - - private - - # This method is supposed to instantiate a new instance of the given class. - # The class is always a descendant from SyntaxTree::Node, so we can make - # certain assumptions about the way the initialize method is set up. If it - # needs to be special-cased, it's done so at the end of this method. - def instantiate(klass) - params = {} - - # Set up all of the keyword parameters for the class. - klass - .instance_method(:initialize) - .parameters - .each { |(type, name)| params[name] = nil if type.start_with?("key") } - - # Set up any default values that have to be arrays. - %i[ - assocs - comments - elements - keywords - locals - optionals - parts - posts - requireds - symbols - values - ].each { |key| params[key] = [] if params.key?(key) } - - # Set up a default location for the node. - params[:location] = Location.fixed(line: 0, char: 0, column: 0) - - case klass.name - when "SyntaxTree::Binary" - klass.new(**params, operator: :+) - when "SyntaxTree::Kw" - klass.new(**params, value: "kw") - when "SyntaxTree::Label" - klass.new(**params, value: "label:") - when "SyntaxTree::Op" - klass.new(**params, value: "+") - when "SyntaxTree::RegexpLiteral" - klass.new(**params, ending: "/") - when "SyntaxTree::Statements" - klass.new(nil, **params, body: []) - else - klass.new(**params) - end - end - end -end diff --git a/test/test_helper.rb b/test/test_helper.rb index 77627e26..b307db3d 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -11,6 +11,36 @@ require "syntax_tree" require "syntax_tree/cli" +require "syntax_tree/reflection" + +SyntaxTree::Reflection.nodes.each do |name, node| + next if name == :Statements + + clazz = SyntaxTree.const_get(name) + parameters = clazz.instance_method(:initialize).parameters + + # First, verify that all of the parameters listed in the list of attributes. + # If there are any parameters that aren't listed in the attributes, then + # something went wrong with the parsing in the reflection module. + raise unless (parameters.map(&:last) - node.attributes.keys).empty? + + # Now we're going to use an alias chain to redefine the initialize method to + # include type checking. + clazz.alias_method(:initialize_without_verify, :initialize) + clazz.define_method(:initialize) do |**kwargs| + kwargs.each do |kwarg, value| + attribute = node.attributes.fetch(kwarg) + + unless attribute.type === value + raise TypeError, "invalid type for #{name}##{kwarg}, expected " \ + "#{attribute.type.inspect}, got #{value.inspect}" + end + end + + initialize_without_verify(**kwargs) + end +end + require "json" require "tempfile" require "pp" From 42572ac17ad319b27cb63dc340f3e7354c83f1f6 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 9 Feb 2023 14:46:49 -0500 Subject: [PATCH 378/536] Generate sorbet types in a rake task --- Rakefile | 11 +- lib/syntax_tree/dsl.rb | 32 ++- lib/syntax_tree/parser.rb | 15 +- lib/syntax_tree/reflection.rb | 21 +- lib/syntax_tree/translation/parser.rb | 3 +- lib/syntax_tree/yarv/compiler.rb | 14 +- tasks/sorbet.rake | 277 ++++++++++++++++++++++++++ test/test_helper.rb | 5 +- 8 files changed, 349 insertions(+), 29 deletions(-) create mode 100644 tasks/sorbet.rake diff --git a/Rakefile b/Rakefile index 22d7d1fe..fb4f8847 100644 --- a/Rakefile +++ b/Rakefile @@ -16,7 +16,16 @@ task default: :test configure = ->(task) do task.source_files = - FileList[%w[Gemfile Rakefile syntax_tree.gemspec lib/**/*.rb test/*.rb]] + FileList[ + %w[ + Gemfile + Rakefile + syntax_tree.gemspec + lib/**/*.rb + tasks/*.rake + test/*.rb + ] + ] # Since Syntax Tree supports back to Ruby 2.7.0, we need to make sure that we # format our code such that it's compatible with that version. This actually diff --git a/lib/syntax_tree/dsl.rb b/lib/syntax_tree/dsl.rb index 860a1fe5..1af19644 100644 --- a/lib/syntax_tree/dsl.rb +++ b/lib/syntax_tree/dsl.rb @@ -210,12 +210,17 @@ def RAssign(value, operator, pattern) end # Create a new ClassDeclaration node. - def ClassDeclaration(constant, superclass, bodystmt) + def ClassDeclaration( + constant, + superclass, + bodystmt, + location = Location.default + ) ClassDeclaration.new( constant: constant, superclass: superclass, bodystmt: bodystmt, - location: Location.default + location: location ) end @@ -225,12 +230,12 @@ def Comma(value) end # Create a new Command node. - def Command(message, arguments, block) + def Command(message, arguments, block, location = Location.default) Command.new( message: message, arguments: arguments, block: block, - location: Location.default + location: location ) end @@ -247,8 +252,8 @@ def CommandCall(receiver, operator, message, arguments, block) end # Create a new Comment node. - def Comment(value, inline) - Comment.new(value: value, inline: inline, location: Location.default) + def Comment(value, inline, location = Location.default) + Comment.new(value: value, inline: inline, location: location) end # Create a new Const node. @@ -285,14 +290,21 @@ def CVar(value) end # Create a new DefNode node. - def DefNode(target, operator, name, params, bodystmt) + def DefNode( + target, + operator, + name, + params, + bodystmt, + location = Location.default + ) DefNode.new( target: target, operator: operator, name: name, params: params, bodystmt: bodystmt, - location: Location.default + location: location ) end @@ -565,8 +577,8 @@ def MAssign(target, value) end # Create a new MethodAddBlock node. - def MethodAddBlock(call, block) - MethodAddBlock.new(call: call, block: block, location: Location.default) + def MethodAddBlock(call, block, location = Location.default) + MethodAddBlock.new(call: call, block: block, location: location) end # Create a new MLHS node. diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index ca006c31..c15a0339 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -912,7 +912,12 @@ def on_bodystmt(statements, rescue_clause, else_clause, ensure_clause) # in the case that we're inside of an endless method definition. In this # case we'll wrap it in a Statements node to be consistent. unless statements.is_a?(Statements) - statements = Statements.new(self, body: [statements], location: statements.location) + statements = + Statements.new( + self, + body: [statements], + location: statements.location + ) end parts = [statements, rescue_clause, else_clause, ensure_clause].compact @@ -1894,14 +1899,16 @@ def on_hshptn(constant, keywords, keyword_rest) else tstring_beg_index = tokens.rindex do |token| - token.is_a?(TStringBeg) && token.location.start_char < label.location.start_char + token.is_a?(TStringBeg) && + token.location.start_char < label.location.start_char end tstring_beg = tokens.delete_at(tstring_beg_index) label_end_index = tokens.rindex do |token| - token.is_a?(LabelEnd) && token.location.start_char == label.location.end_char + token.is_a?(LabelEnd) && + token.location.start_char == label.location.end_char end label_end = tokens.delete_at(label_end_index) @@ -1913,7 +1920,7 @@ def on_hshptn(constant, keywords, keyword_rest) location: tstring_beg.location.to(label_end.location) ), value - ] + ] end end diff --git a/lib/syntax_tree/reflection.rb b/lib/syntax_tree/reflection.rb index ec4345e1..bf4b95f3 100644 --- a/lib/syntax_tree/reflection.rb +++ b/lib/syntax_tree/reflection.rb @@ -34,10 +34,10 @@ class TupleType def initialize(types) @types = types end - + def ===(value) value.is_a?(Array) && value.length == types.length && - value.zip(types).all? { _2 === _1 } + value.zip(types).all? { |item, type| type === item } end def inspect @@ -64,16 +64,20 @@ def inspect class << self def parse(comment) + comment = comment.gsub(/\n/, " ") + unless comment.start_with?("[") raise "Comment does not start with a bracket: #{comment.inspect}" end count = 1 found = - comment.chars[1..].find.with_index(1) do |char, index| - count += { "[" => 1, "]" => -1 }.fetch(char, 0) - break index if count == 0 - end + comment.chars[1..] + .find + .with_index(1) do |char, index| + count += { "[" => 1, "]" => -1 }.fetch(char, 0) + break index if count == 0 + end # If we weren't able to find the end of the balanced brackets, then # the comment is malformed. @@ -209,7 +213,7 @@ def parse_comments(statements, index) attribute = Attribute.new( statement.arguments.parts.first.value.value.to_sym, - parse_comments(statements, statement_index).join(" ") + "#{parse_comments(statements, statement_index).join("\n")}\n" ) # Ensure that we don't already have an attribute named the same as @@ -223,10 +227,11 @@ def parse_comments(statements, index) end # Finally, set it up in the hash of nodes so that we can use it later. + comments = parse_comments(main_statements, main_statement_index) node = Node.new( main_statement.constant.constant.value.to_sym, - parse_comments(main_statements, main_statement_index).join("\n"), + "#{comments.join("\n")}\n", attributes ) diff --git a/lib/syntax_tree/translation/parser.rb b/lib/syntax_tree/translation/parser.rb index 65bf918d..184bb165 100644 --- a/lib/syntax_tree/translation/parser.rb +++ b/lib/syntax_tree/translation/parser.rb @@ -2364,8 +2364,7 @@ def visit_statements(node) # Visit a StringConcat node. def visit_string_concat(node) - location = - source_map_collection(expression: source_range_node(node)) + location = source_map_collection(expression: source_range_node(node)) s(:dstr, [visit(node.left), visit(node.right)], location) end diff --git a/lib/syntax_tree/yarv/compiler.rb b/lib/syntax_tree/yarv/compiler.rb index 1899140a..3aff3fe5 100644 --- a/lib/syntax_tree/yarv/compiler.rb +++ b/lib/syntax_tree/yarv/compiler.rb @@ -1050,11 +1050,21 @@ def visit_if_op(node) visit_if( IfNode.new( predicate: node.predicate, - statements: Statements.new(nil, body: [node.truthy], location: Location.default), + statements: + Statements.new( + nil, + body: [node.truthy], + location: Location.default + ), consequent: Else.new( keyword: Kw.new(value: "else", location: Location.default), - statements: Statements.new(nil, body: [node.falsy], location: Location.default), + statements: + Statements.new( + nil, + body: [node.falsy], + location: Location.default + ), location: Location.default ), location: Location.default diff --git a/tasks/sorbet.rake b/tasks/sorbet.rake new file mode 100644 index 00000000..e4152664 --- /dev/null +++ b/tasks/sorbet.rake @@ -0,0 +1,277 @@ +# frozen_string_literal: true + +module SyntaxTree + class RBI + include DSL + + attr_reader :body, :line + + def initialize + @body = [] + @line = 1 + end + + def generate + require "syntax_tree/reflection" + + body << Comment("# typed: strict", false, location) + @line += 2 + + generate_parent + Reflection.nodes.sort.each { |(_, node)| generate_node(node) } + + Formatter.format(nil, Program(Statements(body))) + end + + private + + def generate_comments(comment) + comment + .lines(chomp: true) + .map { |line| Comment("# #{line}", false, location).tap { @line += 1 } } + end + + def generate_parent + attribute = Reflection.nodes[:Program].attributes[:location] + class_location = location + + node_body = generate_comments(attribute.comment) + node_body << sig_block { sig_returns { sig_type_for(attribute.type) } } + @line += 1 + + node_body << Command( + Ident("attr_reader"), + Args([SymbolLiteral(Ident("location"))]), + nil, + location + ) + @line += 1 + + body << ClassDeclaration( + ConstPathRef(VarRef(Const("SyntaxTree")), Const("Node")), + nil, + BodyStmt(Statements(node_body), nil, nil, nil, nil), + class_location + ) + @line += 2 + end + + def generate_node(node) + body.concat(generate_comments(node.comment)) + class_location = location + @line += 2 + + body << ClassDeclaration( + ConstPathRef(VarRef(Const("SyntaxTree")), Const(node.name.to_s)), + ConstPathRef(VarRef(Const("SyntaxTree")), Const("Node")), + BodyStmt(Statements(generate_node_body(node)), nil, nil, nil, nil), + class_location + ) + + @line += 2 + end + + def generate_node_body(node) + node_body = [] + node.attributes.sort.each do |(name, attribute)| + next if name == :location + + node_body.concat(generate_comments(attribute.comment)) + node_body << sig_block { sig_returns { sig_type_for(attribute.type) } } + @line += 1 + + node_body << Command( + Ident("attr_reader"), + Args([SymbolLiteral(Ident(attribute.name.to_s))]), + nil, + location + ) + @line += 2 + end + + node_body.concat(generate_initialize(node)) + + node_body << sig_block do + CallNode( + sig_params do + BareAssocHash( + [Assoc(Label("visitor:"), sig_type_for(BasicVisitor))] + ) + end, + Period("."), + Ident("returns"), + ArgParen( + Args( + [CallNode(VarRef(Const("T")), Period("."), Ident("untyped"), nil)] + ) + ) + ) + end + @line += 1 + + node_body << generate_def_node( + "accept", + Paren( + LParen("("), + Params.new(requireds: [Ident("visitor")], location: location) + ) + ) + @line += 2 + + node_body << generate_child_nodes + @line += 1 + + node_body << generate_def_node("child_nodes", nil) + @line += 1 + + node_body + end + + def generate_initialize(node) + parameters = + SyntaxTree.const_get(node.name).instance_method(:initialize).parameters + + assocs = + parameters.map do |(_, name)| + Assoc(Label("#{name}:"), sig_type_for(node.attributes[name].type)) + end + + node_body = [] + node_body << sig_block do + CallNode( + sig_params { BareAssocHash(assocs) }, + Period("."), + Ident("void"), + nil + ) + end + @line += 1 + + params = Params.new(location: location) + parameters.each do |(type, name)| + case type + when :req + params.requireds << Ident(name.to_s) + when :keyreq + params.keywords << [Label("#{name}:"), nil] + when :key + params.keywords << [ + Label("#{name}:"), + CallNode( + VarRef(Const("T")), + Period("."), + Ident("unsafe"), + ArgParen(Args([VarRef(Kw("nil"))])) + ) + ] + else + raise + end + end + + node_body << generate_def_node("initialize", Paren(LParen("("), params)) + @line += 2 + + node_body + end + + def generate_child_nodes + type = + Reflection::Type::ArrayType.new( + Reflection::Type::UnionType.new([NilClass, Node]) + ) + + sig_block { sig_returns { sig_type_for(type) } } + end + + def generate_def_node(name, params) + DefNode( + nil, + nil, + Ident(name), + params, + BodyStmt(Statements([VoidStmt()]), nil, nil, nil, nil), + location + ) + end + + def sig_block + MethodAddBlock( + CallNode(nil, nil, Ident("sig"), nil), + BlockNode( + LBrace("{"), + nil, + BodyStmt(Statements([yield]), nil, nil, nil, nil) + ), + location + ) + end + + def sig_params + CallNode(nil, nil, Ident("params"), ArgParen(Args([yield]))) + end + + def sig_returns + CallNode(nil, nil, Ident("returns"), ArgParen(Args([yield]))) + end + + def sig_type_for(type) + case type + when Reflection::Type::ArrayType + ARef( + ConstPathRef(VarRef(Const("T")), Const("Array")), + sig_type_for(type.type) + ) + when Reflection::Type::TupleType + ArrayLiteral(LBracket("["), Args(type.types.map { sig_type_for(_1) })) + when Reflection::Type::UnionType + if type.types.include?(NilClass) + selected = type.types.reject { _1 == NilClass } + subtype = + if selected.size == 1 + selected.first + else + Reflection::Type::UnionType.new(selected) + end + + CallNode( + VarRef(Const("T")), + Period("."), + Ident("nilable"), + ArgParen(Args([sig_type_for(subtype)])) + ) + else + CallNode( + VarRef(Const("T")), + Period("."), + Ident("any"), + ArgParen(Args(type.types.map { sig_type_for(_1) })) + ) + end + when Symbol + ConstRef(Const("Symbol")) + else + *parents, constant = type.name.split("::").map { Const(_1) } + + if parents.empty? + ConstRef(constant) + else + [*parents[1..], constant].inject( + VarRef(parents.first) + ) { |accum, const| ConstPathRef(accum, const) } + end + end + end + + def location + Location.fixed(line: line, char: 0, column: 0) + end + end +end + +namespace :sorbet do + desc "Generate RBI files for Sorbet" + task :rbi do + puts SyntaxTree::RBI.new.generate + end +end diff --git a/test/test_helper.rb b/test/test_helper.rb index b307db3d..18159fab 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -32,8 +32,9 @@ attribute = node.attributes.fetch(kwarg) unless attribute.type === value - raise TypeError, "invalid type for #{name}##{kwarg}, expected " \ - "#{attribute.type.inspect}, got #{value.inspect}" + raise TypeError, + "invalid type for #{name}##{kwarg}, expected " \ + "#{attribute.type.inspect}, got #{value.inspect}" end end From e0be5793aeecc1d0c44a3ff118dd24c653a2e8af Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 9 Feb 2023 16:15:56 -0500 Subject: [PATCH 379/536] More documentation in the test helper --- test/test_helper.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/test_helper.rb b/test/test_helper.rb index 18159fab..e4452e3d 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -11,8 +11,10 @@ require "syntax_tree" require "syntax_tree/cli" +# Here we are going to establish type verification whenever a new node is +# created. We do this through the reflection module, which in turn parses the +# source code of the node classes. require "syntax_tree/reflection" - SyntaxTree::Reflection.nodes.each do |name, node| next if name == :Statements From da19f6a2dc787411e34e4ec90547b136467e7149 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 8 Feb 2023 11:01:27 -0500 Subject: [PATCH 380/536] Location information for parser nodes --- lib/syntax_tree/formatter.rb | 2 +- lib/syntax_tree/node.rb | 8 + lib/syntax_tree/parser.rb | 197 +++- lib/syntax_tree/translation/parser.rb | 1529 ++++++++++--------------- test/syntax_tree_test.rb | 2 +- test/translation/parser_test.rb | 2 +- 6 files changed, 774 insertions(+), 966 deletions(-) diff --git a/lib/syntax_tree/formatter.rb b/lib/syntax_tree/formatter.rb index c64cf7d1..60858bf2 100644 --- a/lib/syntax_tree/formatter.rb +++ b/lib/syntax_tree/formatter.rb @@ -138,7 +138,7 @@ def format(node, stackable: true) # going to just print out the node as it was seen in the source. doc = if last_leading&.ignore? - range = source[node.location.start_char...node.location.end_char] + range = source[node.start_char...node.end_char] first = true range.each_line(chomp: true) do |line| diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 4a98dae4..627deab1 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -126,6 +126,14 @@ def format(q) raise NotImplementedError end + def start_char + location.start_char + end + + def end_char + location.end_char + end + def pretty_print(q) accept(Visitor::PrettyPrintVisitor.new(q)) end diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index c15a0339..cf3982f9 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -256,11 +256,37 @@ def find_token(type) tokens[index] if index end + def find_token_between(type, left, right) + bounds = left.location.end_char...right.location.start_char + index = + tokens.rindex do |token| + char = token.location.start_char + break if char < bounds.begin + + token.is_a?(type) && bounds.cover?(char) + end + + tokens[index] if index + end + def find_keyword(name) index = tokens.rindex { |token| token.is_a?(Kw) && (token.name == name) } tokens[index] if index end + def find_keyword_between(name, left, right) + bounds = left.location.end_char...right.location.start_char + index = + tokens.rindex do |token| + char = token.location.start_char + break if char < bounds.begin + + token.is_a?(Kw) && (token.name == name) && bounds.cover?(char) + end + + tokens[index] if index + end + def find_operator(name) index = tokens.rindex { |token| token.is_a?(Op) && (token.name == name) } tokens[index] if index @@ -645,7 +671,7 @@ def visit_var_ref(node) end def self.visit(node, tokens) - start_char = node.location.start_char + start_char = node.start_char allocated = [] tokens.reverse_each do |token| @@ -874,13 +900,34 @@ def on_binary(left, operator, right) # on_block_var: (Params params, (nil | Array[Ident]) locals) -> BlockVar def on_block_var(params, locals) index = - tokens.rindex do |node| - node.is_a?(Op) && %w[| ||].include?(node.value) && - node.location.start_char < params.location.start_char - end + tokens.rindex { |node| node.is_a?(Op) && %w[| ||].include?(node.value) } + + ending = tokens.delete_at(index) + beginning = ending.value == "||" ? ending : consume_operator(:|) + + # If there are no parameters, then we didn't have anything to base the + # location information of off. Now that we have an opening of the + # block, we can correct this. + if params.empty? + start_line = params.location.start_line + start_char = + ( + if beginning.value == "||" + beginning.location.start_char + else + find_next_statement_start(beginning.location.end_char) + end + ) + + location = + Location.fixed( + line: start_line, + char: start_char, + column: start_char - line_counts[start_line - 1].start + ) - beginning = tokens[index] - ending = tokens[-1] + params = params.copy(location: location) + end BlockVar.new( params: params, @@ -1762,15 +1809,13 @@ def on_for(index, collection, statements) # Consume the do keyword if it exists so that it doesn't get confused for # some other block - keyword = find_keyword(:do) - if keyword && - keyword.location.start_char > collection.location.end_char && - keyword.location.end_char < ending.location.start_char + if (keyword = find_keyword_between(:do, collection, ending)) tokens.delete(keyword) end start_char = find_next_statement_start((keyword || collection).location.end_char) + statements.bind( start_char, start_char - @@ -1984,7 +2029,12 @@ def on_if(predicate, statements, consequent) beginning = consume_keyword(:if) ending = consequent || consume_keyword(:end) - start_char = find_next_statement_start(predicate.location.end_char) + if (keyword = find_keyword_between(:then, predicate, ending)) + tokens.delete(keyword) + end + + start_char = + find_next_statement_start((keyword || predicate).location.end_char) statements.bind( start_char, start_char - line_counts[predicate.location.end_line - 1].start, @@ -2068,7 +2118,8 @@ def on_in(pattern, statements, consequent) statements_start = token end - start_char = find_next_statement_start(statements_start.location.end_char) + start_char = + find_next_statement_start((token || statements_start).location.end_char) statements.bind( start_char, start_char - @@ -2194,12 +2245,19 @@ def on_lambda(params, statements) token.location.start_char > beginning.location.start_char end + if braces + opening = consume_token(TLamBeg) + closing = consume_token(RBrace) + else + opening = consume_keyword(:do) + closing = consume_keyword(:end) + end + # We need to do some special mapping here. Since ripper doesn't support - # capturing lambda var until 3.2, we need to normalize all of that here. + # capturing lambda vars, we need to normalize all of that here. params = - case params - when Paren - # In this case we've gotten to the <3.2 parentheses wrapping a set of + if params.is_a?(Paren) + # In this case we've gotten to the parentheses wrapping a set of # parameters case. Here we need to manually scan for lambda locals. range = (params.location.start_char + 1)...params.location.end_char locals = lambda_locals(source[range]) @@ -2221,25 +2279,28 @@ def on_lambda(params, statements) node.comments.concat(params.comments) node - when Params - # In this case we've gotten to the <3.2 plain set of parameters. In - # this case there cannot be lambda locals, so we will wrap the - # parameters into a lambda var that has no locals. + else + # If there are no parameters, then we didn't have anything to base the + # location information of off. Now that we have an opening of the + # block, we can correct this. + if params.empty? + opening_location = opening.location + location = + Location.fixed( + line: opening_location.start_line, + char: opening_location.start_char, + column: opening_location.start_column + ) + + params = params.copy(location: location) + end + + # In this case we've gotten to the plain set of parameters. In this + # case there cannot be lambda locals, so we will wrap the parameters + # into a lambda var that has no locals. LambdaVar.new(params: params, locals: [], location: params.location) - when LambdaVar - # In this case we've gotten to 3.2+ lambda var. In this case we don't - # need to do anything and can just the value as given. - params end - if braces - opening = consume_token(TLamBeg) - closing = consume_token(RBrace) - else - opening = consume_keyword(:do) - closing = consume_keyword(:end) - end - start_char = find_next_statement_start(opening.location.end_char) statements.bind( start_char, @@ -3134,7 +3195,7 @@ def on_rescue(exceptions, variable, statements, consequent) exceptions = exceptions[0] if exceptions.is_a?(Array) last_node = variable || exceptions || keyword - start_char = find_next_statement_start(last_node.location.end_char) + start_char = find_next_statement_start(last_node.end_char) statements.bind( start_char, start_char - line_counts[last_node.location.start_line - 1].start, @@ -3156,7 +3217,7 @@ def on_rescue(exceptions, variable, statements, consequent) start_char: keyword.location.end_char + 1, start_column: keyword.location.end_column + 1, end_line: last_node.location.end_line, - end_char: last_node.location.end_char, + end_char: last_node.end_char, end_column: last_node.location.end_column ) ) @@ -3267,9 +3328,27 @@ def on_sclass(target, bodystmt) ) end - # def on_semicolon(value) - # value - # end + class Semicolon + attr_reader :location + + def initialize(location:) + @location = location + end + end + + # :call-seq: + # on_semicolon: (String value) -> Semicolon + def on_semicolon(value) + tokens << Semicolon.new( + location: + Location.token( + line: lineno, + char: char_pos, + column: current_column, + size: value.size + ) + ) + end # def on_sp(value) # value @@ -3706,7 +3785,12 @@ def on_unless(predicate, statements, consequent) beginning = consume_keyword(:unless) ending = consequent || consume_keyword(:end) - start_char = find_next_statement_start(predicate.location.end_char) + if (keyword = find_keyword_between(:then, predicate, ending)) + tokens.delete(keyword) + end + + start_char = + find_next_statement_start((keyword || predicate).location.end_char) statements.bind( start_char, start_char - line_counts[predicate.location.end_line - 1].start, @@ -3742,16 +3826,16 @@ def on_until(predicate, statements) beginning = consume_keyword(:until) ending = consume_keyword(:end) - # Consume the do keyword if it exists so that it doesn't get confused for - # some other block - keyword = find_keyword(:do) - if keyword && keyword.location.start_char > predicate.location.end_char && - keyword.location.end_char < ending.location.start_char - tokens.delete(keyword) - end + delimiter = + find_keyword_between(:do, predicate, statements) || + find_token_between(Semicolon, predicate, statements) + + tokens.delete(delimiter) if delimiter # Update the Statements location information - start_char = find_next_statement_start(predicate.location.end_char) + start_char = + find_next_statement_start((delimiter || predicate).location.end_char) + statements.bind( start_char, start_char - line_counts[predicate.location.end_line - 1].start, @@ -3845,7 +3929,8 @@ def on_when(arguments, statements, consequent) statements_start = token end - start_char = find_next_statement_start(statements_start.location.end_char) + start_char = + find_next_statement_start((token || statements_start).location.end_char) statements.bind( start_char, @@ -3869,16 +3954,16 @@ def on_while(predicate, statements) beginning = consume_keyword(:while) ending = consume_keyword(:end) - # Consume the do keyword if it exists so that it doesn't get confused for - # some other block - keyword = find_keyword(:do) - if keyword && keyword.location.start_char > predicate.location.end_char && - keyword.location.end_char < ending.location.start_char - tokens.delete(keyword) - end + delimiter = + find_keyword_between(:do, predicate, statements) || + find_token_between(Semicolon, predicate, statements) + + tokens.delete(delimiter) if delimiter # Update the Statements location information - start_char = find_next_statement_start(predicate.location.end_char) + start_char = + find_next_statement_start((delimiter || predicate).location.end_char) + statements.bind( start_char, start_char - line_counts[predicate.location.end_line - 1].start, diff --git a/lib/syntax_tree/translation/parser.rb b/lib/syntax_tree/translation/parser.rb index 184bb165..b9e91e5f 100644 --- a/lib/syntax_tree/translation/parser.rb +++ b/lib/syntax_tree/translation/parser.rb @@ -27,9 +27,9 @@ def visit_alias(node) s( :alias, [visit(node.left), visit(node.right)], - source_map_keyword_bare( - source_range_length(node.location.start_char, 5), - source_range_node(node) + smap_keyword_bare( + srange_length(node.start_char, 5), + srange_node(node) ) ) end @@ -41,26 +41,20 @@ def visit_aref(node) s( :index, [visit(node.collection)], - source_map_index( - begin_token: - source_range_find( - node.collection.location.end_char, - node.location.end_char, - "[" - ), - end_token: source_range_length(node.location.end_char, -1), - expression: source_range_node(node) + smap_index( + srange_find(node.collection.end_char, node.end_char, "["), + srange_length(node.end_char, -1), + srange_node(node) ) ) else s( :index, [visit(node.collection)].concat(visit_all(node.index.parts)), - source_map_index( - begin_token: - source_range_find_between(node.collection, node.index, "["), - end_token: source_range_length(node.location.end_char, -1), - expression: source_range_node(node) + smap_index( + srange_find_between(node.collection, node.index, "["), + srange_length(node.end_char, -1), + srange_node(node) ) ) end @@ -69,31 +63,25 @@ def visit_aref(node) s( :send, [visit(node.collection), :[]], - source_map_send( - selector: - source_range_find( - node.collection.location.end_char, - node.location.end_char, - "[]" - ), - expression: source_range_node(node) + smap_send_bare( + srange_find(node.collection.end_char, node.end_char, "[]"), + srange_node(node) ) ) else s( :send, [visit(node.collection), :[], *visit_all(node.index.parts)], - source_map_send( - selector: - source_range( - source_range_find_between( - node.collection, - node.index, - "[" - ).begin_pos, - node.location.end_char - ), - expression: source_range_node(node) + smap_send_bare( + srange( + srange_find_between( + node.collection, + node.index, + "[" + ).begin_pos, + node.end_char + ), + srange_node(node) ) ) end @@ -107,26 +95,20 @@ def visit_aref_field(node) s( :indexasgn, [visit(node.collection)], - source_map_index( - begin_token: - source_range_find( - node.collection.location.end_char, - node.location.end_char, - "[" - ), - end_token: source_range_length(node.location.end_char, -1), - expression: source_range_node(node) + smap_index( + srange_find(node.collection.end_char, node.end_char, "["), + srange_length(node.end_char, -1), + srange_node(node) ) ) else s( :indexasgn, [visit(node.collection)].concat(visit_all(node.index.parts)), - source_map_index( - begin_token: - source_range_find_between(node.collection, node.index, "["), - end_token: source_range_length(node.location.end_char, -1), - expression: source_range_node(node) + smap_index( + srange_find_between(node.collection, node.index, "["), + srange_length(node.end_char, -1), + srange_node(node) ) ) end @@ -135,14 +117,9 @@ def visit_aref_field(node) s( :send, [visit(node.collection), :[]=], - source_map_send( - selector: - source_range_find( - node.collection.location.end_char, - node.location.end_char, - "[]" - ), - expression: source_range_node(node) + smap_send_bare( + srange_find(node.collection.end_char, node.end_char, "[]"), + srange_node(node) ) ) else @@ -151,17 +128,16 @@ def visit_aref_field(node) [visit(node.collection), :[]=].concat( visit_all(node.index.parts) ), - source_map_send( - selector: - source_range( - source_range_find_between( - node.collection, - node.index, - "[" - ).begin_pos, - node.location.end_char - ), - expression: source_range_node(node) + smap_send_bare( + srange( + srange_find_between( + node.collection, + node.index, + "[" + ).begin_pos, + node.end_char + ), + srange_node(node) ) ) end @@ -173,10 +149,7 @@ def visit_arg_block(node) s( :block_pass, [visit(node.value)], - source_map_operator( - source_range_length(node.location.start_char, 1), - source_range_node(node) - ) + smap_operator(srange_length(node.start_char, 1), srange_node(node)) ) end @@ -184,32 +157,26 @@ def visit_arg_block(node) def visit_arg_star(node) if stack[-3].is_a?(MLHSParen) && stack[-3].contents.is_a?(MLHS) if node.value.nil? - s(:restarg, [], source_map_variable(nil, source_range_node(node))) + s(:restarg, [], smap_variable(nil, srange_node(node))) else s( :restarg, [node.value.value.to_sym], - source_map_variable( - source_range_node(node.value), - source_range_node(node) - ) + smap_variable(srange_node(node.value), srange_node(node)) ) end else s( :splat, node.value.nil? ? [] : [visit(node.value)], - source_map_operator( - source_range_length(node.location.start_char, 1), - source_range_node(node) - ) + smap_operator(srange_length(node.start_char, 1), srange_node(node)) ) end end # Visit an ArgsForward node. def visit_args_forward(node) - s(:forwarded_args, [], source_map(expression: source_range_node(node))) + s(:forwarded_args, [], smap(srange_node(node))) end # Visit an ArrayLiteral node. @@ -218,12 +185,12 @@ def visit_array(node) :array, node.contents ? visit_all(node.contents.parts) : [], if node.lbracket.nil? - source_map_collection(expression: source_range_node(node)) + smap_collection_bare(srange_node(node)) else - source_map_collection( - begin_token: source_range_node(node.lbracket), - end_token: source_range_length(node.location.end_char, -1), - expression: source_range_node(node) + smap_collection( + srange_node(node.lbracket), + srange_length(node.end_char, -1), + srange_node(node) ) end ) @@ -237,8 +204,7 @@ def visit_aryptn(node) if node.rest.is_a?(VarField) if !node.rest.value.nil? children << s(:match_rest, [visit(node.rest)], nil) - elsif node.posts.empty? && - node.rest.location.start_char == node.rest.location.end_char + elsif node.posts.empty? && node.rest.start_char == node.rest.end_char # Here we have an implicit rest, as in [foo,]. parser has a specific # type for these patterns. type = :array_pattern_with_tail @@ -255,34 +221,29 @@ def visit_aryptn(node) s( type, children + visit_all(node.posts), - source_map_collection( - expression: - source_range( - node.constant.location.end_char + 1, - node.location.end_char - 1 - ) + smap_collection_bare( + srange(node.constant.end_char + 1, node.end_char - 1) ) ) ], - source_map_collection( - begin_token: - source_range_length(node.constant.location.end_char, 1), - end_token: source_range_length(node.location.end_char, -1), - expression: source_range_node(node) + smap_collection( + srange_length(node.constant.end_char, 1), + srange_length(node.end_char, -1), + srange_node(node) ) ) else s( type, children + visit_all(node.posts), - if buffer.source[node.location.start_char] == "[" - source_map_collection( - begin_token: source_range_length(node.location.start_char, 1), - end_token: source_range_length(node.location.end_char, -1), - expression: source_range_node(node) + if buffer.source[node.start_char] == "[" + smap_collection( + srange_length(node.start_char, 1), + srange_length(node.end_char, -1), + srange_node(node) ) else - source_map_collection(expression: source_range_node(node)) + smap_collection_bare(srange_node(node)) end ) end @@ -294,10 +255,8 @@ def visit_assign(node) location = target .location - .with_operator( - source_range_find_between(node.target, node.value, "=") - ) - .with_expression(source_range_node(node)) + .with_operator(srange_find_between(node.target, node.value, "=")) + .with_expression(srange_node(node)) s(target.type, target.children + [visit(node.value)], location) end @@ -305,17 +264,13 @@ def visit_assign(node) # Visit an Assoc node. def visit_assoc(node) if node.value.nil? - expression = - source_range(node.location.start_char, node.location.end_char - 1) + expression = srange(node.start_char, node.end_char - 1) type, location = if node.key.value.start_with?(/[A-Z]/) - [:const, source_map_constant(nil, expression, expression)] + [:const, smap_constant(nil, expression, expression)] else - [ - :send, - source_map_send(selector: expression, expression: expression) - ] + [:send, smap_send_bare(expression, expression)] end s( @@ -324,19 +279,19 @@ def visit_assoc(node) visit(node.key), s(type, [nil, node.key.value.chomp(":").to_sym], location) ], - source_map_operator( - source_range_length(node.key.location.end_char, -1), - source_range_node(node) + smap_operator( + srange_length(node.key.end_char, -1), + srange_node(node) ) ) else s( :pair, [visit(node.key), visit(node.value)], - source_map_operator( - source_range_search_between(node.key, node.value, "=>") || - source_range_length(node.key.location.end_char, -1), - source_range_node(node) + smap_operator( + srange_search_between(node.key, node.value, "=>") || + srange_length(node.key.end_char, -1), + srange_node(node) ) ) end @@ -347,16 +302,13 @@ def visit_assoc_splat(node) s( :kwsplat, [visit(node.value)], - source_map_operator( - source_range_length(node.location.start_char, 2), - source_range_node(node) - ) + smap_operator(srange_length(node.start_char, 2), srange_node(node)) ) end # Visit a Backref node. def visit_backref(node) - location = source_map(expression: source_range_node(node)) + location = smap(srange_node(node)) if node.value.match?(/^\$\d+$/) s(:nth_ref, [node.value[1..].to_i], location) @@ -375,7 +327,7 @@ def visit_bare_assoc_hash(node) :hash end, visit_all(node.assocs), - source_map_collection(expression: source_range_node(node)) + smap_collection_bare(srange_node(node)) ) end @@ -384,15 +336,11 @@ def visit_BEGIN(node) s( :preexe, [visit(node.statements)], - source_map_keyword( - source_range_length(node.location.start_char, 5), - source_range_find( - node.location.start_char + 5, - node.statements.location.start_char, - "{" - ), - source_range_length(node.location.end_char, -1), - source_range_node(node) + smap_keyword( + srange_length(node.start_char, 5), + srange_find(node.start_char + 5, node.statements.start_char, "{"), + srange_length(node.end_char, -1), + srange_node(node) ) ) end @@ -400,10 +348,10 @@ def visit_BEGIN(node) # Visit a Begin node. def visit_begin(node) location = - source_map_collection( - begin_token: source_range_length(node.location.start_char, 5), - end_token: source_range_length(node.location.end_char, -3), - expression: source_range_node(node) + smap_collection( + srange_length(node.start_char, 5), + srange_length(node.end_char, -3), + srange_node(node) ) if node.bodystmt.empty? @@ -439,13 +387,9 @@ def visit_binary(node) node.operator ), [visit(node.left), visit(node.right)], - source_map_operator( - source_range_find_between( - node.left, - node.right, - node.operator.to_s - ), - source_range_node(node) + smap_operator( + srange_find_between(node.left, node.right, node.operator.to_s), + srange_node(node) ) ) when :=~ @@ -459,13 +403,9 @@ def visit_binary(node) s( :match_with_lvasgn, [visit(node.left), visit(node.right)], - source_map_operator( - source_range_find_between( - node.left, - node.right, - node.operator.to_s - ), - source_range_node(node) + smap_operator( + srange_find_between(node.left, node.right, node.operator.to_s), + srange_node(node) ) ) else @@ -479,15 +419,12 @@ def visit_binary(node) # Visit a BlockArg node. def visit_blockarg(node) if node.name.nil? - s(:blockarg, [nil], source_map_variable(nil, source_range_node(node))) + s(:blockarg, [nil], smap_variable(nil, srange_node(node))) else s( :blockarg, [node.name.value.to_sym], - source_map_variable( - source_range_node(node.name), - source_range_node(node) - ) + smap_variable(srange_node(node.name), srange_node(node)) ) end end @@ -499,10 +436,7 @@ def visit_block_var(node) s( :shadowarg, [local.value.to_sym], - source_map_variable( - source_range_node(local), - source_range_node(local) - ) + smap_variable(srange_node(local), srange_node(local)) ) end @@ -522,13 +456,13 @@ def visit_block_var(node) s( :arg, [required.value.to_sym], - source_map_variable( - source_range_node(required), - source_range_node(required) + smap_variable( + srange_node(required), + srange_node(required) ) ) ], - source_map_collection(expression: source_range_node(required)) + smap_collection_bare(srange_node(required)) ) else child = visit(required) @@ -543,10 +477,10 @@ def visit_block_var(node) s( :args, children + shadowargs, - source_map_collection( - begin_token: source_range_length(node.location.start_char, 1), - end_token: source_range_length(node.location.end_char, -1), - expression: source_range_node(node) + smap_collection( + srange_length(node.start_char, 1), + srange_length(node.end_char, -1), + srange_node(node) ) ) end @@ -566,17 +500,12 @@ def visit_bodystmt(node) children << visit(node.else_clause) location = - source_map_condition( - else_token: - source_range_length( - node.else_clause.location.start_char - 3, - -4 - ), - expression: - source_range( - location.expression.begin_pos, - node.else_clause.location.end_char - ) + smap_condition( + nil, + nil, + srange_length(node.else_clause.start_char - 3, -4), + nil, + srange(location.expression.begin_pos, node.else_clause.end_char) ) end @@ -608,9 +537,9 @@ def visit_break(node) s( :break, visit_all(node.arguments.parts), - source_map_keyword_bare( - source_range_length(node.location.start_char, 5), - source_range_node(node) + smap_keyword_bare( + srange_length(node.start_char, 5), + srange_node(node) ) ) end @@ -638,17 +567,18 @@ def visit_case(node) else_token = if clauses.last.is_a?(Else) - source_range_length(clauses.last.location.start_char, 4) + srange_length(clauses.last.start_char, 4) end s( node.consequent.is_a?(In) ? :case_match : :case, [visit(node.value)] + clauses.map { |clause| visit(clause) }, - source_map_condition( - keyword: source_range_length(node.location.start_char, 4), - else_token: else_token, - end_token: source_range_length(node.location.end_char, -3), - expression: source_range_node(node) + smap_condition( + srange_length(node.start_char, 4), + nil, + else_token, + srange_length(node.end_char, -3), + srange_node(node) ) ) end @@ -658,9 +588,10 @@ def visit_CHAR(node) s( :str, [node.value[1..]], - source_map_collection( - begin_token: source_range_length(node.location.start_char, 1), - expression: source_range_node(node) + smap_collection( + srange_length(node.start_char, 1), + nil, + srange_node(node) ) ) end @@ -669,18 +600,18 @@ def visit_CHAR(node) def visit_class(node) operator = if node.superclass - source_range_find_between(node.constant, node.superclass, "<") + srange_find_between(node.constant, node.superclass, "<") end s( :class, [visit(node.constant), visit(node.superclass), visit(node.bodystmt)], - source_map_definition( - keyword: source_range_length(node.location.start_char, 5), - operator: operator, - name: source_range_node(node.constant), - end_token: source_range_length(node.location.end_char, -3) - ).with_expression(source_range_node(node)) + smap_definition( + srange_length(node.start_char, 5), + operator, + srange_node(node.constant), + srange_length(node.end_char, -3) + ).with_expression(srange_node(node)) ) end @@ -721,18 +652,17 @@ def visit_command_call(node) children += visit_all(node.arguments.arguments.parts) end - begin_token = - source_range_length(node.arguments.location.start_char, 1) - end_token = source_range_length(node.arguments.location.end_char, -1) + begin_token = srange_length(node.arguments.start_char, 1) + end_token = srange_length(node.arguments.end_char, -1) end dot_bound = if node.arguments - node.arguments.location.start_char + node.arguments.start_char elsif node.block - node.block.location.start_char + node.block.start_char else - node.location.end_char + node.end_char end call = @@ -743,37 +673,31 @@ def visit_command_call(node) :send end, children, - source_map_send( - dot: - if node.operator == :"::" - source_range_find( - node.receiver.location.end_char, - if node.message == :call - dot_bound - else - node.message.location.start_char - end, - "::" - ) - elsif node.operator - source_range_node(node.operator) - end, - begin_token: begin_token, - end_token: end_token, - selector: - node.message == :call ? nil : source_range_node(node.message), - expression: - if node.arguments.is_a?(ArgParen) || - (node.arguments.is_a?(Args) && node.arguments.parts.any?) - source_range( - node.location.start_char, - node.arguments.location.end_char - ) - elsif node.block - source_range_node(node.message) - else - source_range_node(node) - end + smap_send( + if node.operator == :"::" + srange_find( + node.receiver.end_char, + if node.message == :call + dot_bound + else + node.message.start_char + end, + "::" + ) + elsif node.operator + srange_node(node.operator) + end, + node.message == :call ? nil : srange_node(node.message), + begin_token, + end_token, + if node.arguments.is_a?(ArgParen) || + (node.arguments.is_a?(Args) && node.arguments.parts.any?) + srange(node.start_char, node.arguments.end_char) + elsif node.block + srange_node(node.message) + else + srange_node(node) + end ) ) @@ -783,14 +707,13 @@ def visit_command_call(node) s( type, [call, arguments, visit(node.block.bodystmt)], - source_map_collection( - begin_token: source_range_node(node.block.opening), - end_token: - source_range_length( - node.location.end_char, - node.block.opening.is_a?(Kw) ? -3 : -1 - ), - expression: source_range_node(node) + smap_collection( + srange_node(node.block.opening), + srange_length( + node.end_char, + node.block.opening.is_a?(Kw) ? -3 : -1 + ), + srange_node(node) ) ) else @@ -803,11 +726,7 @@ def visit_const(node) s( :const, [nil, node.value.to_sym], - source_map_constant( - nil, - source_range_node(node), - source_range_node(node) - ) + smap_constant(nil, srange_node(node), srange_node(node)) ) end @@ -820,10 +739,10 @@ def visit_const_path_field(node) s( :casgn, [visit(node.parent), node.constant.value.to_sym], - source_map_constant( - source_range_find_between(node.parent, node.constant, "::"), - source_range_node(node.constant), - source_range_node(node) + smap_constant( + srange_find_between(node.parent, node.constant, "::"), + srange_node(node.constant), + srange_node(node) ) ) end @@ -834,10 +753,10 @@ def visit_const_path_ref(node) s( :const, [visit(node.parent), node.constant.value.to_sym], - source_map_constant( - source_range_find_between(node.parent, node.constant, "::"), - source_range_node(node.constant), - source_range_node(node) + smap_constant( + srange_find_between(node.parent, node.constant, "::"), + srange_node(node.constant), + srange_node(node) ) ) end @@ -847,11 +766,7 @@ def visit_const_ref(node) s( :const, [nil, node.constant.value.to_sym], - source_map_constant( - nil, - source_range_node(node.constant), - source_range_node(node) - ) + smap_constant(nil, srange_node(node.constant), srange_node(node)) ) end @@ -860,7 +775,7 @@ def visit_cvar(node) s( :cvar, [node.value.to_sym], - source_map_variable(source_range_node(node), source_range_node(node)) + smap_variable(srange_node(node), srange_node(node)) ) end @@ -875,7 +790,7 @@ def visit_def(node) s( child.type, child.children, - source_map_collection(expression: nil) + smap_collection_bare(child.location&.expression) ) when Paren child = visit(node.params.contents) @@ -883,37 +798,38 @@ def visit_def(node) s( child.type, child.children, - source_map_collection( - begin_token: - source_range_length(node.params.location.start_char, 1), - end_token: - source_range_length(node.params.location.end_char, -1), - expression: source_range_node(node.params) + smap_collection( + srange_length(node.params.start_char, 1), + srange_length(node.params.end_char, -1), + srange_node(node.params) ) ) else - s(:args, [], source_map_collection(expression: nil)) + s(:args, [], smap_collection_bare(nil)) end location = if node.endless? - source_map_method_definition( - keyword: source_range_length(node.location.start_char, 3), - assignment: - source_range_find_between( - (node.params || node.name), - node.bodystmt, - "=" - ), - name: source_range_node(node.name), - expression: source_range_node(node) + smap_method_definition( + srange_length(node.start_char, 3), + nil, + srange_node(node.name), + nil, + srange_find_between( + (node.params || node.name), + node.bodystmt, + "=" + ), + srange_node(node) ) else - source_map_method_definition( - keyword: source_range_length(node.location.start_char, 3), - name: source_range_node(node.name), - end_token: source_range_length(node.location.end_char, -3), - expression: source_range_node(node) + smap_method_definition( + srange_length(node.start_char, 3), + nil, + srange_node(node.name), + srange_length(node.end_char, -3), + nil, + srange_node(node) ) end @@ -923,13 +839,13 @@ def visit_def(node) s( :defs, [visit(target), name, args, visit(node.bodystmt)], - source_map_method_definition( - keyword: location.keyword, - assignment: location.assignment, - operator: source_range_node(node.operator), - name: location.name, - end_token: location.end, - expression: location.expression + smap_method_definition( + location.keyword, + srange_node(node.operator), + location.name, + location.end, + location.assignment, + location.expression ) ) else @@ -939,23 +855,23 @@ def visit_def(node) # Visit a Defined node. def visit_defined(node) - paren_range = (node.location.start_char + 8)...node.location.end_char + paren_range = (node.start_char + 8)...node.end_char begin_token, end_token = if buffer.source[paren_range].include?("(") [ - source_range_find(paren_range.begin, paren_range.end, "("), - source_range_length(node.location.end_char, -1) + srange_find(paren_range.begin, paren_range.end, "("), + srange_length(node.end_char, -1) ] end s( :defined?, [visit(node.value)], - source_map_keyword( - source_range_length(node.location.start_char, 8), + smap_keyword( + srange_length(node.start_char, 8), begin_token, end_token, - source_range_node(node) + srange_node(node) ) ) end @@ -964,17 +880,13 @@ def visit_defined(node) def visit_dyna_symbol(node) location = if node.quote - source_map_collection( - begin_token: - source_range_length( - node.location.start_char, - node.quote.length - ), - end_token: source_range_length(node.location.end_char, -1), - expression: source_range_node(node) + smap_collection( + srange_length(node.start_char, node.quote.length), + srange_length(node.end_char, -1), + srange_node(node) ) else - source_map_collection(expression: source_range_node(node)) + smap_collection_bare(srange_node(node)) end if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) @@ -998,16 +910,12 @@ def visit_elsif(node) else_token = case node.consequent when Elsif - source_range_length(node.consequent.location.start_char, 5) + srange_length(node.consequent.start_char, 5) when Else - source_range_length(node.consequent.location.start_char, 4) + srange_length(node.consequent.start_char, 4) end - expression = - source_range( - node.location.start_char, - node.statements.location.end_char - 1 - ) + expression = srange(node.start_char, node.statements.end_char - 1) s( :if, @@ -1016,10 +924,12 @@ def visit_elsif(node) visit(node.statements), visit(node.consequent) ], - source_map_condition( - keyword: source_range_length(node.location.start_char, 5), - else_token: else_token, - expression: expression + smap_condition( + srange_length(node.start_char, 5), + nil, + else_token, + nil, + expression ) ) end @@ -1029,35 +939,34 @@ def visit_END(node) s( :postexe, [visit(node.statements)], - source_map_keyword( - source_range_length(node.location.start_char, 3), - source_range_find( - node.location.start_char + 3, - node.statements.location.start_char, - "{" - ), - source_range_length(node.location.end_char, -1), - source_range_node(node) + smap_keyword( + srange_length(node.start_char, 3), + srange_find(node.start_char + 3, node.statements.start_char, "{"), + srange_length(node.end_char, -1), + srange_node(node) ) ) end # Visit an Ensure node. def visit_ensure(node) - start_char = node.location.start_char + start_char = node.start_char end_char = if node.statements.empty? start_char + 6 else - node.statements.body.last.location.end_char + node.statements.body.last.end_char end s( :ensure, [visit(node.statements)], - source_map_condition( - keyword: source_range_length(start_char, 6), - expression: source_range(start_char, end_char) + smap_condition( + srange_length(start_char, 6), + nil, + nil, + nil, + srange(start_char, end_char) ) ) end @@ -1090,15 +999,11 @@ def visit_field(node) # Visit a FloatLiteral node. def visit_float(node) operator = - if %w[+ -].include?(buffer.source[node.location.start_char]) - source_range_length(node.location.start_char, 1) + if %w[+ -].include?(buffer.source[node.start_char]) + srange_length(node.start_char, 1) end - s( - :float, - [node.value.to_f], - source_map_operator(operator, source_range_node(node)) - ) + s(:float, [node.value.to_f], smap_operator(operator, srange_node(node))) end # Visit a FndPtn node. @@ -1106,9 +1011,9 @@ def visit_fndptn(node) left, right = [node.left, node.right].map do |child| location = - source_map_operator( - source_range_length(child.location.start_char, 1), - source_range_node(child) + smap_operator( + srange_length(child.start_char, 1), + srange_node(child) ) if child.is_a?(VarField) && child.value.nil? @@ -1122,10 +1027,10 @@ def visit_fndptn(node) s( :find_pattern, [left, *visit_all(node.values), right], - source_map_collection( - begin_token: source_range_length(node.location.start_char, 1), - end_token: source_range_length(node.location.end_char, -1), - expression: source_range_node(node) + smap_collection( + srange_length(node.start_char, 1), + srange_length(node.end_char, -1), + srange_node(node) ) ) @@ -1141,12 +1046,12 @@ def visit_for(node) s( :for, [visit(node.index), visit(node.collection), visit(node.statements)], - source_map_for( - source_range_length(node.location.start_char, 3), - source_range_find_between(node.index, node.collection, "in"), - source_range_search_between(node.collection, node.statements, "do"), - source_range_length(node.location.end_char, -3), - source_range_node(node) + smap_for( + srange_length(node.start_char, 3), + srange_find_between(node.index, node.collection, "in"), + srange_search_between(node.collection, node.statements, "do"), + srange_length(node.end_char, -3), + srange_node(node) ) ) end @@ -1156,7 +1061,7 @@ def visit_gvar(node) s( :gvar, [node.value.to_sym], - source_map_variable(source_range_node(node), source_range_node(node)) + smap_variable(srange_node(node), srange_node(node)) ) end @@ -1165,10 +1070,10 @@ def visit_hash(node) s( :hash, visit_all(node.assocs), - source_map_collection( - begin_token: source_range_length(node.location.start_char, 1), - end_token: source_range_length(node.location.end_char, -1), - expression: source_range_node(node) + smap_collection( + srange_length(node.start_char, 1), + srange_length(node.end_char, -1), + srange_node(node) ) ) end @@ -1260,20 +1165,17 @@ def visit_heredoc(node) heredoc_segments.trim! location = - source_map_heredoc( - source_range_node(node.beginning), - source_range( + smap_heredoc( + srange_node(node.beginning), + srange( if node.parts.empty? - node.beginning.location.end_char + node.beginning.end_char else - node.parts.first.location.start_char + node.parts.first.start_char end, - node.ending.location.start_char + node.ending.start_char ), - source_range( - node.ending.location.start_char, - node.ending.location.end_char - 1 - ) + srange(node.ending.start_char, node.ending.end_char - 1) ) if node.beginning.value.match?(/`\w+`\z/) @@ -1326,7 +1228,7 @@ def visit_ident(node) s( :lvar, [node.value.to_sym], - source_map_variable(source_range_node(node), source_range_node(node)) + smap_variable(srange_node(node), srange_node(node)) ) end @@ -1359,40 +1261,40 @@ def visit_if(node) :if, [predicate, visit(node.statements), visit(node.consequent)], if node.modifier? - source_map_keyword_bare( - source_range_find_between(node.statements, node.predicate, "if"), - source_range_node(node) + smap_keyword_bare( + srange_find_between(node.statements, node.predicate, "if"), + srange_node(node) ) else - begin_start = node.predicate.location.end_char + begin_start = node.predicate.end_char begin_end = if node.statements.empty? - node.statements.location.end_char + node.statements.end_char else - node.statements.body.first.location.start_char + node.statements.body.first.start_char end begin_token = if buffer.source[begin_start...begin_end].include?("then") - source_range_find(begin_start, begin_end, "then") + srange_find(begin_start, begin_end, "then") elsif buffer.source[begin_start...begin_end].include?(";") - source_range_find(begin_start, begin_end, ";") + srange_find(begin_start, begin_end, ";") end else_token = case node.consequent when Elsif - source_range_length(node.consequent.location.start_char, 5) + srange_length(node.consequent.start_char, 5) when Else - source_range_length(node.consequent.location.start_char, 4) + srange_length(node.consequent.start_char, 4) end - source_map_condition( - keyword: source_range_length(node.location.start_char, 2), - begin_token: begin_token, - else_token: else_token, - end_token: source_range_length(node.location.end_char, -3), - expression: source_range_node(node) + smap_condition( + srange_length(node.start_char, 2), + begin_token, + else_token, + srange_length(node.end_char, -3), + srange_node(node) ) end ) @@ -1403,7 +1305,11 @@ def visit_if_op(node) s( :if, [visit(node.predicate), visit(node.truthy), visit(node.falsy)], - nil + smap_ternary( + srange_find_between(node.predicate, node.truthy, "?"), + srange_find_between(node.truthy, node.falsy, ":"), + srange_node(node) + ) ) end @@ -1417,7 +1323,7 @@ def visit_imaginary(node) # case. Maybe there's an API for this but I can't find it. eval(node.value) ], - source_map_operator(nil, source_range_node(node)) + smap_operator(nil, srange_node(node)) ) end @@ -1446,23 +1352,23 @@ def visit_in(node) ) else begin_token = - source_range_search_between(node.pattern, node.statements, "then") + srange_search_between(node.pattern, node.statements, "then") end_char = if begin_token || node.statements.empty? - node.statements.location.end_char - 1 + node.statements.end_char - 1 else - node.statements.body.last.location.start_char + node.statements.body.last.start_char end s( :in_pattern, [visit(node.pattern), nil, visit(node.statements)], - source_map_keyword( - source_range_length(node.location.start_char, 2), + smap_keyword( + srange_length(node.start_char, 2), begin_token, nil, - source_range(node.location.start_char, end_char) + srange(node.start_char, end_char) ) ) end @@ -1471,15 +1377,11 @@ def visit_in(node) # Visit an Int node. def visit_int(node) operator = - if %w[+ -].include?(buffer.source[node.location.start_char]) - source_range_length(node.location.start_char, 1) + if %w[+ -].include?(buffer.source[node.start_char]) + srange_length(node.start_char, 1) end - s( - :int, - [node.value.to_i], - source_map_operator(operator, source_range_node(node)) - ) + s(:int, [node.value.to_i], smap_operator(operator, srange_node(node))) end # Visit an IVar node. @@ -1487,13 +1389,13 @@ def visit_ivar(node) s( :ivar, [node.value.to_sym], - source_map_variable(source_range_node(node), source_range_node(node)) + smap_variable(srange_node(node), srange_node(node)) ) end # Visit a Kw node. def visit_kw(node) - location = source_map(expression: source_range_node(node)) + location = smap(srange_node(node)) case node.value when "__FILE__" @@ -1514,15 +1416,12 @@ def visit_kw(node) # Visit a KwRestParam node. def visit_kwrest_param(node) if node.name.nil? - s(:kwrestarg, [], source_map_variable(nil, source_range_node(node))) + s(:kwrestarg, [], smap_variable(nil, srange_node(node))) else s( :kwrestarg, [node.name.value.to_sym], - source_map_variable( - source_range_node(node.name), - source_range_node(node) - ) + smap_variable(srange_node(node.name), srange_node(node)) ) end end @@ -1532,10 +1431,7 @@ def visit_label(node) s( :sym, [node.value.chomp(":").to_sym], - source_map_collection( - expression: - source_range(node.location.start_char, node.location.end_char - 1) - ) + smap_collection_bare(srange(node.start_char, node.end_char - 1)) ) end @@ -1550,42 +1446,30 @@ def visit_lambda(node) args_node = maximum end - begin_start = node.params.location.end_char begin_token, end_token = - if buffer.source[begin_start - 1] == "{" - [ - source_range_length(begin_start, -1), - source_range_length(node.location.end_char, -1) - ] + if (srange = srange_search_between(node.params, node.statements, "{")) + [srange, srange_length(node.end_char, -1)] else [ - source_range_length(begin_start, -2), - source_range_length(node.location.end_char, -3) + srange_find_between(node.params, node.statements, "do"), + srange_length(node.end_char, -3) ] end - selector = source_range_length(node.location.start_char, 2) + selector = srange_length(node.start_char, 2) s( type, [ if ::Parser::Builders::Default.emit_lambda - s(:lambda, [], source_map(expression: selector)) + s(:lambda, [], smap(selector)) else - s( - :send, - [nil, :lambda], - source_map_send(selector: selector, expression: selector) - ) + s(:send, [nil, :lambda], smap_send_bare(selector, selector)) end, args_node, visit(node.statements) ], - source_map_collection( - begin_token: begin_token, - end_token: end_token, - expression: source_range_node(node) - ) + smap_collection(begin_token, end_token, srange_node(node)) ) end @@ -1596,21 +1480,18 @@ def visit_lambda_var(node) s( :shadowarg, [local.value.to_sym], - source_map_variable( - source_range_node(local), - source_range_node(local) - ) + smap_variable(srange_node(local), srange_node(local)) ) end location = - if node.location.start_char == node.location.end_char - source_map_collection(expression: nil) + if node.start_char == node.end_char + smap_collection_bare(nil) else - source_map_collection( - begin_token: source_range_length(node.location.start_char, 1), - end_token: source_range_length(node.location.end_char, -1), - expression: source_range_node(node) + smap_collection( + srange_length(node.start_char, 1), + srange_length(node.end_char, -1), + srange_node(node) ) end @@ -1622,9 +1503,9 @@ def visit_massign(node) s( :masgn, [visit(node.target), visit(node.value)], - source_map_operator( - source_range_find_between(node.target, node.value, "="), - source_range_node(node) + smap_operator( + srange_find_between(node.target, node.value, "="), + srange_node(node) ) ) end @@ -1678,16 +1559,13 @@ def visit_mlhs(node) s( :arg, [part.value.to_sym], - source_map_variable( - source_range_node(part), - source_range_node(part) - ) + smap_variable(srange_node(part), srange_node(part)) ) else visit(part) end end, - source_map_collection(expression: source_range_node(node)) + smap_collection_bare(srange_node(node)) ) end @@ -1698,10 +1576,10 @@ def visit_mlhs_paren(node) s( child.type, child.children, - source_map_collection( - begin_token: source_range_length(node.location.start_char, 1), - end_token: source_range_length(node.location.end_char, -1), - expression: source_range_node(node) + smap_collection( + srange_length(node.start_char, 1), + srange_length(node.end_char, -1), + srange_node(node) ) ) end @@ -1711,11 +1589,12 @@ def visit_module(node) s( :module, [visit(node.constant), visit(node.bodystmt)], - source_map_definition( - keyword: source_range_length(node.location.start_char, 6), - name: source_range_node(node.constant), - end_token: source_range_length(node.location.end_char, -3) - ).with_expression(source_range_node(node)) + smap_definition( + srange_length(node.start_char, 6), + nil, + srange_node(node.constant), + srange_length(node.end_char, -3) + ).with_expression(srange_node(node)) ) end @@ -1735,9 +1614,9 @@ def visit_next(node) s( :next, visit_all(node.arguments.parts), - source_map_keyword_bare( - source_range_length(node.location.start_char, 4), - source_range_node(node) + smap_keyword_bare( + srange_length(node.start_char, 4), + srange_node(node) ) ) end @@ -1745,8 +1624,8 @@ def visit_next(node) # Visit a Not node. def visit_not(node) if node.statement.nil? - begin_token = source_range_find(node.location.start_char, nil, "(") - end_token = source_range_find(node.location.start_char, nil, ")") + begin_token = srange_find(node.start_char, nil, "(") + end_token = srange_find(node.start_char, nil, ")") s( :send, @@ -1754,40 +1633,38 @@ def visit_not(node) s( :begin, [], - source_map_collection( - begin_token: begin_token, - end_token: end_token, - expression: begin_token.join(end_token) + smap_collection( + begin_token, + end_token, + begin_token.join(end_token) ) ), :! ], - source_map_send( - selector: source_range_length(node.location.start_char, 3), - expression: source_range_node(node) - ) + smap_send_bare(srange_length(node.start_char, 3), srange_node(node)) ) else begin_token, end_token = if node.parentheses? [ - source_range_find( - node.location.start_char + 3, - node.statement.location.start_char, + srange_find( + node.start_char + 3, + node.statement.start_char, "(" ), - source_range_length(node.location.end_char, -1) + srange_length(node.end_char, -1) ] end s( :send, [visit(node.statement), :!], - source_map_send( - begin_token: begin_token, - end_token: end_token, - selector: source_range_length(node.location.start_char, 3), - expression: source_range_node(node) + smap_send( + nil, + srange_length(node.start_char, 3), + begin_token, + end_token, + srange_node(node) ) ) end @@ -1795,60 +1672,22 @@ def visit_not(node) # Visit an OpAssign node. def visit_opassign(node) + target = visit(node.target) location = - case node.target - when ARefField - source_map_index( - begin_token: - source_range_find( - node.target.collection.location.end_char, - if node.target.index - node.target.index.location.start_char - else - node.target.location.end_char - end, - "[" - ), - end_token: source_range_length(node.target.location.end_char, -1), - expression: source_range_node(node) - ) - when Field - source_map_send( - dot: - if node.target.operator == :"::" - source_range_find_between( - node.target.parent, - node.target.name, - "::" - ) - else - source_range_node(node.target.operator) - end, - selector: source_range_node(node.target.name), - expression: source_range_node(node) - ) - else - source_map_variable( - source_range_node(node.target), - source_range_node(node) - ) - end - - location = location.with_operator(source_range_node(node.operator)) + target + .location + .with_expression(srange_node(node)) + .with_operator(srange_node(node.operator)) case node.operator.value when "||=" - s(:or_asgn, [visit(node.target), visit(node.value)], location) + s(:or_asgn, [target, visit(node.value)], location) when "&&=" - s(:and_asgn, [visit(node.target), visit(node.value)], location) + s(:and_asgn, [target, visit(node.value)], location) else s( :op_asgn, - [ - visit(node.target), - node.operator.value.chomp("=").to_sym, - visit(node.value) - ], + [target, node.operator.value.chomp("=").to_sym, visit(node.value)], location ) end @@ -1867,10 +1706,7 @@ def visit_params(node) s( :arg, [required.value.to_sym], - source_map_variable( - source_range_node(required), - source_range_node(required) - ) + smap_variable(srange_node(required), srange_node(required)) ) end end @@ -1880,10 +1716,10 @@ def visit_params(node) s( :optarg, [name.value.to_sym, visit(value)], - source_map_variable( - source_range_node(name), - source_range_node(name).join(source_range_node(value)) - ).with_operator(source_range_find_between(name, value, "=")) + smap_variable( + srange_node(name), + srange_node(name).join(srange_node(value)) + ).with_operator(srange_find_between(name, value, "=")) ) end @@ -1896,10 +1732,7 @@ def visit_params(node) s( :arg, [post.value.to_sym], - source_map_variable( - source_range_node(post), - source_range_node(post) - ) + smap_variable(srange_node(post), srange_node(post)) ) end @@ -1911,24 +1744,18 @@ def visit_params(node) s( :kwoptarg, [key, visit(value)], - source_map_variable( - source_range( - name.location.start_char, - name.location.end_char - 1 - ), - source_range_node(name).join(source_range_node(value)) + smap_variable( + srange(name.start_char, name.end_char - 1), + srange_node(name).join(srange_node(value)) ) ) else s( :kwarg, [key], - source_map_variable( - source_range( - name.location.start_char, - name.location.end_char - 1 - ), - source_range_node(name) + smap_variable( + srange(name.start_char, name.end_char - 1), + srange_node(name) ) ) end @@ -1941,10 +1768,7 @@ def visit_params(node) children << s( :kwnilarg, [], - source_map_variable( - source_range_length(node.location.end_char, -3), - source_range_node(node) - ) + smap_variable(srange_length(node.end_char, -3), srange_node(node)) ) else children << visit(node.keyword_rest) @@ -1953,8 +1777,7 @@ def visit_params(node) children << visit(node.block) if node.block if node.keyword_rest.is_a?(ArgsForward) - location = - source_map(expression: source_range_node(node.keyword_rest)) + location = smap(srange_node(node.keyword_rest)) # If there are no other arguments and we have the emit_forward_arg # option enabled, then the entire argument list is represented by a @@ -1970,16 +1793,23 @@ def visit_params(node) children.insert(index, s(:forward_arg, [], location)) end - s(:args, children, nil) + location = + unless children.empty? + first = children.first.location.expression + last = children.last.location.expression + smap_collection_bare(first.join(last)) + end + + s(:args, children, location) end # Visit a Paren node. def visit_paren(node) location = - source_map_collection( - begin_token: source_range_length(node.location.start_char, 1), - end_token: source_range_length(node.location.end_char, -1), - expression: source_range_node(node) + smap_collection( + srange_length(node.start_char, 1), + srange_length(node.end_char, -1), + srange_node(node) ) if node.contents.nil? || @@ -1999,22 +1829,14 @@ def visit_pinned_begin(node) s( :begin, [visit(node.statement)], - source_map_collection( - begin_token: - source_range_length(node.location.start_char + 1, 1), - end_token: source_range_length(node.location.end_char, -1), - expression: - source_range( - node.location.start_char + 1, - node.location.end_char - ) + smap_collection( + srange_length(node.start_char + 1, 1), + srange_length(node.end_char, -1), + srange(node.start_char + 1, node.end_char) ) ) ], - source_map_send( - selector: source_range_length(node.location.start_char, 1), - expression: source_range_node(node) - ) + smap_send_bare(srange_length(node.start_char, 1), srange_node(node)) ) end @@ -2023,10 +1845,7 @@ def visit_pinned_var_ref(node) s( :pin, [visit(node.value)], - source_map_send( - selector: source_range_length(node.location.start_char, 1), - expression: source_range_node(node) - ) + smap_send_bare(srange_length(node.start_char, 1), srange_node(node)) ) end @@ -2067,10 +1886,7 @@ def visit_range(node) s( node.operator.value == ".." ? :irange : :erange, [visit(node.left), visit(node.right)], - source_map_operator( - source_range_node(node.operator), - source_range_node(node) - ) + smap_operator(srange_node(node.operator), srange_node(node)) ) end @@ -2079,32 +1895,18 @@ def visit_rassign(node) s( node.operator.value == "=>" ? :match_pattern : :match_pattern_p, [visit(node.value), visit(node.pattern)], - source_map_operator( - source_range_node(node.operator), - source_range_node(node) - ) + smap_operator(srange_node(node.operator), srange_node(node)) ) end # Visit a Rational node. def visit_rational(node) - s( - :rational, - [node.value.to_r], - source_map_operator(nil, source_range_node(node)) - ) + s(:rational, [node.value.to_r], smap_operator(nil, srange_node(node))) end # Visit a Redo node. def visit_redo(node) - s( - :redo, - [], - source_map_keyword_bare( - source_range_node(node), - source_range_node(node) - ) - ) + s(:redo, [], smap_keyword_bare(srange_node(node), srange_node(node))) end # Visit a RegexpLiteral node. @@ -2115,27 +1917,13 @@ def visit_regexp_literal(node) s( :regopt, node.ending.scan(/[a-z]/).sort.map(&:to_sym), - source_map( - expression: - source_range_length( - node.location.end_char, - -(node.ending.length - 1) - ) - ) + smap(srange_length(node.end_char, -(node.ending.length - 1))) ) ), - source_map_collection( - begin_token: - source_range_length( - node.location.start_char, - node.beginning.length - ), - end_token: - source_range_length( - node.location.end_char - node.ending.length, - 1 - ), - expression: source_range_node(node) + smap_collection( + srange_length(node.start_char, node.beginning.length), + srange_length(node.end_char - node.ending.length, 1), + srange_node(node) ) ) end @@ -2145,13 +1933,13 @@ def visit_rescue(node) # In the parser gem, there is a separation between the rescue node and # the rescue body. They have different bounds, so we have to calculate # those here. - start_char = node.location.start_char + start_char = node.start_char body_end_char = if node.statements.empty? start_char + 6 else - node.statements.body.last.location.end_char + node.statements.body.last.end_char end end_char = @@ -2162,16 +1950,16 @@ def visit_rescue(node) if end_node.statements.empty? start_char + 6 else - end_node.statements.body.last.location.end_char + end_node.statements.body.last.end_char end else body_end_char end # These locations are reused for multiple children. - keyword = source_range_length(start_char, 6) - body_expression = source_range(start_char, body_end_char) - expression = source_range(start_char, end_char) + keyword = srange_length(start_char, 6) + body_expression = srange(start_char, body_end_char) + expression = srange(start_char, end_char) exceptions = case node.exception&.exceptions @@ -2208,19 +1996,13 @@ def visit_rescue(node) s( :resbody, [nil, nil, visit(node.statements)], - source_map_rescue_body( - keyword: keyword, - expression: body_expression - ) + smap_rescue_body(keyword, nil, nil, body_expression) ) elsif node.exception.variable.nil? s( :resbody, [exceptions, nil, visit(node.statements)], - source_map_rescue_body( - keyword: keyword, - expression: body_expression - ) + smap_rescue_body(keyword, nil, nil, body_expression) ) else s( @@ -2230,15 +2012,15 @@ def visit_rescue(node) visit(node.exception.variable), visit(node.statements) ], - source_map_rescue_body( - keyword: keyword, - assoc: - source_range_find( - node.location.start_char + 6, - node.exception.variable.location.start_char, - "=>" - ), - expression: body_expression + smap_rescue_body( + keyword, + srange_find( + node.start_char + 6, + node.exception.variable.start_char, + "=>" + ), + nil, + body_expression ) ) end @@ -2250,13 +2032,12 @@ def visit_rescue(node) children << nil end - s(:rescue, children, source_map_condition(expression: expression)) + s(:rescue, children, smap_condition_bare(expression)) end # Visit a RescueMod node. def visit_rescue_mod(node) - keyword = - source_range_find_between(node.statement, node.value, "rescue") + keyword = srange_find_between(node.statement, node.value, "rescue") s( :rescue, @@ -2265,14 +2046,16 @@ def visit_rescue_mod(node) s( :resbody, [nil, nil, visit(node.value)], - source_map_rescue_body( - keyword: keyword, - expression: keyword.join(source_range_node(node.value)) + smap_rescue_body( + keyword, + nil, + nil, + keyword.join(srange_node(node.value)) ) ), nil ], - source_map_condition(expression: source_range_node(node)) + smap_condition_bare(srange_node(node)) ) end @@ -2282,26 +2065,16 @@ def visit_rest_param(node) s( :restarg, [node.name.value.to_sym], - source_map_variable( - source_range_node(node.name), - source_range_node(node) - ) + smap_variable(srange_node(node.name), srange_node(node)) ) else - s(:restarg, [], source_map_variable(nil, source_range_node(node))) + s(:restarg, [], smap_variable(nil, srange_node(node))) end end # Visit a Retry node. def visit_retry(node) - s( - :retry, - [], - source_map_keyword_bare( - source_range_node(node), - source_range_node(node) - ) - ) + s(:retry, [], smap_keyword_bare(srange_node(node), srange_node(node))) end # Visit a ReturnNode node. @@ -2309,9 +2082,9 @@ def visit_return(node) s( :return, node.arguments ? visit_all(node.arguments.parts) : [], - source_map_keyword_bare( - source_range_length(node.location.start_char, 6), - source_range_node(node) + smap_keyword_bare( + srange_length(node.start_char, 6), + srange_node(node) ) ) end @@ -2321,16 +2094,12 @@ def visit_sclass(node) s( :sclass, [visit(node.target), visit(node.bodystmt)], - source_map_definition( - keyword: source_range_length(node.location.start_char, 5), - operator: - source_range_find( - node.location.start_char + 5, - node.target.location.start_char, - "<<" - ), - end_token: source_range_length(node.location.end_char, -3) - ).with_expression(source_range_node(node)) + smap_definition( + srange_length(node.start_char, 5), + srange_find(node.start_char + 5, node.target.start_char, "<<"), + nil, + srange_length(node.end_char, -3) + ).with_expression(srange_node(node)) ) end @@ -2351,12 +2120,8 @@ def visit_statements(node) s( :begin, visit_all(children), - source_map_collection( - expression: - source_range( - children.first.location.start_char, - children.last.location.end_char - ) + smap_collection_bare( + srange(children.first.start_char, children.last.end_char) ) ) end @@ -2364,15 +2129,11 @@ def visit_statements(node) # Visit a StringConcat node. def visit_string_concat(node) - location = source_map_collection(expression: source_range_node(node)) - - s(:dstr, [visit(node.left), visit(node.right)], location) - end - - # Visit a StringContent node. - def visit_string_content(node) - # Can get here if you're inside a hash pattern, e.g., in "a": 1 - s(:sym, [node.parts.first.value.to_sym], nil) + s( + :dstr, + [visit(node.left), visit(node.right)], + smap_collection_bare(srange_node(node)) + ) end # Visit a StringDVar node. @@ -2385,10 +2146,10 @@ def visit_string_embexpr(node) s( :begin, visit(node.statements).then { |child| child ? [child] : [] }, - source_map_collection( - begin_token: source_range_length(node.location.start_char, 2), - end_token: source_range_length(node.location.end_char, -1), - expression: source_range_node(node) + smap_collection( + srange_length(node.start_char, 2), + srange_length(node.end_char, -1), + srange_node(node) ) ) end @@ -2397,17 +2158,13 @@ def visit_string_embexpr(node) def visit_string_literal(node) location = if node.quote - source_map_collection( - begin_token: - source_range_length( - node.location.start_char, - node.quote.length - ), - end_token: source_range_length(node.location.end_char, -1), - expression: source_range_node(node) + smap_collection( + srange_length(node.start_char, node.quote.length), + srange_length(node.end_char, -1), + srange_node(node) ) else - source_map_collection(expression: source_range_node(node)) + smap_collection_bare(srange_node(node)) end if node.parts.empty? @@ -2426,9 +2183,9 @@ def visit_super(node) s( :super, visit_all(node.arguments.parts), - source_map_keyword_bare( - source_range_length(node.location.start_char, 5), - source_range_node(node) + smap_keyword_bare( + srange_length(node.start_char, 5), + srange_node(node) ) ) else @@ -2437,15 +2194,11 @@ def visit_super(node) s( :super, [], - source_map_keyword( - source_range_length(node.location.start_char, 5), - source_range_find( - node.location.start_char + 5, - node.location.end_char, - "(" - ), - source_range_length(node.location.end_char, -1), - source_range_node(node) + smap_keyword( + srange_length(node.start_char, 5), + srange_find(node.start_char + 5, node.end_char, "("), + srange_length(node.end_char, -1), + srange_node(node) ) ) when ArgsForward @@ -2454,15 +2207,11 @@ def visit_super(node) s( :super, visit_all(node.arguments.arguments.parts), - source_map_keyword( - source_range_length(node.location.start_char, 5), - source_range_find( - node.location.start_char + 5, - node.location.end_char, - "(" - ), - source_range_length(node.location.end_char, -1), - source_range_node(node) + smap_keyword( + srange_length(node.start_char, 5), + srange_find(node.start_char + 5, node.end_char, "("), + srange_length(node.end_char, -1), + srange_node(node) ) ) end @@ -2472,17 +2221,14 @@ def visit_super(node) # Visit a SymbolLiteral node. def visit_symbol_literal(node) begin_token = - if buffer.source[node.location.start_char] == ":" - source_range_length(node.location.start_char, 1) + if buffer.source[node.start_char] == ":" + srange_length(node.start_char, 1) end s( :sym, [node.value.value.to_sym], - source_map_collection( - begin_token: begin_token, - expression: source_range_node(node) - ) + smap_collection(begin_token, nil, srange_node(node)) ) end @@ -2517,19 +2263,13 @@ def visit_top_const_field(node) s( :casgn, [ - s( - :cbase, - [], - source_map( - expression: source_range_length(node.location.start_char, 2) - ) - ), + s(:cbase, [], smap(srange_length(node.start_char, 2))), node.constant.value.to_sym ], - source_map_constant( - source_range_length(node.location.start_char, 2), - source_range_node(node.constant), - source_range_node(node) + smap_constant( + srange_length(node.start_char, 2), + srange_node(node.constant), + srange_node(node) ) ) end @@ -2539,19 +2279,13 @@ def visit_top_const_ref(node) s( :const, [ - s( - :cbase, - [], - source_map( - expression: source_range_length(node.location.start_char, 2) - ) - ), + s(:cbase, [], smap(srange_length(node.start_char, 2))), node.constant.value.to_sym ], - source_map_constant( - source_range_length(node.location.start_char, 2), - source_range_node(node.constant), - source_range_node(node) + smap_constant( + srange_length(node.start_char, 2), + srange_node(node.constant), + srange_node(node) ) ) end @@ -2563,7 +2297,7 @@ def visit_tstring_content(node) s( :str, ["\"#{dumped}\"".undump], - source_map_collection(expression: source_range_node(node)) + smap_collection_bare(srange_node(node)) ) end @@ -2593,9 +2327,9 @@ def visit_undef(node) s( :undef, visit_all(node.symbols), - source_map_keyword_bare( - source_range_length(node.location.start_char, 5), - source_range_node(node) + smap_keyword_bare( + srange_length(node.start_char, 5), + srange_node(node) ) ) end @@ -2625,19 +2359,17 @@ def visit_unless(node) :if, [predicate, visit(node.consequent), visit(node.statements)], if node.modifier? - source_map_keyword_bare( - source_range_find_between( - node.statements, - node.predicate, - "unless" - ), - source_range_node(node) + smap_keyword_bare( + srange_find_between(node.statements, node.predicate, "unless"), + srange_node(node) ) else - source_map_condition( - keyword: source_range_length(node.location.start_char, 6), - end_token: source_range_length(node.location.end_char, -3), - expression: source_range_node(node) + smap_condition( + srange_length(node.start_char, 6), + srange_search_between(node.predicate, node.statements, "then"), + nil, + srange_length(node.end_char, -3), + srange_node(node) ) end ) @@ -2649,20 +2381,17 @@ def visit_until(node) loop_post?(node) ? :until_post : :until, [visit(node.predicate), visit(node.statements)], if node.modifier? - source_map_keyword_bare( - source_range_find_between( - node.statements, - node.predicate, - "until" - ), - source_range_node(node) + smap_keyword_bare( + srange_find_between(node.statements, node.predicate, "until"), + srange_node(node) ) else - source_map_keyword( - source_range_length(node.location.start_char, 5), - nil, - source_range_length(node.location.end_char, -3), - source_range_node(node) + smap_keyword( + srange_length(node.start_char, 5), + srange_search_between(node.predicate, node.statements, "do") || + srange_search_between(node.predicate, node.statements, ";"), + srange_length(node.end_char, -3), + srange_node(node) ) end ) @@ -2687,27 +2416,16 @@ def visit_var_field(node) s( :match_var, [name], - source_map_variable( - source_range_node(node.value), - source_range_node(node.value) - ) + smap_variable(srange_node(node.value), srange_node(node.value)) ) elsif node.value.is_a?(Const) s( :casgn, [nil, name], - source_map_constant( - nil, - source_range_node(node.value), - source_range_node(node) - ) + smap_constant(nil, srange_node(node.value), srange_node(node)) ) else - location = - source_map_variable( - source_range_node(node), - source_range_node(node) - ) + location = smap_variable(srange_node(node), srange_node(node)) case node.value when CVar @@ -2747,27 +2465,27 @@ def visit_vcall(node) # Visit a When node. def visit_when(node) - keyword = source_range_length(node.location.start_char, 4) + keyword = srange_length(node.start_char, 4) begin_token = - if buffer.source[node.statements.location.start_char] == ";" - source_range_length(node.statements.location.start_char, 1) + if buffer.source[node.statements.start_char] == ";" + srange_length(node.statements.start_char, 1) end end_char = if node.statements.body.empty? - node.statements.location.end_char + node.statements.end_char else - node.statements.body.last.location.end_char + node.statements.body.last.end_char end s( :when, visit_all(node.arguments.parts) + [visit(node.statements)], - source_map_keyword( + smap_keyword( keyword, begin_token, nil, - source_range(keyword.begin_pos, end_char) + srange(keyword.begin_pos, end_char) ) ) end @@ -2778,20 +2496,17 @@ def visit_while(node) loop_post?(node) ? :while_post : :while, [visit(node.predicate), visit(node.statements)], if node.modifier? - source_map_keyword_bare( - source_range_find_between( - node.statements, - node.predicate, - "while" - ), - source_range_node(node) + smap_keyword_bare( + srange_find_between(node.statements, node.predicate, "while"), + srange_node(node) ) else - source_map_keyword( - source_range_length(node.location.start_char, 5), - nil, - source_range_length(node.location.end_char, -3), - source_range_node(node) + smap_keyword( + srange_length(node.start_char, 5), + srange_search_between(node.predicate, node.statements, "do") || + srange_search_between(node.predicate, node.statements, ";"), + srange_length(node.end_char, -3), + srange_node(node) ) end ) @@ -2824,10 +2539,13 @@ def visit_xstring_literal(node) s( :xstr, visit_all(node.parts), - source_map_collection( - begin_token: source_range_length(node.location.start_char, 1), - end_token: source_range_length(node.location.end_char, -1), - expression: source_range_node(node) + smap_collection( + srange_length( + node.start_char, + buffer.source[node.start_char] == "%" ? 3 : 1 + ), + srange_length(node.end_char, -1), + srange_node(node) ) ) end @@ -2838,29 +2556,29 @@ def visit_yield(node) s( :yield, [], - source_map_keyword_bare( - source_range_length(node.location.start_char, 5), - source_range_node(node) + smap_keyword_bare( + srange_length(node.start_char, 5), + srange_node(node) ) ) when Args s( :yield, visit_all(node.arguments.parts), - source_map_keyword_bare( - source_range_length(node.location.start_char, 5), - source_range_node(node) + smap_keyword_bare( + srange_length(node.start_char, 5), + srange_node(node) ) ) else s( :yield, visit_all(node.arguments.contents.parts), - source_map_keyword( - source_range_length(node.location.start_char, 5), - source_range_length(node.arguments.location.start_char, 1), - source_range_length(node.location.end_char, -1), - source_range_node(node) + smap_keyword( + srange_length(node.start_char, 5), + srange_length(node.arguments.start_char, 1), + srange_length(node.end_char, -1), + srange_node(node) ) ) end @@ -2871,9 +2589,9 @@ def visit_zsuper(node) s( :zsuper, [], - source_map_keyword_bare( - source_range_length(node.location.start_char, 5), - source_range_node(node) + smap_keyword_bare( + srange_length(node.start_char, 5), + srange_node(node) ) ) end @@ -2885,7 +2603,7 @@ def block_children(node) if node.block_var visit(node.block_var) else - s(:args, [], source_map_collection(expression: nil)) + s(:args, [], smap_collection_bare(nil)) end type = :block @@ -2923,10 +2641,10 @@ def canonical_unary(node) location: Location.new( start_line: node.location.start_line, - start_char: node.location.start_char, + start_char: node.start_char, start_column: node.location.start_column, end_line: node.location.start_line, - end_char: node.location.start_char + length, + end_char: node.start_char + length, end_column: node.location.start_column + length ) ), @@ -2940,8 +2658,8 @@ def canonical_unary(node) def canonical_binary(node) operator = node.operator.to_s - start_char = node.left.location.end_char - end_char = node.right.location.start_char + start_char = node.left.end_char + end_char = node.right.start_char index = buffer.source[start_char...end_char].index(operator) start_line = @@ -3007,12 +2725,12 @@ def s(type, children, location) end # Constructs a plain source map just for an expression. - def source_map(expression:) + def smap(expression) ::Parser::Source::Map.new(expression) end # Constructs a new source map for a collection. - def source_map_collection(begin_token: nil, end_token: nil, expression:) + def smap_collection(begin_token, end_token, expression) ::Parser::Source::Map::Collection.new( begin_token, end_token, @@ -3020,13 +2738,18 @@ def source_map_collection(begin_token: nil, end_token: nil, expression:) ) end + # Constructs a new source map for a collection without a begin or end. + def smap_collection_bare(expression) + smap_collection(nil, nil, expression) + end + # Constructs a new source map for a conditional expression. - def source_map_condition( - keyword: nil, - begin_token: nil, - else_token: nil, - end_token: nil, - expression: + def smap_condition( + keyword, + begin_token, + else_token, + end_token, + expression ) ::Parser::Source::Map::Condition.new( keyword, @@ -3037,18 +2760,19 @@ def source_map_condition( ) end + # Constructs a new source map for a conditional expression with no begin + # or end. + def smap_condition_bare(expression) + smap_condition(nil, nil, nil, nil, expression) + end + # Constructs a new source map for a constant reference. - def source_map_constant(double_colon, name, expression) + def smap_constant(double_colon, name, expression) ::Parser::Source::Map::Constant.new(double_colon, name, expression) end # Constructs a new source map for a class definition. - def source_map_definition( - keyword: nil, - operator: nil, - name: nil, - end_token: nil - ) + def smap_definition(keyword, operator, name, end_token) ::Parser::Source::Map::Definition.new( keyword, operator, @@ -3058,7 +2782,7 @@ def source_map_definition( end # Constructs a new source map for a for loop. - def source_map_for(keyword, in_token, begin_token, end_token, expression) + def smap_for(keyword, in_token, begin_token, end_token, expression) ::Parser::Source::Map::For.new( keyword, in_token, @@ -3069,7 +2793,7 @@ def source_map_for(keyword, in_token, begin_token, end_token, expression) end # Constructs a new source map for a heredoc. - def source_map_heredoc(expression, heredoc_body, heredoc_end) + def smap_heredoc(expression, heredoc_body, heredoc_end) ::Parser::Source::Map::Heredoc.new( expression, heredoc_body, @@ -3078,12 +2802,12 @@ def source_map_heredoc(expression, heredoc_body, heredoc_end) end # Construct a source map for an index operation. - def source_map_index(begin_token: nil, end_token: nil, expression:) + def smap_index(begin_token, end_token, expression) ::Parser::Source::Map::Index.new(begin_token, end_token, expression) end # Constructs a new source map for the use of a keyword. - def source_map_keyword(keyword, begin_token, end_token, expression) + def smap_keyword(keyword, begin_token, end_token, expression) ::Parser::Source::Map::Keyword.new( keyword, begin_token, @@ -3094,18 +2818,18 @@ def source_map_keyword(keyword, begin_token, end_token, expression) # Constructs a new source map for the use of a keyword without a begin or # end token. - def source_map_keyword_bare(keyword, expression) - source_map_keyword(keyword, nil, nil, expression) + def smap_keyword_bare(keyword, expression) + smap_keyword(keyword, nil, nil, expression) end # Constructs a new source map for a method definition. - def source_map_method_definition( - keyword: nil, - operator: nil, - name: nil, - end_token: nil, - assignment: nil, - expression: + def smap_method_definition( + keyword, + operator, + name, + end_token, + assignment, + expression ) ::Parser::Source::Map::MethodDefinition.new( keyword, @@ -3118,17 +2842,12 @@ def source_map_method_definition( end # Constructs a new source map for an operator. - def source_map_operator(operator, expression) + def smap_operator(operator, expression) ::Parser::Source::Map::Operator.new(operator, expression) end # Constructs a source map for the body of a rescue clause. - def source_map_rescue_body( - keyword: nil, - assoc: nil, - begin_token: nil, - expression: - ) + def smap_rescue_body(keyword, assoc, begin_token, expression) ::Parser::Source::Map::RescueBody.new( keyword, assoc, @@ -3138,13 +2857,7 @@ def source_map_rescue_body( end # Constructs a new source map for a method call. - def source_map_send( - dot: nil, - selector: nil, - begin_token: nil, - end_token: nil, - expression: - ) + def smap_send(dot, selector, begin_token, end_token, expression) ::Parser::Source::Map::Send.new( dot, selector, @@ -3154,74 +2867,76 @@ def source_map_send( ) end + # Constructs a new source map for a method call without a begin or end. + def smap_send_bare(selector, expression) + smap_send(nil, selector, nil, nil, expression) + end + + # Constructs a new source map for a ternary expression. + def smap_ternary(question, colon, expression) + ::Parser::Source::Map::Ternary.new(question, colon, expression) + end + # Constructs a new source map for a variable. - def source_map_variable(name, expression) + def smap_variable(name, expression) ::Parser::Source::Map::Variable.new(name, expression) end # Constructs a new source range from the given start and end offsets. - def source_range(start_char, end_char) + def srange(start_char, end_char) ::Parser::Source::Range.new(buffer, start_char, end_char) end # Constructs a new source range by finding the given needle in the given # range of the source. If the needle is not found, returns nil. - def source_range_search(start_char, end_char, needle) + def srange_search(start_char, end_char, needle) index = buffer.source[start_char...end_char].index(needle) return unless index offset = start_char + index - source_range(offset, offset + needle.length) + srange(offset, offset + needle.length) end # Constructs a new source range by searching for the given needle between # the end location of the start node and the start location of the end # node. If the needle is not found, returns nil. - def source_range_search_between(start_node, end_node, needle) - source_range_search( - start_node.location.end_char, - end_node.location.start_char, - needle - ) + def srange_search_between(start_node, end_node, needle) + srange_search(start_node.end_char, end_node.start_char, needle) end # Constructs a new source range by finding the given needle in the given # range of the source. If it needle is not found, raises an error. - def source_range_find(start_char, end_char, needle) - source_range = source_range_search(start_char, end_char, needle) + def srange_find(start_char, end_char, needle) + srange = srange_search(start_char, end_char, needle) - unless source_range + unless srange slice = buffer.source[start_char...end_char].inspect raise "Could not find #{needle.inspect} in #{slice}" end - source_range + srange end # Constructs a new source range by finding the given needle between the # end location of the start node and the start location of the end node. # If the needle is not found, returns raises an error. - def source_range_find_between(start_node, end_node, needle) - source_range_find( - start_node.location.end_char, - end_node.location.start_char, - needle - ) + def srange_find_between(start_node, end_node, needle) + srange_find(start_node.end_char, end_node.start_char, needle) end # Constructs a new source range from the given start offset and length. - def source_range_length(start_char, length) + def srange_length(start_char, length) if length > 0 - source_range(start_char, start_char + length) + srange(start_char, start_char + length) else - source_range(start_char + length, start_char) + srange(start_char + length, start_char) end end # Constructs a new source range using the given node's location. - def source_range_node(node) + def srange_node(node) location = node.location - source_range(location.start_char, location.end_char) + srange(location.start_char, location.end_char) end end end diff --git a/test/syntax_tree_test.rb b/test/syntax_tree_test.rb index 05242d94..f12065b8 100644 --- a/test/syntax_tree_test.rb +++ b/test/syntax_tree_test.rb @@ -22,7 +22,7 @@ def method # comment SOURCE bodystmt = SyntaxTree.parse(source).statements.body.first.bodystmt - assert_equal(20, bodystmt.location.start_char) + assert_equal(20, bodystmt.start_char) end def test_parse_error diff --git a/test/translation/parser_test.rb b/test/translation/parser_test.rb index 576d4ac1..ad87d8c6 100644 --- a/test/translation/parser_test.rb +++ b/test/translation/parser_test.rb @@ -113,7 +113,7 @@ class ParserTest < Minitest::Test name = prefix[4..] next if all_failures.any? { |pattern| File.fnmatch?(pattern, name) } - define_method(name) { assert_parses(lines.join("\n")) } + define_method(name) { assert_parses("#{lines.join("\n")}\n") } end private From 52f44038ca66a4542d97aff05b85e1e6e84b002a Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 9 Feb 2023 16:25:29 -0500 Subject: [PATCH 381/536] Add a rubocop ast translator --- lib/syntax_tree/parser.rb | 34 ++-- lib/syntax_tree/translation.rb | 11 ++ lib/syntax_tree/translation/parser.rb | 213 ++++++++++++--------- lib/syntax_tree/translation/rubocop_ast.rb | 21 ++ 4 files changed, 169 insertions(+), 110 deletions(-) create mode 100644 lib/syntax_tree/translation/rubocop_ast.rb diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index cf3982f9..be6265d1 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -275,7 +275,7 @@ def find_keyword(name) end def find_keyword_between(name, left, right) - bounds = left.location.end_char...right.location.start_char + bounds = left.end_char...right.start_char index = tokens.rindex do |token| char = token.location.start_char @@ -1807,19 +1807,19 @@ def on_for(index, collection, statements) in_keyword = consume_keyword(:in) ending = consume_keyword(:end) - # Consume the do keyword if it exists so that it doesn't get confused for - # some other block - if (keyword = find_keyword_between(:do, collection, ending)) - tokens.delete(keyword) - end + delimiter = + find_keyword_between(:do, collection, ending) || + find_token_between(Semicolon, collection, ending) + + tokens.delete(delimiter) if delimiter start_char = - find_next_statement_start((keyword || collection).location.end_char) + find_next_statement_start((delimiter || collection).location.end_char) statements.bind( start_char, start_char - - line_counts[(keyword || collection).location.end_line - 1].start, + line_counts[(delimiter || collection).location.end_line - 1].start, ending.location.start_char, ending.location.start_column ) @@ -3328,10 +3328,13 @@ def on_sclass(target, bodystmt) ) end + # Semicolons are tokens that get added to the token list but never get + # attached to the AST. Because of this they only need to track their + # associated location so they can be used for computing bounds. class Semicolon attr_reader :location - def initialize(location:) + def initialize(location) @location = location end end @@ -3340,13 +3343,12 @@ def initialize(location:) # on_semicolon: (String value) -> Semicolon def on_semicolon(value) tokens << Semicolon.new( - location: - Location.token( - line: lineno, - char: char_pos, - column: current_column, - size: value.size - ) + Location.token( + line: lineno, + char: char_pos, + column: current_column, + size: value.size + ) ) end diff --git a/lib/syntax_tree/translation.rb b/lib/syntax_tree/translation.rb index d3f2e56f..6fc96f00 100644 --- a/lib/syntax_tree/translation.rb +++ b/lib/syntax_tree/translation.rb @@ -13,5 +13,16 @@ def self.to_parser(node, buffer) node.accept(Parser.new(buffer)) end + + # This method translates the given node into the representation defined by + # the rubocop/rubocop-ast gem. We don't explicitly list it as a dependency + # because it's not required for the core functionality of Syntax Tree. + def self.to_rubocop_ast(node, buffer) + require "rubocop/ast" + require_relative "translation/parser" + require_relative "translation/rubocop_ast" + + node.accept(RuboCopAST.new(buffer)) + end end end diff --git a/lib/syntax_tree/translation/parser.rb b/lib/syntax_tree/translation/parser.rb index b9e91e5f..70c98336 100644 --- a/lib/syntax_tree/translation/parser.rb +++ b/lib/syntax_tree/translation/parser.rb @@ -5,6 +5,73 @@ module Translation # This visitor is responsible for converting the syntax tree produced by # Syntax Tree into the syntax tree produced by the whitequark/parser gem. class Parser < BasicVisitor + # Heredocs are represented _very_ differently in the parser gem from how + # they are represented in the Syntax Tree AST. This class is responsible + # for handling the translation. + class HeredocBuilder + Line = Struct.new(:value, :segments) + + attr_reader :node, :segments + + def initialize(node) + @node = node + @segments = [] + end + + def <<(segment) + if segment.type == :str && segments.last && + segments.last.type == :str && + !segments.last.children.first.end_with?("\n") + segments.last.children.first << segment.children.first + else + segments << segment + end + end + + def trim! + return unless node.beginning.value[2] == "~" + lines = [Line.new(+"", [])] + + segments.each do |segment| + lines.last.segments << segment + + if segment.type == :str + lines.last.value << segment.children.first + lines << Line.new(+"", []) if lines.last.value.end_with?("\n") + end + end + + lines.pop if lines.last.value.empty? + return if lines.empty? + + segments.clear + lines.each do |line| + remaining = node.dedent + + line.segments.each do |segment| + if segment.type == :str + if remaining > 0 + whitespace = segment.children.first[/^\s{0,#{remaining}}/] + segment.children.first.sub!(/^#{whitespace}/, "") + remaining -= whitespace.length + end + + if node.beginning.value[3] != "'" && segments.any? && + segments.last.type == :str && + segments.last.children.first.end_with?("\\\n") + segments.last.children.first.gsub!(/\\\n\z/, "") + segments.last.children.first.concat(segment.children.first) + elsif !segment.children.first.empty? + segments << segment + end + else + segments << segment + end + end + end + end + end + attr_reader :buffer, :stack def initialize(buffer) @@ -665,6 +732,25 @@ def visit_command_call(node) node.end_char end + expression = + if node.arguments.is_a?(ArgParen) + srange(node.start_char, node.arguments.end_char) + elsif node.arguments.is_a?(Args) && node.arguments.parts.any? + last_part = node.arguments.parts.last + end_char = + if last_part.is_a?(Heredoc) + last_part.beginning.end_char + else + last_part.end_char + end + + srange(node.start_char, end_char) + elsif node.block + srange_node(node.message) + else + srange_node(node) + end + call = s( if node.operator.is_a?(Op) && node.operator.value == "&." @@ -690,14 +776,7 @@ def visit_command_call(node) node.message == :call ? nil : srange_node(node.message), begin_token, end_token, - if node.arguments.is_a?(ArgParen) || - (node.arguments.is_a?(Args) && node.arguments.parts.any?) - srange(node.start_char, node.arguments.end_char) - elsif node.block - srange_node(node.message) - else - srange_node(node) - end + expression ) ) @@ -1049,7 +1128,8 @@ def visit_for(node) smap_for( srange_length(node.start_char, 3), srange_find_between(node.index, node.collection, "in"), - srange_search_between(node.collection, node.statements, "do"), + srange_search_between(node.collection, node.statements, "do") || + srange_search_between(node.collection, node.statements, ";"), srange_length(node.end_char, -3), srange_node(node) ) @@ -1078,98 +1158,43 @@ def visit_hash(node) ) end - # Heredocs are represented _very_ differently in the parser gem from how - # they are represented in the Syntax Tree AST. This class is responsible - # for handling the translation. - class HeredocSegments - HeredocLine = Struct.new(:value, :segments) - - attr_reader :node, :segments - - def initialize(node) - @node = node - @segments = [] - end - - def <<(segment) - if segment.type == :str && segments.last && - segments.last.type == :str && - !segments.last.children.first.end_with?("\n") - segments.last.children.first << segment.children.first - else - segments << segment - end - end - - def trim! - return unless node.beginning.value[2] == "~" - lines = [HeredocLine.new(+"", [])] - - segments.each do |segment| - lines.last.segments << segment - - if segment.type == :str - lines.last.value << segment.children.first - - if lines.last.value.end_with?("\n") - lines << HeredocLine.new(+"", []) - end - end - end - - lines.pop if lines.last.value.empty? - return if lines.empty? - - segments.clear - lines.each do |line| - remaining = node.dedent - - line.segments.each do |segment| - if segment.type == :str - if remaining > 0 - whitespace = segment.children.first[/^\s{0,#{remaining}}/] - segment.children.first.sub!(/^#{whitespace}/, "") - remaining -= whitespace.length - end - - if node.beginning.value[3] != "'" && segments.any? && - segments.last.type == :str && - segments.last.children.first.end_with?("\\\n") - segments.last.children.first.gsub!(/\\\n\z/, "") - segments.last.children.first.concat(segment.children.first) - elsif !segment.children.first.empty? - segments << segment - end - else - segments << segment - end - end - end - end - end - # Visit a Heredoc node. def visit_heredoc(node) - heredoc_segments = HeredocSegments.new(node) + heredoc = HeredocBuilder.new(node) + # For each part of the heredoc, if it's a string content node, split it + # into multiple string content nodes, one for each line. Otherwise, + # visit the node as normal. node.parts.each do |part| if part.is_a?(TStringContent) && part.value.count("\n") > 1 - part - .value - .split("\n") - .each { |line| heredoc_segments << s(:str, ["#{line}\n"], nil) } + index = part.start_char + lines = part.value.split("\n") + + lines.each do |line| + length = line.length + 1 + location = smap_collection_bare(srange_length(index, length)) + + heredoc << s(:str, ["#{line}\n"], location) + index += length + end else - heredoc_segments << visit(part) + heredoc << visit(part) end end - heredoc_segments.trim! + # Now that we have all of the pieces on the heredoc, we can trim it if + # it is a heredoc that supports trimming (i.e., it has a ~ on the + # declaration). + heredoc.trim! + + # Generate the location for the heredoc, which goes from the declaration + # to the ending delimiter. location = smap_heredoc( srange_node(node.beginning), srange( if node.parts.empty? - node.beginning.end_char + node.beginning.end_char + 1 else node.parts.first.start_char end, @@ -1178,15 +1203,15 @@ def visit_heredoc(node) srange(node.ending.start_char, node.ending.end_char - 1) ) + # Finally, decide which kind of heredoc node to generate based on its + # declaration and contents. if node.beginning.value.match?(/`\w+`\z/) - s(:xstr, heredoc_segments.segments, location) - elsif heredoc_segments.segments.length > 1 - s(:dstr, heredoc_segments.segments, location) - elsif heredoc_segments.segments.empty? - s(:dstr, [], location) - else - segment = heredoc_segments.segments.first + s(:xstr, heredoc.segments, location) + elsif heredoc.segments.length == 1 + segment = heredoc.segments.first s(segment.type, segment.children, location) + else + s(:dstr, heredoc.segments, location) end end diff --git a/lib/syntax_tree/translation/rubocop_ast.rb b/lib/syntax_tree/translation/rubocop_ast.rb new file mode 100644 index 00000000..53c6737b --- /dev/null +++ b/lib/syntax_tree/translation/rubocop_ast.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module SyntaxTree + module Translation + # This visitor is responsible for converting the syntax tree produced by + # Syntax Tree into the syntax tree produced by the rubocop/rubocop-ast gem. + class RuboCopAST < Parser + private + + # This method is effectively the same thing as the parser gem except that + # it uses the rubocop-ast specializations of the nodes. + def s(type, children, location) + ::RuboCop::AST::Builder::NODE_MAP.fetch(type, ::RuboCop::AST::Node).new( + type, + children, + location: location + ) + end + end + end +end From cd882e8f621a37887d8c16540f1491a5591c70fe Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 9 Feb 2023 17:15:51 -0500 Subject: [PATCH 382/536] Remove the parser from the statements node --- lib/syntax_tree/dsl.rb | 2 +- lib/syntax_tree/node.rb | 18 +++++-------- lib/syntax_tree/parser.rb | 46 +++++++++++++++++++++----------- lib/syntax_tree/yarv/compiler.rb | 7 +---- 4 files changed, 39 insertions(+), 34 deletions(-) diff --git a/lib/syntax_tree/dsl.rb b/lib/syntax_tree/dsl.rb index 1af19644..4506aa04 100644 --- a/lib/syntax_tree/dsl.rb +++ b/lib/syntax_tree/dsl.rb @@ -791,7 +791,7 @@ def SClass(target, bodystmt) # Create a new Statements node. def Statements(body) - Statements.new(nil, body: body, location: Location.default) + Statements.new(body: body, location: Location.default) end # Create a new StringContent node. diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 627deab1..0a495890 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -2275,7 +2275,7 @@ def initialize( @comments = [] end - def bind(start_char, start_column, end_char, end_column) + def bind(parser, start_char, start_column, end_char, end_column) @location = Location.new( start_line: location.start_line, @@ -2289,6 +2289,7 @@ def bind(start_char, start_column, end_char, end_column) # Here we're going to determine the bounds for the statements consequent = rescue_clause || else_clause || ensure_clause statements.bind( + parser, start_char, start_column, consequent ? consequent.location.start_char : end_char, @@ -9816,23 +9817,19 @@ def ===(other) # propagate that onto void_stmt nodes inside the stmts in order to make sure # all comments get printed appropriately. class Statements < Node - # [Parser] the parser that is generating this node - attr_reader :parser - # [Array[ Node ]] the list of expressions contained within this node attr_reader :body # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(parser, body:, location:) - @parser = parser + def initialize(body:, location:) @body = body @location = location @comments = [] end - def bind(start_char, start_column, end_char, end_column) + def bind(parser, start_char, start_column, end_char, end_column) @location = Location.new( start_line: location.start_line, @@ -9858,7 +9855,7 @@ def bind(start_char, start_column, end_char, end_column) body[0] = VoidStmt.new(location: location) end - attach_comments(start_char, end_char) + attach_comments(parser, start_char, end_char) end def bind_end(end_char, end_column) @@ -9890,7 +9887,6 @@ def child_nodes def copy(body: nil, location: nil) node = Statements.new( - parser, body: body || self.body, location: location || self.location ) @@ -9902,7 +9898,7 @@ def copy(body: nil, location: nil) alias deconstruct child_nodes def deconstruct_keys(_keys) - { parser: parser, body: body, location: location, comments: comments } + { body: body, location: location, comments: comments } end def format(q) @@ -9962,7 +9958,7 @@ def ===(other) # As efficiently as possible, gather up all of the comments that have been # found while this statements list was being parsed and add them into the # body. - def attach_comments(start_char, end_char) + def attach_comments(parser, start_char, end_char) parser_comments = parser.comments comment_index = 0 diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index be6265d1..8059b18c 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -374,6 +374,7 @@ def on_BEGIN(statements) start_char = find_next_statement_start(lbrace.location.end_char) statements.bind( + self, start_char, start_char - line_counts[lbrace.location.start_line - 1].start, rbrace.location.start_char, @@ -412,6 +413,7 @@ def on_END(statements) start_char = find_next_statement_start(lbrace.location.end_char) statements.bind( + self, start_char, start_char - line_counts[lbrace.location.start_line - 1].start, rbrace.location.start_char, @@ -849,6 +851,7 @@ def on_begin(bodystmt) end bodystmt.bind( + self, find_next_statement_start(keyword.location.end_char), keyword.location.end_column, end_location.end_char, @@ -960,11 +963,7 @@ def on_bodystmt(statements, rescue_clause, else_clause, ensure_clause) # case we'll wrap it in a Statements node to be consistent. unless statements.is_a?(Statements) statements = - Statements.new( - self, - body: [statements], - location: statements.location - ) + Statements.new(body: [statements], location: statements.location) end parts = [statements, rescue_clause, else_clause, ensure_clause].compact @@ -991,6 +990,7 @@ def on_brace_block(block_var, statements) start_char = find_next_statement_start(location.end_char) statements.bind( + self, start_char, start_char - line_counts[location.start_line - 1].start, rbrace.location.start_char, @@ -1098,6 +1098,7 @@ def on_class(constant, superclass, bodystmt) start_char = find_next_statement_start(location.end_char) bodystmt.bind( + self, start_char, start_char - line_counts[location.start_line - 1].start, ending.location.start_char, @@ -1307,6 +1308,7 @@ def on_def(name, params, bodystmt) start_char = find_next_statement_start(params.location.end_char) bodystmt.bind( + self, start_char, start_char - line_counts[params.location.start_line - 1].start, ending.location.start_char, @@ -1395,6 +1397,7 @@ def on_defs(target, operator, name, params, bodystmt) start_char = find_next_statement_start(params.location.end_char) bodystmt.bind( + self, start_char, start_char - line_counts[params.location.start_line - 1].start, ending.location.start_char, @@ -1434,6 +1437,7 @@ def on_do_block(block_var, bodystmt) start_char = find_next_statement_start(location.end_char) bodystmt.bind( + self, start_char, start_char - line_counts[location.start_line - 1].start, ending.location.start_char, @@ -1529,6 +1533,7 @@ def on_else(statements) start_char = find_next_statement_start(keyword.location.end_char) statements.bind( + self, start_char, start_char - line_counts[keyword.location.start_line - 1].start, ending.location.start_char, @@ -1554,6 +1559,7 @@ def on_elsif(predicate, statements, consequent) start_char = find_next_statement_start(predicate.location.end_char) statements.bind( + self, start_char, start_char - line_counts[predicate.location.start_line - 1].start, ending.location.start_char, @@ -1677,6 +1683,7 @@ def on_ensure(statements) ending = find_keyword(:end) start_char = find_next_statement_start(keyword.location.end_char) statements.bind( + self, start_char, start_char - line_counts[keyword.location.start_line - 1].start, ending.location.start_char, @@ -1817,6 +1824,7 @@ def on_for(index, collection, statements) find_next_statement_start((delimiter || collection).location.end_char) statements.bind( + self, start_char, start_char - line_counts[(delimiter || collection).location.end_line - 1].start, @@ -2036,6 +2044,7 @@ def on_if(predicate, statements, consequent) start_char = find_next_statement_start((keyword || predicate).location.end_char) statements.bind( + self, start_char, start_char - line_counts[predicate.location.end_line - 1].start, ending.location.start_char, @@ -2069,7 +2078,7 @@ def on_if_mod(predicate, statement) IfNode.new( predicate: predicate, statements: - Statements.new(self, body: [statement], location: statement.location), + Statements.new(body: [statement], location: statement.location), consequent: nil, location: statement.location.to(predicate.location) ) @@ -2121,6 +2130,7 @@ def on_in(pattern, statements, consequent) start_char = find_next_statement_start((token || statements_start).location.end_char) statements.bind( + self, start_char, start_char - line_counts[statements_start.location.start_line - 1].start, @@ -2303,6 +2313,7 @@ def on_lambda(params, statements) start_char = find_next_statement_start(opening.location.end_char) statements.bind( + self, start_char, start_char - line_counts[opening.location.end_line - 1].start, closing.location.start_char, @@ -2587,6 +2598,7 @@ def on_module(constant, bodystmt) start_char = find_next_statement_start(constant.location.end_char) bodystmt.bind( + self, start_char, start_char - line_counts[constant.location.start_line - 1].start, ending.location.start_char, @@ -2863,7 +2875,7 @@ def on_program(statements) ) statements.body << @__end__ if @__end__ - statements.bind(0, 0, source.length, last_column) + statements.bind(self, 0, 0, source.length, last_column) program = Program.new(statements: statements, location: location) attach_comments(program, @comments) @@ -3197,6 +3209,7 @@ def on_rescue(exceptions, variable, statements, consequent) last_node = variable || exceptions || keyword start_char = find_next_statement_start(last_node.end_char) statements.bind( + self, start_char, start_char - line_counts[last_node.location.start_line - 1].start, char_pos, @@ -3315,6 +3328,7 @@ def on_sclass(target, bodystmt) start_char = find_next_statement_start(target.location.end_char) bodystmt.bind( + self, start_char, start_char - line_counts[target.location.start_line - 1].start, ending.location.start_char, @@ -3368,18 +3382,13 @@ def on_stmts_add(statements, statement) statements.location.to(statement.location) end - Statements.new( - self, - body: statements.body << statement, - location: location - ) + Statements.new(body: statements.body << statement, location: location) end # :call-seq: # on_stmts_new: () -> Statements def on_stmts_new Statements.new( - self, body: [], location: Location.fixed(line: lineno, char: char_pos, column: current_column) @@ -3444,6 +3453,7 @@ def on_string_embexpr(statements) embexpr_end = consume_token(EmbExprEnd) statements.bind( + self, embexpr_beg.location.end_char, embexpr_beg.location.end_column, embexpr_end.location.start_char, @@ -3794,6 +3804,7 @@ def on_unless(predicate, statements, consequent) start_char = find_next_statement_start((keyword || predicate).location.end_char) statements.bind( + self, start_char, start_char - line_counts[predicate.location.end_line - 1].start, ending.location.start_char, @@ -3816,7 +3827,7 @@ def on_unless_mod(predicate, statement) UnlessNode.new( predicate: predicate, statements: - Statements.new(self, body: [statement], location: statement.location), + Statements.new(body: [statement], location: statement.location), consequent: nil, location: statement.location.to(predicate.location) ) @@ -3839,6 +3850,7 @@ def on_until(predicate, statements) find_next_statement_start((delimiter || predicate).location.end_char) statements.bind( + self, start_char, start_char - line_counts[predicate.location.end_line - 1].start, ending.location.start_char, @@ -3860,7 +3872,7 @@ def on_until_mod(predicate, statement) UntilNode.new( predicate: predicate, statements: - Statements.new(self, body: [statement], location: statement.location), + Statements.new(body: [statement], location: statement.location), location: statement.location.to(predicate.location) ) end @@ -3935,6 +3947,7 @@ def on_when(arguments, statements, consequent) find_next_statement_start((token || statements_start).location.end_char) statements.bind( + self, start_char, start_char - line_counts[statements_start.location.start_line - 1].start, @@ -3967,6 +3980,7 @@ def on_while(predicate, statements) find_next_statement_start((delimiter || predicate).location.end_char) statements.bind( + self, start_char, start_char - line_counts[predicate.location.end_line - 1].start, ending.location.start_char, @@ -3988,7 +4002,7 @@ def on_while_mod(predicate, statement) WhileNode.new( predicate: predicate, statements: - Statements.new(self, body: [statement], location: statement.location), + Statements.new(body: [statement], location: statement.location), location: statement.location.to(predicate.location) ) end diff --git a/lib/syntax_tree/yarv/compiler.rb b/lib/syntax_tree/yarv/compiler.rb index 3aff3fe5..e1a8544a 100644 --- a/lib/syntax_tree/yarv/compiler.rb +++ b/lib/syntax_tree/yarv/compiler.rb @@ -1051,17 +1051,12 @@ def visit_if_op(node) IfNode.new( predicate: node.predicate, statements: - Statements.new( - nil, - body: [node.truthy], - location: Location.default - ), + Statements.new(body: [node.truthy], location: Location.default), consequent: Else.new( keyword: Kw.new(value: "else", location: Location.default), statements: Statements.new( - nil, body: [node.falsy], location: Location.default ), From 05401daab1fc49fc7a940c293e45b858851c9176 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 9 Feb 2023 17:26:36 -0500 Subject: [PATCH 383/536] Test that the syntax tree is marshalable --- test/syntax_tree_test.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/syntax_tree_test.rb b/test/syntax_tree_test.rb index f12065b8..27aa6851 100644 --- a/test/syntax_tree_test.rb +++ b/test/syntax_tree_test.rb @@ -29,6 +29,11 @@ def test_parse_error assert_raises(Parser::ParseError) { SyntaxTree.parse("<>") } end + def test_marshalable + node = SyntaxTree.parse("1 + 2") + assert_operator(node, :===, Marshal.load(Marshal.dump(node))) + end + def test_maxwidth_format assert_equal("foo +\n bar\n", SyntaxTree.format("foo + bar", 5)) end From 7f4fe77b58e930106d391e4e91f055e7e0bf0e74 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 10 Feb 2023 10:11:40 -0500 Subject: [PATCH 384/536] Move mermaid rendering into its own file --- lib/syntax_tree.rb | 60 ++++++++------- lib/syntax_tree/mermaid.rb | 85 ++++++++++++++++++++++ lib/syntax_tree/visitor/mermaid_visitor.rb | 37 ++++------ 3 files changed, 130 insertions(+), 52 deletions(-) create mode 100644 lib/syntax_tree/mermaid.rb diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index e5bc5ab5..edf7688e 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -1,6 +1,5 @@ # frozen_string_literal: true -require "cgi" require "etc" require "json" require "pp" @@ -71,19 +70,6 @@ module SyntaxTree # that Syntax Tree can format arbitrary parts of a document. DEFAULT_INDENTATION = 0 - # This is a hook provided so that plugins can register themselves as the - # handler for a particular file type. - def self.register_handler(extension, handler) - HANDLERS[extension] = handler - end - - # Parses the given source and returns the syntax tree. - def self.parse(source) - parser = Parser.new(source) - response = parser.parse - response unless parser.error? - end - # Parses the given source and returns the formatted source. def self.format( source, @@ -98,6 +84,20 @@ def self.format( formatter.output.join end + # Indexes the given source code to return a list of all class, module, and + # method definitions. Used to quickly provide indexing capability for IDEs or + # documentation generation. + def self.index(source) + Index.index(source) + end + + # Indexes the given file to return a list of all class, module, and method + # definitions. Used to quickly provide indexing capability for IDEs or + # documentation generation. + def self.index_file(filepath) + Index.index_file(filepath) + end + # A convenience method for creating a new mutation visitor. def self.mutation visitor = Visitor::MutationVisitor.new @@ -105,6 +105,18 @@ def self.mutation visitor end + # Parses the given source and returns the syntax tree. + def self.parse(source) + parser = Parser.new(source) + response = parser.parse + response unless parser.error? + end + + # Parses the given file and returns the syntax tree. + def self.parse_file(filepath) + parse(read(filepath)) + end + # Returns the source from the given filepath taking into account any potential # magic encoding comments. def self.read(filepath) @@ -120,23 +132,15 @@ def self.read(filepath) File.read(filepath, encoding: encoding) end + # This is a hook provided so that plugins can register themselves as the + # handler for a particular file type. + def self.register_handler(extension, handler) + HANDLERS[extension] = handler + end + # Searches through the given source using the given pattern and yields each # node in the tree that matches the pattern to the given block. def self.search(source, query, &block) Search.new(Pattern.new(query).compile).scan(parse(source), &block) end - - # Indexes the given source code to return a list of all class, module, and - # method definitions. Used to quickly provide indexing capability for IDEs or - # documentation generation. - def self.index(source) - Index.index(source) - end - - # Indexes the given file to return a list of all class, module, and method - # definitions. Used to quickly provide indexing capability for IDEs or - # documentation generation. - def self.index_file(filepath) - Index.index_file(filepath) - end end diff --git a/lib/syntax_tree/mermaid.rb b/lib/syntax_tree/mermaid.rb new file mode 100644 index 00000000..fa923876 --- /dev/null +++ b/lib/syntax_tree/mermaid.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +require "cgi" + +module SyntaxTree + # This module is responsible for rendering mermaid flow charts. + module Mermaid + class Node + SHAPES = %i[circle rectangle stadium].freeze + + attr_reader :id, :label, :shape + + def initialize(id, label, shape) + raise unless SHAPES.include?(shape) + + @id = id + @label = label + @shape = shape + end + + def render + left_bound, right_bound = + case shape + when :circle + ["((", "))"] + when :rectangle + ["[", "]"] + when :stadium + ["([", "])"] + end + + " #{id}#{left_bound}\"#{CGI.escapeHTML(label)}\"#{right_bound}" + end + end + + class Edge + TYPES = %i[directed].freeze + + attr_reader :from, :to, :label, :type + + def initialize(from, to, label, type) + raise unless TYPES.include?(type) + + @from = from + @to = to + @label = label + @type = type + end + + def render + case type + when :directed + " #{from.id} -- \"#{CGI.escapeHTML(label)}\" --> #{to.id}" + end + end + end + + class FlowChart + attr_reader :nodes, :edges + + def initialize + @nodes = {} + @edges = [] + end + + def edge(from, to, label, type = :directed) + edges << Edge.new(from, to, label, type) + end + + def node(id, label, shape = :rectangle) + nodes[id] = Node.new(id, label, shape) + end + + def render + output = StringIO.new + output.puts("flowchart TD") + + nodes.each_value { |node| output.puts(node.render) } + edges.each { |edge| output.puts(edge.render) } + + output.string + end + end + end +end diff --git a/lib/syntax_tree/visitor/mermaid_visitor.rb b/lib/syntax_tree/visitor/mermaid_visitor.rb index 2b06049a..e63ee2a6 100644 --- a/lib/syntax_tree/visitor/mermaid_visitor.rb +++ b/lib/syntax_tree/visitor/mermaid_visitor.rb @@ -4,18 +4,16 @@ module SyntaxTree class Visitor # This visitor transforms the AST into a mermaid flow chart. class MermaidVisitor < FieldVisitor - attr_reader :output, :target + attr_reader :flowchart, :target def initialize - @output = StringIO.new - @output.puts("flowchart TD") - + @flowchart = Mermaid::FlowChart.new @target = nil end def visit_program(node) super - output.string + flowchart.render end private @@ -26,19 +24,13 @@ def comments(node) def field(name, value) case value - when Node - node_id = visit(value) - output.puts(" #{target} -- \"#{name}\" --> #{node_id}") - when String - node_id = "#{target}_#{name}" - output.puts(" #{node_id}([#{CGI.escapeHTML(value.inspect)}])") - output.puts(" #{target} -- \"#{name}\" --> #{node_id}") when nil # skip + when Node + flowchart.edge(target, visit(value), name) else - node_id = "#{target}_#{name}" - output.puts(" #{node_id}([\"#{CGI.escapeHTML(value.inspect)}\"])") - output.puts(" #{target} -- \"#{name}\" --> #{node_id}") + to = flowchart.node("#{target.id}_#{name}", value.inspect, :stadium) + flowchart.edge(target, to, name) end end @@ -52,11 +44,8 @@ def node(node, type) previous_target = target begin - @target = "node_#{node.object_id}" - + @target = flowchart.node("node_#{node.object_id}", type) yield - - output.puts(" #{@target}[\"#{type}\"]") @target ensure @target = previous_target @@ -65,11 +54,11 @@ def node(node, type) def pairs(name, values) values.each_with_index do |(key, value), index| - node_id = "#{target}_#{name}_#{index}" - output.puts(" #{node_id}((\" \"))") - output.puts(" #{target} -- \"#{name}[#{index}]\" --> #{node_id}") - output.puts(" #{node_id} -- \"[0]\" --> #{visit(key)}") - output.puts(" #{node_id} -- \"[1]\" --> #{visit(value)}") if value + to = flowchart.node("#{target.id}_#{name}_#{index}", " ", :circle) + + flowchart.edge(target, to, "#{name}[#{index}]") + flowchart.edge(to, visit(key), "[0]") + flowchart.edge(to, visit(value), "[1]") if value end end From 103236bb822f7cb7a449a559321e82f0bef75e4c Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 10 Feb 2023 10:26:34 -0500 Subject: [PATCH 385/536] Render CFG using new mermaid code --- lib/syntax_tree.rb | 1 + lib/syntax_tree/mermaid.rb | 75 +++++++++++++++------- lib/syntax_tree/visitor/mermaid_visitor.rb | 4 +- lib/syntax_tree/yarv/control_flow_graph.rb | 37 +++++------ 4 files changed, 74 insertions(+), 43 deletions(-) diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index edf7688e..9cbd49c7 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -23,6 +23,7 @@ require_relative "syntax_tree/visitor/environment" require_relative "syntax_tree/visitor/with_environment" +require_relative "syntax_tree/mermaid" require_relative "syntax_tree/parser" require_relative "syntax_tree/pattern" require_relative "syntax_tree/search" diff --git a/lib/syntax_tree/mermaid.rb b/lib/syntax_tree/mermaid.rb index fa923876..f5c85f2f 100644 --- a/lib/syntax_tree/mermaid.rb +++ b/lib/syntax_tree/mermaid.rb @@ -6,7 +6,7 @@ module SyntaxTree # This module is responsible for rendering mermaid flow charts. module Mermaid class Node - SHAPES = %i[circle rectangle stadium].freeze + SHAPES = %i[circle rectangle rounded stadium].freeze attr_reader :id, :label, :shape @@ -19,17 +19,23 @@ def initialize(id, label, shape) end def render - left_bound, right_bound = - case shape - when :circle - ["((", "))"] - when :rectangle - ["[", "]"] - when :stadium - ["([", "])"] - end + left_bound, right_bound = bounds + "#{id}#{left_bound}\"#{CGI.escapeHTML(label)}\"#{right_bound}" + end - " #{id}#{left_bound}\"#{CGI.escapeHTML(label)}\"#{right_bound}" + private + + def bounds + case shape + when :circle + ["((", "))"] + when :rectangle + ["[", "]"] + when :rounded + ["(", ")"] + when :stadium + ["([", "])"] + end end end @@ -50,34 +56,57 @@ def initialize(from, to, label, type) def render case type when :directed - " #{from.id} -- \"#{CGI.escapeHTML(label)}\" --> #{to.id}" + if label + "#{from.id} -- \"#{CGI.escapeHTML(label)}\" --> #{to.id}" + else + "#{from.id} --> #{to.id}" + end end end end class FlowChart - attr_reader :nodes, :edges + attr_reader :output, :prefix, :nodes def initialize + @output = StringIO.new + @output.puts("flowchart TD") + @prefix = " " @nodes = {} - @edges = [] end - def edge(from, to, label, type = :directed) - edges << Edge.new(from, to, label, type) + def edge(from, to, label = nil, type: :directed) + edge = Edge.new(from, to, label, type) + output.puts("#{prefix}#{edge.render}") end - def node(id, label, shape = :rectangle) - nodes[id] = Node.new(id, label, shape) + def fetch(id) + nodes.fetch(id) end - def render - output = StringIO.new - output.puts("flowchart TD") + def node(id, label, shape: :rectangle) + node = Node.new(id, label, shape) + nodes[id] = node + + output.puts("#{prefix}#{nodes[id].render}") + node + end + + def subgraph(id) + output.puts("#{prefix}subgraph #{id}") + + previous = prefix + @prefix = "#{prefix} " - nodes.each_value { |node| output.puts(node.render) } - edges.each { |edge| output.puts(edge.render) } + begin + yield + ensure + @prefix = previous + output.puts("#{prefix}end") + end + end + def render output.string end end diff --git a/lib/syntax_tree/visitor/mermaid_visitor.rb b/lib/syntax_tree/visitor/mermaid_visitor.rb index e63ee2a6..1694952d 100644 --- a/lib/syntax_tree/visitor/mermaid_visitor.rb +++ b/lib/syntax_tree/visitor/mermaid_visitor.rb @@ -29,7 +29,7 @@ def field(name, value) when Node flowchart.edge(target, visit(value), name) else - to = flowchart.node("#{target.id}_#{name}", value.inspect, :stadium) + to = flowchart.node("#{target.id}_#{name}", value.inspect, shape: :stadium) flowchart.edge(target, to, name) end end @@ -54,7 +54,7 @@ def node(node, type) def pairs(name, values) values.each_with_index do |(key, value), index| - to = flowchart.node("#{target.id}_#{name}_#{index}", " ", :circle) + to = flowchart.node("#{target.id}_#{name}_#{index}", " ", shape: :circle) flowchart.edge(target, to, "#{name}[#{index}]") flowchart.edge(to, visit(key), "[0]") diff --git a/lib/syntax_tree/yarv/control_flow_graph.rb b/lib/syntax_tree/yarv/control_flow_graph.rb index 73d30208..927f535a 100644 --- a/lib/syntax_tree/yarv/control_flow_graph.rb +++ b/lib/syntax_tree/yarv/control_flow_graph.rb @@ -208,25 +208,24 @@ def to_son end def to_mermaid - output = StringIO.new - output.puts("flowchart TD") + flowchart = Mermaid::FlowChart.new + disasm = Disassembler::Mermaid.new - fmt = Disassembler::Mermaid.new blocks.each do |block| - output.puts(" subgraph #{block.id}") - previous = nil - - block.each_with_length do |insn, length| - node_id = "node_#{length}" - label = "%04d %s" % [length, insn.disasm(fmt)] - - output.puts(" #{node_id}(\"#{CGI.escapeHTML(label)}\")") - output.puts(" #{previous} --> #{node_id}") if previous - - previous = node_id + flowchart.subgraph(block.id) do + previous = nil + + block.each_with_length do |insn, length| + node = + flowchart.node( + "node_#{length}", + "%04d %s" % [length, insn.disasm(disasm)] + ) + + flowchart.edge(previous, node) if previous + previous = node + end end - - output.puts(" end") end blocks.each do |block| @@ -235,11 +234,13 @@ def to_mermaid block.block_start + block.insns.sum(&:length) - block.insns.last.length - output.puts(" node_#{offset} --> node_#{outgoing.block_start}") + from = flowchart.fetch("node_#{offset}") + to = flowchart.fetch("node_#{outgoing.block_start}") + flowchart.edge(from, to) end end - output.string + flowchart.render end # This method is used to verify that the control flow graph is well From 6dbe713baf4dd6fd87183d77dfc38340d7bbbf6f Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 10 Feb 2023 10:29:21 -0500 Subject: [PATCH 386/536] Fix up data flow mermaid rendering --- lib/syntax_tree/yarv/data_flow_graph.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/syntax_tree/yarv/data_flow_graph.rb b/lib/syntax_tree/yarv/data_flow_graph.rb index ace40296..185eeee5 100644 --- a/lib/syntax_tree/yarv/data_flow_graph.rb +++ b/lib/syntax_tree/yarv/data_flow_graph.rb @@ -155,8 +155,8 @@ def to_mermaid end insn_flows[length].in.each do |input| - if input.is_a?(Integer) - output.puts(" node_#{input} --> #{node_id}") + if input.is_a?(LocalArgument) + output.puts(" node_#{input.length} --> #{node_id}") links << "green" end end From 72619fb4469786b62a3e97d63c30d62c404f31b3 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 10 Feb 2023 10:38:17 -0500 Subject: [PATCH 387/536] Render DFG with new mermaid renderer --- lib/syntax_tree/mermaid.rb | 88 +++++++++++++--------- lib/syntax_tree/visitor/mermaid_visitor.rb | 10 +-- lib/syntax_tree/yarv/control_flow_graph.rb | 4 +- lib/syntax_tree/yarv/data_flow_graph.rb | 57 ++++++-------- 4 files changed, 84 insertions(+), 75 deletions(-) diff --git a/lib/syntax_tree/mermaid.rb b/lib/syntax_tree/mermaid.rb index f5c85f2f..28cc095a 100644 --- a/lib/syntax_tree/mermaid.rb +++ b/lib/syntax_tree/mermaid.rb @@ -5,6 +5,39 @@ module SyntaxTree # This module is responsible for rendering mermaid flow charts. module Mermaid + def self.escape(label) + "\"#{CGI.escapeHTML(label)}\"" + end + + class Link + TYPES = %i[directed].freeze + COLORS = %i[green red].freeze + + attr_reader :from, :to, :label, :type, :color + + def initialize(from, to, label, type, color) + raise if !TYPES.include?(type) + raise if color && !COLORS.include?(color) + + @from = from + @to = to + @label = label + @type = type + @color = color + end + + def render + case type + when :directed + if label + "#{from.id} -- #{Mermaid.escape(label)} --> #{to.id}" + else + "#{from.id} --> #{to.id}" + end + end + end + end + class Node SHAPES = %i[circle rectangle rounded stadium].freeze @@ -20,7 +53,7 @@ def initialize(id, label, shape) def render left_bound, right_bound = bounds - "#{id}#{left_bound}\"#{CGI.escapeHTML(label)}\"#{right_bound}" + "#{id}#{left_bound}#{Mermaid.escape(label)}#{right_bound}" end private @@ -39,51 +72,30 @@ def bounds end end - class Edge - TYPES = %i[directed].freeze - - attr_reader :from, :to, :label, :type - - def initialize(from, to, label, type) - raise unless TYPES.include?(type) - - @from = from - @to = to - @label = label - @type = type - end - - def render - case type - when :directed - if label - "#{from.id} -- \"#{CGI.escapeHTML(label)}\" --> #{to.id}" - else - "#{from.id} --> #{to.id}" - end - end - end - end - class FlowChart - attr_reader :output, :prefix, :nodes + attr_reader :output, :prefix, :nodes, :links def initialize @output = StringIO.new @output.puts("flowchart TD") @prefix = " " - @nodes = {} - end - def edge(from, to, label = nil, type: :directed) - edge = Edge.new(from, to, label, type) - output.puts("#{prefix}#{edge.render}") + @nodes = {} + @links = [] end def fetch(id) nodes.fetch(id) end + def link(from, to, label = nil, type: :directed, color: nil) + link = Link.new(from, to, label, type, color) + links << link + + output.puts("#{prefix}#{link.render}") + link + end + def node(id, label, shape: :rectangle) node = Node.new(id, label, shape) nodes[id] = node @@ -92,8 +104,8 @@ def node(id, label, shape: :rectangle) node end - def subgraph(id) - output.puts("#{prefix}subgraph #{id}") + def subgraph(label) + output.puts("#{prefix}subgraph #{Mermaid.escape(label)}") previous = prefix @prefix = "#{prefix} " @@ -107,6 +119,12 @@ def subgraph(id) end def render + links.each_with_index do |link, index| + if link.color + output.puts("#{prefix}linkStyle #{index} stroke:#{link.color}") + end + end + output.string end end diff --git a/lib/syntax_tree/visitor/mermaid_visitor.rb b/lib/syntax_tree/visitor/mermaid_visitor.rb index 1694952d..542fe192 100644 --- a/lib/syntax_tree/visitor/mermaid_visitor.rb +++ b/lib/syntax_tree/visitor/mermaid_visitor.rb @@ -27,10 +27,10 @@ def field(name, value) when nil # skip when Node - flowchart.edge(target, visit(value), name) + flowchart.link(target, visit(value), name) else to = flowchart.node("#{target.id}_#{name}", value.inspect, shape: :stadium) - flowchart.edge(target, to, name) + flowchart.link(target, to, name) end end @@ -56,9 +56,9 @@ def pairs(name, values) values.each_with_index do |(key, value), index| to = flowchart.node("#{target.id}_#{name}_#{index}", " ", shape: :circle) - flowchart.edge(target, to, "#{name}[#{index}]") - flowchart.edge(to, visit(key), "[0]") - flowchart.edge(to, visit(value), "[1]") if value + flowchart.link(target, to, "#{name}[#{index}]") + flowchart.link(to, visit(key), "[0]") + flowchart.link(to, visit(value), "[1]") if value end end diff --git a/lib/syntax_tree/yarv/control_flow_graph.rb b/lib/syntax_tree/yarv/control_flow_graph.rb index 927f535a..5da2cc14 100644 --- a/lib/syntax_tree/yarv/control_flow_graph.rb +++ b/lib/syntax_tree/yarv/control_flow_graph.rb @@ -222,7 +222,7 @@ def to_mermaid "%04d %s" % [length, insn.disasm(disasm)] ) - flowchart.edge(previous, node) if previous + flowchart.link(previous, node) if previous previous = node end end @@ -236,7 +236,7 @@ def to_mermaid from = flowchart.fetch("node_#{offset}") to = flowchart.fetch("node_#{outgoing.block_start}") - flowchart.edge(from, to) + flowchart.link(from, to) end end diff --git a/lib/syntax_tree/yarv/data_flow_graph.rb b/lib/syntax_tree/yarv/data_flow_graph.rb index 185eeee5..4adf2bcf 100644 --- a/lib/syntax_tree/yarv/data_flow_graph.rb +++ b/lib/syntax_tree/yarv/data_flow_graph.rb @@ -125,11 +125,8 @@ def to_son end def to_mermaid - output = StringIO.new - output.puts("flowchart TD") - - fmt = Disassembler::Mermaid.new - links = [] + flowchart = Mermaid::FlowChart.new + disasm = Disassembler::Mermaid.new blocks.each do |block| block_flow = block_flows.fetch(block.id) @@ -140,31 +137,28 @@ def to_mermaid block.id end - output.puts(" subgraph \"#{CGI.escapeHTML(graph_name)}\"") - previous = nil - - block.each_with_length do |insn, length| - node_id = "node_#{length}" - label = "%04d %s" % [length, insn.disasm(fmt)] - - output.puts(" #{node_id}(\"#{CGI.escapeHTML(label)}\")") + flowchart.subgraph(graph_name) do + previous = nil - if previous - output.puts(" #{previous} --> #{node_id}") - links << "red" - end - - insn_flows[length].in.each do |input| - if input.is_a?(LocalArgument) - output.puts(" node_#{input.length} --> #{node_id}") - links << "green" + block.each_with_length do |insn, length| + node = + flowchart.node( + "node_#{length}", + "%04d %s" % [length, insn.disasm(disasm)], + shape: :rounded + ) + + flowchart.link(previous, node, color: :red) if previous + insn_flows[length].in.each do |input| + if input.is_a?(LocalArgument) + from = flowchart.fetch("node_#{input.length}") + flowchart.link(from, node, color: :green) + end end - end - previous = node_id + previous = node + end end - - output.puts(" end") end blocks.each do |block| @@ -173,16 +167,13 @@ def to_mermaid block.block_start + block.insns.sum(&:length) - block.insns.last.length - output.puts(" node_#{offset} --> node_#{outgoing.block_start}") - links << "red" + from = flowchart.fetch("node_#{offset}") + to = flowchart.fetch("node_#{outgoing.block_start}") + flowchart.link(from, to, color: :red) end end - links.each_with_index do |color, index| - output.puts(" linkStyle #{index} stroke:#{color}") - end - - output.string + flowchart.render end # Verify that we constructed the data flow graph correctly. From a8fd78b0c6e4070fdf92d17bb4de834946e154df Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 10 Feb 2023 10:47:00 -0500 Subject: [PATCH 388/536] Render sea of nodes to mermaid using new API --- .rubocop.yml | 3 + lib/syntax_tree/mermaid.rb | 170 +++++++++++++-------- lib/syntax_tree/visitor/mermaid_visitor.rb | 11 +- lib/syntax_tree/yarv/control_flow_graph.rb | 55 ++++--- lib/syntax_tree/yarv/data_flow_graph.rb | 79 +++++----- lib/syntax_tree/yarv/disassembler.rb | 6 +- lib/syntax_tree/yarv/sea_of_nodes.rb | 67 +++----- test/yarv_test.rb | 100 ++++++------ 8 files changed, 261 insertions(+), 230 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index 33636c44..21beca1b 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -117,6 +117,9 @@ Style/FormatStringToken: Style/GuardClause: Enabled: false +Style/HashLikeCase: + Enabled: false + Style/IdenticalConditionalBranches: Enabled: false diff --git a/lib/syntax_tree/mermaid.rb b/lib/syntax_tree/mermaid.rb index 28cc095a..70cbc054 100644 --- a/lib/syntax_tree/mermaid.rb +++ b/lib/syntax_tree/mermaid.rb @@ -3,20 +3,85 @@ require "cgi" module SyntaxTree - # This module is responsible for rendering mermaid flow charts. + # This module is responsible for rendering mermaid (https://mermaid.js.org/) + # flow charts. module Mermaid - def self.escape(label) - "\"#{CGI.escapeHTML(label)}\"" + # This is the main class that handles rendering a flowchart. It keeps track + # of its nodes and links and renders them according to the mermaid syntax. + class FlowChart + attr_reader :output, :prefix, :nodes, :links + + def initialize + @output = StringIO.new + @output.puts("flowchart TD") + @prefix = " " + + @nodes = {} + @links = [] + end + + # Retrieve a node that has already been added to the flowchart by its id. + def fetch(id) + nodes.fetch(id) + end + + # Add a link to the flowchart between two nodes with an optional label. + def link(from, to, label = nil, type: :directed, color: nil) + link = Link.new(from, to, label, type, color) + links << link + + output.puts("#{prefix}#{link.render}") + link + end + + # Add a node to the flowchart with an optional label. + def node(id, label = " ", shape: :rectangle) + node = Node.new(id, label, shape) + nodes[id] = node + + output.puts("#{prefix}#{nodes[id].render}") + node + end + + # Add a subgraph to the flowchart. Within the given block, all of the + # nodes will be rendered within the subgraph. + def subgraph(label) + output.puts("#{prefix}subgraph #{Mermaid.escape(label)}") + + previous = prefix + @prefix = "#{prefix} " + + begin + yield + ensure + @prefix = previous + output.puts("#{prefix}end") + end + end + + # Return the rendered flowchart. + def render + links.each_with_index do |link, index| + if link.color + output.puts("#{prefix}linkStyle #{index} stroke:#{link.color}") + end + end + + output.string + end end + # This class represents a link between two nodes in a flowchart. It is not + # meant to be interacted with directly, but rather used as a data structure + # by the FlowChart class. class Link - TYPES = %i[directed].freeze + TYPES = %i[directed dotted].freeze COLORS = %i[green red].freeze attr_reader :from, :to, :label, :type, :color def initialize(from, to, label, type, color) - raise if !TYPES.include?(type) + raise unless TYPES.include?(type) raise if color && !COLORS.include?(color) @from = from @@ -27,17 +92,31 @@ def initialize(from, to, label, type, color) end def render + left_side, right_side, full_side = sides + + if label + escaped = Mermaid.escape(label) + "#{from.id} #{left_side} #{escaped} #{right_side} #{to.id}" + else + "#{from.id} #{full_side} #{to.id}" + end + end + + private + + def sides case type when :directed - if label - "#{from.id} -- #{Mermaid.escape(label)} --> #{to.id}" - else - "#{from.id} --> #{to.id}" - end + %w[-- --> -->] + when :dotted + %w[-. .-> -.->] end end end + # This class represents a node in a flowchart. Unlike the Link class, it can + # be used directly. It is the return value of the #node method, and is meant + # to be passed around to #link methods to create links between nodes. class Node SHAPES = %i[circle rectangle rounded stadium].freeze @@ -61,72 +140,37 @@ def render def bounds case shape when :circle - ["((", "))"] + %w[(( ))] when :rectangle ["[", "]"] when :rounded - ["(", ")"] + %w[( )] when :stadium ["([", "])"] end end end - class FlowChart - attr_reader :output, :prefix, :nodes, :links - - def initialize - @output = StringIO.new - @output.puts("flowchart TD") - @prefix = " " - - @nodes = {} - @links = [] - end - - def fetch(id) - nodes.fetch(id) - end - - def link(from, to, label = nil, type: :directed, color: nil) - link = Link.new(from, to, label, type, color) - links << link - - output.puts("#{prefix}#{link.render}") - link + class << self + # Escape a label to be used in the mermaid syntax. This is used to escape + # HTML entities such that they render properly within the quotes. + def escape(label) + "\"#{CGI.escapeHTML(label)}\"" end - def node(id, label, shape: :rectangle) - node = Node.new(id, label, shape) - nodes[id] = node - - output.puts("#{prefix}#{nodes[id].render}") - node - end - - def subgraph(label) - output.puts("#{prefix}subgraph #{Mermaid.escape(label)}") - - previous = prefix - @prefix = "#{prefix} " - - begin - yield - ensure - @prefix = previous - output.puts("#{prefix}end") + # Create a new flowchart. If a block is given, it will be yielded to and + # the flowchart will be rendered. Otherwise, the flowchart will be + # returned. + def flowchart + flowchart = FlowChart.new + + if block_given? + yield flowchart + flowchart.render + else + flowchart end end - - def render - links.each_with_index do |link, index| - if link.color - output.puts("#{prefix}linkStyle #{index} stroke:#{link.color}") - end - end - - output.string - end end end end diff --git a/lib/syntax_tree/visitor/mermaid_visitor.rb b/lib/syntax_tree/visitor/mermaid_visitor.rb index 542fe192..504e2fb0 100644 --- a/lib/syntax_tree/visitor/mermaid_visitor.rb +++ b/lib/syntax_tree/visitor/mermaid_visitor.rb @@ -7,7 +7,7 @@ class MermaidVisitor < FieldVisitor attr_reader :flowchart, :target def initialize - @flowchart = Mermaid::FlowChart.new + @flowchart = Mermaid.flowchart @target = nil end @@ -29,7 +29,12 @@ def field(name, value) when Node flowchart.link(target, visit(value), name) else - to = flowchart.node("#{target.id}_#{name}", value.inspect, shape: :stadium) + to = + flowchart.node( + "#{target.id}_#{name}", + value.inspect, + shape: :stadium + ) flowchart.link(target, to, name) end end @@ -54,7 +59,7 @@ def node(node, type) def pairs(name, values) values.each_with_index do |(key, value), index| - to = flowchart.node("#{target.id}_#{name}_#{index}", " ", shape: :circle) + to = flowchart.node("#{target.id}_#{name}_#{index}", shape: :circle) flowchart.link(target, to, "#{name}[#{index}]") flowchart.link(to, visit(key), "[0]") diff --git a/lib/syntax_tree/yarv/control_flow_graph.rb b/lib/syntax_tree/yarv/control_flow_graph.rb index 5da2cc14..2829bb21 100644 --- a/lib/syntax_tree/yarv/control_flow_graph.rb +++ b/lib/syntax_tree/yarv/control_flow_graph.rb @@ -208,39 +208,38 @@ def to_son end def to_mermaid - flowchart = Mermaid::FlowChart.new - disasm = Disassembler::Mermaid.new - - blocks.each do |block| - flowchart.subgraph(block.id) do - previous = nil - - block.each_with_length do |insn, length| - node = - flowchart.node( - "node_#{length}", - "%04d %s" % [length, insn.disasm(disasm)] - ) - - flowchart.link(previous, node) if previous - previous = node + Mermaid.flowchart do |flowchart| + disasm = Disassembler::Squished.new + + blocks.each do |block| + flowchart.subgraph(block.id) do + previous = nil + + block.each_with_length do |insn, length| + node = + flowchart.node( + "node_#{length}", + "%04d %s" % [length, insn.disasm(disasm)] + ) + + flowchart.link(previous, node) if previous + previous = node + end end end - end - blocks.each do |block| - block.outgoing_blocks.each do |outgoing| - offset = - block.block_start + block.insns.sum(&:length) - - block.insns.last.length - - from = flowchart.fetch("node_#{offset}") - to = flowchart.fetch("node_#{outgoing.block_start}") - flowchart.link(from, to) + blocks.each do |block| + block.outgoing_blocks.each do |outgoing| + offset = + block.block_start + block.insns.sum(&:length) - + block.insns.last.length + + from = flowchart.fetch("node_#{offset}") + to = flowchart.fetch("node_#{outgoing.block_start}") + flowchart.link(from, to) + end end end - - flowchart.render end # This method is used to verify that the control flow graph is well diff --git a/lib/syntax_tree/yarv/data_flow_graph.rb b/lib/syntax_tree/yarv/data_flow_graph.rb index 4adf2bcf..aedee9ba 100644 --- a/lib/syntax_tree/yarv/data_flow_graph.rb +++ b/lib/syntax_tree/yarv/data_flow_graph.rb @@ -125,55 +125,54 @@ def to_son end def to_mermaid - flowchart = Mermaid::FlowChart.new - disasm = Disassembler::Mermaid.new + Mermaid.flowchart do |flowchart| + disasm = Disassembler::Squished.new - blocks.each do |block| - block_flow = block_flows.fetch(block.id) - graph_name = - if block_flow.in.any? - "#{block.id} #{block_flows[block.id].in.join(", ")}" - else - block.id - end - - flowchart.subgraph(graph_name) do - previous = nil + blocks.each do |block| + block_flow = block_flows.fetch(block.id) + graph_name = + if block_flow.in.any? + "#{block.id} #{block_flows[block.id].in.join(", ")}" + else + block.id + end - block.each_with_length do |insn, length| - node = - flowchart.node( - "node_#{length}", - "%04d %s" % [length, insn.disasm(disasm)], - shape: :rounded - ) - - flowchart.link(previous, node, color: :red) if previous - insn_flows[length].in.each do |input| - if input.is_a?(LocalArgument) - from = flowchart.fetch("node_#{input.length}") - flowchart.link(from, node, color: :green) + flowchart.subgraph(graph_name) do + previous = nil + + block.each_with_length do |insn, length| + node = + flowchart.node( + "node_#{length}", + "%04d %s" % [length, insn.disasm(disasm)], + shape: :rounded + ) + + flowchart.link(previous, node, color: :red) if previous + insn_flows[length].in.each do |input| + if input.is_a?(LocalArgument) + from = flowchart.fetch("node_#{input.length}") + flowchart.link(from, node, color: :green) + end end - end - previous = node + previous = node + end end end - end - blocks.each do |block| - block.outgoing_blocks.each do |outgoing| - offset = - block.block_start + block.insns.sum(&:length) - - block.insns.last.length - - from = flowchart.fetch("node_#{offset}") - to = flowchart.fetch("node_#{outgoing.block_start}") - flowchart.link(from, to, color: :red) + blocks.each do |block| + block.outgoing_blocks.each do |outgoing| + offset = + block.block_start + block.insns.sum(&:length) - + block.insns.last.length + + from = flowchart.fetch("node_#{offset}") + to = flowchart.fetch("node_#{outgoing.block_start}") + flowchart.link(from, to, color: :red) + end end end - - flowchart.render end # Verify that we constructed the data flow graph correctly. diff --git a/lib/syntax_tree/yarv/disassembler.rb b/lib/syntax_tree/yarv/disassembler.rb index f60af0fd..dac220fd 100644 --- a/lib/syntax_tree/yarv/disassembler.rb +++ b/lib/syntax_tree/yarv/disassembler.rb @@ -4,9 +4,9 @@ module SyntaxTree module YARV class Disassembler # This class is another object that handles disassembling a YARV - # instruction sequence but it does so in order to provide a label for a - # mermaid diagram. - class Mermaid + # instruction sequence but it renders it without any of the extra spacing + # or alignment. + class Squished def calldata(value) value.inspect end diff --git a/lib/syntax_tree/yarv/sea_of_nodes.rb b/lib/syntax_tree/yarv/sea_of_nodes.rb index 181d729c..33ef14f7 100644 --- a/lib/syntax_tree/yarv/sea_of_nodes.rb +++ b/lib/syntax_tree/yarv/sea_of_nodes.rb @@ -27,7 +27,7 @@ def id end def label - "%04d %s" % [offset, insn.disasm(Disassembler::Mermaid.new)] + "%04d %s" % [offset, insn.disasm(Disassembler::Squished.new)] end end @@ -466,53 +466,34 @@ def initialize(dfg, nodes, local_graphs) end def to_mermaid - output = StringIO.new - output.puts("flowchart TD") - - nodes.each do |node| - escaped = "\"#{CGI.escapeHTML(node.label)}\"" - output.puts(" node_#{node.id}(#{escaped})") - end - - link_counter = 0 - nodes.each do |producer| - producer.outputs.each do |consumer_edge| - case consumer_edge.type - when :data - edge = "-->" - edge_style = "stroke:green;" - when :control - edge = "-->" - edge_style = "stroke:red;" - when :info - edge = "-.->" - else - raise - end - - label = - if !consumer_edge.label - "" - elsif consumer_edge.to.is_a?(PhiNode) - # Edges into phi nodes are labelled by the offset of the - # instruction going into the merge. - "|%04d| " % consumer_edge.label - else - "|#{consumer_edge.label}| " - end + Mermaid.flowchart do |flowchart| + nodes.each do |node| + flowchart.node("node_#{node.id}", node.label, shape: :rounded) + end - to_id = "node_#{consumer_edge.to.id}" - output.puts(" node_#{producer.id} #{edge} #{label}#{to_id}") + nodes.each do |producer| + producer.outputs.each do |consumer_edge| + label = + if !consumer_edge.label + # No label. + elsif consumer_edge.to.is_a?(PhiNode) + # Edges into phi nodes are labelled by the offset of the + # instruction going into the merge. + "%04d" % consumer_edge.label + else + consumer_edge.label.to_s + end - if edge_style - output.puts(" linkStyle #{link_counter} #{edge_style}") + flowchart.link( + flowchart.fetch("node_#{producer.id}"), + flowchart.fetch("node_#{consumer_edge.to.id}"), + label, + type: consumer_edge.type == :info ? :dotted : :directed, + color: { data: :green, control: :red }[consumer_edge.type] + ) end - - link_counter += 1 end end - - output.string end def verify diff --git a/test/yarv_test.rb b/test/yarv_test.rb index a1e89568..78622434 100644 --- a/test/yarv_test.rb +++ b/test/yarv_test.rb @@ -386,35 +386,35 @@ def test_son node_16("0016 leave") node_1000("1000 ψ") node_1001("1001 φ") - node_0 --> |0| node_3 - linkStyle 0 stroke:green; - node_2 --> |1| node_3 - linkStyle 1 stroke:green; + node_0 -- "0" --> node_3 + node_2 -- "1" --> node_3 node_3 --> node_5 - linkStyle 2 stroke:red; - node_3 --> |0| node_5 - linkStyle 3 stroke:green; - node_5 --> |branch0| node_11 - linkStyle 4 stroke:red; - node_5 --> |fallthrough| node_1000 - linkStyle 5 stroke:red; - node_7 --> |0009| node_1001 - linkStyle 6 stroke:green; - node_11 --> |branch0| node_1000 - linkStyle 7 stroke:red; - node_11 --> |0011| node_1001 - linkStyle 8 stroke:green; - node_12 --> |1| node_14 - linkStyle 9 stroke:green; + node_3 -- "0" --> node_5 + node_5 -- "branch0" --> node_11 + node_5 -- "fallthrough" --> node_1000 + node_7 -- "0009" --> node_1001 + node_11 -- "branch0" --> node_1000 + node_11 -- "0011" --> node_1001 + node_12 -- "1" --> node_14 node_14 --> node_16 - linkStyle 10 stroke:red; - node_14 --> |0| node_16 - linkStyle 11 stroke:green; + node_14 -- "0" --> node_16 node_1000 --> node_14 - linkStyle 12 stroke:red; node_1001 -.-> node_1000 - node_1001 --> |0| node_14 - linkStyle 14 stroke:green; + node_1001 -- "0" --> node_14 + linkStyle 0 stroke:green + linkStyle 1 stroke:green + linkStyle 2 stroke:red + linkStyle 3 stroke:green + linkStyle 4 stroke:red + linkStyle 5 stroke:red + linkStyle 6 stroke:green + linkStyle 7 stroke:red + linkStyle 8 stroke:green + linkStyle 9 stroke:green + linkStyle 10 stroke:red + linkStyle 11 stroke:green + linkStyle 12 stroke:red + linkStyle 14 stroke:green MERMAID end @@ -438,35 +438,35 @@ def test_son_indirect_basic_block_argument node_16("0016 leave") node_1002("1002 ψ") node_1004("1004 φ") - node_0 --> |0| node_14 - linkStyle 0 stroke:green; - node_2 --> |0| node_5 - linkStyle 1 stroke:green; - node_4 --> |1| node_5 - linkStyle 2 stroke:green; + node_0 -- "0" --> node_14 + node_2 -- "0" --> node_5 + node_4 -- "1" --> node_5 node_5 --> node_7 - linkStyle 3 stroke:red; - node_5 --> |0| node_7 - linkStyle 4 stroke:green; - node_7 --> |branch0| node_13 - linkStyle 5 stroke:red; - node_7 --> |fallthrough| node_1002 - linkStyle 6 stroke:red; - node_9 --> |0011| node_1004 - linkStyle 7 stroke:green; - node_13 --> |branch0| node_1002 - linkStyle 8 stroke:red; - node_13 --> |0013| node_1004 - linkStyle 9 stroke:green; + node_5 -- "0" --> node_7 + node_7 -- "branch0" --> node_13 + node_7 -- "fallthrough" --> node_1002 + node_9 -- "0011" --> node_1004 + node_13 -- "branch0" --> node_1002 + node_13 -- "0013" --> node_1004 node_14 --> node_16 - linkStyle 10 stroke:red; - node_14 --> |0| node_16 - linkStyle 11 stroke:green; + node_14 -- "0" --> node_16 node_1002 --> node_14 - linkStyle 12 stroke:red; node_1004 -.-> node_1002 - node_1004 --> |1| node_14 - linkStyle 14 stroke:green; + node_1004 -- "1" --> node_14 + linkStyle 0 stroke:green + linkStyle 1 stroke:green + linkStyle 2 stroke:green + linkStyle 3 stroke:red + linkStyle 4 stroke:green + linkStyle 5 stroke:red + linkStyle 6 stroke:red + linkStyle 7 stroke:green + linkStyle 8 stroke:red + linkStyle 9 stroke:green + linkStyle 10 stroke:red + linkStyle 11 stroke:green + linkStyle 12 stroke:red + linkStyle 14 stroke:green MERMAID end From db06d7ebe75f4fb68202435c06f81a56c82526b3 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 10 Feb 2023 11:06:53 -0500 Subject: [PATCH 389/536] Start autoloading more things --- lib/syntax_tree.rb | 28 ++++++++-------------------- lib/syntax_tree/yarv.rb | 16 ++++++++++++++++ 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index 9cbd49c7..220389cb 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -9,7 +9,6 @@ require_relative "syntax_tree/formatter" require_relative "syntax_tree/node" -require_relative "syntax_tree/dsl" require_relative "syntax_tree/version" require_relative "syntax_tree/basic_visitor" @@ -23,29 +22,10 @@ require_relative "syntax_tree/visitor/environment" require_relative "syntax_tree/visitor/with_environment" -require_relative "syntax_tree/mermaid" require_relative "syntax_tree/parser" require_relative "syntax_tree/pattern" require_relative "syntax_tree/search" require_relative "syntax_tree/index" - -require_relative "syntax_tree/yarv" -require_relative "syntax_tree/yarv/basic_block" -require_relative "syntax_tree/yarv/bf" -require_relative "syntax_tree/yarv/calldata" -require_relative "syntax_tree/yarv/compiler" -require_relative "syntax_tree/yarv/control_flow_graph" -require_relative "syntax_tree/yarv/data_flow_graph" -require_relative "syntax_tree/yarv/decompiler" -require_relative "syntax_tree/yarv/disassembler" -require_relative "syntax_tree/yarv/instruction_sequence" -require_relative "syntax_tree/yarv/instructions" -require_relative "syntax_tree/yarv/legacy" -require_relative "syntax_tree/yarv/local_table" -require_relative "syntax_tree/yarv/sea_of_nodes" -require_relative "syntax_tree/yarv/assembler" -require_relative "syntax_tree/yarv/vm" - require_relative "syntax_tree/translation" # Syntax Tree is a suite of tools built on top of the internal CRuby parser. It @@ -53,6 +33,14 @@ # tools necessary to inspect and manipulate that syntax tree. It can be used to # build formatters, linters, language servers, and more. module SyntaxTree + # Syntax Tree the library has many features that aren't always used by the + # CLI. Requiring those features takes time, so we autoload as many constants + # as possible in order to keep the CLI as fast as possible. + + autoload :DSL, "syntax_tree/dsl" + autoload :Mermaid, "syntax_tree/mermaid" + autoload :YARV, "syntax_tree/yarv" + # This holds references to objects that respond to both #parse and #format # so that we can use them in the CLI. HANDLERS = {} diff --git a/lib/syntax_tree/yarv.rb b/lib/syntax_tree/yarv.rb index 7e4da7bb..ff8d3801 100644 --- a/lib/syntax_tree/yarv.rb +++ b/lib/syntax_tree/yarv.rb @@ -1,5 +1,21 @@ # frozen_string_literal: true +require_relative "yarv/basic_block" +require_relative "yarv/bf" +require_relative "yarv/calldata" +require_relative "yarv/compiler" +require_relative "yarv/control_flow_graph" +require_relative "yarv/data_flow_graph" +require_relative "yarv/decompiler" +require_relative "yarv/disassembler" +require_relative "yarv/instruction_sequence" +require_relative "yarv/instructions" +require_relative "yarv/legacy" +require_relative "yarv/local_table" +require_relative "yarv/sea_of_nodes" +require_relative "yarv/assembler" +require_relative "yarv/vm" + module SyntaxTree # This module provides an object representation of the YARV bytecode. module YARV From 0cf3e858b2dc3cee1af05a6ee3c0913d261727be Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 10 Feb 2023 11:26:23 -0500 Subject: [PATCH 390/536] Autoload a bunch of stuff --- README.md | 4 +- lib/syntax_tree.rb | 40 +- lib/syntax_tree/cli.rb | 4 +- lib/syntax_tree/field_visitor.rb | 1028 ++++++++++++++++ lib/syntax_tree/json_visitor.rb | 55 + lib/syntax_tree/language_server.rb | 157 ++- .../language_server/inlay_hints.rb | 159 --- lib/syntax_tree/match_visitor.rb | 120 ++ lib/syntax_tree/mermaid.rb | 1 + lib/syntax_tree/mermaid_visitor.rb | 73 ++ lib/syntax_tree/mutation_visitor.rb | 922 +++++++++++++++ lib/syntax_tree/node.rb | 8 +- lib/syntax_tree/pretty_print_visitor.rb | 83 ++ lib/syntax_tree/visitor/environment.rb | 84 -- lib/syntax_tree/visitor/field_visitor.rb | 1031 ----------------- lib/syntax_tree/visitor/json_visitor.rb | 55 - lib/syntax_tree/visitor/match_visitor.rb | 122 -- lib/syntax_tree/visitor/mermaid_visitor.rb | 75 -- lib/syntax_tree/visitor/mutation_visitor.rb | 924 --------------- .../visitor/pretty_print_visitor.rb | 85 -- .../{visitor => }/with_environment.rb | 81 ++ lib/syntax_tree/yarv.rb | 2 + lib/syntax_tree/yarv/compiler.rb | 2 +- test/test_helper.rb | 2 +- 24 files changed, 2549 insertions(+), 2568 deletions(-) create mode 100644 lib/syntax_tree/field_visitor.rb create mode 100644 lib/syntax_tree/json_visitor.rb delete mode 100644 lib/syntax_tree/language_server/inlay_hints.rb create mode 100644 lib/syntax_tree/match_visitor.rb create mode 100644 lib/syntax_tree/mermaid_visitor.rb create mode 100644 lib/syntax_tree/mutation_visitor.rb create mode 100644 lib/syntax_tree/pretty_print_visitor.rb delete mode 100644 lib/syntax_tree/visitor/environment.rb delete mode 100644 lib/syntax_tree/visitor/field_visitor.rb delete mode 100644 lib/syntax_tree/visitor/json_visitor.rb delete mode 100644 lib/syntax_tree/visitor/match_visitor.rb delete mode 100644 lib/syntax_tree/visitor/mermaid_visitor.rb delete mode 100644 lib/syntax_tree/visitor/mutation_visitor.rb delete mode 100644 lib/syntax_tree/visitor/pretty_print_visitor.rb rename lib/syntax_tree/{visitor => }/with_environment.rb (58%) diff --git a/README.md b/README.md index 6ca9b01a..5f447ad8 100644 --- a/README.md +++ b/README.md @@ -341,7 +341,7 @@ This function takes an input string containing Ruby code, parses it into its und ### SyntaxTree.mutation(&block) -This function yields a new mutation visitor to the block, and then returns the initialized visitor. It's effectively a shortcut for creating a `SyntaxTree::Visitor::MutationVisitor` without having to remember the class name. For more information on that visitor, see the definition below. +This function yields a new mutation visitor to the block, and then returns the initialized visitor. It's effectively a shortcut for creating a `SyntaxTree::MutationVisitor` without having to remember the class name. For more information on that visitor, see the definition below. ### SyntaxTree.search(source, query, &block) @@ -558,7 +558,7 @@ The `MutationVisitor` is a visitor that can be used to mutate the tree. It works ```ruby # Create a new visitor -visitor = SyntaxTree::Visitor::MutationVisitor.new +visitor = SyntaxTree::MutationVisitor.new # Specify that it should mutate If nodes with assignments in their predicates visitor.mutate("IfNode[predicate: Assign | OpAssign]") do |node| diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index 220389cb..0bdc4827 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -1,32 +1,15 @@ # frozen_string_literal: true -require "etc" -require "json" -require "pp" require "prettier_print" require "ripper" -require "stringio" -require_relative "syntax_tree/formatter" require_relative "syntax_tree/node" -require_relative "syntax_tree/version" - require_relative "syntax_tree/basic_visitor" require_relative "syntax_tree/visitor" -require_relative "syntax_tree/visitor/field_visitor" -require_relative "syntax_tree/visitor/json_visitor" -require_relative "syntax_tree/visitor/match_visitor" -require_relative "syntax_tree/visitor/mermaid_visitor" -require_relative "syntax_tree/visitor/mutation_visitor" -require_relative "syntax_tree/visitor/pretty_print_visitor" -require_relative "syntax_tree/visitor/environment" -require_relative "syntax_tree/visitor/with_environment" +require_relative "syntax_tree/formatter" require_relative "syntax_tree/parser" -require_relative "syntax_tree/pattern" -require_relative "syntax_tree/search" -require_relative "syntax_tree/index" -require_relative "syntax_tree/translation" +require_relative "syntax_tree/version" # Syntax Tree is a suite of tools built on top of the internal CRuby parser. It # provides the ability to generate a syntax tree from source, as well as the @@ -38,7 +21,19 @@ module SyntaxTree # as possible in order to keep the CLI as fast as possible. autoload :DSL, "syntax_tree/dsl" + autoload :FieldVisitor, "syntax_tree/field_visitor" + autoload :Index, "syntax_tree/index" + autoload :JSONVisitor, "syntax_tree/json_visitor" + autoload :LanguageServer, "syntax_tree/language_server" + autoload :MatchVisitor, "syntax_tree/match_visitor" autoload :Mermaid, "syntax_tree/mermaid" + autoload :MermaidVisitor, "syntax_tree/mermaid_visitor" + autoload :MutationVisitor, "syntax_tree/mutation_visitor" + autoload :Pattern, "syntax_tree/pattern" + autoload :PrettyPrintVisitor, "syntax_tree/pretty_print_visitor" + autoload :Search, "syntax_tree/search" + autoload :Translation, "syntax_tree/translation" + autoload :WithEnvironment, "syntax_tree/with_environment" autoload :YARV, "syntax_tree/yarv" # This holds references to objects that respond to both #parse and #format @@ -89,7 +84,7 @@ def self.index_file(filepath) # A convenience method for creating a new mutation visitor. def self.mutation - visitor = Visitor::MutationVisitor.new + visitor = MutationVisitor.new yield visitor visitor end @@ -130,6 +125,9 @@ def self.register_handler(extension, handler) # Searches through the given source using the given pattern and yields each # node in the tree that matches the pattern to the given block. def self.search(source, query, &block) - Search.new(Pattern.new(query).compile).scan(parse(source), &block) + pattern = Pattern.new(query).compile + program = parse(source) + + Search.new(pattern).scan(program, &block) end end diff --git a/lib/syntax_tree/cli.rb b/lib/syntax_tree/cli.rb index 7e6f4067..cbe10446 100644 --- a/lib/syntax_tree/cli.rb +++ b/lib/syntax_tree/cli.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +require "etc" require "optparse" module SyntaxTree @@ -238,7 +239,7 @@ def run(item) # representation. class Json < Action def run(item) - object = Visitor::JSONVisitor.new.visit(item.handler.parse(item.source)) + object = item.handler.parse(item.source).accept(JSONVisitor.new) puts JSON.pretty_generate(object) end end @@ -501,7 +502,6 @@ def run(argv) when "j", "json" Json.new(options) when "lsp" - require "syntax_tree/language_server" LanguageServer.new(print_width: options.print_width).run return 0 when "m", "match" diff --git a/lib/syntax_tree/field_visitor.rb b/lib/syntax_tree/field_visitor.rb new file mode 100644 index 00000000..f4fc00e3 --- /dev/null +++ b/lib/syntax_tree/field_visitor.rb @@ -0,0 +1,1028 @@ +# frozen_string_literal: true + +module SyntaxTree + # This is the parent class of a lot of built-in visitors for Syntax Tree. It + # reflects visiting each of the fields on every node in turn. It itself does + # not do anything with these fields, it leaves that behavior up to the + # subclass to implement. + # + # In order to properly use this class, you will need to subclass it and + # implement #comments, #field, #list, #node, #pairs, and #text. Those are + # documented here. + # + # == comments(node) + # + # This accepts the node that is being visited and does something depending on + # the comments attached to the node. + # + # == field(name, value) + # + # This accepts the name of the field being visited as a string (like "value") + # and the actual value of that field. The value can be a subclass of Node or + # any other type that can be held within the tree. + # + # == list(name, values) + # + # This accepts the name of the field being visited as well as a list of + # values. This is used, for example, when visiting something like the body of + # a Statements node. + # + # == node(name, node) + # + # This is the parent serialization method for each node. It is called with the + # node itself, as well as the type of the node as a string. The type is an + # internally used value that usually resembles the name of the ripper event + # that generated the node. The method should yield to the given block which + # then calls through to visit each of the fields on the node. + # + # == text(name, value) + # + # This accepts the name of the field being visited as well as a string value + # representing the value of the field. + # + # == pairs(name, values) + # + # This accepts the name of the field being visited as well as a list of pairs + # that represent the value of the field. It is used only in a couple of + # circumstances, like when visiting the list of optional parameters defined on + # a method. + # + class FieldVisitor < BasicVisitor + def visit_aref(node) + node(node, "aref") do + field("collection", node.collection) + field("index", node.index) + comments(node) + end + end + + def visit_aref_field(node) + node(node, "aref_field") do + field("collection", node.collection) + field("index", node.index) + comments(node) + end + end + + def visit_alias(node) + node(node, "alias") do + field("left", node.left) + field("right", node.right) + comments(node) + end + end + + def visit_arg_block(node) + node(node, "arg_block") do + field("value", node.value) if node.value + comments(node) + end + end + + def visit_arg_paren(node) + node(node, "arg_paren") do + field("arguments", node.arguments) + comments(node) + end + end + + def visit_arg_star(node) + node(node, "arg_star") do + field("value", node.value) + comments(node) + end + end + + def visit_args(node) + node(node, "args") do + list("parts", node.parts) + comments(node) + end + end + + def visit_args_forward(node) + node(node, "args_forward") { comments(node) } + end + + def visit_array(node) + node(node, "array") do + field("contents", node.contents) + comments(node) + end + end + + def visit_aryptn(node) + node(node, "aryptn") do + field("constant", node.constant) if node.constant + list("requireds", node.requireds) if node.requireds.any? + field("rest", node.rest) if node.rest + list("posts", node.posts) if node.posts.any? + comments(node) + end + end + + def visit_assign(node) + node(node, "assign") do + field("target", node.target) + field("value", node.value) + comments(node) + end + end + + def visit_assoc(node) + node(node, "assoc") do + field("key", node.key) + field("value", node.value) if node.value + comments(node) + end + end + + def visit_assoc_splat(node) + node(node, "assoc_splat") do + field("value", node.value) + comments(node) + end + end + + def visit_backref(node) + visit_token(node, "backref") + end + + def visit_backtick(node) + visit_token(node, "backtick") + end + + def visit_bare_assoc_hash(node) + node(node, "bare_assoc_hash") do + list("assocs", node.assocs) + comments(node) + end + end + + def visit_BEGIN(node) + node(node, "BEGIN") do + field("statements", node.statements) + comments(node) + end + end + + def visit_begin(node) + node(node, "begin") do + field("bodystmt", node.bodystmt) + comments(node) + end + end + + def visit_binary(node) + node(node, "binary") do + field("left", node.left) + text("operator", node.operator) + field("right", node.right) + comments(node) + end + end + + def visit_block(node) + node(node, "block") do + field("block_var", node.block_var) if node.block_var + field("bodystmt", node.bodystmt) + comments(node) + end + end + + def visit_blockarg(node) + node(node, "blockarg") do + field("name", node.name) if node.name + comments(node) + end + end + + def visit_block_var(node) + node(node, "block_var") do + field("params", node.params) + list("locals", node.locals) if node.locals.any? + comments(node) + end + end + + def visit_bodystmt(node) + node(node, "bodystmt") do + field("statements", node.statements) + field("rescue_clause", node.rescue_clause) if node.rescue_clause + field("else_clause", node.else_clause) if node.else_clause + field("ensure_clause", node.ensure_clause) if node.ensure_clause + comments(node) + end + end + + def visit_break(node) + node(node, "break") do + field("arguments", node.arguments) + comments(node) + end + end + + def visit_call(node) + node(node, "call") do + field("receiver", node.receiver) + field("operator", node.operator) + field("message", node.message) + field("arguments", node.arguments) if node.arguments + comments(node) + end + end + + def visit_case(node) + node(node, "case") do + field("keyword", node.keyword) + field("value", node.value) if node.value + field("consequent", node.consequent) + comments(node) + end + end + + def visit_CHAR(node) + visit_token(node, "CHAR") + end + + def visit_class(node) + node(node, "class") do + field("constant", node.constant) + field("superclass", node.superclass) if node.superclass + field("bodystmt", node.bodystmt) + comments(node) + end + end + + def visit_comma(node) + node(node, "comma") { field("value", node.value) } + end + + def visit_command(node) + node(node, "command") do + field("message", node.message) + field("arguments", node.arguments) + comments(node) + end + end + + def visit_command_call(node) + node(node, "command_call") do + field("receiver", node.receiver) + field("operator", node.operator) + field("message", node.message) + field("arguments", node.arguments) if node.arguments + comments(node) + end + end + + def visit_comment(node) + node(node, "comment") { field("value", node.value) } + end + + def visit_const(node) + visit_token(node, "const") + end + + def visit_const_path_field(node) + node(node, "const_path_field") do + field("parent", node.parent) + field("constant", node.constant) + comments(node) + end + end + + def visit_const_path_ref(node) + node(node, "const_path_ref") do + field("parent", node.parent) + field("constant", node.constant) + comments(node) + end + end + + def visit_const_ref(node) + node(node, "const_ref") do + field("constant", node.constant) + comments(node) + end + end + + def visit_cvar(node) + visit_token(node, "cvar") + end + + def visit_def(node) + node(node, "def") do + field("target", node.target) + field("operator", node.operator) + field("name", node.name) + field("params", node.params) + field("bodystmt", node.bodystmt) + comments(node) + end + end + + def visit_defined(node) + node(node, "defined") do + field("value", node.value) + comments(node) + end + end + + def visit_dyna_symbol(node) + node(node, "dyna_symbol") do + list("parts", node.parts) + comments(node) + end + end + + def visit_END(node) + node(node, "END") do + field("statements", node.statements) + comments(node) + end + end + + def visit_else(node) + node(node, "else") do + field("statements", node.statements) + comments(node) + end + end + + def visit_elsif(node) + node(node, "elsif") do + field("predicate", node.predicate) + field("statements", node.statements) + field("consequent", node.consequent) if node.consequent + comments(node) + end + end + + def visit_embdoc(node) + node(node, "embdoc") { field("value", node.value) } + end + + def visit_embexpr_beg(node) + node(node, "embexpr_beg") { field("value", node.value) } + end + + def visit_embexpr_end(node) + node(node, "embexpr_end") { field("value", node.value) } + end + + def visit_embvar(node) + node(node, "embvar") { field("value", node.value) } + end + + def visit_ensure(node) + node(node, "ensure") do + field("statements", node.statements) + comments(node) + end + end + + def visit_excessed_comma(node) + visit_token(node, "excessed_comma") + end + + def visit_field(node) + node(node, "field") do + field("parent", node.parent) + field("operator", node.operator) + field("name", node.name) + comments(node) + end + end + + def visit_float(node) + visit_token(node, "float") + end + + def visit_fndptn(node) + node(node, "fndptn") do + field("constant", node.constant) if node.constant + field("left", node.left) + list("values", node.values) + field("right", node.right) + comments(node) + end + end + + def visit_for(node) + node(node, "for") do + field("index", node.index) + field("collection", node.collection) + field("statements", node.statements) + comments(node) + end + end + + def visit_gvar(node) + visit_token(node, "gvar") + end + + def visit_hash(node) + node(node, "hash") do + list("assocs", node.assocs) if node.assocs.any? + comments(node) + end + end + + def visit_heredoc(node) + node(node, "heredoc") do + list("parts", node.parts) + comments(node) + end + end + + def visit_heredoc_beg(node) + visit_token(node, "heredoc_beg") + end + + def visit_heredoc_end(node) + visit_token(node, "heredoc_end") + end + + def visit_hshptn(node) + node(node, "hshptn") do + field("constant", node.constant) if node.constant + pairs("keywords", node.keywords) if node.keywords.any? + field("keyword_rest", node.keyword_rest) if node.keyword_rest + comments(node) + end + end + + def visit_ident(node) + visit_token(node, "ident") + end + + def visit_if(node) + node(node, "if") do + field("predicate", node.predicate) + field("statements", node.statements) + field("consequent", node.consequent) if node.consequent + comments(node) + end + end + + def visit_if_op(node) + node(node, "if_op") do + field("predicate", node.predicate) + field("truthy", node.truthy) + field("falsy", node.falsy) + comments(node) + end + end + + def visit_imaginary(node) + visit_token(node, "imaginary") + end + + def visit_in(node) + node(node, "in") do + field("pattern", node.pattern) + field("statements", node.statements) + field("consequent", node.consequent) if node.consequent + comments(node) + end + end + + def visit_int(node) + visit_token(node, "int") + end + + def visit_ivar(node) + visit_token(node, "ivar") + end + + def visit_kw(node) + visit_token(node, "kw") + end + + def visit_kwrest_param(node) + node(node, "kwrest_param") do + field("name", node.name) + comments(node) + end + end + + def visit_label(node) + visit_token(node, "label") + end + + def visit_label_end(node) + node(node, "label_end") { field("value", node.value) } + end + + def visit_lambda(node) + node(node, "lambda") do + field("params", node.params) + field("statements", node.statements) + comments(node) + end + end + + def visit_lambda_var(node) + node(node, "lambda_var") do + field("params", node.params) + list("locals", node.locals) if node.locals.any? + comments(node) + end + end + + def visit_lbrace(node) + visit_token(node, "lbrace") + end + + def visit_lbracket(node) + visit_token(node, "lbracket") + end + + def visit_lparen(node) + visit_token(node, "lparen") + end + + def visit_massign(node) + node(node, "massign") do + field("target", node.target) + field("value", node.value) + comments(node) + end + end + + def visit_method_add_block(node) + node(node, "method_add_block") do + field("call", node.call) + field("block", node.block) + comments(node) + end + end + + def visit_mlhs(node) + node(node, "mlhs") do + list("parts", node.parts) + comments(node) + end + end + + def visit_mlhs_paren(node) + node(node, "mlhs_paren") do + field("contents", node.contents) + comments(node) + end + end + + def visit_module(node) + node(node, "module") do + field("constant", node.constant) + field("bodystmt", node.bodystmt) + comments(node) + end + end + + def visit_mrhs(node) + node(node, "mrhs") do + list("parts", node.parts) + comments(node) + end + end + + def visit_next(node) + node(node, "next") do + field("arguments", node.arguments) + comments(node) + end + end + + def visit_not(node) + node(node, "not") do + field("statement", node.statement) + comments(node) + end + end + + def visit_op(node) + visit_token(node, "op") + end + + def visit_opassign(node) + node(node, "opassign") do + field("target", node.target) + field("operator", node.operator) + field("value", node.value) + comments(node) + end + end + + def visit_params(node) + node(node, "params") do + list("requireds", node.requireds) if node.requireds.any? + pairs("optionals", node.optionals) if node.optionals.any? + field("rest", node.rest) if node.rest + list("posts", node.posts) if node.posts.any? + pairs("keywords", node.keywords) if node.keywords.any? + field("keyword_rest", node.keyword_rest) if node.keyword_rest + field("block", node.block) if node.block + comments(node) + end + end + + def visit_paren(node) + node(node, "paren") do + field("contents", node.contents) + comments(node) + end + end + + def visit_period(node) + visit_token(node, "period") + end + + def visit_pinned_begin(node) + node(node, "pinned_begin") do + field("statement", node.statement) + comments(node) + end + end + + def visit_pinned_var_ref(node) + node(node, "pinned_var_ref") do + field("value", node.value) + comments(node) + end + end + + def visit_program(node) + node(node, "program") do + field("statements", node.statements) + comments(node) + end + end + + def visit_qsymbols(node) + node(node, "qsymbols") do + list("elements", node.elements) + comments(node) + end + end + + def visit_qsymbols_beg(node) + node(node, "qsymbols_beg") { field("value", node.value) } + end + + def visit_qwords(node) + node(node, "qwords") do + list("elements", node.elements) + comments(node) + end + end + + def visit_qwords_beg(node) + node(node, "qwords_beg") { field("value", node.value) } + end + + def visit_range(node) + node(node, "range") do + field("left", node.left) if node.left + field("operator", node.operator) + field("right", node.right) if node.right + comments(node) + end + end + + def visit_rassign(node) + node(node, "rassign") do + field("value", node.value) + field("operator", node.operator) + field("pattern", node.pattern) + comments(node) + end + end + + def visit_rational(node) + visit_token(node, "rational") + end + + def visit_rbrace(node) + node(node, "rbrace") { field("value", node.value) } + end + + def visit_rbracket(node) + node(node, "rbracket") { field("value", node.value) } + end + + def visit_redo(node) + node(node, "redo") { comments(node) } + end + + def visit_regexp_beg(node) + node(node, "regexp_beg") { field("value", node.value) } + end + + def visit_regexp_content(node) + node(node, "regexp_content") { list("parts", node.parts) } + end + + def visit_regexp_end(node) + node(node, "regexp_end") { field("value", node.value) } + end + + def visit_regexp_literal(node) + node(node, "regexp_literal") do + list("parts", node.parts) + field("options", node.options) + comments(node) + end + end + + def visit_rescue(node) + node(node, "rescue") do + field("exception", node.exception) if node.exception + field("statements", node.statements) + field("consequent", node.consequent) if node.consequent + comments(node) + end + end + + def visit_rescue_ex(node) + node(node, "rescue_ex") do + field("exceptions", node.exceptions) + field("variable", node.variable) + comments(node) + end + end + + def visit_rescue_mod(node) + node(node, "rescue_mod") do + field("statement", node.statement) + field("value", node.value) + comments(node) + end + end + + def visit_rest_param(node) + node(node, "rest_param") do + field("name", node.name) + comments(node) + end + end + + def visit_retry(node) + node(node, "retry") { comments(node) } + end + + def visit_return(node) + node(node, "return") do + field("arguments", node.arguments) + comments(node) + end + end + + def visit_rparen(node) + node(node, "rparen") { field("value", node.value) } + end + + def visit_sclass(node) + node(node, "sclass") do + field("target", node.target) + field("bodystmt", node.bodystmt) + comments(node) + end + end + + def visit_statements(node) + node(node, "statements") do + list("body", node.body) + comments(node) + end + end + + def visit_string_concat(node) + node(node, "string_concat") do + field("left", node.left) + field("right", node.right) + comments(node) + end + end + + def visit_string_content(node) + node(node, "string_content") { list("parts", node.parts) } + end + + def visit_string_dvar(node) + node(node, "string_dvar") do + field("variable", node.variable) + comments(node) + end + end + + def visit_string_embexpr(node) + node(node, "string_embexpr") do + field("statements", node.statements) + comments(node) + end + end + + def visit_string_literal(node) + node(node, "string_literal") do + list("parts", node.parts) + comments(node) + end + end + + def visit_super(node) + node(node, "super") do + field("arguments", node.arguments) + comments(node) + end + end + + def visit_symbeg(node) + node(node, "symbeg") { field("value", node.value) } + end + + def visit_symbol_content(node) + node(node, "symbol_content") { field("value", node.value) } + end + + def visit_symbol_literal(node) + node(node, "symbol_literal") do + field("value", node.value) + comments(node) + end + end + + def visit_symbols(node) + node(node, "symbols") do + list("elements", node.elements) + comments(node) + end + end + + def visit_symbols_beg(node) + node(node, "symbols_beg") { field("value", node.value) } + end + + def visit_tlambda(node) + node(node, "tlambda") { field("value", node.value) } + end + + def visit_tlambeg(node) + node(node, "tlambeg") { field("value", node.value) } + end + + def visit_top_const_field(node) + node(node, "top_const_field") do + field("constant", node.constant) + comments(node) + end + end + + def visit_top_const_ref(node) + node(node, "top_const_ref") do + field("constant", node.constant) + comments(node) + end + end + + def visit_tstring_beg(node) + node(node, "tstring_beg") { field("value", node.value) } + end + + def visit_tstring_content(node) + visit_token(node, "tstring_content") + end + + def visit_tstring_end(node) + node(node, "tstring_end") { field("value", node.value) } + end + + def visit_unary(node) + node(node, "unary") do + field("operator", node.operator) + field("statement", node.statement) + comments(node) + end + end + + def visit_undef(node) + node(node, "undef") do + list("symbols", node.symbols) + comments(node) + end + end + + def visit_unless(node) + node(node, "unless") do + field("predicate", node.predicate) + field("statements", node.statements) + field("consequent", node.consequent) if node.consequent + comments(node) + end + end + + def visit_until(node) + node(node, "until") do + field("predicate", node.predicate) + field("statements", node.statements) + comments(node) + end + end + + def visit_var_field(node) + node(node, "var_field") do + field("value", node.value) + comments(node) + end + end + + def visit_var_ref(node) + node(node, "var_ref") do + field("value", node.value) + comments(node) + end + end + + def visit_vcall(node) + node(node, "vcall") do + field("value", node.value) + comments(node) + end + end + + def visit_void_stmt(node) + node(node, "void_stmt") { comments(node) } + end + + def visit_when(node) + node(node, "when") do + field("arguments", node.arguments) + field("statements", node.statements) + field("consequent", node.consequent) if node.consequent + comments(node) + end + end + + def visit_while(node) + node(node, "while") do + field("predicate", node.predicate) + field("statements", node.statements) + comments(node) + end + end + + def visit_word(node) + node(node, "word") do + list("parts", node.parts) + comments(node) + end + end + + def visit_words(node) + node(node, "words") do + list("elements", node.elements) + comments(node) + end + end + + def visit_words_beg(node) + node(node, "words_beg") { field("value", node.value) } + end + + def visit_xstring(node) + node(node, "xstring") { list("parts", node.parts) } + end + + def visit_xstring_literal(node) + node(node, "xstring_literal") do + list("parts", node.parts) + comments(node) + end + end + + def visit_yield(node) + node(node, "yield") do + field("arguments", node.arguments) + comments(node) + end + end + + def visit_zsuper(node) + node(node, "zsuper") { comments(node) } + end + + def visit___end__(node) + visit_token(node, "__end__") + end + + private + + def visit_token(node, type) + node(node, type) do + field("value", node.value) + comments(node) + end + end + end +end diff --git a/lib/syntax_tree/json_visitor.rb b/lib/syntax_tree/json_visitor.rb new file mode 100644 index 00000000..7ad3fba0 --- /dev/null +++ b/lib/syntax_tree/json_visitor.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +require "json" + +module SyntaxTree + # This visitor transforms the AST into a hash that contains only primitives + # that can be easily serialized into JSON. + class JSONVisitor < FieldVisitor + attr_reader :target + + def initialize + @target = nil + end + + private + + def comments(node) + target[:comments] = visit_all(node.comments) + end + + def field(name, value) + target[name] = value.is_a?(Node) ? visit(value) : value + end + + def list(name, values) + target[name] = visit_all(values) + end + + def node(node, type) + previous = @target + @target = { type: type, location: visit_location(node.location) } + yield + @target + ensure + @target = previous + end + + def pairs(name, values) + target[name] = values.map { |(key, value)| [visit(key), visit(value)] } + end + + def text(name, value) + target[name] = value + end + + def visit_location(location) + [ + location.start_line, + location.start_char, + location.end_line, + location.end_char + ] + end + end +end diff --git a/lib/syntax_tree/language_server.rb b/lib/syntax_tree/language_server.rb index a7b23664..afb1540e 100644 --- a/lib/syntax_tree/language_server.rb +++ b/lib/syntax_tree/language_server.rb @@ -2,10 +2,9 @@ require "cgi" require "json" +require "pp" require "uri" -require_relative "language_server/inlay_hints" - module SyntaxTree # Syntax Tree additionally ships with a language server conforming to the # language server protocol. It can be invoked through the CLI by running: @@ -13,6 +12,160 @@ module SyntaxTree # stree lsp # class LanguageServer + # This class provides inlay hints for the language server. For more + # information, see the spec here: + # https://github.com/microsoft/language-server-protocol/issues/956. + class InlayHints < Visitor + # This represents a hint that is going to be displayed in the editor. + class Hint + attr_reader :line, :character, :label + + def initialize(line:, character:, label:) + @line = line + @character = character + @label = label + end + + # This is the shape that the LSP expects. + def to_json(*opts) + { + position: { + line: line, + character: character + }, + label: label + }.to_json(*opts) + end + end + + attr_reader :stack, :hints + + def initialize + @stack = [] + @hints = [] + end + + def visit(node) + stack << node + result = super + stack.pop + result + end + + # Adds parentheses around assignments contained within the default values + # of parameters. For example, + # + # def foo(a = b = c) + # end + # + # becomes + # + # def foo(a = ₍b = c₎) + # end + # + def visit_assign(node) + parentheses(node.location) if stack[-2].is_a?(Params) + super + end + + # Adds parentheses around binary expressions to make it clear which + # subexpression will be evaluated first. For example, + # + # a + b * c + # + # becomes + # + # a + ₍b * c₎ + # + def visit_binary(node) + case stack[-2] + when Assign, OpAssign + parentheses(node.location) + when Binary + parentheses(node.location) if stack[-2].operator != node.operator + end + + super + end + + # Adds parentheses around ternary operators contained within certain + # expressions where it could be confusing which subexpression will get + # evaluated first. For example, + # + # a ? b : c ? d : e + # + # becomes + # + # a ? b : ₍c ? d : e₎ + # + def visit_if_op(node) + case stack[-2] + when Assign, Binary, IfOp, OpAssign + parentheses(node.location) + end + + super + end + + # Adds the implicitly rescued StandardError into a bare rescue clause. For + # example, + # + # begin + # rescue + # end + # + # becomes + # + # begin + # rescue StandardError + # end + # + def visit_rescue(node) + if node.exception.nil? + hints << Hint.new( + line: node.location.start_line - 1, + character: node.location.start_column + "rescue".length, + label: " StandardError" + ) + end + + super + end + + # Adds parentheses around unary statements using the - operator that are + # contained within Binary nodes. For example, + # + # -a + b + # + # becomes + # + # ₍-a₎ + b + # + def visit_unary(node) + if stack[-2].is_a?(Binary) && (node.operator == "-") + parentheses(node.location) + end + + super + end + + private + + def parentheses(location) + hints << Hint.new( + line: location.start_line - 1, + character: location.start_column, + label: "₍" + ) + + hints << Hint.new( + line: location.end_line - 1, + character: location.end_column, + label: "₎" + ) + end + end + # This is a small module that effectively mirrors pattern matching. We're # using it so that we can support truffleruby without having to ignore the # language server. diff --git a/lib/syntax_tree/language_server/inlay_hints.rb b/lib/syntax_tree/language_server/inlay_hints.rb deleted file mode 100644 index dfd63b8d..00000000 --- a/lib/syntax_tree/language_server/inlay_hints.rb +++ /dev/null @@ -1,159 +0,0 @@ -# frozen_string_literal: true - -module SyntaxTree - class LanguageServer - # This class provides inlay hints for the language server. For more - # information, see the spec here: - # https://github.com/microsoft/language-server-protocol/issues/956. - class InlayHints < Visitor - # This represents a hint that is going to be displayed in the editor. - class Hint - attr_reader :line, :character, :label - - def initialize(line:, character:, label:) - @line = line - @character = character - @label = label - end - - # This is the shape that the LSP expects. - def to_json(*opts) - { - position: { - line: line, - character: character - }, - label: label - }.to_json(*opts) - end - end - - attr_reader :stack, :hints - - def initialize - @stack = [] - @hints = [] - end - - def visit(node) - stack << node - result = super - stack.pop - result - end - - # Adds parentheses around assignments contained within the default values - # of parameters. For example, - # - # def foo(a = b = c) - # end - # - # becomes - # - # def foo(a = ₍b = c₎) - # end - # - def visit_assign(node) - parentheses(node.location) if stack[-2].is_a?(Params) - super - end - - # Adds parentheses around binary expressions to make it clear which - # subexpression will be evaluated first. For example, - # - # a + b * c - # - # becomes - # - # a + ₍b * c₎ - # - def visit_binary(node) - case stack[-2] - when Assign, OpAssign - parentheses(node.location) - when Binary - parentheses(node.location) if stack[-2].operator != node.operator - end - - super - end - - # Adds parentheses around ternary operators contained within certain - # expressions where it could be confusing which subexpression will get - # evaluated first. For example, - # - # a ? b : c ? d : e - # - # becomes - # - # a ? b : ₍c ? d : e₎ - # - def visit_if_op(node) - case stack[-2] - when Assign, Binary, IfOp, OpAssign - parentheses(node.location) - end - - super - end - - # Adds the implicitly rescued StandardError into a bare rescue clause. For - # example, - # - # begin - # rescue - # end - # - # becomes - # - # begin - # rescue StandardError - # end - # - def visit_rescue(node) - if node.exception.nil? - hints << Hint.new( - line: node.location.start_line - 1, - character: node.location.start_column + "rescue".length, - label: " StandardError" - ) - end - - super - end - - # Adds parentheses around unary statements using the - operator that are - # contained within Binary nodes. For example, - # - # -a + b - # - # becomes - # - # ₍-a₎ + b - # - def visit_unary(node) - if stack[-2].is_a?(Binary) && (node.operator == "-") - parentheses(node.location) - end - - super - end - - private - - def parentheses(location) - hints << Hint.new( - line: location.start_line - 1, - character: location.start_column, - label: "₍" - ) - - hints << Hint.new( - line: location.end_line - 1, - character: location.end_column, - label: "₎" - ) - end - end - end -end diff --git a/lib/syntax_tree/match_visitor.rb b/lib/syntax_tree/match_visitor.rb new file mode 100644 index 00000000..ca5bf234 --- /dev/null +++ b/lib/syntax_tree/match_visitor.rb @@ -0,0 +1,120 @@ +# frozen_string_literal: true + +module SyntaxTree + # This visitor transforms the AST into a Ruby pattern matching expression that + # would match correctly against the AST. + class MatchVisitor < FieldVisitor + attr_reader :q + + def initialize(q) + @q = q + end + + def visit(node) + case node + when Node + super + when String + # pp will split up a string on newlines and concat them together using a + # "+" operator. This breaks the pattern matching expression. So instead + # we're going to check here for strings and manually put the entire + # value into the output buffer. + q.text(node.inspect) + else + node.pretty_print(q) + end + end + + private + + def comments(node) + return if node.comments.empty? + + q.nest(0) do + q.text("comments: [") + q.indent do + q.breakable("") + q.seplist(node.comments) { |comment| visit(comment) } + end + q.breakable("") + q.text("]") + end + end + + def field(name, value) + q.nest(0) do + q.text(name) + q.text(": ") + visit(value) + end + end + + def list(name, values) + q.group do + q.text(name) + q.text(": [") + q.indent do + q.breakable("") + q.seplist(values) { |value| visit(value) } + end + q.breakable("") + q.text("]") + end + end + + def node(node, _type) + items = [] + q.with_target(items) { yield } + + if items.empty? + q.text(node.class.name) + return + end + + q.group do + q.text(node.class.name) + q.text("[") + q.indent do + q.breakable("") + q.seplist(items) { |item| q.target << item } + end + q.breakable("") + q.text("]") + end + end + + def pairs(name, values) + q.group do + q.text(name) + q.text(": [") + q.indent do + q.breakable("") + q.seplist(values) do |(key, value)| + q.group do + q.text("[") + q.indent do + q.breakable("") + visit(key) + q.text(",") + q.breakable + visit(value || nil) + end + q.breakable("") + q.text("]") + end + end + end + q.breakable("") + q.text("]") + end + end + + def text(name, value) + q.nest(0) do + q.text(name) + q.text(": ") + value.pretty_print(q) + end + end + end +end diff --git a/lib/syntax_tree/mermaid.rb b/lib/syntax_tree/mermaid.rb index 70cbc054..68ea4734 100644 --- a/lib/syntax_tree/mermaid.rb +++ b/lib/syntax_tree/mermaid.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require "cgi" +require "stringio" module SyntaxTree # This module is responsible for rendering mermaid (https://mermaid.js.org/) diff --git a/lib/syntax_tree/mermaid_visitor.rb b/lib/syntax_tree/mermaid_visitor.rb new file mode 100644 index 00000000..52d1b5c6 --- /dev/null +++ b/lib/syntax_tree/mermaid_visitor.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +module SyntaxTree + # This visitor transforms the AST into a mermaid flow chart. + class MermaidVisitor < FieldVisitor + attr_reader :flowchart, :target + + def initialize + @flowchart = Mermaid.flowchart + @target = nil + end + + def visit_program(node) + super + flowchart.render + end + + private + + def comments(node) + # Ignore + end + + def field(name, value) + case value + when nil + # skip + when Node + flowchart.link(target, visit(value), name) + else + to = + flowchart.node( + "#{target.id}_#{name}", + value.inspect, + shape: :stadium + ) + flowchart.link(target, to, name) + end + end + + def list(name, values) + values.each_with_index do |value, index| + field("#{name}[#{index}]", value) + end + end + + def node(node, type) + previous_target = target + + begin + @target = flowchart.node("node_#{node.object_id}", type) + yield + @target + ensure + @target = previous_target + end + end + + def pairs(name, values) + values.each_with_index do |(key, value), index| + to = flowchart.node("#{target.id}_#{name}_#{index}", shape: :circle) + + flowchart.link(target, to, "#{name}[#{index}]") + flowchart.link(to, visit(key), "[0]") + flowchart.link(to, visit(value), "[1]") if value + end + end + + def text(name, value) + field(name, value) + end + end +end diff --git a/lib/syntax_tree/mutation_visitor.rb b/lib/syntax_tree/mutation_visitor.rb new file mode 100644 index 00000000..2d96620d --- /dev/null +++ b/lib/syntax_tree/mutation_visitor.rb @@ -0,0 +1,922 @@ +# frozen_string_literal: true + +module SyntaxTree + # This visitor walks through the tree and copies each node as it is being + # visited. This is useful for mutating the tree before it is formatted. + class MutationVisitor < BasicVisitor + attr_reader :mutations + + def initialize + @mutations = [] + end + + # Create a new mutation based on the given query that will mutate the node + # using the given block. The block should return a new node that will take + # the place of the given node in the tree. These blocks frequently make use + # of the `copy` method on nodes to create a new node with the same + # properties as the original node. + def mutate(query, &block) + mutations << [Pattern.new(query).compile, block] + end + + # This is the base visit method for each node in the tree. It first creates + # a copy of the node using the visit_* methods defined below. Then it checks + # each mutation in sequence and calls it if it finds a match. + def visit(node) + return unless node + result = node.accept(self) + + mutations.each do |(pattern, mutation)| + result = mutation.call(result) if pattern.call(result) + end + + result + end + + # Visit a BEGINBlock node. + def visit_BEGIN(node) + node.copy( + lbrace: visit(node.lbrace), + statements: visit(node.statements) + ) + end + + # Visit a CHAR node. + def visit_CHAR(node) + node.copy + end + + # Visit a ENDBlock node. + def visit_END(node) + node.copy( + lbrace: visit(node.lbrace), + statements: visit(node.statements) + ) + end + + # Visit a EndContent node. + def visit___end__(node) + node.copy + end + + # Visit a AliasNode node. + def visit_alias(node) + node.copy(left: visit(node.left), right: visit(node.right)) + end + + # Visit a ARef node. + def visit_aref(node) + node.copy(index: visit(node.index)) + end + + # Visit a ARefField node. + def visit_aref_field(node) + node.copy(index: visit(node.index)) + end + + # Visit a ArgParen node. + def visit_arg_paren(node) + node.copy(arguments: visit(node.arguments)) + end + + # Visit a Args node. + def visit_args(node) + node.copy(parts: visit_all(node.parts)) + end + + # Visit a ArgBlock node. + def visit_arg_block(node) + node.copy(value: visit(node.value)) + end + + # Visit a ArgStar node. + def visit_arg_star(node) + node.copy(value: visit(node.value)) + end + + # Visit a ArgsForward node. + def visit_args_forward(node) + node.copy + end + + # Visit a ArrayLiteral node. + def visit_array(node) + node.copy( + lbracket: visit(node.lbracket), + contents: visit(node.contents) + ) + end + + # Visit a AryPtn node. + def visit_aryptn(node) + node.copy( + constant: visit(node.constant), + requireds: visit_all(node.requireds), + rest: visit(node.rest), + posts: visit_all(node.posts) + ) + end + + # Visit a Assign node. + def visit_assign(node) + node.copy(target: visit(node.target)) + end + + # Visit a Assoc node. + def visit_assoc(node) + node.copy + end + + # Visit a AssocSplat node. + def visit_assoc_splat(node) + node.copy + end + + # Visit a Backref node. + def visit_backref(node) + node.copy + end + + # Visit a Backtick node. + def visit_backtick(node) + node.copy + end + + # Visit a BareAssocHash node. + def visit_bare_assoc_hash(node) + node.copy(assocs: visit_all(node.assocs)) + end + + # Visit a Begin node. + def visit_begin(node) + node.copy(bodystmt: visit(node.bodystmt)) + end + + # Visit a PinnedBegin node. + def visit_pinned_begin(node) + node.copy + end + + # Visit a Binary node. + def visit_binary(node) + node.copy + end + + # Visit a BlockVar node. + def visit_block_var(node) + node.copy(params: visit(node.params), locals: visit_all(node.locals)) + end + + # Visit a BlockArg node. + def visit_blockarg(node) + node.copy(name: visit(node.name)) + end + + # Visit a BodyStmt node. + def visit_bodystmt(node) + node.copy( + statements: visit(node.statements), + rescue_clause: visit(node.rescue_clause), + else_clause: visit(node.else_clause), + ensure_clause: visit(node.ensure_clause) + ) + end + + # Visit a Break node. + def visit_break(node) + node.copy(arguments: visit(node.arguments)) + end + + # Visit a Call node. + def visit_call(node) + node.copy( + receiver: visit(node.receiver), + operator: node.operator == :"::" ? :"::" : visit(node.operator), + message: node.message == :call ? :call : visit(node.message), + arguments: visit(node.arguments) + ) + end + + # Visit a Case node. + def visit_case(node) + node.copy( + keyword: visit(node.keyword), + value: visit(node.value), + consequent: visit(node.consequent) + ) + end + + # Visit a RAssign node. + def visit_rassign(node) + node.copy(operator: visit(node.operator)) + end + + # Visit a ClassDeclaration node. + def visit_class(node) + node.copy( + constant: visit(node.constant), + superclass: visit(node.superclass), + bodystmt: visit(node.bodystmt) + ) + end + + # Visit a Comma node. + def visit_comma(node) + node.copy + end + + # Visit a Command node. + def visit_command(node) + node.copy( + message: visit(node.message), + arguments: visit(node.arguments), + block: visit(node.block) + ) + end + + # Visit a CommandCall node. + def visit_command_call(node) + node.copy( + operator: node.operator == :"::" ? :"::" : visit(node.operator), + message: visit(node.message), + arguments: visit(node.arguments), + block: visit(node.block) + ) + end + + # Visit a Comment node. + def visit_comment(node) + node.copy + end + + # Visit a Const node. + def visit_const(node) + node.copy + end + + # Visit a ConstPathField node. + def visit_const_path_field(node) + node.copy(constant: visit(node.constant)) + end + + # Visit a ConstPathRef node. + def visit_const_path_ref(node) + node.copy(constant: visit(node.constant)) + end + + # Visit a ConstRef node. + def visit_const_ref(node) + node.copy(constant: visit(node.constant)) + end + + # Visit a CVar node. + def visit_cvar(node) + node.copy + end + + # Visit a Def node. + def visit_def(node) + node.copy( + target: visit(node.target), + operator: visit(node.operator), + name: visit(node.name), + params: visit(node.params), + bodystmt: visit(node.bodystmt) + ) + end + + # Visit a Defined node. + def visit_defined(node) + node.copy + end + + # Visit a Block node. + def visit_block(node) + node.copy( + opening: visit(node.opening), + block_var: visit(node.block_var), + bodystmt: visit(node.bodystmt) + ) + end + + # Visit a RangeNode node. + def visit_range(node) + node.copy( + left: visit(node.left), + operator: visit(node.operator), + right: visit(node.right) + ) + end + + # Visit a DynaSymbol node. + def visit_dyna_symbol(node) + node.copy(parts: visit_all(node.parts)) + end + + # Visit a Else node. + def visit_else(node) + node.copy( + keyword: visit(node.keyword), + statements: visit(node.statements) + ) + end + + # Visit a Elsif node. + def visit_elsif(node) + node.copy( + statements: visit(node.statements), + consequent: visit(node.consequent) + ) + end + + # Visit a EmbDoc node. + def visit_embdoc(node) + node.copy + end + + # Visit a EmbExprBeg node. + def visit_embexpr_beg(node) + node.copy + end + + # Visit a EmbExprEnd node. + def visit_embexpr_end(node) + node.copy + end + + # Visit a EmbVar node. + def visit_embvar(node) + node.copy + end + + # Visit a Ensure node. + def visit_ensure(node) + node.copy( + keyword: visit(node.keyword), + statements: visit(node.statements) + ) + end + + # Visit a ExcessedComma node. + def visit_excessed_comma(node) + node.copy + end + + # Visit a Field node. + def visit_field(node) + node.copy( + operator: node.operator == :"::" ? :"::" : visit(node.operator), + name: visit(node.name) + ) + end + + # Visit a FloatLiteral node. + def visit_float(node) + node.copy + end + + # Visit a FndPtn node. + def visit_fndptn(node) + node.copy( + constant: visit(node.constant), + left: visit(node.left), + values: visit_all(node.values), + right: visit(node.right) + ) + end + + # Visit a For node. + def visit_for(node) + node.copy(index: visit(node.index), statements: visit(node.statements)) + end + + # Visit a GVar node. + def visit_gvar(node) + node.copy + end + + # Visit a HashLiteral node. + def visit_hash(node) + node.copy(lbrace: visit(node.lbrace), assocs: visit_all(node.assocs)) + end + + # Visit a Heredoc node. + def visit_heredoc(node) + node.copy( + beginning: visit(node.beginning), + ending: visit(node.ending), + parts: visit_all(node.parts) + ) + end + + # Visit a HeredocBeg node. + def visit_heredoc_beg(node) + node.copy + end + + # Visit a HeredocEnd node. + def visit_heredoc_end(node) + node.copy + end + + # Visit a HshPtn node. + def visit_hshptn(node) + node.copy( + constant: visit(node.constant), + keywords: + node.keywords.map { |label, value| [visit(label), visit(value)] }, + keyword_rest: visit(node.keyword_rest) + ) + end + + # Visit a Ident node. + def visit_ident(node) + node.copy + end + + # Visit a IfNode node. + def visit_if(node) + node.copy( + predicate: visit(node.predicate), + statements: visit(node.statements), + consequent: visit(node.consequent) + ) + end + + # Visit a IfOp node. + def visit_if_op(node) + node.copy + end + + # Visit a Imaginary node. + def visit_imaginary(node) + node.copy + end + + # Visit a In node. + def visit_in(node) + node.copy( + statements: visit(node.statements), + consequent: visit(node.consequent) + ) + end + + # Visit a Int node. + def visit_int(node) + node.copy + end + + # Visit a IVar node. + def visit_ivar(node) + node.copy + end + + # Visit a Kw node. + def visit_kw(node) + node.copy + end + + # Visit a KwRestParam node. + def visit_kwrest_param(node) + node.copy(name: visit(node.name)) + end + + # Visit a Label node. + def visit_label(node) + node.copy + end + + # Visit a LabelEnd node. + def visit_label_end(node) + node.copy + end + + # Visit a Lambda node. + def visit_lambda(node) + node.copy( + params: visit(node.params), + statements: visit(node.statements) + ) + end + + # Visit a LambdaVar node. + def visit_lambda_var(node) + node.copy(params: visit(node.params), locals: visit_all(node.locals)) + end + + # Visit a LBrace node. + def visit_lbrace(node) + node.copy + end + + # Visit a LBracket node. + def visit_lbracket(node) + node.copy + end + + # Visit a LParen node. + def visit_lparen(node) + node.copy + end + + # Visit a MAssign node. + def visit_massign(node) + node.copy(target: visit(node.target)) + end + + # Visit a MethodAddBlock node. + def visit_method_add_block(node) + node.copy(call: visit(node.call), block: visit(node.block)) + end + + # Visit a MLHS node. + def visit_mlhs(node) + node.copy(parts: visit_all(node.parts)) + end + + # Visit a MLHSParen node. + def visit_mlhs_paren(node) + node.copy(contents: visit(node.contents)) + end + + # Visit a ModuleDeclaration node. + def visit_module(node) + node.copy( + constant: visit(node.constant), + bodystmt: visit(node.bodystmt) + ) + end + + # Visit a MRHS node. + def visit_mrhs(node) + node.copy(parts: visit_all(node.parts)) + end + + # Visit a Next node. + def visit_next(node) + node.copy(arguments: visit(node.arguments)) + end + + # Visit a Op node. + def visit_op(node) + node.copy + end + + # Visit a OpAssign node. + def visit_opassign(node) + node.copy(target: visit(node.target), operator: visit(node.operator)) + end + + # Visit a Params node. + def visit_params(node) + node.copy( + requireds: visit_all(node.requireds), + optionals: + node.optionals.map { |ident, value| [visit(ident), visit(value)] }, + rest: visit(node.rest), + posts: visit_all(node.posts), + keywords: + node.keywords.map { |ident, value| [visit(ident), visit(value)] }, + keyword_rest: + node.keyword_rest == :nil ? :nil : visit(node.keyword_rest), + block: visit(node.block) + ) + end + + # Visit a Paren node. + def visit_paren(node) + node.copy(lparen: visit(node.lparen), contents: visit(node.contents)) + end + + # Visit a Period node. + def visit_period(node) + node.copy + end + + # Visit a Program node. + def visit_program(node) + node.copy(statements: visit(node.statements)) + end + + # Visit a QSymbols node. + def visit_qsymbols(node) + node.copy( + beginning: visit(node.beginning), + elements: visit_all(node.elements) + ) + end + + # Visit a QSymbolsBeg node. + def visit_qsymbols_beg(node) + node.copy + end + + # Visit a QWords node. + def visit_qwords(node) + node.copy( + beginning: visit(node.beginning), + elements: visit_all(node.elements) + ) + end + + # Visit a QWordsBeg node. + def visit_qwords_beg(node) + node.copy + end + + # Visit a RationalLiteral node. + def visit_rational(node) + node.copy + end + + # Visit a RBrace node. + def visit_rbrace(node) + node.copy + end + + # Visit a RBracket node. + def visit_rbracket(node) + node.copy + end + + # Visit a Redo node. + def visit_redo(node) + node.copy + end + + # Visit a RegexpContent node. + def visit_regexp_content(node) + node.copy(parts: visit_all(node.parts)) + end + + # Visit a RegexpBeg node. + def visit_regexp_beg(node) + node.copy + end + + # Visit a RegexpEnd node. + def visit_regexp_end(node) + node.copy + end + + # Visit a RegexpLiteral node. + def visit_regexp_literal(node) + node.copy(parts: visit_all(node.parts)) + end + + # Visit a RescueEx node. + def visit_rescue_ex(node) + node.copy(variable: visit(node.variable)) + end + + # Visit a Rescue node. + def visit_rescue(node) + node.copy( + keyword: visit(node.keyword), + exception: visit(node.exception), + statements: visit(node.statements), + consequent: visit(node.consequent) + ) + end + + # Visit a RescueMod node. + def visit_rescue_mod(node) + node.copy + end + + # Visit a RestParam node. + def visit_rest_param(node) + node.copy(name: visit(node.name)) + end + + # Visit a Retry node. + def visit_retry(node) + node.copy + end + + # Visit a Return node. + def visit_return(node) + node.copy(arguments: visit(node.arguments)) + end + + # Visit a RParen node. + def visit_rparen(node) + node.copy + end + + # Visit a SClass node. + def visit_sclass(node) + node.copy(bodystmt: visit(node.bodystmt)) + end + + # Visit a Statements node. + def visit_statements(node) + node.copy(body: visit_all(node.body)) + end + + # Visit a StringContent node. + def visit_string_content(node) + node.copy(parts: visit_all(node.parts)) + end + + # Visit a StringConcat node. + def visit_string_concat(node) + node.copy(left: visit(node.left), right: visit(node.right)) + end + + # Visit a StringDVar node. + def visit_string_dvar(node) + node.copy(variable: visit(node.variable)) + end + + # Visit a StringEmbExpr node. + def visit_string_embexpr(node) + node.copy(statements: visit(node.statements)) + end + + # Visit a StringLiteral node. + def visit_string_literal(node) + node.copy(parts: visit_all(node.parts)) + end + + # Visit a Super node. + def visit_super(node) + node.copy(arguments: visit(node.arguments)) + end + + # Visit a SymBeg node. + def visit_symbeg(node) + node.copy + end + + # Visit a SymbolContent node. + def visit_symbol_content(node) + node.copy(value: visit(node.value)) + end + + # Visit a SymbolLiteral node. + def visit_symbol_literal(node) + node.copy(value: visit(node.value)) + end + + # Visit a Symbols node. + def visit_symbols(node) + node.copy( + beginning: visit(node.beginning), + elements: visit_all(node.elements) + ) + end + + # Visit a SymbolsBeg node. + def visit_symbols_beg(node) + node.copy + end + + # Visit a TLambda node. + def visit_tlambda(node) + node.copy + end + + # Visit a TLamBeg node. + def visit_tlambeg(node) + node.copy + end + + # Visit a TopConstField node. + def visit_top_const_field(node) + node.copy(constant: visit(node.constant)) + end + + # Visit a TopConstRef node. + def visit_top_const_ref(node) + node.copy(constant: visit(node.constant)) + end + + # Visit a TStringBeg node. + def visit_tstring_beg(node) + node.copy + end + + # Visit a TStringContent node. + def visit_tstring_content(node) + node.copy + end + + # Visit a TStringEnd node. + def visit_tstring_end(node) + node.copy + end + + # Visit a Not node. + def visit_not(node) + node.copy(statement: visit(node.statement)) + end + + # Visit a Unary node. + def visit_unary(node) + node.copy + end + + # Visit a Undef node. + def visit_undef(node) + node.copy(symbols: visit_all(node.symbols)) + end + + # Visit a UnlessNode node. + def visit_unless(node) + node.copy( + predicate: visit(node.predicate), + statements: visit(node.statements), + consequent: visit(node.consequent) + ) + end + + # Visit a UntilNode node. + def visit_until(node) + node.copy( + predicate: visit(node.predicate), + statements: visit(node.statements) + ) + end + + # Visit a VarField node. + def visit_var_field(node) + node.copy(value: visit(node.value)) + end + + # Visit a VarRef node. + def visit_var_ref(node) + node.copy(value: visit(node.value)) + end + + # Visit a PinnedVarRef node. + def visit_pinned_var_ref(node) + node.copy(value: visit(node.value)) + end + + # Visit a VCall node. + def visit_vcall(node) + node.copy(value: visit(node.value)) + end + + # Visit a VoidStmt node. + def visit_void_stmt(node) + node.copy + end + + # Visit a When node. + def visit_when(node) + node.copy( + arguments: visit(node.arguments), + statements: visit(node.statements), + consequent: visit(node.consequent) + ) + end + + # Visit a WhileNode node. + def visit_while(node) + node.copy( + predicate: visit(node.predicate), + statements: visit(node.statements) + ) + end + + # Visit a Word node. + def visit_word(node) + node.copy(parts: visit_all(node.parts)) + end + + # Visit a Words node. + def visit_words(node) + node.copy( + beginning: visit(node.beginning), + elements: visit_all(node.elements) + ) + end + + # Visit a WordsBeg node. + def visit_words_beg(node) + node.copy + end + + # Visit a XString node. + def visit_xstring(node) + node.copy(parts: visit_all(node.parts)) + end + + # Visit a XStringLiteral node. + def visit_xstring_literal(node) + node.copy(parts: visit_all(node.parts)) + end + + # Visit a YieldNode node. + def visit_yield(node) + node.copy(arguments: visit(node.arguments)) + end + + # Visit a ZSuper node. + def visit_zsuper(node) + node.copy + end + end +end diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 0a495890..567ec0c8 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -135,19 +135,19 @@ def end_char end def pretty_print(q) - accept(Visitor::PrettyPrintVisitor.new(q)) + accept(PrettyPrintVisitor.new(q)) end def to_json(*opts) - accept(Visitor::JSONVisitor.new).to_json(*opts) + accept(JSONVisitor.new).to_json(*opts) end def to_mermaid - accept(Visitor::MermaidVisitor.new) + accept(MermaidVisitor.new) end def construct_keys - PrettierPrint.format(+"") { |q| accept(Visitor::MatchVisitor.new(q)) } + PrettierPrint.format(+"") { |q| accept(MatchVisitor.new(q)) } end end diff --git a/lib/syntax_tree/pretty_print_visitor.rb b/lib/syntax_tree/pretty_print_visitor.rb new file mode 100644 index 00000000..894e0cf4 --- /dev/null +++ b/lib/syntax_tree/pretty_print_visitor.rb @@ -0,0 +1,83 @@ +# frozen_string_literal: true + +module SyntaxTree + # This visitor pretty-prints the AST into an equivalent s-expression. + class PrettyPrintVisitor < FieldVisitor + attr_reader :q + + def initialize(q) + @q = q + end + + # This is here because we need to make sure the operator is cast to a string + # before we print it out. + def visit_binary(node) + node(node, "binary") do + field("left", node.left) + text("operator", node.operator.to_s) + field("right", node.right) + comments(node) + end + end + + # This is here to make it a little nicer to look at labels since they + # typically have their : at the end of the value. + def visit_label(node) + node(node, "label") do + q.breakable + q.text(":") + q.text(node.value[0...-1]) + comments(node) + end + end + + private + + def comments(node) + return if node.comments.empty? + + q.breakable + q.group(2, "(", ")") do + q.seplist(node.comments) { |comment| q.pp(comment) } + end + end + + def field(_name, value) + q.breakable + q.pp(value) + end + + def list(_name, values) + q.breakable + q.group(2, "(", ")") { q.seplist(values) { |value| q.pp(value) } } + end + + def node(_node, type) + q.group(2, "(", ")") do + q.text(type) + yield + end + end + + def pairs(_name, values) + q.group(2, "(", ")") do + q.seplist(values) do |(key, value)| + q.pp(key) + + if value + q.text("=") + q.group(2) do + q.breakable("") + q.pp(value) + end + end + end + end + end + + def text(_name, value) + q.breakable + q.text(value) + end + end +end diff --git a/lib/syntax_tree/visitor/environment.rb b/lib/syntax_tree/visitor/environment.rb deleted file mode 100644 index b07a5203..00000000 --- a/lib/syntax_tree/visitor/environment.rb +++ /dev/null @@ -1,84 +0,0 @@ -# frozen_string_literal: true - -module SyntaxTree - # The environment class is used to keep track of local variables and arguments - # inside a particular scope - class Environment - # This class tracks the occurrences of a local variable or argument - class Local - # [Symbol] The type of the local (e.g. :argument, :variable) - attr_reader :type - - # [Array[Location]] The locations of all definitions and assignments of - # this local - attr_reader :definitions - - # [Array[Location]] The locations of all usages of this local - attr_reader :usages - - # initialize: (Symbol type) -> void - def initialize(type) - @type = type - @definitions = [] - @usages = [] - end - - # add_definition: (Location location) -> void - def add_definition(location) - @definitions << location - end - - # add_usage: (Location location) -> void - def add_usage(location) - @usages << location - end - end - - # [Array[Local]] The local variables and arguments defined in this - # environment - attr_reader :locals - - # [Environment | nil] The parent environment - attr_reader :parent - - # initialize: (Environment | nil parent) -> void - def initialize(parent = nil) - @locals = {} - @parent = parent - end - - # Adding a local definition will either insert a new entry in the locals - # hash or append a new definition location to an existing local. Notice that - # it's not possible to change the type of a local after it has been - # registered - # add_local_definition: (Ident | Label identifier, Symbol type) -> void - def add_local_definition(identifier, type) - name = identifier.value.delete_suffix(":") - - @locals[name] ||= Local.new(type) - @locals[name].add_definition(identifier.location) - end - - # Adding a local usage will either insert a new entry in the locals - # hash or append a new usage location to an existing local. Notice that - # it's not possible to change the type of a local after it has been - # registered - # add_local_usage: (Ident | Label identifier, Symbol type) -> void - def add_local_usage(identifier, type) - name = identifier.value.delete_suffix(":") - - @locals[name] ||= Local.new(type) - @locals[name].add_usage(identifier.location) - end - - # Try to find the local given its name in this environment or any of its - # parents - # find_local: (String name) -> Local | nil - def find_local(name) - local = @locals[name] - return local unless local.nil? - - @parent&.find_local(name) - end - end -end diff --git a/lib/syntax_tree/visitor/field_visitor.rb b/lib/syntax_tree/visitor/field_visitor.rb deleted file mode 100644 index 6e643e09..00000000 --- a/lib/syntax_tree/visitor/field_visitor.rb +++ /dev/null @@ -1,1031 +0,0 @@ -# frozen_string_literal: true - -module SyntaxTree - class Visitor - # This is the parent class of a lot of built-in visitors for Syntax Tree. It - # reflects visiting each of the fields on every node in turn. It itself does - # not do anything with these fields, it leaves that behavior up to the - # subclass to implement. - # - # In order to properly use this class, you will need to subclass it and - # implement #comments, #field, #list, #node, #pairs, and #text. Those are - # documented here. - # - # == comments(node) - # - # This accepts the node that is being visited and does something depending - # on the comments attached to the node. - # - # == field(name, value) - # - # This accepts the name of the field being visited as a string (like - # "value") and the actual value of that field. The value can be a subclass - # of Node or any other type that can be held within the tree. - # - # == list(name, values) - # - # This accepts the name of the field being visited as well as a list of - # values. This is used, for example, when visiting something like the body - # of a Statements node. - # - # == node(name, node) - # - # This is the parent serialization method for each node. It is called with - # the node itself, as well as the type of the node as a string. The type - # is an internally used value that usually resembles the name of the - # ripper event that generated the node. The method should yield to the - # given block which then calls through to visit each of the fields on the - # node. - # - # == text(name, value) - # - # This accepts the name of the field being visited as well as a string - # value representing the value of the field. - # - # == pairs(name, values) - # - # This accepts the name of the field being visited as well as a list of - # pairs that represent the value of the field. It is used only in a couple - # of circumstances, like when visiting the list of optional parameters - # defined on a method. - # - class FieldVisitor < BasicVisitor - def visit_aref(node) - node(node, "aref") do - field("collection", node.collection) - field("index", node.index) - comments(node) - end - end - - def visit_aref_field(node) - node(node, "aref_field") do - field("collection", node.collection) - field("index", node.index) - comments(node) - end - end - - def visit_alias(node) - node(node, "alias") do - field("left", node.left) - field("right", node.right) - comments(node) - end - end - - def visit_arg_block(node) - node(node, "arg_block") do - field("value", node.value) if node.value - comments(node) - end - end - - def visit_arg_paren(node) - node(node, "arg_paren") do - field("arguments", node.arguments) - comments(node) - end - end - - def visit_arg_star(node) - node(node, "arg_star") do - field("value", node.value) - comments(node) - end - end - - def visit_args(node) - node(node, "args") do - list("parts", node.parts) - comments(node) - end - end - - def visit_args_forward(node) - node(node, "args_forward") { comments(node) } - end - - def visit_array(node) - node(node, "array") do - field("contents", node.contents) - comments(node) - end - end - - def visit_aryptn(node) - node(node, "aryptn") do - field("constant", node.constant) if node.constant - list("requireds", node.requireds) if node.requireds.any? - field("rest", node.rest) if node.rest - list("posts", node.posts) if node.posts.any? - comments(node) - end - end - - def visit_assign(node) - node(node, "assign") do - field("target", node.target) - field("value", node.value) - comments(node) - end - end - - def visit_assoc(node) - node(node, "assoc") do - field("key", node.key) - field("value", node.value) if node.value - comments(node) - end - end - - def visit_assoc_splat(node) - node(node, "assoc_splat") do - field("value", node.value) - comments(node) - end - end - - def visit_backref(node) - visit_token(node, "backref") - end - - def visit_backtick(node) - visit_token(node, "backtick") - end - - def visit_bare_assoc_hash(node) - node(node, "bare_assoc_hash") do - list("assocs", node.assocs) - comments(node) - end - end - - def visit_BEGIN(node) - node(node, "BEGIN") do - field("statements", node.statements) - comments(node) - end - end - - def visit_begin(node) - node(node, "begin") do - field("bodystmt", node.bodystmt) - comments(node) - end - end - - def visit_binary(node) - node(node, "binary") do - field("left", node.left) - text("operator", node.operator) - field("right", node.right) - comments(node) - end - end - - def visit_block(node) - node(node, "block") do - field("block_var", node.block_var) if node.block_var - field("bodystmt", node.bodystmt) - comments(node) - end - end - - def visit_blockarg(node) - node(node, "blockarg") do - field("name", node.name) if node.name - comments(node) - end - end - - def visit_block_var(node) - node(node, "block_var") do - field("params", node.params) - list("locals", node.locals) if node.locals.any? - comments(node) - end - end - - def visit_bodystmt(node) - node(node, "bodystmt") do - field("statements", node.statements) - field("rescue_clause", node.rescue_clause) if node.rescue_clause - field("else_clause", node.else_clause) if node.else_clause - field("ensure_clause", node.ensure_clause) if node.ensure_clause - comments(node) - end - end - - def visit_break(node) - node(node, "break") do - field("arguments", node.arguments) - comments(node) - end - end - - def visit_call(node) - node(node, "call") do - field("receiver", node.receiver) - field("operator", node.operator) - field("message", node.message) - field("arguments", node.arguments) if node.arguments - comments(node) - end - end - - def visit_case(node) - node(node, "case") do - field("keyword", node.keyword) - field("value", node.value) if node.value - field("consequent", node.consequent) - comments(node) - end - end - - def visit_CHAR(node) - visit_token(node, "CHAR") - end - - def visit_class(node) - node(node, "class") do - field("constant", node.constant) - field("superclass", node.superclass) if node.superclass - field("bodystmt", node.bodystmt) - comments(node) - end - end - - def visit_comma(node) - node(node, "comma") { field("value", node.value) } - end - - def visit_command(node) - node(node, "command") do - field("message", node.message) - field("arguments", node.arguments) - comments(node) - end - end - - def visit_command_call(node) - node(node, "command_call") do - field("receiver", node.receiver) - field("operator", node.operator) - field("message", node.message) - field("arguments", node.arguments) if node.arguments - comments(node) - end - end - - def visit_comment(node) - node(node, "comment") { field("value", node.value) } - end - - def visit_const(node) - visit_token(node, "const") - end - - def visit_const_path_field(node) - node(node, "const_path_field") do - field("parent", node.parent) - field("constant", node.constant) - comments(node) - end - end - - def visit_const_path_ref(node) - node(node, "const_path_ref") do - field("parent", node.parent) - field("constant", node.constant) - comments(node) - end - end - - def visit_const_ref(node) - node(node, "const_ref") do - field("constant", node.constant) - comments(node) - end - end - - def visit_cvar(node) - visit_token(node, "cvar") - end - - def visit_def(node) - node(node, "def") do - field("target", node.target) - field("operator", node.operator) - field("name", node.name) - field("params", node.params) - field("bodystmt", node.bodystmt) - comments(node) - end - end - - def visit_defined(node) - node(node, "defined") do - field("value", node.value) - comments(node) - end - end - - def visit_dyna_symbol(node) - node(node, "dyna_symbol") do - list("parts", node.parts) - comments(node) - end - end - - def visit_END(node) - node(node, "END") do - field("statements", node.statements) - comments(node) - end - end - - def visit_else(node) - node(node, "else") do - field("statements", node.statements) - comments(node) - end - end - - def visit_elsif(node) - node(node, "elsif") do - field("predicate", node.predicate) - field("statements", node.statements) - field("consequent", node.consequent) if node.consequent - comments(node) - end - end - - def visit_embdoc(node) - node(node, "embdoc") { field("value", node.value) } - end - - def visit_embexpr_beg(node) - node(node, "embexpr_beg") { field("value", node.value) } - end - - def visit_embexpr_end(node) - node(node, "embexpr_end") { field("value", node.value) } - end - - def visit_embvar(node) - node(node, "embvar") { field("value", node.value) } - end - - def visit_ensure(node) - node(node, "ensure") do - field("statements", node.statements) - comments(node) - end - end - - def visit_excessed_comma(node) - visit_token(node, "excessed_comma") - end - - def visit_field(node) - node(node, "field") do - field("parent", node.parent) - field("operator", node.operator) - field("name", node.name) - comments(node) - end - end - - def visit_float(node) - visit_token(node, "float") - end - - def visit_fndptn(node) - node(node, "fndptn") do - field("constant", node.constant) if node.constant - field("left", node.left) - list("values", node.values) - field("right", node.right) - comments(node) - end - end - - def visit_for(node) - node(node, "for") do - field("index", node.index) - field("collection", node.collection) - field("statements", node.statements) - comments(node) - end - end - - def visit_gvar(node) - visit_token(node, "gvar") - end - - def visit_hash(node) - node(node, "hash") do - list("assocs", node.assocs) if node.assocs.any? - comments(node) - end - end - - def visit_heredoc(node) - node(node, "heredoc") do - list("parts", node.parts) - comments(node) - end - end - - def visit_heredoc_beg(node) - visit_token(node, "heredoc_beg") - end - - def visit_heredoc_end(node) - visit_token(node, "heredoc_end") - end - - def visit_hshptn(node) - node(node, "hshptn") do - field("constant", node.constant) if node.constant - pairs("keywords", node.keywords) if node.keywords.any? - field("keyword_rest", node.keyword_rest) if node.keyword_rest - comments(node) - end - end - - def visit_ident(node) - visit_token(node, "ident") - end - - def visit_if(node) - node(node, "if") do - field("predicate", node.predicate) - field("statements", node.statements) - field("consequent", node.consequent) if node.consequent - comments(node) - end - end - - def visit_if_op(node) - node(node, "if_op") do - field("predicate", node.predicate) - field("truthy", node.truthy) - field("falsy", node.falsy) - comments(node) - end - end - - def visit_imaginary(node) - visit_token(node, "imaginary") - end - - def visit_in(node) - node(node, "in") do - field("pattern", node.pattern) - field("statements", node.statements) - field("consequent", node.consequent) if node.consequent - comments(node) - end - end - - def visit_int(node) - visit_token(node, "int") - end - - def visit_ivar(node) - visit_token(node, "ivar") - end - - def visit_kw(node) - visit_token(node, "kw") - end - - def visit_kwrest_param(node) - node(node, "kwrest_param") do - field("name", node.name) - comments(node) - end - end - - def visit_label(node) - visit_token(node, "label") - end - - def visit_label_end(node) - node(node, "label_end") { field("value", node.value) } - end - - def visit_lambda(node) - node(node, "lambda") do - field("params", node.params) - field("statements", node.statements) - comments(node) - end - end - - def visit_lambda_var(node) - node(node, "lambda_var") do - field("params", node.params) - list("locals", node.locals) if node.locals.any? - comments(node) - end - end - - def visit_lbrace(node) - visit_token(node, "lbrace") - end - - def visit_lbracket(node) - visit_token(node, "lbracket") - end - - def visit_lparen(node) - visit_token(node, "lparen") - end - - def visit_massign(node) - node(node, "massign") do - field("target", node.target) - field("value", node.value) - comments(node) - end - end - - def visit_method_add_block(node) - node(node, "method_add_block") do - field("call", node.call) - field("block", node.block) - comments(node) - end - end - - def visit_mlhs(node) - node(node, "mlhs") do - list("parts", node.parts) - comments(node) - end - end - - def visit_mlhs_paren(node) - node(node, "mlhs_paren") do - field("contents", node.contents) - comments(node) - end - end - - def visit_module(node) - node(node, "module") do - field("constant", node.constant) - field("bodystmt", node.bodystmt) - comments(node) - end - end - - def visit_mrhs(node) - node(node, "mrhs") do - list("parts", node.parts) - comments(node) - end - end - - def visit_next(node) - node(node, "next") do - field("arguments", node.arguments) - comments(node) - end - end - - def visit_not(node) - node(node, "not") do - field("statement", node.statement) - comments(node) - end - end - - def visit_op(node) - visit_token(node, "op") - end - - def visit_opassign(node) - node(node, "opassign") do - field("target", node.target) - field("operator", node.operator) - field("value", node.value) - comments(node) - end - end - - def visit_params(node) - node(node, "params") do - list("requireds", node.requireds) if node.requireds.any? - pairs("optionals", node.optionals) if node.optionals.any? - field("rest", node.rest) if node.rest - list("posts", node.posts) if node.posts.any? - pairs("keywords", node.keywords) if node.keywords.any? - field("keyword_rest", node.keyword_rest) if node.keyword_rest - field("block", node.block) if node.block - comments(node) - end - end - - def visit_paren(node) - node(node, "paren") do - field("contents", node.contents) - comments(node) - end - end - - def visit_period(node) - visit_token(node, "period") - end - - def visit_pinned_begin(node) - node(node, "pinned_begin") do - field("statement", node.statement) - comments(node) - end - end - - def visit_pinned_var_ref(node) - node(node, "pinned_var_ref") do - field("value", node.value) - comments(node) - end - end - - def visit_program(node) - node(node, "program") do - field("statements", node.statements) - comments(node) - end - end - - def visit_qsymbols(node) - node(node, "qsymbols") do - list("elements", node.elements) - comments(node) - end - end - - def visit_qsymbols_beg(node) - node(node, "qsymbols_beg") { field("value", node.value) } - end - - def visit_qwords(node) - node(node, "qwords") do - list("elements", node.elements) - comments(node) - end - end - - def visit_qwords_beg(node) - node(node, "qwords_beg") { field("value", node.value) } - end - - def visit_range(node) - node(node, "range") do - field("left", node.left) if node.left - field("operator", node.operator) - field("right", node.right) if node.right - comments(node) - end - end - - def visit_rassign(node) - node(node, "rassign") do - field("value", node.value) - field("operator", node.operator) - field("pattern", node.pattern) - comments(node) - end - end - - def visit_rational(node) - visit_token(node, "rational") - end - - def visit_rbrace(node) - node(node, "rbrace") { field("value", node.value) } - end - - def visit_rbracket(node) - node(node, "rbracket") { field("value", node.value) } - end - - def visit_redo(node) - node(node, "redo") { comments(node) } - end - - def visit_regexp_beg(node) - node(node, "regexp_beg") { field("value", node.value) } - end - - def visit_regexp_content(node) - node(node, "regexp_content") { list("parts", node.parts) } - end - - def visit_regexp_end(node) - node(node, "regexp_end") { field("value", node.value) } - end - - def visit_regexp_literal(node) - node(node, "regexp_literal") do - list("parts", node.parts) - field("options", node.options) - comments(node) - end - end - - def visit_rescue(node) - node(node, "rescue") do - field("exception", node.exception) if node.exception - field("statements", node.statements) - field("consequent", node.consequent) if node.consequent - comments(node) - end - end - - def visit_rescue_ex(node) - node(node, "rescue_ex") do - field("exceptions", node.exceptions) - field("variable", node.variable) - comments(node) - end - end - - def visit_rescue_mod(node) - node(node, "rescue_mod") do - field("statement", node.statement) - field("value", node.value) - comments(node) - end - end - - def visit_rest_param(node) - node(node, "rest_param") do - field("name", node.name) - comments(node) - end - end - - def visit_retry(node) - node(node, "retry") { comments(node) } - end - - def visit_return(node) - node(node, "return") do - field("arguments", node.arguments) - comments(node) - end - end - - def visit_rparen(node) - node(node, "rparen") { field("value", node.value) } - end - - def visit_sclass(node) - node(node, "sclass") do - field("target", node.target) - field("bodystmt", node.bodystmt) - comments(node) - end - end - - def visit_statements(node) - node(node, "statements") do - list("body", node.body) - comments(node) - end - end - - def visit_string_concat(node) - node(node, "string_concat") do - field("left", node.left) - field("right", node.right) - comments(node) - end - end - - def visit_string_content(node) - node(node, "string_content") { list("parts", node.parts) } - end - - def visit_string_dvar(node) - node(node, "string_dvar") do - field("variable", node.variable) - comments(node) - end - end - - def visit_string_embexpr(node) - node(node, "string_embexpr") do - field("statements", node.statements) - comments(node) - end - end - - def visit_string_literal(node) - node(node, "string_literal") do - list("parts", node.parts) - comments(node) - end - end - - def visit_super(node) - node(node, "super") do - field("arguments", node.arguments) - comments(node) - end - end - - def visit_symbeg(node) - node(node, "symbeg") { field("value", node.value) } - end - - def visit_symbol_content(node) - node(node, "symbol_content") { field("value", node.value) } - end - - def visit_symbol_literal(node) - node(node, "symbol_literal") do - field("value", node.value) - comments(node) - end - end - - def visit_symbols(node) - node(node, "symbols") do - list("elements", node.elements) - comments(node) - end - end - - def visit_symbols_beg(node) - node(node, "symbols_beg") { field("value", node.value) } - end - - def visit_tlambda(node) - node(node, "tlambda") { field("value", node.value) } - end - - def visit_tlambeg(node) - node(node, "tlambeg") { field("value", node.value) } - end - - def visit_top_const_field(node) - node(node, "top_const_field") do - field("constant", node.constant) - comments(node) - end - end - - def visit_top_const_ref(node) - node(node, "top_const_ref") do - field("constant", node.constant) - comments(node) - end - end - - def visit_tstring_beg(node) - node(node, "tstring_beg") { field("value", node.value) } - end - - def visit_tstring_content(node) - visit_token(node, "tstring_content") - end - - def visit_tstring_end(node) - node(node, "tstring_end") { field("value", node.value) } - end - - def visit_unary(node) - node(node, "unary") do - field("operator", node.operator) - field("statement", node.statement) - comments(node) - end - end - - def visit_undef(node) - node(node, "undef") do - list("symbols", node.symbols) - comments(node) - end - end - - def visit_unless(node) - node(node, "unless") do - field("predicate", node.predicate) - field("statements", node.statements) - field("consequent", node.consequent) if node.consequent - comments(node) - end - end - - def visit_until(node) - node(node, "until") do - field("predicate", node.predicate) - field("statements", node.statements) - comments(node) - end - end - - def visit_var_field(node) - node(node, "var_field") do - field("value", node.value) - comments(node) - end - end - - def visit_var_ref(node) - node(node, "var_ref") do - field("value", node.value) - comments(node) - end - end - - def visit_vcall(node) - node(node, "vcall") do - field("value", node.value) - comments(node) - end - end - - def visit_void_stmt(node) - node(node, "void_stmt") { comments(node) } - end - - def visit_when(node) - node(node, "when") do - field("arguments", node.arguments) - field("statements", node.statements) - field("consequent", node.consequent) if node.consequent - comments(node) - end - end - - def visit_while(node) - node(node, "while") do - field("predicate", node.predicate) - field("statements", node.statements) - comments(node) - end - end - - def visit_word(node) - node(node, "word") do - list("parts", node.parts) - comments(node) - end - end - - def visit_words(node) - node(node, "words") do - list("elements", node.elements) - comments(node) - end - end - - def visit_words_beg(node) - node(node, "words_beg") { field("value", node.value) } - end - - def visit_xstring(node) - node(node, "xstring") { list("parts", node.parts) } - end - - def visit_xstring_literal(node) - node(node, "xstring_literal") do - list("parts", node.parts) - comments(node) - end - end - - def visit_yield(node) - node(node, "yield") do - field("arguments", node.arguments) - comments(node) - end - end - - def visit_zsuper(node) - node(node, "zsuper") { comments(node) } - end - - def visit___end__(node) - visit_token(node, "__end__") - end - - private - - def visit_token(node, type) - node(node, type) do - field("value", node.value) - comments(node) - end - end - end - end -end diff --git a/lib/syntax_tree/visitor/json_visitor.rb b/lib/syntax_tree/visitor/json_visitor.rb deleted file mode 100644 index b516980c..00000000 --- a/lib/syntax_tree/visitor/json_visitor.rb +++ /dev/null @@ -1,55 +0,0 @@ -# frozen_string_literal: true - -module SyntaxTree - class Visitor - # This visitor transforms the AST into a hash that contains only primitives - # that can be easily serialized into JSON. - class JSONVisitor < FieldVisitor - attr_reader :target - - def initialize - @target = nil - end - - private - - def comments(node) - target[:comments] = visit_all(node.comments) - end - - def field(name, value) - target[name] = value.is_a?(Node) ? visit(value) : value - end - - def list(name, values) - target[name] = visit_all(values) - end - - def node(node, type) - previous = @target - @target = { type: type, location: visit_location(node.location) } - yield - @target - ensure - @target = previous - end - - def pairs(name, values) - target[name] = values.map { |(key, value)| [visit(key), visit(value)] } - end - - def text(name, value) - target[name] = value - end - - def visit_location(location) - [ - location.start_line, - location.start_char, - location.end_line, - location.end_char - ] - end - end - end -end diff --git a/lib/syntax_tree/visitor/match_visitor.rb b/lib/syntax_tree/visitor/match_visitor.rb deleted file mode 100644 index e0bdaf08..00000000 --- a/lib/syntax_tree/visitor/match_visitor.rb +++ /dev/null @@ -1,122 +0,0 @@ -# frozen_string_literal: true - -module SyntaxTree - class Visitor - # This visitor transforms the AST into a Ruby pattern matching expression - # that would match correctly against the AST. - class MatchVisitor < FieldVisitor - attr_reader :q - - def initialize(q) - @q = q - end - - def visit(node) - case node - when Node - super - when String - # pp will split up a string on newlines and concat them together using - # a "+" operator. This breaks the pattern matching expression. So - # instead we're going to check here for strings and manually put the - # entire value into the output buffer. - q.text(node.inspect) - else - node.pretty_print(q) - end - end - - private - - def comments(node) - return if node.comments.empty? - - q.nest(0) do - q.text("comments: [") - q.indent do - q.breakable("") - q.seplist(node.comments) { |comment| visit(comment) } - end - q.breakable("") - q.text("]") - end - end - - def field(name, value) - q.nest(0) do - q.text(name) - q.text(": ") - visit(value) - end - end - - def list(name, values) - q.group do - q.text(name) - q.text(": [") - q.indent do - q.breakable("") - q.seplist(values) { |value| visit(value) } - end - q.breakable("") - q.text("]") - end - end - - def node(node, _type) - items = [] - q.with_target(items) { yield } - - if items.empty? - q.text(node.class.name) - return - end - - q.group do - q.text(node.class.name) - q.text("[") - q.indent do - q.breakable("") - q.seplist(items) { |item| q.target << item } - end - q.breakable("") - q.text("]") - end - end - - def pairs(name, values) - q.group do - q.text(name) - q.text(": [") - q.indent do - q.breakable("") - q.seplist(values) do |(key, value)| - q.group do - q.text("[") - q.indent do - q.breakable("") - visit(key) - q.text(",") - q.breakable - visit(value || nil) - end - q.breakable("") - q.text("]") - end - end - end - q.breakable("") - q.text("]") - end - end - - def text(name, value) - q.nest(0) do - q.text(name) - q.text(": ") - value.pretty_print(q) - end - end - end - end -end diff --git a/lib/syntax_tree/visitor/mermaid_visitor.rb b/lib/syntax_tree/visitor/mermaid_visitor.rb deleted file mode 100644 index 504e2fb0..00000000 --- a/lib/syntax_tree/visitor/mermaid_visitor.rb +++ /dev/null @@ -1,75 +0,0 @@ -# frozen_string_literal: true - -module SyntaxTree - class Visitor - # This visitor transforms the AST into a mermaid flow chart. - class MermaidVisitor < FieldVisitor - attr_reader :flowchart, :target - - def initialize - @flowchart = Mermaid.flowchart - @target = nil - end - - def visit_program(node) - super - flowchart.render - end - - private - - def comments(node) - # Ignore - end - - def field(name, value) - case value - when nil - # skip - when Node - flowchart.link(target, visit(value), name) - else - to = - flowchart.node( - "#{target.id}_#{name}", - value.inspect, - shape: :stadium - ) - flowchart.link(target, to, name) - end - end - - def list(name, values) - values.each_with_index do |value, index| - field("#{name}[#{index}]", value) - end - end - - def node(node, type) - previous_target = target - - begin - @target = flowchart.node("node_#{node.object_id}", type) - yield - @target - ensure - @target = previous_target - end - end - - def pairs(name, values) - values.each_with_index do |(key, value), index| - to = flowchart.node("#{target.id}_#{name}_#{index}", shape: :circle) - - flowchart.link(target, to, "#{name}[#{index}]") - flowchart.link(to, visit(key), "[0]") - flowchart.link(to, visit(value), "[1]") if value - end - end - - def text(name, value) - field(name, value) - end - end - end -end diff --git a/lib/syntax_tree/visitor/mutation_visitor.rb b/lib/syntax_tree/visitor/mutation_visitor.rb deleted file mode 100644 index 65f8c5ba..00000000 --- a/lib/syntax_tree/visitor/mutation_visitor.rb +++ /dev/null @@ -1,924 +0,0 @@ -# frozen_string_literal: true - -module SyntaxTree - class Visitor - # This visitor walks through the tree and copies each node as it is being - # visited. This is useful for mutating the tree before it is formatted. - class MutationVisitor < BasicVisitor - attr_reader :mutations - - def initialize - @mutations = [] - end - - # Create a new mutation based on the given query that will mutate the node - # using the given block. The block should return a new node that will take - # the place of the given node in the tree. These blocks frequently make - # use of the `copy` method on nodes to create a new node with the same - # properties as the original node. - def mutate(query, &block) - mutations << [Pattern.new(query).compile, block] - end - - # This is the base visit method for each node in the tree. It first - # creates a copy of the node using the visit_* methods defined below. Then - # it checks each mutation in sequence and calls it if it finds a match. - def visit(node) - return unless node - result = node.accept(self) - - mutations.each do |(pattern, mutation)| - result = mutation.call(result) if pattern.call(result) - end - - result - end - - # Visit a BEGINBlock node. - def visit_BEGIN(node) - node.copy( - lbrace: visit(node.lbrace), - statements: visit(node.statements) - ) - end - - # Visit a CHAR node. - def visit_CHAR(node) - node.copy - end - - # Visit a ENDBlock node. - def visit_END(node) - node.copy( - lbrace: visit(node.lbrace), - statements: visit(node.statements) - ) - end - - # Visit a EndContent node. - def visit___end__(node) - node.copy - end - - # Visit a AliasNode node. - def visit_alias(node) - node.copy(left: visit(node.left), right: visit(node.right)) - end - - # Visit a ARef node. - def visit_aref(node) - node.copy(index: visit(node.index)) - end - - # Visit a ARefField node. - def visit_aref_field(node) - node.copy(index: visit(node.index)) - end - - # Visit a ArgParen node. - def visit_arg_paren(node) - node.copy(arguments: visit(node.arguments)) - end - - # Visit a Args node. - def visit_args(node) - node.copy(parts: visit_all(node.parts)) - end - - # Visit a ArgBlock node. - def visit_arg_block(node) - node.copy(value: visit(node.value)) - end - - # Visit a ArgStar node. - def visit_arg_star(node) - node.copy(value: visit(node.value)) - end - - # Visit a ArgsForward node. - def visit_args_forward(node) - node.copy - end - - # Visit a ArrayLiteral node. - def visit_array(node) - node.copy( - lbracket: visit(node.lbracket), - contents: visit(node.contents) - ) - end - - # Visit a AryPtn node. - def visit_aryptn(node) - node.copy( - constant: visit(node.constant), - requireds: visit_all(node.requireds), - rest: visit(node.rest), - posts: visit_all(node.posts) - ) - end - - # Visit a Assign node. - def visit_assign(node) - node.copy(target: visit(node.target)) - end - - # Visit a Assoc node. - def visit_assoc(node) - node.copy - end - - # Visit a AssocSplat node. - def visit_assoc_splat(node) - node.copy - end - - # Visit a Backref node. - def visit_backref(node) - node.copy - end - - # Visit a Backtick node. - def visit_backtick(node) - node.copy - end - - # Visit a BareAssocHash node. - def visit_bare_assoc_hash(node) - node.copy(assocs: visit_all(node.assocs)) - end - - # Visit a Begin node. - def visit_begin(node) - node.copy(bodystmt: visit(node.bodystmt)) - end - - # Visit a PinnedBegin node. - def visit_pinned_begin(node) - node.copy - end - - # Visit a Binary node. - def visit_binary(node) - node.copy - end - - # Visit a BlockVar node. - def visit_block_var(node) - node.copy(params: visit(node.params), locals: visit_all(node.locals)) - end - - # Visit a BlockArg node. - def visit_blockarg(node) - node.copy(name: visit(node.name)) - end - - # Visit a BodyStmt node. - def visit_bodystmt(node) - node.copy( - statements: visit(node.statements), - rescue_clause: visit(node.rescue_clause), - else_clause: visit(node.else_clause), - ensure_clause: visit(node.ensure_clause) - ) - end - - # Visit a Break node. - def visit_break(node) - node.copy(arguments: visit(node.arguments)) - end - - # Visit a Call node. - def visit_call(node) - node.copy( - receiver: visit(node.receiver), - operator: node.operator == :"::" ? :"::" : visit(node.operator), - message: node.message == :call ? :call : visit(node.message), - arguments: visit(node.arguments) - ) - end - - # Visit a Case node. - def visit_case(node) - node.copy( - keyword: visit(node.keyword), - value: visit(node.value), - consequent: visit(node.consequent) - ) - end - - # Visit a RAssign node. - def visit_rassign(node) - node.copy(operator: visit(node.operator)) - end - - # Visit a ClassDeclaration node. - def visit_class(node) - node.copy( - constant: visit(node.constant), - superclass: visit(node.superclass), - bodystmt: visit(node.bodystmt) - ) - end - - # Visit a Comma node. - def visit_comma(node) - node.copy - end - - # Visit a Command node. - def visit_command(node) - node.copy( - message: visit(node.message), - arguments: visit(node.arguments), - block: visit(node.block) - ) - end - - # Visit a CommandCall node. - def visit_command_call(node) - node.copy( - operator: node.operator == :"::" ? :"::" : visit(node.operator), - message: visit(node.message), - arguments: visit(node.arguments), - block: visit(node.block) - ) - end - - # Visit a Comment node. - def visit_comment(node) - node.copy - end - - # Visit a Const node. - def visit_const(node) - node.copy - end - - # Visit a ConstPathField node. - def visit_const_path_field(node) - node.copy(constant: visit(node.constant)) - end - - # Visit a ConstPathRef node. - def visit_const_path_ref(node) - node.copy(constant: visit(node.constant)) - end - - # Visit a ConstRef node. - def visit_const_ref(node) - node.copy(constant: visit(node.constant)) - end - - # Visit a CVar node. - def visit_cvar(node) - node.copy - end - - # Visit a Def node. - def visit_def(node) - node.copy( - target: visit(node.target), - operator: visit(node.operator), - name: visit(node.name), - params: visit(node.params), - bodystmt: visit(node.bodystmt) - ) - end - - # Visit a Defined node. - def visit_defined(node) - node.copy - end - - # Visit a Block node. - def visit_block(node) - node.copy( - opening: visit(node.opening), - block_var: visit(node.block_var), - bodystmt: visit(node.bodystmt) - ) - end - - # Visit a RangeNode node. - def visit_range(node) - node.copy( - left: visit(node.left), - operator: visit(node.operator), - right: visit(node.right) - ) - end - - # Visit a DynaSymbol node. - def visit_dyna_symbol(node) - node.copy(parts: visit_all(node.parts)) - end - - # Visit a Else node. - def visit_else(node) - node.copy( - keyword: visit(node.keyword), - statements: visit(node.statements) - ) - end - - # Visit a Elsif node. - def visit_elsif(node) - node.copy( - statements: visit(node.statements), - consequent: visit(node.consequent) - ) - end - - # Visit a EmbDoc node. - def visit_embdoc(node) - node.copy - end - - # Visit a EmbExprBeg node. - def visit_embexpr_beg(node) - node.copy - end - - # Visit a EmbExprEnd node. - def visit_embexpr_end(node) - node.copy - end - - # Visit a EmbVar node. - def visit_embvar(node) - node.copy - end - - # Visit a Ensure node. - def visit_ensure(node) - node.copy( - keyword: visit(node.keyword), - statements: visit(node.statements) - ) - end - - # Visit a ExcessedComma node. - def visit_excessed_comma(node) - node.copy - end - - # Visit a Field node. - def visit_field(node) - node.copy( - operator: node.operator == :"::" ? :"::" : visit(node.operator), - name: visit(node.name) - ) - end - - # Visit a FloatLiteral node. - def visit_float(node) - node.copy - end - - # Visit a FndPtn node. - def visit_fndptn(node) - node.copy( - constant: visit(node.constant), - left: visit(node.left), - values: visit_all(node.values), - right: visit(node.right) - ) - end - - # Visit a For node. - def visit_for(node) - node.copy(index: visit(node.index), statements: visit(node.statements)) - end - - # Visit a GVar node. - def visit_gvar(node) - node.copy - end - - # Visit a HashLiteral node. - def visit_hash(node) - node.copy(lbrace: visit(node.lbrace), assocs: visit_all(node.assocs)) - end - - # Visit a Heredoc node. - def visit_heredoc(node) - node.copy( - beginning: visit(node.beginning), - ending: visit(node.ending), - parts: visit_all(node.parts) - ) - end - - # Visit a HeredocBeg node. - def visit_heredoc_beg(node) - node.copy - end - - # Visit a HeredocEnd node. - def visit_heredoc_end(node) - node.copy - end - - # Visit a HshPtn node. - def visit_hshptn(node) - node.copy( - constant: visit(node.constant), - keywords: - node.keywords.map { |label, value| [visit(label), visit(value)] }, - keyword_rest: visit(node.keyword_rest) - ) - end - - # Visit a Ident node. - def visit_ident(node) - node.copy - end - - # Visit a IfNode node. - def visit_if(node) - node.copy( - predicate: visit(node.predicate), - statements: visit(node.statements), - consequent: visit(node.consequent) - ) - end - - # Visit a IfOp node. - def visit_if_op(node) - node.copy - end - - # Visit a Imaginary node. - def visit_imaginary(node) - node.copy - end - - # Visit a In node. - def visit_in(node) - node.copy( - statements: visit(node.statements), - consequent: visit(node.consequent) - ) - end - - # Visit a Int node. - def visit_int(node) - node.copy - end - - # Visit a IVar node. - def visit_ivar(node) - node.copy - end - - # Visit a Kw node. - def visit_kw(node) - node.copy - end - - # Visit a KwRestParam node. - def visit_kwrest_param(node) - node.copy(name: visit(node.name)) - end - - # Visit a Label node. - def visit_label(node) - node.copy - end - - # Visit a LabelEnd node. - def visit_label_end(node) - node.copy - end - - # Visit a Lambda node. - def visit_lambda(node) - node.copy( - params: visit(node.params), - statements: visit(node.statements) - ) - end - - # Visit a LambdaVar node. - def visit_lambda_var(node) - node.copy(params: visit(node.params), locals: visit_all(node.locals)) - end - - # Visit a LBrace node. - def visit_lbrace(node) - node.copy - end - - # Visit a LBracket node. - def visit_lbracket(node) - node.copy - end - - # Visit a LParen node. - def visit_lparen(node) - node.copy - end - - # Visit a MAssign node. - def visit_massign(node) - node.copy(target: visit(node.target)) - end - - # Visit a MethodAddBlock node. - def visit_method_add_block(node) - node.copy(call: visit(node.call), block: visit(node.block)) - end - - # Visit a MLHS node. - def visit_mlhs(node) - node.copy(parts: visit_all(node.parts)) - end - - # Visit a MLHSParen node. - def visit_mlhs_paren(node) - node.copy(contents: visit(node.contents)) - end - - # Visit a ModuleDeclaration node. - def visit_module(node) - node.copy( - constant: visit(node.constant), - bodystmt: visit(node.bodystmt) - ) - end - - # Visit a MRHS node. - def visit_mrhs(node) - node.copy(parts: visit_all(node.parts)) - end - - # Visit a Next node. - def visit_next(node) - node.copy(arguments: visit(node.arguments)) - end - - # Visit a Op node. - def visit_op(node) - node.copy - end - - # Visit a OpAssign node. - def visit_opassign(node) - node.copy(target: visit(node.target), operator: visit(node.operator)) - end - - # Visit a Params node. - def visit_params(node) - node.copy( - requireds: visit_all(node.requireds), - optionals: - node.optionals.map { |ident, value| [visit(ident), visit(value)] }, - rest: visit(node.rest), - posts: visit_all(node.posts), - keywords: - node.keywords.map { |ident, value| [visit(ident), visit(value)] }, - keyword_rest: - node.keyword_rest == :nil ? :nil : visit(node.keyword_rest), - block: visit(node.block) - ) - end - - # Visit a Paren node. - def visit_paren(node) - node.copy(lparen: visit(node.lparen), contents: visit(node.contents)) - end - - # Visit a Period node. - def visit_period(node) - node.copy - end - - # Visit a Program node. - def visit_program(node) - node.copy(statements: visit(node.statements)) - end - - # Visit a QSymbols node. - def visit_qsymbols(node) - node.copy( - beginning: visit(node.beginning), - elements: visit_all(node.elements) - ) - end - - # Visit a QSymbolsBeg node. - def visit_qsymbols_beg(node) - node.copy - end - - # Visit a QWords node. - def visit_qwords(node) - node.copy( - beginning: visit(node.beginning), - elements: visit_all(node.elements) - ) - end - - # Visit a QWordsBeg node. - def visit_qwords_beg(node) - node.copy - end - - # Visit a RationalLiteral node. - def visit_rational(node) - node.copy - end - - # Visit a RBrace node. - def visit_rbrace(node) - node.copy - end - - # Visit a RBracket node. - def visit_rbracket(node) - node.copy - end - - # Visit a Redo node. - def visit_redo(node) - node.copy - end - - # Visit a RegexpContent node. - def visit_regexp_content(node) - node.copy(parts: visit_all(node.parts)) - end - - # Visit a RegexpBeg node. - def visit_regexp_beg(node) - node.copy - end - - # Visit a RegexpEnd node. - def visit_regexp_end(node) - node.copy - end - - # Visit a RegexpLiteral node. - def visit_regexp_literal(node) - node.copy(parts: visit_all(node.parts)) - end - - # Visit a RescueEx node. - def visit_rescue_ex(node) - node.copy(variable: visit(node.variable)) - end - - # Visit a Rescue node. - def visit_rescue(node) - node.copy( - keyword: visit(node.keyword), - exception: visit(node.exception), - statements: visit(node.statements), - consequent: visit(node.consequent) - ) - end - - # Visit a RescueMod node. - def visit_rescue_mod(node) - node.copy - end - - # Visit a RestParam node. - def visit_rest_param(node) - node.copy(name: visit(node.name)) - end - - # Visit a Retry node. - def visit_retry(node) - node.copy - end - - # Visit a Return node. - def visit_return(node) - node.copy(arguments: visit(node.arguments)) - end - - # Visit a RParen node. - def visit_rparen(node) - node.copy - end - - # Visit a SClass node. - def visit_sclass(node) - node.copy(bodystmt: visit(node.bodystmt)) - end - - # Visit a Statements node. - def visit_statements(node) - node.copy(body: visit_all(node.body)) - end - - # Visit a StringContent node. - def visit_string_content(node) - node.copy(parts: visit_all(node.parts)) - end - - # Visit a StringConcat node. - def visit_string_concat(node) - node.copy(left: visit(node.left), right: visit(node.right)) - end - - # Visit a StringDVar node. - def visit_string_dvar(node) - node.copy(variable: visit(node.variable)) - end - - # Visit a StringEmbExpr node. - def visit_string_embexpr(node) - node.copy(statements: visit(node.statements)) - end - - # Visit a StringLiteral node. - def visit_string_literal(node) - node.copy(parts: visit_all(node.parts)) - end - - # Visit a Super node. - def visit_super(node) - node.copy(arguments: visit(node.arguments)) - end - - # Visit a SymBeg node. - def visit_symbeg(node) - node.copy - end - - # Visit a SymbolContent node. - def visit_symbol_content(node) - node.copy(value: visit(node.value)) - end - - # Visit a SymbolLiteral node. - def visit_symbol_literal(node) - node.copy(value: visit(node.value)) - end - - # Visit a Symbols node. - def visit_symbols(node) - node.copy( - beginning: visit(node.beginning), - elements: visit_all(node.elements) - ) - end - - # Visit a SymbolsBeg node. - def visit_symbols_beg(node) - node.copy - end - - # Visit a TLambda node. - def visit_tlambda(node) - node.copy - end - - # Visit a TLamBeg node. - def visit_tlambeg(node) - node.copy - end - - # Visit a TopConstField node. - def visit_top_const_field(node) - node.copy(constant: visit(node.constant)) - end - - # Visit a TopConstRef node. - def visit_top_const_ref(node) - node.copy(constant: visit(node.constant)) - end - - # Visit a TStringBeg node. - def visit_tstring_beg(node) - node.copy - end - - # Visit a TStringContent node. - def visit_tstring_content(node) - node.copy - end - - # Visit a TStringEnd node. - def visit_tstring_end(node) - node.copy - end - - # Visit a Not node. - def visit_not(node) - node.copy(statement: visit(node.statement)) - end - - # Visit a Unary node. - def visit_unary(node) - node.copy - end - - # Visit a Undef node. - def visit_undef(node) - node.copy(symbols: visit_all(node.symbols)) - end - - # Visit a UnlessNode node. - def visit_unless(node) - node.copy( - predicate: visit(node.predicate), - statements: visit(node.statements), - consequent: visit(node.consequent) - ) - end - - # Visit a UntilNode node. - def visit_until(node) - node.copy( - predicate: visit(node.predicate), - statements: visit(node.statements) - ) - end - - # Visit a VarField node. - def visit_var_field(node) - node.copy(value: visit(node.value)) - end - - # Visit a VarRef node. - def visit_var_ref(node) - node.copy(value: visit(node.value)) - end - - # Visit a PinnedVarRef node. - def visit_pinned_var_ref(node) - node.copy(value: visit(node.value)) - end - - # Visit a VCall node. - def visit_vcall(node) - node.copy(value: visit(node.value)) - end - - # Visit a VoidStmt node. - def visit_void_stmt(node) - node.copy - end - - # Visit a When node. - def visit_when(node) - node.copy( - arguments: visit(node.arguments), - statements: visit(node.statements), - consequent: visit(node.consequent) - ) - end - - # Visit a WhileNode node. - def visit_while(node) - node.copy( - predicate: visit(node.predicate), - statements: visit(node.statements) - ) - end - - # Visit a Word node. - def visit_word(node) - node.copy(parts: visit_all(node.parts)) - end - - # Visit a Words node. - def visit_words(node) - node.copy( - beginning: visit(node.beginning), - elements: visit_all(node.elements) - ) - end - - # Visit a WordsBeg node. - def visit_words_beg(node) - node.copy - end - - # Visit a XString node. - def visit_xstring(node) - node.copy(parts: visit_all(node.parts)) - end - - # Visit a XStringLiteral node. - def visit_xstring_literal(node) - node.copy(parts: visit_all(node.parts)) - end - - # Visit a YieldNode node. - def visit_yield(node) - node.copy(arguments: visit(node.arguments)) - end - - # Visit a ZSuper node. - def visit_zsuper(node) - node.copy - end - end - end -end diff --git a/lib/syntax_tree/visitor/pretty_print_visitor.rb b/lib/syntax_tree/visitor/pretty_print_visitor.rb deleted file mode 100644 index 674e3aac..00000000 --- a/lib/syntax_tree/visitor/pretty_print_visitor.rb +++ /dev/null @@ -1,85 +0,0 @@ -# frozen_string_literal: true - -module SyntaxTree - class Visitor - # This visitor pretty-prints the AST into an equivalent s-expression. - class PrettyPrintVisitor < FieldVisitor - attr_reader :q - - def initialize(q) - @q = q - end - - # This is here because we need to make sure the operator is cast to a - # string before we print it out. - def visit_binary(node) - node(node, "binary") do - field("left", node.left) - text("operator", node.operator.to_s) - field("right", node.right) - comments(node) - end - end - - # This is here to make it a little nicer to look at labels since they - # typically have their : at the end of the value. - def visit_label(node) - node(node, "label") do - q.breakable - q.text(":") - q.text(node.value[0...-1]) - comments(node) - end - end - - private - - def comments(node) - return if node.comments.empty? - - q.breakable - q.group(2, "(", ")") do - q.seplist(node.comments) { |comment| q.pp(comment) } - end - end - - def field(_name, value) - q.breakable - q.pp(value) - end - - def list(_name, values) - q.breakable - q.group(2, "(", ")") { q.seplist(values) { |value| q.pp(value) } } - end - - def node(_node, type) - q.group(2, "(", ")") do - q.text(type) - yield - end - end - - def pairs(_name, values) - q.group(2, "(", ")") do - q.seplist(values) do |(key, value)| - q.pp(key) - - if value - q.text("=") - q.group(2) do - q.breakable("") - q.pp(value) - end - end - end - end - end - - def text(_name, value) - q.breakable - q.text(value) - end - end - end -end diff --git a/lib/syntax_tree/visitor/with_environment.rb b/lib/syntax_tree/with_environment.rb similarity index 58% rename from lib/syntax_tree/visitor/with_environment.rb rename to lib/syntax_tree/with_environment.rb index 59033d50..60301390 100644 --- a/lib/syntax_tree/visitor/with_environment.rb +++ b/lib/syntax_tree/with_environment.rb @@ -22,6 +22,87 @@ module SyntaxTree # end # end module WithEnvironment + # The environment class is used to keep track of local variables and + # arguments inside a particular scope + class Environment + # This class tracks the occurrences of a local variable or argument + class Local + # [Symbol] The type of the local (e.g. :argument, :variable) + attr_reader :type + + # [Array[Location]] The locations of all definitions and assignments of + # this local + attr_reader :definitions + + # [Array[Location]] The locations of all usages of this local + attr_reader :usages + + # initialize: (Symbol type) -> void + def initialize(type) + @type = type + @definitions = [] + @usages = [] + end + + # add_definition: (Location location) -> void + def add_definition(location) + @definitions << location + end + + # add_usage: (Location location) -> void + def add_usage(location) + @usages << location + end + end + + # [Array[Local]] The local variables and arguments defined in this + # environment + attr_reader :locals + + # [Environment | nil] The parent environment + attr_reader :parent + + # initialize: (Environment | nil parent) -> void + def initialize(parent = nil) + @locals = {} + @parent = parent + end + + # Adding a local definition will either insert a new entry in the locals + # hash or append a new definition location to an existing local. Notice that + # it's not possible to change the type of a local after it has been + # registered + # add_local_definition: (Ident | Label identifier, Symbol type) -> void + def add_local_definition(identifier, type) + name = identifier.value.delete_suffix(":") + + @locals[name] ||= Local.new(type) + @locals[name].add_definition(identifier.location) + end + + # Adding a local usage will either insert a new entry in the locals + # hash or append a new usage location to an existing local. Notice that + # it's not possible to change the type of a local after it has been + # registered + # add_local_usage: (Ident | Label identifier, Symbol type) -> void + def add_local_usage(identifier, type) + name = identifier.value.delete_suffix(":") + + @locals[name] ||= Local.new(type) + @locals[name].add_usage(identifier.location) + end + + # Try to find the local given its name in this environment or any of its + # parents + # find_local: (String name) -> Local | nil + def find_local(name) + local = @locals[name] + return local unless local.nil? + + @parent&.find_local(name) + end + end + def current_environment @current_environment ||= Environment.new end diff --git a/lib/syntax_tree/yarv.rb b/lib/syntax_tree/yarv.rb index ff8d3801..bd5c54b9 100644 --- a/lib/syntax_tree/yarv.rb +++ b/lib/syntax_tree/yarv.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require "stringio" + require_relative "yarv/basic_block" require_relative "yarv/bf" require_relative "yarv/calldata" diff --git a/lib/syntax_tree/yarv/compiler.rb b/lib/syntax_tree/yarv/compiler.rb index e1a8544a..a8044faf 100644 --- a/lib/syntax_tree/yarv/compiler.rb +++ b/lib/syntax_tree/yarv/compiler.rb @@ -8,7 +8,7 @@ module YARV # # You use this as with any other visitor. First you parse code into a tree, # then you visit it with this compiler. Visiting the root node of the tree - # will return a SyntaxTree::Visitor::Compiler::InstructionSequence object. + # will return a SyntaxTree::YARV::Compiler::InstructionSequence object. # With that object you can call #to_a on it, which will return a serialized # form of the instruction sequence as an array. This array _should_ mirror # the array given by RubyVM::InstructionSequence#to_a. diff --git a/test/test_helper.rb b/test/test_helper.rb index e4452e3d..2c8f6466 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -94,7 +94,7 @@ def assert_syntax_tree(node) assert_includes(pretty, type) # Assert that we can get back a new tree by using the mutation visitor. - assert_operator node, :===, node.accept(Visitor::MutationVisitor.new) + assert_operator node, :===, node.accept(MutationVisitor.new) # Serialize the node to JSON, parse it back out, and assert that we have # found the expected type. From 0dd027671e860975d85fd8af3cf8e2e2c117a59a Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 10 Feb 2023 11:37:12 -0500 Subject: [PATCH 391/536] More utility functions --- lib/syntax_tree.rb | 35 ++++++++++++++++++- lib/syntax_tree/mermaid_visitor.rb | 6 +--- lib/syntax_tree/mutation_visitor.rb | 25 +++----------- lib/syntax_tree/with_environment.rb | 52 +++++++++++++---------------- 4 files changed, 64 insertions(+), 54 deletions(-) diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index 0bdc4827..70126b14 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -60,9 +60,36 @@ def self.format( maxwidth = DEFAULT_PRINT_WIDTH, base_indentation = DEFAULT_INDENTATION, options: Formatter::Options.new + ) + format_node( + source, + parse(source), + maxwidth, + base_indentation, + options: options + ) + end + + # Parses the given file and returns the formatted source. + def self.format_file( + filepath, + maxwidth = DEFAULT_PRINT_WIDTH, + base_indentation = DEFAULT_INDENTATION, + options: Formatter::Options.new + ) + format(read(filepath), maxwidth, base_indentation, options: options) + end + + # Accepts a node in the tree and returns the formatted source. + def self.format_node( + source, + node, + maxwidth = DEFAULT_PRINT_WIDTH, + base_indentation = DEFAULT_INDENTATION, + options: Formatter::Options.new ) formatter = Formatter.new(source, [], maxwidth, options: options) - parse(source).format(formatter) + node.format(formatter) formatter.flush(base_indentation) formatter.output.join @@ -130,4 +157,10 @@ def self.search(source, query, &block) Search.new(pattern).scan(program, &block) end + + # Searches through the given file using the given pattern and yields each + # node in the tree that matches the pattern to the given block. + def self.search_file(filepath, query, &block) + search(read(filepath), query, &block) + end end diff --git a/lib/syntax_tree/mermaid_visitor.rb b/lib/syntax_tree/mermaid_visitor.rb index 52d1b5c6..fc9f6706 100644 --- a/lib/syntax_tree/mermaid_visitor.rb +++ b/lib/syntax_tree/mermaid_visitor.rb @@ -29,11 +29,7 @@ def field(name, value) flowchart.link(target, visit(value), name) else to = - flowchart.node( - "#{target.id}_#{name}", - value.inspect, - shape: :stadium - ) + flowchart.node("#{target.id}_#{name}", value.inspect, shape: :stadium) flowchart.link(target, to, name) end end diff --git a/lib/syntax_tree/mutation_visitor.rb b/lib/syntax_tree/mutation_visitor.rb index 2d96620d..f96e442f 100644 --- a/lib/syntax_tree/mutation_visitor.rb +++ b/lib/syntax_tree/mutation_visitor.rb @@ -35,10 +35,7 @@ def visit(node) # Visit a BEGINBlock node. def visit_BEGIN(node) - node.copy( - lbrace: visit(node.lbrace), - statements: visit(node.statements) - ) + node.copy(lbrace: visit(node.lbrace), statements: visit(node.statements)) end # Visit a CHAR node. @@ -48,10 +45,7 @@ def visit_CHAR(node) # Visit a ENDBlock node. def visit_END(node) - node.copy( - lbrace: visit(node.lbrace), - statements: visit(node.statements) - ) + node.copy(lbrace: visit(node.lbrace), statements: visit(node.statements)) end # Visit a EndContent node. @@ -101,10 +95,7 @@ def visit_args_forward(node) # Visit a ArrayLiteral node. def visit_array(node) - node.copy( - lbracket: visit(node.lbracket), - contents: visit(node.contents) - ) + node.copy(lbracket: visit(node.lbracket), contents: visit(node.contents)) end # Visit a AryPtn node. @@ -493,10 +484,7 @@ def visit_label_end(node) # Visit a Lambda node. def visit_lambda(node) - node.copy( - params: visit(node.params), - statements: visit(node.statements) - ) + node.copy(params: visit(node.params), statements: visit(node.statements)) end # Visit a LambdaVar node. @@ -541,10 +529,7 @@ def visit_mlhs_paren(node) # Visit a ModuleDeclaration node. def visit_module(node) - node.copy( - constant: visit(node.constant), - bodystmt: visit(node.bodystmt) - ) + node.copy(constant: visit(node.constant), bodystmt: visit(node.bodystmt)) end # Visit a MRHS node. diff --git a/lib/syntax_tree/with_environment.rb b/lib/syntax_tree/with_environment.rb index 60301390..13f5e080 100644 --- a/lib/syntax_tree/with_environment.rb +++ b/lib/syntax_tree/with_environment.rb @@ -5,22 +5,25 @@ module SyntaxTree # from Visitor. The module overrides a few visit methods to automatically keep # track of local variables and arguments defined in the current environment. # Example usage: - # class MyVisitor < Visitor - # include WithEnvironment # - # def visit_ident(node) - # # Check if we're visiting an identifier for an argument, a local - # variable or something else - # local = current_environment.find_local(node) + # class MyVisitor < Visitor + # include WithEnvironment # - # if local.type == :argument - # # handle identifiers for arguments - # elsif local.type == :variable - # # handle identifiers for variables - # else - # # handle other identifiers, such as method names + # def visit_ident(node) + # # Check if we're visiting an identifier for an argument, a local + # # variable or something else + # local = current_environment.find_local(node) + # + # if local.type == :argument + # # handle identifiers for arguments + # elsif local.type == :variable + # # handle identifiers for variables + # else + # # handle other identifiers, such as method names + # end # end - # end + # end + # module WithEnvironment # The environment class is used to keep track of local variables and # arguments inside a particular scope @@ -37,19 +40,16 @@ class Local # [Array[Location]] The locations of all usages of this local attr_reader :usages - # initialize: (Symbol type) -> void def initialize(type) @type = type @definitions = [] @usages = [] end - # add_definition: (Location location) -> void def add_definition(location) @definitions << location end - # add_usage: (Location location) -> void def add_usage(location) @usages << location end @@ -62,17 +62,15 @@ def add_usage(location) # [Environment | nil] The parent environment attr_reader :parent - # initialize: (Environment | nil parent) -> void def initialize(parent = nil) @locals = {} @parent = parent end # Adding a local definition will either insert a new entry in the locals - # hash or append a new definition location to an existing local. Notice that - # it's not possible to change the type of a local after it has been - # registered - # add_local_definition: (Ident | Label identifier, Symbol type) -> void + # hash or append a new definition location to an existing local. Notice + # that it's not possible to change the type of a local after it has been + # registered. def add_local_definition(identifier, type) name = identifier.value.delete_suffix(":") @@ -83,8 +81,7 @@ def add_local_definition(identifier, type) # Adding a local usage will either insert a new entry in the locals # hash or append a new usage location to an existing local. Notice that # it's not possible to change the type of a local after it has been - # registered - # add_local_usage: (Ident | Label identifier, Symbol type) -> void + # registered. def add_local_usage(identifier, type) name = identifier.value.delete_suffix(":") @@ -93,8 +90,7 @@ def add_local_usage(identifier, type) end # Try to find the local given its name in this environment or any of its - # parents - # find_local: (String name) -> Local | nil + # parents. def find_local(name) local = @locals[name] return local unless local.nil? @@ -116,7 +112,7 @@ def with_new_environment end # Visits for nodes that create new environments, such as classes, modules - # and method definitions + # and method definitions. def visit_class(node) with_new_environment { super } end @@ -127,7 +123,7 @@ def visit_module(node) # When we find a method invocation with a block, only the code that happens # inside of the block needs a fresh environment. The method invocation - # itself happens in the same environment + # itself happens in the same environment. def visit_method_add_block(node) visit(node.call) with_new_environment { visit(node.block) } @@ -138,7 +134,7 @@ def visit_def(node) end # Visit for keeping track of local arguments, such as method and block - # arguments + # arguments. def visit_params(node) add_argument_definitions(node.requireds) From 1a202316e4919eef70ed6f2945d0135686982ad9 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 10 Feb 2023 11:56:41 -0500 Subject: [PATCH 392/536] Use visit_methods {} --- .rubocop.yml | 3 + lib/syntax_tree/field_visitor.rb | 1444 ++++----- lib/syntax_tree/index.rb | 110 +- lib/syntax_tree/language_server.rb | 170 +- lib/syntax_tree/mutation_visitor.rb | 1457 ++++----- lib/syntax_tree/parser.rb | 6 +- lib/syntax_tree/translation/parser.rb | 4231 +++++++++++++------------ lib/syntax_tree/with_environment.rb | 6 +- lib/syntax_tree/yarv/compiler.rb | 199 +- test/visitor_test.rb | 14 +- test/visitor_with_environment_test.rb | 50 +- 11 files changed, 3890 insertions(+), 3800 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index 21beca1b..e5a3fe96 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -84,6 +84,9 @@ Security/Eval: Style/AccessorGrouping: Enabled: false +Style/Alias: + Enabled: false + Style/CaseEquality: Enabled: false diff --git a/lib/syntax_tree/field_visitor.rb b/lib/syntax_tree/field_visitor.rb index f4fc00e3..ca1df55b 100644 --- a/lib/syntax_tree/field_visitor.rb +++ b/lib/syntax_tree/field_visitor.rb @@ -48,972 +48,974 @@ module SyntaxTree # a method. # class FieldVisitor < BasicVisitor - def visit_aref(node) - node(node, "aref") do - field("collection", node.collection) - field("index", node.index) - comments(node) + visit_methods do + def visit_aref(node) + node(node, "aref") do + field("collection", node.collection) + field("index", node.index) + comments(node) + end end - end - def visit_aref_field(node) - node(node, "aref_field") do - field("collection", node.collection) - field("index", node.index) - comments(node) + def visit_aref_field(node) + node(node, "aref_field") do + field("collection", node.collection) + field("index", node.index) + comments(node) + end end - end - def visit_alias(node) - node(node, "alias") do - field("left", node.left) - field("right", node.right) - comments(node) + def visit_alias(node) + node(node, "alias") do + field("left", node.left) + field("right", node.right) + comments(node) + end end - end - def visit_arg_block(node) - node(node, "arg_block") do - field("value", node.value) if node.value - comments(node) + def visit_arg_block(node) + node(node, "arg_block") do + field("value", node.value) if node.value + comments(node) + end end - end - def visit_arg_paren(node) - node(node, "arg_paren") do - field("arguments", node.arguments) - comments(node) + def visit_arg_paren(node) + node(node, "arg_paren") do + field("arguments", node.arguments) + comments(node) + end end - end - def visit_arg_star(node) - node(node, "arg_star") do - field("value", node.value) - comments(node) + def visit_arg_star(node) + node(node, "arg_star") do + field("value", node.value) + comments(node) + end end - end - def visit_args(node) - node(node, "args") do - list("parts", node.parts) - comments(node) + def visit_args(node) + node(node, "args") do + list("parts", node.parts) + comments(node) + end end - end - def visit_args_forward(node) - node(node, "args_forward") { comments(node) } - end + def visit_args_forward(node) + node(node, "args_forward") { comments(node) } + end - def visit_array(node) - node(node, "array") do - field("contents", node.contents) - comments(node) + def visit_array(node) + node(node, "array") do + field("contents", node.contents) + comments(node) + end end - end - def visit_aryptn(node) - node(node, "aryptn") do - field("constant", node.constant) if node.constant - list("requireds", node.requireds) if node.requireds.any? - field("rest", node.rest) if node.rest - list("posts", node.posts) if node.posts.any? - comments(node) + def visit_aryptn(node) + node(node, "aryptn") do + field("constant", node.constant) if node.constant + list("requireds", node.requireds) if node.requireds.any? + field("rest", node.rest) if node.rest + list("posts", node.posts) if node.posts.any? + comments(node) + end end - end - def visit_assign(node) - node(node, "assign") do - field("target", node.target) - field("value", node.value) - comments(node) + def visit_assign(node) + node(node, "assign") do + field("target", node.target) + field("value", node.value) + comments(node) + end end - end - def visit_assoc(node) - node(node, "assoc") do - field("key", node.key) - field("value", node.value) if node.value - comments(node) + def visit_assoc(node) + node(node, "assoc") do + field("key", node.key) + field("value", node.value) if node.value + comments(node) + end end - end - def visit_assoc_splat(node) - node(node, "assoc_splat") do - field("value", node.value) - comments(node) + def visit_assoc_splat(node) + node(node, "assoc_splat") do + field("value", node.value) + comments(node) + end end - end - def visit_backref(node) - visit_token(node, "backref") - end + def visit_backref(node) + visit_token(node, "backref") + end - def visit_backtick(node) - visit_token(node, "backtick") - end + def visit_backtick(node) + visit_token(node, "backtick") + end - def visit_bare_assoc_hash(node) - node(node, "bare_assoc_hash") do - list("assocs", node.assocs) - comments(node) + def visit_bare_assoc_hash(node) + node(node, "bare_assoc_hash") do + list("assocs", node.assocs) + comments(node) + end end - end - def visit_BEGIN(node) - node(node, "BEGIN") do - field("statements", node.statements) - comments(node) + def visit_BEGIN(node) + node(node, "BEGIN") do + field("statements", node.statements) + comments(node) + end end - end - def visit_begin(node) - node(node, "begin") do - field("bodystmt", node.bodystmt) - comments(node) + def visit_begin(node) + node(node, "begin") do + field("bodystmt", node.bodystmt) + comments(node) + end end - end - def visit_binary(node) - node(node, "binary") do - field("left", node.left) - text("operator", node.operator) - field("right", node.right) - comments(node) + def visit_binary(node) + node(node, "binary") do + field("left", node.left) + text("operator", node.operator) + field("right", node.right) + comments(node) + end end - end - def visit_block(node) - node(node, "block") do - field("block_var", node.block_var) if node.block_var - field("bodystmt", node.bodystmt) - comments(node) + def visit_block(node) + node(node, "block") do + field("block_var", node.block_var) if node.block_var + field("bodystmt", node.bodystmt) + comments(node) + end end - end - def visit_blockarg(node) - node(node, "blockarg") do - field("name", node.name) if node.name - comments(node) + def visit_blockarg(node) + node(node, "blockarg") do + field("name", node.name) if node.name + comments(node) + end end - end - def visit_block_var(node) - node(node, "block_var") do - field("params", node.params) - list("locals", node.locals) if node.locals.any? - comments(node) + def visit_block_var(node) + node(node, "block_var") do + field("params", node.params) + list("locals", node.locals) if node.locals.any? + comments(node) + end end - end - def visit_bodystmt(node) - node(node, "bodystmt") do - field("statements", node.statements) - field("rescue_clause", node.rescue_clause) if node.rescue_clause - field("else_clause", node.else_clause) if node.else_clause - field("ensure_clause", node.ensure_clause) if node.ensure_clause - comments(node) + def visit_bodystmt(node) + node(node, "bodystmt") do + field("statements", node.statements) + field("rescue_clause", node.rescue_clause) if node.rescue_clause + field("else_clause", node.else_clause) if node.else_clause + field("ensure_clause", node.ensure_clause) if node.ensure_clause + comments(node) + end end - end - def visit_break(node) - node(node, "break") do - field("arguments", node.arguments) - comments(node) + def visit_break(node) + node(node, "break") do + field("arguments", node.arguments) + comments(node) + end end - end - def visit_call(node) - node(node, "call") do - field("receiver", node.receiver) - field("operator", node.operator) - field("message", node.message) - field("arguments", node.arguments) if node.arguments - comments(node) + def visit_call(node) + node(node, "call") do + field("receiver", node.receiver) + field("operator", node.operator) + field("message", node.message) + field("arguments", node.arguments) if node.arguments + comments(node) + end end - end - def visit_case(node) - node(node, "case") do - field("keyword", node.keyword) - field("value", node.value) if node.value - field("consequent", node.consequent) - comments(node) + def visit_case(node) + node(node, "case") do + field("keyword", node.keyword) + field("value", node.value) if node.value + field("consequent", node.consequent) + comments(node) + end end - end - def visit_CHAR(node) - visit_token(node, "CHAR") - end + def visit_CHAR(node) + visit_token(node, "CHAR") + end - def visit_class(node) - node(node, "class") do - field("constant", node.constant) - field("superclass", node.superclass) if node.superclass - field("bodystmt", node.bodystmt) - comments(node) + def visit_class(node) + node(node, "class") do + field("constant", node.constant) + field("superclass", node.superclass) if node.superclass + field("bodystmt", node.bodystmt) + comments(node) + end end - end - def visit_comma(node) - node(node, "comma") { field("value", node.value) } - end + def visit_comma(node) + node(node, "comma") { field("value", node.value) } + end - def visit_command(node) - node(node, "command") do - field("message", node.message) - field("arguments", node.arguments) - comments(node) + def visit_command(node) + node(node, "command") do + field("message", node.message) + field("arguments", node.arguments) + comments(node) + end end - end - def visit_command_call(node) - node(node, "command_call") do - field("receiver", node.receiver) - field("operator", node.operator) - field("message", node.message) - field("arguments", node.arguments) if node.arguments - comments(node) + def visit_command_call(node) + node(node, "command_call") do + field("receiver", node.receiver) + field("operator", node.operator) + field("message", node.message) + field("arguments", node.arguments) if node.arguments + comments(node) + end end - end - def visit_comment(node) - node(node, "comment") { field("value", node.value) } - end + def visit_comment(node) + node(node, "comment") { field("value", node.value) } + end - def visit_const(node) - visit_token(node, "const") - end + def visit_const(node) + visit_token(node, "const") + end - def visit_const_path_field(node) - node(node, "const_path_field") do - field("parent", node.parent) - field("constant", node.constant) - comments(node) + def visit_const_path_field(node) + node(node, "const_path_field") do + field("parent", node.parent) + field("constant", node.constant) + comments(node) + end end - end - def visit_const_path_ref(node) - node(node, "const_path_ref") do - field("parent", node.parent) - field("constant", node.constant) - comments(node) + def visit_const_path_ref(node) + node(node, "const_path_ref") do + field("parent", node.parent) + field("constant", node.constant) + comments(node) + end end - end - def visit_const_ref(node) - node(node, "const_ref") do - field("constant", node.constant) - comments(node) + def visit_const_ref(node) + node(node, "const_ref") do + field("constant", node.constant) + comments(node) + end end - end - def visit_cvar(node) - visit_token(node, "cvar") - end + def visit_cvar(node) + visit_token(node, "cvar") + end - def visit_def(node) - node(node, "def") do - field("target", node.target) - field("operator", node.operator) - field("name", node.name) - field("params", node.params) - field("bodystmt", node.bodystmt) - comments(node) + def visit_def(node) + node(node, "def") do + field("target", node.target) + field("operator", node.operator) + field("name", node.name) + field("params", node.params) + field("bodystmt", node.bodystmt) + comments(node) + end end - end - def visit_defined(node) - node(node, "defined") do - field("value", node.value) - comments(node) + def visit_defined(node) + node(node, "defined") do + field("value", node.value) + comments(node) + end end - end - def visit_dyna_symbol(node) - node(node, "dyna_symbol") do - list("parts", node.parts) - comments(node) + def visit_dyna_symbol(node) + node(node, "dyna_symbol") do + list("parts", node.parts) + comments(node) + end end - end - def visit_END(node) - node(node, "END") do - field("statements", node.statements) - comments(node) + def visit_END(node) + node(node, "END") do + field("statements", node.statements) + comments(node) + end end - end - def visit_else(node) - node(node, "else") do - field("statements", node.statements) - comments(node) + def visit_else(node) + node(node, "else") do + field("statements", node.statements) + comments(node) + end end - end - def visit_elsif(node) - node(node, "elsif") do - field("predicate", node.predicate) - field("statements", node.statements) - field("consequent", node.consequent) if node.consequent - comments(node) + def visit_elsif(node) + node(node, "elsif") do + field("predicate", node.predicate) + field("statements", node.statements) + field("consequent", node.consequent) if node.consequent + comments(node) + end end - end - def visit_embdoc(node) - node(node, "embdoc") { field("value", node.value) } - end + def visit_embdoc(node) + node(node, "embdoc") { field("value", node.value) } + end - def visit_embexpr_beg(node) - node(node, "embexpr_beg") { field("value", node.value) } - end + def visit_embexpr_beg(node) + node(node, "embexpr_beg") { field("value", node.value) } + end - def visit_embexpr_end(node) - node(node, "embexpr_end") { field("value", node.value) } - end + def visit_embexpr_end(node) + node(node, "embexpr_end") { field("value", node.value) } + end - def visit_embvar(node) - node(node, "embvar") { field("value", node.value) } - end + def visit_embvar(node) + node(node, "embvar") { field("value", node.value) } + end - def visit_ensure(node) - node(node, "ensure") do - field("statements", node.statements) - comments(node) + def visit_ensure(node) + node(node, "ensure") do + field("statements", node.statements) + comments(node) + end end - end - def visit_excessed_comma(node) - visit_token(node, "excessed_comma") - end + def visit_excessed_comma(node) + visit_token(node, "excessed_comma") + end - def visit_field(node) - node(node, "field") do - field("parent", node.parent) - field("operator", node.operator) - field("name", node.name) - comments(node) + def visit_field(node) + node(node, "field") do + field("parent", node.parent) + field("operator", node.operator) + field("name", node.name) + comments(node) + end end - end - def visit_float(node) - visit_token(node, "float") - end + def visit_float(node) + visit_token(node, "float") + end - def visit_fndptn(node) - node(node, "fndptn") do - field("constant", node.constant) if node.constant - field("left", node.left) - list("values", node.values) - field("right", node.right) - comments(node) + def visit_fndptn(node) + node(node, "fndptn") do + field("constant", node.constant) if node.constant + field("left", node.left) + list("values", node.values) + field("right", node.right) + comments(node) + end end - end - def visit_for(node) - node(node, "for") do - field("index", node.index) - field("collection", node.collection) - field("statements", node.statements) - comments(node) + def visit_for(node) + node(node, "for") do + field("index", node.index) + field("collection", node.collection) + field("statements", node.statements) + comments(node) + end end - end - def visit_gvar(node) - visit_token(node, "gvar") - end + def visit_gvar(node) + visit_token(node, "gvar") + end - def visit_hash(node) - node(node, "hash") do - list("assocs", node.assocs) if node.assocs.any? - comments(node) + def visit_hash(node) + node(node, "hash") do + list("assocs", node.assocs) if node.assocs.any? + comments(node) + end end - end - def visit_heredoc(node) - node(node, "heredoc") do - list("parts", node.parts) - comments(node) + def visit_heredoc(node) + node(node, "heredoc") do + list("parts", node.parts) + comments(node) + end end - end - def visit_heredoc_beg(node) - visit_token(node, "heredoc_beg") - end + def visit_heredoc_beg(node) + visit_token(node, "heredoc_beg") + end - def visit_heredoc_end(node) - visit_token(node, "heredoc_end") - end + def visit_heredoc_end(node) + visit_token(node, "heredoc_end") + end - def visit_hshptn(node) - node(node, "hshptn") do - field("constant", node.constant) if node.constant - pairs("keywords", node.keywords) if node.keywords.any? - field("keyword_rest", node.keyword_rest) if node.keyword_rest - comments(node) + def visit_hshptn(node) + node(node, "hshptn") do + field("constant", node.constant) if node.constant + pairs("keywords", node.keywords) if node.keywords.any? + field("keyword_rest", node.keyword_rest) if node.keyword_rest + comments(node) + end end - end - def visit_ident(node) - visit_token(node, "ident") - end + def visit_ident(node) + visit_token(node, "ident") + end - def visit_if(node) - node(node, "if") do - field("predicate", node.predicate) - field("statements", node.statements) - field("consequent", node.consequent) if node.consequent - comments(node) + def visit_if(node) + node(node, "if") do + field("predicate", node.predicate) + field("statements", node.statements) + field("consequent", node.consequent) if node.consequent + comments(node) + end end - end - def visit_if_op(node) - node(node, "if_op") do - field("predicate", node.predicate) - field("truthy", node.truthy) - field("falsy", node.falsy) - comments(node) + def visit_if_op(node) + node(node, "if_op") do + field("predicate", node.predicate) + field("truthy", node.truthy) + field("falsy", node.falsy) + comments(node) + end end - end - def visit_imaginary(node) - visit_token(node, "imaginary") - end + def visit_imaginary(node) + visit_token(node, "imaginary") + end - def visit_in(node) - node(node, "in") do - field("pattern", node.pattern) - field("statements", node.statements) - field("consequent", node.consequent) if node.consequent - comments(node) + def visit_in(node) + node(node, "in") do + field("pattern", node.pattern) + field("statements", node.statements) + field("consequent", node.consequent) if node.consequent + comments(node) + end end - end - def visit_int(node) - visit_token(node, "int") - end + def visit_int(node) + visit_token(node, "int") + end - def visit_ivar(node) - visit_token(node, "ivar") - end + def visit_ivar(node) + visit_token(node, "ivar") + end - def visit_kw(node) - visit_token(node, "kw") - end + def visit_kw(node) + visit_token(node, "kw") + end - def visit_kwrest_param(node) - node(node, "kwrest_param") do - field("name", node.name) - comments(node) + def visit_kwrest_param(node) + node(node, "kwrest_param") do + field("name", node.name) + comments(node) + end end - end - def visit_label(node) - visit_token(node, "label") - end + def visit_label(node) + visit_token(node, "label") + end - def visit_label_end(node) - node(node, "label_end") { field("value", node.value) } - end + def visit_label_end(node) + node(node, "label_end") { field("value", node.value) } + end - def visit_lambda(node) - node(node, "lambda") do - field("params", node.params) - field("statements", node.statements) - comments(node) + def visit_lambda(node) + node(node, "lambda") do + field("params", node.params) + field("statements", node.statements) + comments(node) + end end - end - def visit_lambda_var(node) - node(node, "lambda_var") do - field("params", node.params) - list("locals", node.locals) if node.locals.any? - comments(node) + def visit_lambda_var(node) + node(node, "lambda_var") do + field("params", node.params) + list("locals", node.locals) if node.locals.any? + comments(node) + end end - end - def visit_lbrace(node) - visit_token(node, "lbrace") - end + def visit_lbrace(node) + visit_token(node, "lbrace") + end - def visit_lbracket(node) - visit_token(node, "lbracket") - end + def visit_lbracket(node) + visit_token(node, "lbracket") + end - def visit_lparen(node) - visit_token(node, "lparen") - end + def visit_lparen(node) + visit_token(node, "lparen") + end - def visit_massign(node) - node(node, "massign") do - field("target", node.target) - field("value", node.value) - comments(node) + def visit_massign(node) + node(node, "massign") do + field("target", node.target) + field("value", node.value) + comments(node) + end end - end - def visit_method_add_block(node) - node(node, "method_add_block") do - field("call", node.call) - field("block", node.block) - comments(node) + def visit_method_add_block(node) + node(node, "method_add_block") do + field("call", node.call) + field("block", node.block) + comments(node) + end end - end - def visit_mlhs(node) - node(node, "mlhs") do - list("parts", node.parts) - comments(node) + def visit_mlhs(node) + node(node, "mlhs") do + list("parts", node.parts) + comments(node) + end end - end - def visit_mlhs_paren(node) - node(node, "mlhs_paren") do - field("contents", node.contents) - comments(node) + def visit_mlhs_paren(node) + node(node, "mlhs_paren") do + field("contents", node.contents) + comments(node) + end end - end - def visit_module(node) - node(node, "module") do - field("constant", node.constant) - field("bodystmt", node.bodystmt) - comments(node) + def visit_module(node) + node(node, "module") do + field("constant", node.constant) + field("bodystmt", node.bodystmt) + comments(node) + end end - end - def visit_mrhs(node) - node(node, "mrhs") do - list("parts", node.parts) - comments(node) + def visit_mrhs(node) + node(node, "mrhs") do + list("parts", node.parts) + comments(node) + end end - end - def visit_next(node) - node(node, "next") do - field("arguments", node.arguments) - comments(node) + def visit_next(node) + node(node, "next") do + field("arguments", node.arguments) + comments(node) + end end - end - def visit_not(node) - node(node, "not") do - field("statement", node.statement) - comments(node) + def visit_not(node) + node(node, "not") do + field("statement", node.statement) + comments(node) + end end - end - def visit_op(node) - visit_token(node, "op") - end + def visit_op(node) + visit_token(node, "op") + end - def visit_opassign(node) - node(node, "opassign") do - field("target", node.target) - field("operator", node.operator) - field("value", node.value) - comments(node) + def visit_opassign(node) + node(node, "opassign") do + field("target", node.target) + field("operator", node.operator) + field("value", node.value) + comments(node) + end end - end - def visit_params(node) - node(node, "params") do - list("requireds", node.requireds) if node.requireds.any? - pairs("optionals", node.optionals) if node.optionals.any? - field("rest", node.rest) if node.rest - list("posts", node.posts) if node.posts.any? - pairs("keywords", node.keywords) if node.keywords.any? - field("keyword_rest", node.keyword_rest) if node.keyword_rest - field("block", node.block) if node.block - comments(node) + def visit_params(node) + node(node, "params") do + list("requireds", node.requireds) if node.requireds.any? + pairs("optionals", node.optionals) if node.optionals.any? + field("rest", node.rest) if node.rest + list("posts", node.posts) if node.posts.any? + pairs("keywords", node.keywords) if node.keywords.any? + field("keyword_rest", node.keyword_rest) if node.keyword_rest + field("block", node.block) if node.block + comments(node) + end end - end - def visit_paren(node) - node(node, "paren") do - field("contents", node.contents) - comments(node) + def visit_paren(node) + node(node, "paren") do + field("contents", node.contents) + comments(node) + end end - end - def visit_period(node) - visit_token(node, "period") - end + def visit_period(node) + visit_token(node, "period") + end - def visit_pinned_begin(node) - node(node, "pinned_begin") do - field("statement", node.statement) - comments(node) + def visit_pinned_begin(node) + node(node, "pinned_begin") do + field("statement", node.statement) + comments(node) + end end - end - def visit_pinned_var_ref(node) - node(node, "pinned_var_ref") do - field("value", node.value) - comments(node) + def visit_pinned_var_ref(node) + node(node, "pinned_var_ref") do + field("value", node.value) + comments(node) + end end - end - def visit_program(node) - node(node, "program") do - field("statements", node.statements) - comments(node) + def visit_program(node) + node(node, "program") do + field("statements", node.statements) + comments(node) + end end - end - def visit_qsymbols(node) - node(node, "qsymbols") do - list("elements", node.elements) - comments(node) + def visit_qsymbols(node) + node(node, "qsymbols") do + list("elements", node.elements) + comments(node) + end end - end - def visit_qsymbols_beg(node) - node(node, "qsymbols_beg") { field("value", node.value) } - end + def visit_qsymbols_beg(node) + node(node, "qsymbols_beg") { field("value", node.value) } + end - def visit_qwords(node) - node(node, "qwords") do - list("elements", node.elements) - comments(node) + def visit_qwords(node) + node(node, "qwords") do + list("elements", node.elements) + comments(node) + end end - end - def visit_qwords_beg(node) - node(node, "qwords_beg") { field("value", node.value) } - end + def visit_qwords_beg(node) + node(node, "qwords_beg") { field("value", node.value) } + end - def visit_range(node) - node(node, "range") do - field("left", node.left) if node.left - field("operator", node.operator) - field("right", node.right) if node.right - comments(node) + def visit_range(node) + node(node, "range") do + field("left", node.left) if node.left + field("operator", node.operator) + field("right", node.right) if node.right + comments(node) + end end - end - def visit_rassign(node) - node(node, "rassign") do - field("value", node.value) - field("operator", node.operator) - field("pattern", node.pattern) - comments(node) + def visit_rassign(node) + node(node, "rassign") do + field("value", node.value) + field("operator", node.operator) + field("pattern", node.pattern) + comments(node) + end end - end - def visit_rational(node) - visit_token(node, "rational") - end + def visit_rational(node) + visit_token(node, "rational") + end - def visit_rbrace(node) - node(node, "rbrace") { field("value", node.value) } - end + def visit_rbrace(node) + node(node, "rbrace") { field("value", node.value) } + end - def visit_rbracket(node) - node(node, "rbracket") { field("value", node.value) } - end + def visit_rbracket(node) + node(node, "rbracket") { field("value", node.value) } + end - def visit_redo(node) - node(node, "redo") { comments(node) } - end + def visit_redo(node) + node(node, "redo") { comments(node) } + end - def visit_regexp_beg(node) - node(node, "regexp_beg") { field("value", node.value) } - end + def visit_regexp_beg(node) + node(node, "regexp_beg") { field("value", node.value) } + end - def visit_regexp_content(node) - node(node, "regexp_content") { list("parts", node.parts) } - end + def visit_regexp_content(node) + node(node, "regexp_content") { list("parts", node.parts) } + end - def visit_regexp_end(node) - node(node, "regexp_end") { field("value", node.value) } - end + def visit_regexp_end(node) + node(node, "regexp_end") { field("value", node.value) } + end - def visit_regexp_literal(node) - node(node, "regexp_literal") do - list("parts", node.parts) - field("options", node.options) - comments(node) + def visit_regexp_literal(node) + node(node, "regexp_literal") do + list("parts", node.parts) + field("options", node.options) + comments(node) + end end - end - def visit_rescue(node) - node(node, "rescue") do - field("exception", node.exception) if node.exception - field("statements", node.statements) - field("consequent", node.consequent) if node.consequent - comments(node) + def visit_rescue(node) + node(node, "rescue") do + field("exception", node.exception) if node.exception + field("statements", node.statements) + field("consequent", node.consequent) if node.consequent + comments(node) + end end - end - def visit_rescue_ex(node) - node(node, "rescue_ex") do - field("exceptions", node.exceptions) - field("variable", node.variable) - comments(node) + def visit_rescue_ex(node) + node(node, "rescue_ex") do + field("exceptions", node.exceptions) + field("variable", node.variable) + comments(node) + end end - end - def visit_rescue_mod(node) - node(node, "rescue_mod") do - field("statement", node.statement) - field("value", node.value) - comments(node) + def visit_rescue_mod(node) + node(node, "rescue_mod") do + field("statement", node.statement) + field("value", node.value) + comments(node) + end end - end - def visit_rest_param(node) - node(node, "rest_param") do - field("name", node.name) - comments(node) + def visit_rest_param(node) + node(node, "rest_param") do + field("name", node.name) + comments(node) + end end - end - def visit_retry(node) - node(node, "retry") { comments(node) } - end + def visit_retry(node) + node(node, "retry") { comments(node) } + end - def visit_return(node) - node(node, "return") do - field("arguments", node.arguments) - comments(node) + def visit_return(node) + node(node, "return") do + field("arguments", node.arguments) + comments(node) + end end - end - def visit_rparen(node) - node(node, "rparen") { field("value", node.value) } - end + def visit_rparen(node) + node(node, "rparen") { field("value", node.value) } + end - def visit_sclass(node) - node(node, "sclass") do - field("target", node.target) - field("bodystmt", node.bodystmt) - comments(node) + def visit_sclass(node) + node(node, "sclass") do + field("target", node.target) + field("bodystmt", node.bodystmt) + comments(node) + end end - end - def visit_statements(node) - node(node, "statements") do - list("body", node.body) - comments(node) + def visit_statements(node) + node(node, "statements") do + list("body", node.body) + comments(node) + end end - end - def visit_string_concat(node) - node(node, "string_concat") do - field("left", node.left) - field("right", node.right) - comments(node) + def visit_string_concat(node) + node(node, "string_concat") do + field("left", node.left) + field("right", node.right) + comments(node) + end end - end - def visit_string_content(node) - node(node, "string_content") { list("parts", node.parts) } - end + def visit_string_content(node) + node(node, "string_content") { list("parts", node.parts) } + end - def visit_string_dvar(node) - node(node, "string_dvar") do - field("variable", node.variable) - comments(node) + def visit_string_dvar(node) + node(node, "string_dvar") do + field("variable", node.variable) + comments(node) + end end - end - def visit_string_embexpr(node) - node(node, "string_embexpr") do - field("statements", node.statements) - comments(node) + def visit_string_embexpr(node) + node(node, "string_embexpr") do + field("statements", node.statements) + comments(node) + end end - end - def visit_string_literal(node) - node(node, "string_literal") do - list("parts", node.parts) - comments(node) + def visit_string_literal(node) + node(node, "string_literal") do + list("parts", node.parts) + comments(node) + end end - end - def visit_super(node) - node(node, "super") do - field("arguments", node.arguments) - comments(node) + def visit_super(node) + node(node, "super") do + field("arguments", node.arguments) + comments(node) + end end - end - def visit_symbeg(node) - node(node, "symbeg") { field("value", node.value) } - end + def visit_symbeg(node) + node(node, "symbeg") { field("value", node.value) } + end - def visit_symbol_content(node) - node(node, "symbol_content") { field("value", node.value) } - end + def visit_symbol_content(node) + node(node, "symbol_content") { field("value", node.value) } + end - def visit_symbol_literal(node) - node(node, "symbol_literal") do - field("value", node.value) - comments(node) + def visit_symbol_literal(node) + node(node, "symbol_literal") do + field("value", node.value) + comments(node) + end end - end - def visit_symbols(node) - node(node, "symbols") do - list("elements", node.elements) - comments(node) + def visit_symbols(node) + node(node, "symbols") do + list("elements", node.elements) + comments(node) + end end - end - def visit_symbols_beg(node) - node(node, "symbols_beg") { field("value", node.value) } - end + def visit_symbols_beg(node) + node(node, "symbols_beg") { field("value", node.value) } + end - def visit_tlambda(node) - node(node, "tlambda") { field("value", node.value) } - end + def visit_tlambda(node) + node(node, "tlambda") { field("value", node.value) } + end - def visit_tlambeg(node) - node(node, "tlambeg") { field("value", node.value) } - end + def visit_tlambeg(node) + node(node, "tlambeg") { field("value", node.value) } + end - def visit_top_const_field(node) - node(node, "top_const_field") do - field("constant", node.constant) - comments(node) + def visit_top_const_field(node) + node(node, "top_const_field") do + field("constant", node.constant) + comments(node) + end end - end - def visit_top_const_ref(node) - node(node, "top_const_ref") do - field("constant", node.constant) - comments(node) + def visit_top_const_ref(node) + node(node, "top_const_ref") do + field("constant", node.constant) + comments(node) + end end - end - def visit_tstring_beg(node) - node(node, "tstring_beg") { field("value", node.value) } - end + def visit_tstring_beg(node) + node(node, "tstring_beg") { field("value", node.value) } + end - def visit_tstring_content(node) - visit_token(node, "tstring_content") - end + def visit_tstring_content(node) + visit_token(node, "tstring_content") + end - def visit_tstring_end(node) - node(node, "tstring_end") { field("value", node.value) } - end + def visit_tstring_end(node) + node(node, "tstring_end") { field("value", node.value) } + end - def visit_unary(node) - node(node, "unary") do - field("operator", node.operator) - field("statement", node.statement) - comments(node) + def visit_unary(node) + node(node, "unary") do + field("operator", node.operator) + field("statement", node.statement) + comments(node) + end end - end - def visit_undef(node) - node(node, "undef") do - list("symbols", node.symbols) - comments(node) + def visit_undef(node) + node(node, "undef") do + list("symbols", node.symbols) + comments(node) + end end - end - def visit_unless(node) - node(node, "unless") do - field("predicate", node.predicate) - field("statements", node.statements) - field("consequent", node.consequent) if node.consequent - comments(node) + def visit_unless(node) + node(node, "unless") do + field("predicate", node.predicate) + field("statements", node.statements) + field("consequent", node.consequent) if node.consequent + comments(node) + end end - end - def visit_until(node) - node(node, "until") do - field("predicate", node.predicate) - field("statements", node.statements) - comments(node) + def visit_until(node) + node(node, "until") do + field("predicate", node.predicate) + field("statements", node.statements) + comments(node) + end end - end - def visit_var_field(node) - node(node, "var_field") do - field("value", node.value) - comments(node) + def visit_var_field(node) + node(node, "var_field") do + field("value", node.value) + comments(node) + end end - end - def visit_var_ref(node) - node(node, "var_ref") do - field("value", node.value) - comments(node) + def visit_var_ref(node) + node(node, "var_ref") do + field("value", node.value) + comments(node) + end end - end - def visit_vcall(node) - node(node, "vcall") do - field("value", node.value) - comments(node) + def visit_vcall(node) + node(node, "vcall") do + field("value", node.value) + comments(node) + end end - end - def visit_void_stmt(node) - node(node, "void_stmt") { comments(node) } - end + def visit_void_stmt(node) + node(node, "void_stmt") { comments(node) } + end - def visit_when(node) - node(node, "when") do - field("arguments", node.arguments) - field("statements", node.statements) - field("consequent", node.consequent) if node.consequent - comments(node) + def visit_when(node) + node(node, "when") do + field("arguments", node.arguments) + field("statements", node.statements) + field("consequent", node.consequent) if node.consequent + comments(node) + end end - end - def visit_while(node) - node(node, "while") do - field("predicate", node.predicate) - field("statements", node.statements) - comments(node) + def visit_while(node) + node(node, "while") do + field("predicate", node.predicate) + field("statements", node.statements) + comments(node) + end end - end - def visit_word(node) - node(node, "word") do - list("parts", node.parts) - comments(node) + def visit_word(node) + node(node, "word") do + list("parts", node.parts) + comments(node) + end end - end - def visit_words(node) - node(node, "words") do - list("elements", node.elements) - comments(node) + def visit_words(node) + node(node, "words") do + list("elements", node.elements) + comments(node) + end end - end - def visit_words_beg(node) - node(node, "words_beg") { field("value", node.value) } - end + def visit_words_beg(node) + node(node, "words_beg") { field("value", node.value) } + end - def visit_xstring(node) - node(node, "xstring") { list("parts", node.parts) } - end + def visit_xstring(node) + node(node, "xstring") { list("parts", node.parts) } + end - def visit_xstring_literal(node) - node(node, "xstring_literal") do - list("parts", node.parts) - comments(node) + def visit_xstring_literal(node) + node(node, "xstring_literal") do + list("parts", node.parts) + comments(node) + end end - end - def visit_yield(node) - node(node, "yield") do - field("arguments", node.arguments) - comments(node) + def visit_yield(node) + node(node, "yield") do + field("arguments", node.arguments) + comments(node) + end end - end - def visit_zsuper(node) - node(node, "zsuper") { comments(node) } - end + def visit_zsuper(node) + node(node, "zsuper") { comments(node) } + end - def visit___end__(node) - visit_token(node, "__end__") + def visit___end__(node) + visit_token(node, "__end__") + end end private diff --git a/lib/syntax_tree/index.rb b/lib/syntax_tree/index.rb index 8b33f785..ab2460dd 100644 --- a/lib/syntax_tree/index.rb +++ b/lib/syntax_tree/index.rb @@ -257,74 +257,76 @@ def initialize @statements = nil end - def visit_class(node) - name = visit(node.constant).to_sym - location = - Location.new(node.location.start_line, node.location.start_column) - - results << ClassDefinition.new( - nesting.dup, - name, - location, - comments_for(node) - ) - - nesting << name - super - nesting.pop - end - - def visit_const_ref(node) - node.constant.value - end + visit_methods do + def visit_class(node) + name = visit(node.constant).to_sym + location = + Location.new(node.location.start_line, node.location.start_column) - def visit_def(node) - name = node.name.value.to_sym - location = - Location.new(node.location.start_line, node.location.start_column) - - results << if node.target.nil? - MethodDefinition.new( + results << ClassDefinition.new( nesting.dup, name, location, comments_for(node) ) - else - SingletonMethodDefinition.new( + + nesting << name + super + nesting.pop + end + + def visit_const_ref(node) + node.constant.value + end + + def visit_def(node) + name = node.name.value.to_sym + location = + Location.new(node.location.start_line, node.location.start_column) + + results << if node.target.nil? + MethodDefinition.new( + nesting.dup, + name, + location, + comments_for(node) + ) + else + SingletonMethodDefinition.new( + nesting.dup, + name, + location, + comments_for(node) + ) + end + end + + def visit_module(node) + name = visit(node.constant).to_sym + location = + Location.new(node.location.start_line, node.location.start_column) + + results << ModuleDefinition.new( nesting.dup, name, location, comments_for(node) ) - end - end - - def visit_module(node) - name = visit(node.constant).to_sym - location = - Location.new(node.location.start_line, node.location.start_column) - results << ModuleDefinition.new( - nesting.dup, - name, - location, - comments_for(node) - ) - - nesting << name - super - nesting.pop - end + nesting << name + super + nesting.pop + end - def visit_program(node) - super - results - end + def visit_program(node) + super + results + end - def visit_statements(node) - @statements = node - super + def visit_statements(node) + @statements = node + super + end end private diff --git a/lib/syntax_tree/language_server.rb b/lib/syntax_tree/language_server.rb index afb1540e..6ec81030 100644 --- a/lib/syntax_tree/language_server.rb +++ b/lib/syntax_tree/language_server.rb @@ -52,101 +52,103 @@ def visit(node) result end - # Adds parentheses around assignments contained within the default values - # of parameters. For example, - # - # def foo(a = b = c) - # end - # - # becomes - # - # def foo(a = ₍b = c₎) - # end - # - def visit_assign(node) - parentheses(node.location) if stack[-2].is_a?(Params) - super - end - - # Adds parentheses around binary expressions to make it clear which - # subexpression will be evaluated first. For example, - # - # a + b * c - # - # becomes - # - # a + ₍b * c₎ - # - def visit_binary(node) - case stack[-2] - when Assign, OpAssign - parentheses(node.location) - when Binary - parentheses(node.location) if stack[-2].operator != node.operator + visit_methods do + # Adds parentheses around assignments contained within the default + # values of parameters. For example, + # + # def foo(a = b = c) + # end + # + # becomes + # + # def foo(a = ₍b = c₎) + # end + # + def visit_assign(node) + parentheses(node.location) if stack[-2].is_a?(Params) + super end - super - end + # Adds parentheses around binary expressions to make it clear which + # subexpression will be evaluated first. For example, + # + # a + b * c + # + # becomes + # + # a + ₍b * c₎ + # + def visit_binary(node) + case stack[-2] + when Assign, OpAssign + parentheses(node.location) + when Binary + parentheses(node.location) if stack[-2].operator != node.operator + end - # Adds parentheses around ternary operators contained within certain - # expressions where it could be confusing which subexpression will get - # evaluated first. For example, - # - # a ? b : c ? d : e - # - # becomes - # - # a ? b : ₍c ? d : e₎ - # - def visit_if_op(node) - case stack[-2] - when Assign, Binary, IfOp, OpAssign - parentheses(node.location) + super end - super - end + # Adds parentheses around ternary operators contained within certain + # expressions where it could be confusing which subexpression will get + # evaluated first. For example, + # + # a ? b : c ? d : e + # + # becomes + # + # a ? b : ₍c ? d : e₎ + # + def visit_if_op(node) + case stack[-2] + when Assign, Binary, IfOp, OpAssign + parentheses(node.location) + end - # Adds the implicitly rescued StandardError into a bare rescue clause. For - # example, - # - # begin - # rescue - # end - # - # becomes - # - # begin - # rescue StandardError - # end - # - def visit_rescue(node) - if node.exception.nil? - hints << Hint.new( - line: node.location.start_line - 1, - character: node.location.start_column + "rescue".length, - label: " StandardError" - ) + super end - super - end + # Adds the implicitly rescued StandardError into a bare rescue clause. + # For example, + # + # begin + # rescue + # end + # + # becomes + # + # begin + # rescue StandardError + # end + # + def visit_rescue(node) + if node.exception.nil? + hints << Hint.new( + line: node.location.start_line - 1, + character: node.location.start_column + "rescue".length, + label: " StandardError" + ) + end - # Adds parentheses around unary statements using the - operator that are - # contained within Binary nodes. For example, - # - # -a + b - # - # becomes - # - # ₍-a₎ + b - # - def visit_unary(node) - if stack[-2].is_a?(Binary) && (node.operator == "-") - parentheses(node.location) + super end - super + # Adds parentheses around unary statements using the - operator that are + # contained within Binary nodes. For example, + # + # -a + b + # + # becomes + # + # ₍-a₎ + b + # + def visit_unary(node) + if stack[-2].is_a?(Binary) && (node.operator == "-") + parentheses(node.location) + end + + super + end end private diff --git a/lib/syntax_tree/mutation_visitor.rb b/lib/syntax_tree/mutation_visitor.rb index f96e442f..0b4b9357 100644 --- a/lib/syntax_tree/mutation_visitor.rb +++ b/lib/syntax_tree/mutation_visitor.rb @@ -33,875 +33,892 @@ def visit(node) result end - # Visit a BEGINBlock node. - def visit_BEGIN(node) - node.copy(lbrace: visit(node.lbrace), statements: visit(node.statements)) - end + visit_methods do + # Visit a BEGINBlock node. + def visit_BEGIN(node) + node.copy( + lbrace: visit(node.lbrace), + statements: visit(node.statements) + ) + end - # Visit a CHAR node. - def visit_CHAR(node) - node.copy - end + # Visit a CHAR node. + def visit_CHAR(node) + node.copy + end - # Visit a ENDBlock node. - def visit_END(node) - node.copy(lbrace: visit(node.lbrace), statements: visit(node.statements)) - end + # Visit a ENDBlock node. + def visit_END(node) + node.copy( + lbrace: visit(node.lbrace), + statements: visit(node.statements) + ) + end - # Visit a EndContent node. - def visit___end__(node) - node.copy - end + # Visit a EndContent node. + def visit___end__(node) + node.copy + end - # Visit a AliasNode node. - def visit_alias(node) - node.copy(left: visit(node.left), right: visit(node.right)) - end + # Visit a AliasNode node. + def visit_alias(node) + node.copy(left: visit(node.left), right: visit(node.right)) + end - # Visit a ARef node. - def visit_aref(node) - node.copy(index: visit(node.index)) - end + # Visit a ARef node. + def visit_aref(node) + node.copy(index: visit(node.index)) + end - # Visit a ARefField node. - def visit_aref_field(node) - node.copy(index: visit(node.index)) - end + # Visit a ARefField node. + def visit_aref_field(node) + node.copy(index: visit(node.index)) + end - # Visit a ArgParen node. - def visit_arg_paren(node) - node.copy(arguments: visit(node.arguments)) - end + # Visit a ArgParen node. + def visit_arg_paren(node) + node.copy(arguments: visit(node.arguments)) + end - # Visit a Args node. - def visit_args(node) - node.copy(parts: visit_all(node.parts)) - end + # Visit a Args node. + def visit_args(node) + node.copy(parts: visit_all(node.parts)) + end - # Visit a ArgBlock node. - def visit_arg_block(node) - node.copy(value: visit(node.value)) - end + # Visit a ArgBlock node. + def visit_arg_block(node) + node.copy(value: visit(node.value)) + end - # Visit a ArgStar node. - def visit_arg_star(node) - node.copy(value: visit(node.value)) - end + # Visit a ArgStar node. + def visit_arg_star(node) + node.copy(value: visit(node.value)) + end - # Visit a ArgsForward node. - def visit_args_forward(node) - node.copy - end + # Visit a ArgsForward node. + def visit_args_forward(node) + node.copy + end - # Visit a ArrayLiteral node. - def visit_array(node) - node.copy(lbracket: visit(node.lbracket), contents: visit(node.contents)) - end + # Visit a ArrayLiteral node. + def visit_array(node) + node.copy( + lbracket: visit(node.lbracket), + contents: visit(node.contents) + ) + end - # Visit a AryPtn node. - def visit_aryptn(node) - node.copy( - constant: visit(node.constant), - requireds: visit_all(node.requireds), - rest: visit(node.rest), - posts: visit_all(node.posts) - ) - end + # Visit a AryPtn node. + def visit_aryptn(node) + node.copy( + constant: visit(node.constant), + requireds: visit_all(node.requireds), + rest: visit(node.rest), + posts: visit_all(node.posts) + ) + end - # Visit a Assign node. - def visit_assign(node) - node.copy(target: visit(node.target)) - end + # Visit a Assign node. + def visit_assign(node) + node.copy(target: visit(node.target)) + end - # Visit a Assoc node. - def visit_assoc(node) - node.copy - end + # Visit a Assoc node. + def visit_assoc(node) + node.copy + end - # Visit a AssocSplat node. - def visit_assoc_splat(node) - node.copy - end + # Visit a AssocSplat node. + def visit_assoc_splat(node) + node.copy + end - # Visit a Backref node. - def visit_backref(node) - node.copy - end + # Visit a Backref node. + def visit_backref(node) + node.copy + end - # Visit a Backtick node. - def visit_backtick(node) - node.copy - end + # Visit a Backtick node. + def visit_backtick(node) + node.copy + end - # Visit a BareAssocHash node. - def visit_bare_assoc_hash(node) - node.copy(assocs: visit_all(node.assocs)) - end + # Visit a BareAssocHash node. + def visit_bare_assoc_hash(node) + node.copy(assocs: visit_all(node.assocs)) + end - # Visit a Begin node. - def visit_begin(node) - node.copy(bodystmt: visit(node.bodystmt)) - end + # Visit a Begin node. + def visit_begin(node) + node.copy(bodystmt: visit(node.bodystmt)) + end - # Visit a PinnedBegin node. - def visit_pinned_begin(node) - node.copy - end + # Visit a PinnedBegin node. + def visit_pinned_begin(node) + node.copy + end - # Visit a Binary node. - def visit_binary(node) - node.copy - end + # Visit a Binary node. + def visit_binary(node) + node.copy + end - # Visit a BlockVar node. - def visit_block_var(node) - node.copy(params: visit(node.params), locals: visit_all(node.locals)) - end + # Visit a BlockVar node. + def visit_block_var(node) + node.copy(params: visit(node.params), locals: visit_all(node.locals)) + end - # Visit a BlockArg node. - def visit_blockarg(node) - node.copy(name: visit(node.name)) - end + # Visit a BlockArg node. + def visit_blockarg(node) + node.copy(name: visit(node.name)) + end - # Visit a BodyStmt node. - def visit_bodystmt(node) - node.copy( - statements: visit(node.statements), - rescue_clause: visit(node.rescue_clause), - else_clause: visit(node.else_clause), - ensure_clause: visit(node.ensure_clause) - ) - end + # Visit a BodyStmt node. + def visit_bodystmt(node) + node.copy( + statements: visit(node.statements), + rescue_clause: visit(node.rescue_clause), + else_clause: visit(node.else_clause), + ensure_clause: visit(node.ensure_clause) + ) + end - # Visit a Break node. - def visit_break(node) - node.copy(arguments: visit(node.arguments)) - end + # Visit a Break node. + def visit_break(node) + node.copy(arguments: visit(node.arguments)) + end - # Visit a Call node. - def visit_call(node) - node.copy( - receiver: visit(node.receiver), - operator: node.operator == :"::" ? :"::" : visit(node.operator), - message: node.message == :call ? :call : visit(node.message), - arguments: visit(node.arguments) - ) - end + # Visit a Call node. + def visit_call(node) + node.copy( + receiver: visit(node.receiver), + operator: node.operator == :"::" ? :"::" : visit(node.operator), + message: node.message == :call ? :call : visit(node.message), + arguments: visit(node.arguments) + ) + end - # Visit a Case node. - def visit_case(node) - node.copy( - keyword: visit(node.keyword), - value: visit(node.value), - consequent: visit(node.consequent) - ) - end + # Visit a Case node. + def visit_case(node) + node.copy( + keyword: visit(node.keyword), + value: visit(node.value), + consequent: visit(node.consequent) + ) + end - # Visit a RAssign node. - def visit_rassign(node) - node.copy(operator: visit(node.operator)) - end + # Visit a RAssign node. + def visit_rassign(node) + node.copy(operator: visit(node.operator)) + end - # Visit a ClassDeclaration node. - def visit_class(node) - node.copy( - constant: visit(node.constant), - superclass: visit(node.superclass), - bodystmt: visit(node.bodystmt) - ) - end + # Visit a ClassDeclaration node. + def visit_class(node) + node.copy( + constant: visit(node.constant), + superclass: visit(node.superclass), + bodystmt: visit(node.bodystmt) + ) + end - # Visit a Comma node. - def visit_comma(node) - node.copy - end + # Visit a Comma node. + def visit_comma(node) + node.copy + end - # Visit a Command node. - def visit_command(node) - node.copy( - message: visit(node.message), - arguments: visit(node.arguments), - block: visit(node.block) - ) - end + # Visit a Command node. + def visit_command(node) + node.copy( + message: visit(node.message), + arguments: visit(node.arguments), + block: visit(node.block) + ) + end - # Visit a CommandCall node. - def visit_command_call(node) - node.copy( - operator: node.operator == :"::" ? :"::" : visit(node.operator), - message: visit(node.message), - arguments: visit(node.arguments), - block: visit(node.block) - ) - end + # Visit a CommandCall node. + def visit_command_call(node) + node.copy( + operator: node.operator == :"::" ? :"::" : visit(node.operator), + message: visit(node.message), + arguments: visit(node.arguments), + block: visit(node.block) + ) + end - # Visit a Comment node. - def visit_comment(node) - node.copy - end + # Visit a Comment node. + def visit_comment(node) + node.copy + end - # Visit a Const node. - def visit_const(node) - node.copy - end + # Visit a Const node. + def visit_const(node) + node.copy + end - # Visit a ConstPathField node. - def visit_const_path_field(node) - node.copy(constant: visit(node.constant)) - end + # Visit a ConstPathField node. + def visit_const_path_field(node) + node.copy(constant: visit(node.constant)) + end - # Visit a ConstPathRef node. - def visit_const_path_ref(node) - node.copy(constant: visit(node.constant)) - end + # Visit a ConstPathRef node. + def visit_const_path_ref(node) + node.copy(constant: visit(node.constant)) + end - # Visit a ConstRef node. - def visit_const_ref(node) - node.copy(constant: visit(node.constant)) - end + # Visit a ConstRef node. + def visit_const_ref(node) + node.copy(constant: visit(node.constant)) + end - # Visit a CVar node. - def visit_cvar(node) - node.copy - end + # Visit a CVar node. + def visit_cvar(node) + node.copy + end - # Visit a Def node. - def visit_def(node) - node.copy( - target: visit(node.target), - operator: visit(node.operator), - name: visit(node.name), - params: visit(node.params), - bodystmt: visit(node.bodystmt) - ) - end + # Visit a Def node. + def visit_def(node) + node.copy( + target: visit(node.target), + operator: visit(node.operator), + name: visit(node.name), + params: visit(node.params), + bodystmt: visit(node.bodystmt) + ) + end - # Visit a Defined node. - def visit_defined(node) - node.copy - end + # Visit a Defined node. + def visit_defined(node) + node.copy + end - # Visit a Block node. - def visit_block(node) - node.copy( - opening: visit(node.opening), - block_var: visit(node.block_var), - bodystmt: visit(node.bodystmt) - ) - end + # Visit a Block node. + def visit_block(node) + node.copy( + opening: visit(node.opening), + block_var: visit(node.block_var), + bodystmt: visit(node.bodystmt) + ) + end - # Visit a RangeNode node. - def visit_range(node) - node.copy( - left: visit(node.left), - operator: visit(node.operator), - right: visit(node.right) - ) - end + # Visit a RangeNode node. + def visit_range(node) + node.copy( + left: visit(node.left), + operator: visit(node.operator), + right: visit(node.right) + ) + end - # Visit a DynaSymbol node. - def visit_dyna_symbol(node) - node.copy(parts: visit_all(node.parts)) - end + # Visit a DynaSymbol node. + def visit_dyna_symbol(node) + node.copy(parts: visit_all(node.parts)) + end - # Visit a Else node. - def visit_else(node) - node.copy( - keyword: visit(node.keyword), - statements: visit(node.statements) - ) - end + # Visit a Else node. + def visit_else(node) + node.copy( + keyword: visit(node.keyword), + statements: visit(node.statements) + ) + end - # Visit a Elsif node. - def visit_elsif(node) - node.copy( - statements: visit(node.statements), - consequent: visit(node.consequent) - ) - end + # Visit a Elsif node. + def visit_elsif(node) + node.copy( + statements: visit(node.statements), + consequent: visit(node.consequent) + ) + end - # Visit a EmbDoc node. - def visit_embdoc(node) - node.copy - end + # Visit a EmbDoc node. + def visit_embdoc(node) + node.copy + end - # Visit a EmbExprBeg node. - def visit_embexpr_beg(node) - node.copy - end + # Visit a EmbExprBeg node. + def visit_embexpr_beg(node) + node.copy + end - # Visit a EmbExprEnd node. - def visit_embexpr_end(node) - node.copy - end + # Visit a EmbExprEnd node. + def visit_embexpr_end(node) + node.copy + end - # Visit a EmbVar node. - def visit_embvar(node) - node.copy - end + # Visit a EmbVar node. + def visit_embvar(node) + node.copy + end - # Visit a Ensure node. - def visit_ensure(node) - node.copy( - keyword: visit(node.keyword), - statements: visit(node.statements) - ) - end + # Visit a Ensure node. + def visit_ensure(node) + node.copy( + keyword: visit(node.keyword), + statements: visit(node.statements) + ) + end - # Visit a ExcessedComma node. - def visit_excessed_comma(node) - node.copy - end + # Visit a ExcessedComma node. + def visit_excessed_comma(node) + node.copy + end - # Visit a Field node. - def visit_field(node) - node.copy( - operator: node.operator == :"::" ? :"::" : visit(node.operator), - name: visit(node.name) - ) - end + # Visit a Field node. + def visit_field(node) + node.copy( + operator: node.operator == :"::" ? :"::" : visit(node.operator), + name: visit(node.name) + ) + end - # Visit a FloatLiteral node. - def visit_float(node) - node.copy - end + # Visit a FloatLiteral node. + def visit_float(node) + node.copy + end - # Visit a FndPtn node. - def visit_fndptn(node) - node.copy( - constant: visit(node.constant), - left: visit(node.left), - values: visit_all(node.values), - right: visit(node.right) - ) - end + # Visit a FndPtn node. + def visit_fndptn(node) + node.copy( + constant: visit(node.constant), + left: visit(node.left), + values: visit_all(node.values), + right: visit(node.right) + ) + end - # Visit a For node. - def visit_for(node) - node.copy(index: visit(node.index), statements: visit(node.statements)) - end + # Visit a For node. + def visit_for(node) + node.copy(index: visit(node.index), statements: visit(node.statements)) + end - # Visit a GVar node. - def visit_gvar(node) - node.copy - end + # Visit a GVar node. + def visit_gvar(node) + node.copy + end - # Visit a HashLiteral node. - def visit_hash(node) - node.copy(lbrace: visit(node.lbrace), assocs: visit_all(node.assocs)) - end + # Visit a HashLiteral node. + def visit_hash(node) + node.copy(lbrace: visit(node.lbrace), assocs: visit_all(node.assocs)) + end - # Visit a Heredoc node. - def visit_heredoc(node) - node.copy( - beginning: visit(node.beginning), - ending: visit(node.ending), - parts: visit_all(node.parts) - ) - end + # Visit a Heredoc node. + def visit_heredoc(node) + node.copy( + beginning: visit(node.beginning), + ending: visit(node.ending), + parts: visit_all(node.parts) + ) + end - # Visit a HeredocBeg node. - def visit_heredoc_beg(node) - node.copy - end + # Visit a HeredocBeg node. + def visit_heredoc_beg(node) + node.copy + end - # Visit a HeredocEnd node. - def visit_heredoc_end(node) - node.copy - end + # Visit a HeredocEnd node. + def visit_heredoc_end(node) + node.copy + end - # Visit a HshPtn node. - def visit_hshptn(node) - node.copy( - constant: visit(node.constant), - keywords: - node.keywords.map { |label, value| [visit(label), visit(value)] }, - keyword_rest: visit(node.keyword_rest) - ) - end + # Visit a HshPtn node. + def visit_hshptn(node) + node.copy( + constant: visit(node.constant), + keywords: + node.keywords.map { |label, value| [visit(label), visit(value)] }, + keyword_rest: visit(node.keyword_rest) + ) + end - # Visit a Ident node. - def visit_ident(node) - node.copy - end + # Visit a Ident node. + def visit_ident(node) + node.copy + end - # Visit a IfNode node. - def visit_if(node) - node.copy( - predicate: visit(node.predicate), - statements: visit(node.statements), - consequent: visit(node.consequent) - ) - end + # Visit a IfNode node. + def visit_if(node) + node.copy( + predicate: visit(node.predicate), + statements: visit(node.statements), + consequent: visit(node.consequent) + ) + end - # Visit a IfOp node. - def visit_if_op(node) - node.copy - end + # Visit a IfOp node. + def visit_if_op(node) + node.copy + end - # Visit a Imaginary node. - def visit_imaginary(node) - node.copy - end + # Visit a Imaginary node. + def visit_imaginary(node) + node.copy + end - # Visit a In node. - def visit_in(node) - node.copy( - statements: visit(node.statements), - consequent: visit(node.consequent) - ) - end + # Visit a In node. + def visit_in(node) + node.copy( + statements: visit(node.statements), + consequent: visit(node.consequent) + ) + end - # Visit a Int node. - def visit_int(node) - node.copy - end + # Visit a Int node. + def visit_int(node) + node.copy + end - # Visit a IVar node. - def visit_ivar(node) - node.copy - end + # Visit a IVar node. + def visit_ivar(node) + node.copy + end - # Visit a Kw node. - def visit_kw(node) - node.copy - end + # Visit a Kw node. + def visit_kw(node) + node.copy + end - # Visit a KwRestParam node. - def visit_kwrest_param(node) - node.copy(name: visit(node.name)) - end + # Visit a KwRestParam node. + def visit_kwrest_param(node) + node.copy(name: visit(node.name)) + end - # Visit a Label node. - def visit_label(node) - node.copy - end + # Visit a Label node. + def visit_label(node) + node.copy + end - # Visit a LabelEnd node. - def visit_label_end(node) - node.copy - end + # Visit a LabelEnd node. + def visit_label_end(node) + node.copy + end - # Visit a Lambda node. - def visit_lambda(node) - node.copy(params: visit(node.params), statements: visit(node.statements)) - end + # Visit a Lambda node. + def visit_lambda(node) + node.copy( + params: visit(node.params), + statements: visit(node.statements) + ) + end - # Visit a LambdaVar node. - def visit_lambda_var(node) - node.copy(params: visit(node.params), locals: visit_all(node.locals)) - end + # Visit a LambdaVar node. + def visit_lambda_var(node) + node.copy(params: visit(node.params), locals: visit_all(node.locals)) + end - # Visit a LBrace node. - def visit_lbrace(node) - node.copy - end + # Visit a LBrace node. + def visit_lbrace(node) + node.copy + end - # Visit a LBracket node. - def visit_lbracket(node) - node.copy - end + # Visit a LBracket node. + def visit_lbracket(node) + node.copy + end - # Visit a LParen node. - def visit_lparen(node) - node.copy - end + # Visit a LParen node. + def visit_lparen(node) + node.copy + end - # Visit a MAssign node. - def visit_massign(node) - node.copy(target: visit(node.target)) - end + # Visit a MAssign node. + def visit_massign(node) + node.copy(target: visit(node.target)) + end - # Visit a MethodAddBlock node. - def visit_method_add_block(node) - node.copy(call: visit(node.call), block: visit(node.block)) - end + # Visit a MethodAddBlock node. + def visit_method_add_block(node) + node.copy(call: visit(node.call), block: visit(node.block)) + end - # Visit a MLHS node. - def visit_mlhs(node) - node.copy(parts: visit_all(node.parts)) - end + # Visit a MLHS node. + def visit_mlhs(node) + node.copy(parts: visit_all(node.parts)) + end - # Visit a MLHSParen node. - def visit_mlhs_paren(node) - node.copy(contents: visit(node.contents)) - end + # Visit a MLHSParen node. + def visit_mlhs_paren(node) + node.copy(contents: visit(node.contents)) + end - # Visit a ModuleDeclaration node. - def visit_module(node) - node.copy(constant: visit(node.constant), bodystmt: visit(node.bodystmt)) - end + # Visit a ModuleDeclaration node. + def visit_module(node) + node.copy( + constant: visit(node.constant), + bodystmt: visit(node.bodystmt) + ) + end - # Visit a MRHS node. - def visit_mrhs(node) - node.copy(parts: visit_all(node.parts)) - end + # Visit a MRHS node. + def visit_mrhs(node) + node.copy(parts: visit_all(node.parts)) + end - # Visit a Next node. - def visit_next(node) - node.copy(arguments: visit(node.arguments)) - end + # Visit a Next node. + def visit_next(node) + node.copy(arguments: visit(node.arguments)) + end - # Visit a Op node. - def visit_op(node) - node.copy - end + # Visit a Op node. + def visit_op(node) + node.copy + end - # Visit a OpAssign node. - def visit_opassign(node) - node.copy(target: visit(node.target), operator: visit(node.operator)) - end + # Visit a OpAssign node. + def visit_opassign(node) + node.copy(target: visit(node.target), operator: visit(node.operator)) + end - # Visit a Params node. - def visit_params(node) - node.copy( - requireds: visit_all(node.requireds), - optionals: - node.optionals.map { |ident, value| [visit(ident), visit(value)] }, - rest: visit(node.rest), - posts: visit_all(node.posts), - keywords: - node.keywords.map { |ident, value| [visit(ident), visit(value)] }, - keyword_rest: - node.keyword_rest == :nil ? :nil : visit(node.keyword_rest), - block: visit(node.block) - ) - end + # Visit a Params node. + def visit_params(node) + node.copy( + requireds: visit_all(node.requireds), + optionals: + node.optionals.map { |ident, value| [visit(ident), visit(value)] }, + rest: visit(node.rest), + posts: visit_all(node.posts), + keywords: + node.keywords.map { |ident, value| [visit(ident), visit(value)] }, + keyword_rest: + node.keyword_rest == :nil ? :nil : visit(node.keyword_rest), + block: visit(node.block) + ) + end - # Visit a Paren node. - def visit_paren(node) - node.copy(lparen: visit(node.lparen), contents: visit(node.contents)) - end + # Visit a Paren node. + def visit_paren(node) + node.copy(lparen: visit(node.lparen), contents: visit(node.contents)) + end - # Visit a Period node. - def visit_period(node) - node.copy - end + # Visit a Period node. + def visit_period(node) + node.copy + end - # Visit a Program node. - def visit_program(node) - node.copy(statements: visit(node.statements)) - end + # Visit a Program node. + def visit_program(node) + node.copy(statements: visit(node.statements)) + end - # Visit a QSymbols node. - def visit_qsymbols(node) - node.copy( - beginning: visit(node.beginning), - elements: visit_all(node.elements) - ) - end + # Visit a QSymbols node. + def visit_qsymbols(node) + node.copy( + beginning: visit(node.beginning), + elements: visit_all(node.elements) + ) + end - # Visit a QSymbolsBeg node. - def visit_qsymbols_beg(node) - node.copy - end + # Visit a QSymbolsBeg node. + def visit_qsymbols_beg(node) + node.copy + end - # Visit a QWords node. - def visit_qwords(node) - node.copy( - beginning: visit(node.beginning), - elements: visit_all(node.elements) - ) - end + # Visit a QWords node. + def visit_qwords(node) + node.copy( + beginning: visit(node.beginning), + elements: visit_all(node.elements) + ) + end - # Visit a QWordsBeg node. - def visit_qwords_beg(node) - node.copy - end + # Visit a QWordsBeg node. + def visit_qwords_beg(node) + node.copy + end - # Visit a RationalLiteral node. - def visit_rational(node) - node.copy - end + # Visit a RationalLiteral node. + def visit_rational(node) + node.copy + end - # Visit a RBrace node. - def visit_rbrace(node) - node.copy - end + # Visit a RBrace node. + def visit_rbrace(node) + node.copy + end - # Visit a RBracket node. - def visit_rbracket(node) - node.copy - end + # Visit a RBracket node. + def visit_rbracket(node) + node.copy + end - # Visit a Redo node. - def visit_redo(node) - node.copy - end + # Visit a Redo node. + def visit_redo(node) + node.copy + end - # Visit a RegexpContent node. - def visit_regexp_content(node) - node.copy(parts: visit_all(node.parts)) - end + # Visit a RegexpContent node. + def visit_regexp_content(node) + node.copy(parts: visit_all(node.parts)) + end - # Visit a RegexpBeg node. - def visit_regexp_beg(node) - node.copy - end + # Visit a RegexpBeg node. + def visit_regexp_beg(node) + node.copy + end - # Visit a RegexpEnd node. - def visit_regexp_end(node) - node.copy - end + # Visit a RegexpEnd node. + def visit_regexp_end(node) + node.copy + end - # Visit a RegexpLiteral node. - def visit_regexp_literal(node) - node.copy(parts: visit_all(node.parts)) - end + # Visit a RegexpLiteral node. + def visit_regexp_literal(node) + node.copy(parts: visit_all(node.parts)) + end - # Visit a RescueEx node. - def visit_rescue_ex(node) - node.copy(variable: visit(node.variable)) - end + # Visit a RescueEx node. + def visit_rescue_ex(node) + node.copy(variable: visit(node.variable)) + end - # Visit a Rescue node. - def visit_rescue(node) - node.copy( - keyword: visit(node.keyword), - exception: visit(node.exception), - statements: visit(node.statements), - consequent: visit(node.consequent) - ) - end + # Visit a Rescue node. + def visit_rescue(node) + node.copy( + keyword: visit(node.keyword), + exception: visit(node.exception), + statements: visit(node.statements), + consequent: visit(node.consequent) + ) + end - # Visit a RescueMod node. - def visit_rescue_mod(node) - node.copy - end + # Visit a RescueMod node. + def visit_rescue_mod(node) + node.copy + end - # Visit a RestParam node. - def visit_rest_param(node) - node.copy(name: visit(node.name)) - end + # Visit a RestParam node. + def visit_rest_param(node) + node.copy(name: visit(node.name)) + end - # Visit a Retry node. - def visit_retry(node) - node.copy - end + # Visit a Retry node. + def visit_retry(node) + node.copy + end - # Visit a Return node. - def visit_return(node) - node.copy(arguments: visit(node.arguments)) - end + # Visit a Return node. + def visit_return(node) + node.copy(arguments: visit(node.arguments)) + end - # Visit a RParen node. - def visit_rparen(node) - node.copy - end + # Visit a RParen node. + def visit_rparen(node) + node.copy + end - # Visit a SClass node. - def visit_sclass(node) - node.copy(bodystmt: visit(node.bodystmt)) - end + # Visit a SClass node. + def visit_sclass(node) + node.copy(bodystmt: visit(node.bodystmt)) + end - # Visit a Statements node. - def visit_statements(node) - node.copy(body: visit_all(node.body)) - end + # Visit a Statements node. + def visit_statements(node) + node.copy(body: visit_all(node.body)) + end - # Visit a StringContent node. - def visit_string_content(node) - node.copy(parts: visit_all(node.parts)) - end + # Visit a StringContent node. + def visit_string_content(node) + node.copy(parts: visit_all(node.parts)) + end - # Visit a StringConcat node. - def visit_string_concat(node) - node.copy(left: visit(node.left), right: visit(node.right)) - end + # Visit a StringConcat node. + def visit_string_concat(node) + node.copy(left: visit(node.left), right: visit(node.right)) + end - # Visit a StringDVar node. - def visit_string_dvar(node) - node.copy(variable: visit(node.variable)) - end + # Visit a StringDVar node. + def visit_string_dvar(node) + node.copy(variable: visit(node.variable)) + end - # Visit a StringEmbExpr node. - def visit_string_embexpr(node) - node.copy(statements: visit(node.statements)) - end + # Visit a StringEmbExpr node. + def visit_string_embexpr(node) + node.copy(statements: visit(node.statements)) + end - # Visit a StringLiteral node. - def visit_string_literal(node) - node.copy(parts: visit_all(node.parts)) - end + # Visit a StringLiteral node. + def visit_string_literal(node) + node.copy(parts: visit_all(node.parts)) + end - # Visit a Super node. - def visit_super(node) - node.copy(arguments: visit(node.arguments)) - end + # Visit a Super node. + def visit_super(node) + node.copy(arguments: visit(node.arguments)) + end - # Visit a SymBeg node. - def visit_symbeg(node) - node.copy - end + # Visit a SymBeg node. + def visit_symbeg(node) + node.copy + end - # Visit a SymbolContent node. - def visit_symbol_content(node) - node.copy(value: visit(node.value)) - end + # Visit a SymbolContent node. + def visit_symbol_content(node) + node.copy(value: visit(node.value)) + end - # Visit a SymbolLiteral node. - def visit_symbol_literal(node) - node.copy(value: visit(node.value)) - end + # Visit a SymbolLiteral node. + def visit_symbol_literal(node) + node.copy(value: visit(node.value)) + end - # Visit a Symbols node. - def visit_symbols(node) - node.copy( - beginning: visit(node.beginning), - elements: visit_all(node.elements) - ) - end + # Visit a Symbols node. + def visit_symbols(node) + node.copy( + beginning: visit(node.beginning), + elements: visit_all(node.elements) + ) + end - # Visit a SymbolsBeg node. - def visit_symbols_beg(node) - node.copy - end + # Visit a SymbolsBeg node. + def visit_symbols_beg(node) + node.copy + end - # Visit a TLambda node. - def visit_tlambda(node) - node.copy - end + # Visit a TLambda node. + def visit_tlambda(node) + node.copy + end - # Visit a TLamBeg node. - def visit_tlambeg(node) - node.copy - end + # Visit a TLamBeg node. + def visit_tlambeg(node) + node.copy + end - # Visit a TopConstField node. - def visit_top_const_field(node) - node.copy(constant: visit(node.constant)) - end + # Visit a TopConstField node. + def visit_top_const_field(node) + node.copy(constant: visit(node.constant)) + end - # Visit a TopConstRef node. - def visit_top_const_ref(node) - node.copy(constant: visit(node.constant)) - end + # Visit a TopConstRef node. + def visit_top_const_ref(node) + node.copy(constant: visit(node.constant)) + end - # Visit a TStringBeg node. - def visit_tstring_beg(node) - node.copy - end + # Visit a TStringBeg node. + def visit_tstring_beg(node) + node.copy + end - # Visit a TStringContent node. - def visit_tstring_content(node) - node.copy - end + # Visit a TStringContent node. + def visit_tstring_content(node) + node.copy + end - # Visit a TStringEnd node. - def visit_tstring_end(node) - node.copy - end + # Visit a TStringEnd node. + def visit_tstring_end(node) + node.copy + end - # Visit a Not node. - def visit_not(node) - node.copy(statement: visit(node.statement)) - end + # Visit a Not node. + def visit_not(node) + node.copy(statement: visit(node.statement)) + end - # Visit a Unary node. - def visit_unary(node) - node.copy - end + # Visit a Unary node. + def visit_unary(node) + node.copy + end - # Visit a Undef node. - def visit_undef(node) - node.copy(symbols: visit_all(node.symbols)) - end + # Visit a Undef node. + def visit_undef(node) + node.copy(symbols: visit_all(node.symbols)) + end - # Visit a UnlessNode node. - def visit_unless(node) - node.copy( - predicate: visit(node.predicate), - statements: visit(node.statements), - consequent: visit(node.consequent) - ) - end + # Visit a UnlessNode node. + def visit_unless(node) + node.copy( + predicate: visit(node.predicate), + statements: visit(node.statements), + consequent: visit(node.consequent) + ) + end - # Visit a UntilNode node. - def visit_until(node) - node.copy( - predicate: visit(node.predicate), - statements: visit(node.statements) - ) - end + # Visit a UntilNode node. + def visit_until(node) + node.copy( + predicate: visit(node.predicate), + statements: visit(node.statements) + ) + end - # Visit a VarField node. - def visit_var_field(node) - node.copy(value: visit(node.value)) - end + # Visit a VarField node. + def visit_var_field(node) + node.copy(value: visit(node.value)) + end - # Visit a VarRef node. - def visit_var_ref(node) - node.copy(value: visit(node.value)) - end + # Visit a VarRef node. + def visit_var_ref(node) + node.copy(value: visit(node.value)) + end - # Visit a PinnedVarRef node. - def visit_pinned_var_ref(node) - node.copy(value: visit(node.value)) - end + # Visit a PinnedVarRef node. + def visit_pinned_var_ref(node) + node.copy(value: visit(node.value)) + end - # Visit a VCall node. - def visit_vcall(node) - node.copy(value: visit(node.value)) - end + # Visit a VCall node. + def visit_vcall(node) + node.copy(value: visit(node.value)) + end - # Visit a VoidStmt node. - def visit_void_stmt(node) - node.copy - end + # Visit a VoidStmt node. + def visit_void_stmt(node) + node.copy + end - # Visit a When node. - def visit_when(node) - node.copy( - arguments: visit(node.arguments), - statements: visit(node.statements), - consequent: visit(node.consequent) - ) - end + # Visit a When node. + def visit_when(node) + node.copy( + arguments: visit(node.arguments), + statements: visit(node.statements), + consequent: visit(node.consequent) + ) + end - # Visit a WhileNode node. - def visit_while(node) - node.copy( - predicate: visit(node.predicate), - statements: visit(node.statements) - ) - end + # Visit a WhileNode node. + def visit_while(node) + node.copy( + predicate: visit(node.predicate), + statements: visit(node.statements) + ) + end - # Visit a Word node. - def visit_word(node) - node.copy(parts: visit_all(node.parts)) - end + # Visit a Word node. + def visit_word(node) + node.copy(parts: visit_all(node.parts)) + end - # Visit a Words node. - def visit_words(node) - node.copy( - beginning: visit(node.beginning), - elements: visit_all(node.elements) - ) - end + # Visit a Words node. + def visit_words(node) + node.copy( + beginning: visit(node.beginning), + elements: visit_all(node.elements) + ) + end - # Visit a WordsBeg node. - def visit_words_beg(node) - node.copy - end + # Visit a WordsBeg node. + def visit_words_beg(node) + node.copy + end - # Visit a XString node. - def visit_xstring(node) - node.copy(parts: visit_all(node.parts)) - end + # Visit a XString node. + def visit_xstring(node) + node.copy(parts: visit_all(node.parts)) + end - # Visit a XStringLiteral node. - def visit_xstring_literal(node) - node.copy(parts: visit_all(node.parts)) - end + # Visit a XStringLiteral node. + def visit_xstring_literal(node) + node.copy(parts: visit_all(node.parts)) + end - # Visit a YieldNode node. - def visit_yield(node) - node.copy(arguments: visit(node.arguments)) - end + # Visit a YieldNode node. + def visit_yield(node) + node.copy(arguments: visit(node.arguments)) + end - # Visit a ZSuper node. - def visit_zsuper(node) - node.copy + # Visit a ZSuper node. + def visit_zsuper(node) + node.copy + end end end end diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index 8059b18c..426bd945 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -668,8 +668,10 @@ def visit(node) stack.pop end - def visit_var_ref(node) - node.pin(stack[-2], pins.shift) + visit_methods do + def visit_var_ref(node) + node.pin(stack[-2], pins.shift) + end end def self.visit(node, tokens) diff --git a/lib/syntax_tree/translation/parser.rb b/lib/syntax_tree/translation/parser.rb index 70c98336..ad889478 100644 --- a/lib/syntax_tree/translation/parser.rb +++ b/lib/syntax_tree/translation/parser.rb @@ -89,2538 +89,2589 @@ def visit(node) result end - # Visit an AliasNode node. - def visit_alias(node) - s( - :alias, - [visit(node.left), visit(node.right)], - smap_keyword_bare( - srange_length(node.start_char, 5), - srange_node(node) + visit_methods do + # Visit an AliasNode node. + def visit_alias(node) + s( + :alias, + [visit(node.left), visit(node.right)], + smap_keyword_bare( + srange_length(node.start_char, 5), + srange_node(node) + ) ) - ) - end + end - # Visit an ARefNode. - def visit_aref(node) - if ::Parser::Builders::Default.emit_index - if node.index.nil? - s( - :index, - [visit(node.collection)], - smap_index( - srange_find(node.collection.end_char, node.end_char, "["), - srange_length(node.end_char, -1), - srange_node(node) + # Visit an ARefNode. + def visit_aref(node) + if ::Parser::Builders::Default.emit_index + if node.index.nil? + s( + :index, + [visit(node.collection)], + smap_index( + srange_find(node.collection.end_char, node.end_char, "["), + srange_length(node.end_char, -1), + srange_node(node) + ) ) - ) + else + s( + :index, + [visit(node.collection)].concat(visit_all(node.index.parts)), + smap_index( + srange_find_between(node.collection, node.index, "["), + srange_length(node.end_char, -1), + srange_node(node) + ) + ) + end else - s( - :index, - [visit(node.collection)].concat(visit_all(node.index.parts)), - smap_index( - srange_find_between(node.collection, node.index, "["), - srange_length(node.end_char, -1), - srange_node(node) + if node.index.nil? + s( + :send, + [visit(node.collection), :[]], + smap_send_bare( + srange_find(node.collection.end_char, node.end_char, "[]"), + srange_node(node) + ) ) - ) + else + s( + :send, + [visit(node.collection), :[], *visit_all(node.index.parts)], + smap_send_bare( + srange( + srange_find_between( + node.collection, + node.index, + "[" + ).begin_pos, + node.end_char + ), + srange_node(node) + ) + ) + end end - else - if node.index.nil? - s( - :send, - [visit(node.collection), :[]], - smap_send_bare( - srange_find(node.collection.end_char, node.end_char, "[]"), - srange_node(node) + end + + # Visit an ARefField node. + def visit_aref_field(node) + if ::Parser::Builders::Default.emit_index + if node.index.nil? + s( + :indexasgn, + [visit(node.collection)], + smap_index( + srange_find(node.collection.end_char, node.end_char, "["), + srange_length(node.end_char, -1), + srange_node(node) + ) ) - ) + else + s( + :indexasgn, + [visit(node.collection)].concat(visit_all(node.index.parts)), + smap_index( + srange_find_between(node.collection, node.index, "["), + srange_length(node.end_char, -1), + srange_node(node) + ) + ) + end else - s( - :send, - [visit(node.collection), :[], *visit_all(node.index.parts)], - smap_send_bare( - srange( - srange_find_between( - node.collection, - node.index, - "[" - ).begin_pos, - node.end_char + if node.index.nil? + s( + :send, + [visit(node.collection), :[]=], + smap_send_bare( + srange_find(node.collection.end_char, node.end_char, "[]"), + srange_node(node) + ) + ) + else + s( + :send, + [visit(node.collection), :[]=].concat( + visit_all(node.index.parts) ), + smap_send_bare( + srange( + srange_find_between( + node.collection, + node.index, + "[" + ).begin_pos, + node.end_char + ), + srange_node(node) + ) + ) + end + end + end + + # Visit an ArgBlock node. + def visit_arg_block(node) + s( + :block_pass, + [visit(node.value)], + smap_operator(srange_length(node.start_char, 1), srange_node(node)) + ) + end + + # Visit an ArgStar node. + def visit_arg_star(node) + if stack[-3].is_a?(MLHSParen) && stack[-3].contents.is_a?(MLHS) + if node.value.nil? + s(:restarg, [], smap_variable(nil, srange_node(node))) + else + s( + :restarg, + [node.value.value.to_sym], + smap_variable(srange_node(node.value), srange_node(node)) + ) + end + else + s( + :splat, + node.value.nil? ? [] : [visit(node.value)], + smap_operator( + srange_length(node.start_char, 1), srange_node(node) ) ) end end - end - # Visit an ARefField node. - def visit_aref_field(node) - if ::Parser::Builders::Default.emit_index - if node.index.nil? - s( - :indexasgn, - [visit(node.collection)], - smap_index( - srange_find(node.collection.end_char, node.end_char, "["), + # Visit an ArgsForward node. + def visit_args_forward(node) + s(:forwarded_args, [], smap(srange_node(node))) + end + + # Visit an ArrayLiteral node. + def visit_array(node) + s( + :array, + node.contents ? visit_all(node.contents.parts) : [], + if node.lbracket.nil? + smap_collection_bare(srange_node(node)) + else + smap_collection( + srange_node(node.lbracket), srange_length(node.end_char, -1), srange_node(node) ) - ) - else + end + ) + end + + # Visit an AryPtn node. + def visit_aryptn(node) + type = :array_pattern + children = visit_all(node.requireds) + + if node.rest.is_a?(VarField) + if !node.rest.value.nil? + children << s(:match_rest, [visit(node.rest)], nil) + elsif node.posts.empty? && + node.rest.start_char == node.rest.end_char + # Here we have an implicit rest, as in [foo,]. parser has a + # specific type for these patterns. + type = :array_pattern_with_tail + else + children << s(:match_rest, [], nil) + end + end + + if node.constant s( - :indexasgn, - [visit(node.collection)].concat(visit_all(node.index.parts)), - smap_index( - srange_find_between(node.collection, node.index, "["), + :const_pattern, + [ + visit(node.constant), + s( + type, + children + visit_all(node.posts), + smap_collection_bare( + srange(node.constant.end_char + 1, node.end_char - 1) + ) + ) + ], + smap_collection( + srange_length(node.constant.end_char, 1), srange_length(node.end_char, -1), srange_node(node) ) ) + else + s( + type, + children + visit_all(node.posts), + if buffer.source[node.start_char] == "[" + smap_collection( + srange_length(node.start_char, 1), + srange_length(node.end_char, -1), + srange_node(node) + ) + else + smap_collection_bare(srange_node(node)) + end + ) end - else - if node.index.nil? + end + + # Visit an Assign node. + def visit_assign(node) + target = visit(node.target) + location = + target + .location + .with_operator(srange_find_between(node.target, node.value, "=")) + .with_expression(srange_node(node)) + + s(target.type, target.children + [visit(node.value)], location) + end + + # Visit an Assoc node. + def visit_assoc(node) + if node.value.nil? + expression = srange(node.start_char, node.end_char - 1) + + type, location = + if node.key.value.start_with?(/[A-Z]/) + [:const, smap_constant(nil, expression, expression)] + else + [:send, smap_send_bare(expression, expression)] + end + s( - :send, - [visit(node.collection), :[]=], - smap_send_bare( - srange_find(node.collection.end_char, node.end_char, "[]"), + :pair, + [ + visit(node.key), + s(type, [nil, node.key.value.chomp(":").to_sym], location) + ], + smap_operator( + srange_length(node.key.end_char, -1), srange_node(node) ) ) else s( - :send, - [visit(node.collection), :[]=].concat( - visit_all(node.index.parts) - ), - smap_send_bare( - srange( - srange_find_between( - node.collection, - node.index, - "[" - ).begin_pos, - node.end_char - ), + :pair, + [visit(node.key), visit(node.value)], + smap_operator( + srange_search_between(node.key, node.value, "=>") || + srange_length(node.key.end_char, -1), srange_node(node) ) ) end end - end - # Visit an ArgBlock node. - def visit_arg_block(node) - s( - :block_pass, - [visit(node.value)], - smap_operator(srange_length(node.start_char, 1), srange_node(node)) - ) - end + # Visit an AssocSplat node. + def visit_assoc_splat(node) + s( + :kwsplat, + [visit(node.value)], + smap_operator(srange_length(node.start_char, 2), srange_node(node)) + ) + end - # Visit an ArgStar node. - def visit_arg_star(node) - if stack[-3].is_a?(MLHSParen) && stack[-3].contents.is_a?(MLHS) - if node.value.nil? - s(:restarg, [], smap_variable(nil, srange_node(node))) + # Visit a Backref node. + def visit_backref(node) + location = smap(srange_node(node)) + + if node.value.match?(/^\$\d+$/) + s(:nth_ref, [node.value[1..].to_i], location) else - s( - :restarg, - [node.value.value.to_sym], - smap_variable(srange_node(node.value), srange_node(node)) - ) + s(:back_ref, [node.value.to_sym], location) end - else + end + + # Visit a BareAssocHash node. + def visit_bare_assoc_hash(node) s( - :splat, - node.value.nil? ? [] : [visit(node.value)], - smap_operator(srange_length(node.start_char, 1), srange_node(node)) + if ::Parser::Builders::Default.emit_kwargs && + !stack[-2].is_a?(ArrayLiteral) + :kwargs + else + :hash + end, + visit_all(node.assocs), + smap_collection_bare(srange_node(node)) ) end - end - # Visit an ArgsForward node. - def visit_args_forward(node) - s(:forwarded_args, [], smap(srange_node(node))) - end + # Visit a BEGINBlock node. + def visit_BEGIN(node) + s( + :preexe, + [visit(node.statements)], + smap_keyword( + srange_length(node.start_char, 5), + srange_find(node.start_char + 5, node.statements.start_char, "{"), + srange_length(node.end_char, -1), + srange_node(node) + ) + ) + end - # Visit an ArrayLiteral node. - def visit_array(node) - s( - :array, - node.contents ? visit_all(node.contents.parts) : [], - if node.lbracket.nil? - smap_collection_bare(srange_node(node)) - else + # Visit a Begin node. + def visit_begin(node) + location = smap_collection( - srange_node(node.lbracket), - srange_length(node.end_char, -1), + srange_length(node.start_char, 5), + srange_length(node.end_char, -3), srange_node(node) ) - end - ) - end - # Visit an AryPtn node. - def visit_aryptn(node) - type = :array_pattern - children = visit_all(node.requireds) - - if node.rest.is_a?(VarField) - if !node.rest.value.nil? - children << s(:match_rest, [visit(node.rest)], nil) - elsif node.posts.empty? && node.rest.start_char == node.rest.end_char - # Here we have an implicit rest, as in [foo,]. parser has a specific - # type for these patterns. - type = :array_pattern_with_tail + if node.bodystmt.empty? + s(:kwbegin, [], location) + elsif node.bodystmt.rescue_clause.nil? && + node.bodystmt.ensure_clause.nil? && + node.bodystmt.else_clause.nil? + child = visit(node.bodystmt.statements) + + s( + :kwbegin, + child.type == :begin ? child.children : [child], + location + ) else - children << s(:match_rest, [], nil) + s(:kwbegin, [visit(node.bodystmt)], location) end end - if node.constant - s( - :const_pattern, - [ - visit(node.constant), - s( - type, - children + visit_all(node.posts), - smap_collection_bare( - srange(node.constant.end_char + 1, node.end_char - 1) - ) + # Visit a Binary node. + def visit_binary(node) + case node.operator + when :| + current = -2 + while stack[current].is_a?(Binary) && stack[current].operator == :| + current -= 1 + end + + if stack[current].is_a?(In) + s(:match_alt, [visit(node.left), visit(node.right)], nil) + else + visit(canonical_binary(node)) + end + when :"=>", :"&&", :and, :"||", :or + s( + { "=>": :match_as, "&&": :and, "||": :or }.fetch( + node.operator, + node.operator + ), + [visit(node.left), visit(node.right)], + smap_operator( + srange_find_between(node.left, node.right, node.operator.to_s), + srange_node(node) ) - ], - smap_collection( - srange_length(node.constant.end_char, 1), - srange_length(node.end_char, -1), - srange_node(node) ) - ) - else - s( - type, - children + visit_all(node.posts), - if buffer.source[node.start_char] == "[" - smap_collection( - srange_length(node.start_char, 1), - srange_length(node.end_char, -1), - srange_node(node) + when :=~ + # When you use a regular expression on the left hand side of a =~ + # operator and it doesn't have interpolatoin, then its named capture + # groups introduce local variables into the scope. In this case the + # parser gem has a different node (match_with_lvasgn) instead of the + # regular send. + if node.left.is_a?(RegexpLiteral) && node.left.parts.length == 1 && + node.left.parts.first.is_a?(TStringContent) + s( + :match_with_lvasgn, + [visit(node.left), visit(node.right)], + smap_operator( + srange_find_between( + node.left, + node.right, + node.operator.to_s + ), + srange_node(node) + ) ) else - smap_collection_bare(srange_node(node)) + visit(canonical_binary(node)) end - ) + else + visit(canonical_binary(node)) + end end - end - # Visit an Assign node. - def visit_assign(node) - target = visit(node.target) - location = - target - .location - .with_operator(srange_find_between(node.target, node.value, "=")) - .with_expression(srange_node(node)) + # Visit a BlockArg node. + def visit_blockarg(node) + if node.name.nil? + s(:blockarg, [nil], smap_variable(nil, srange_node(node))) + else + s( + :blockarg, + [node.name.value.to_sym], + smap_variable(srange_node(node.name), srange_node(node)) + ) + end + end - s(target.type, target.children + [visit(node.value)], location) - end + # Visit a BlockVar node. + def visit_block_var(node) + shadowargs = + node.locals.map do |local| + s( + :shadowarg, + [local.value.to_sym], + smap_variable(srange_node(local), srange_node(local)) + ) + end - # Visit an Assoc node. - def visit_assoc(node) - if node.value.nil? - expression = srange(node.start_char, node.end_char - 1) + params = node.params + children = + if ::Parser::Builders::Default.emit_procarg0 && node.arg0? + # There is a special node type in the parser gem for when a single + # required parameter to a block would potentially be expanded + # automatically. We handle that case here. + required = params.requireds.first + procarg0 = + if ::Parser::Builders::Default.emit_arg_inside_procarg0 && + required.is_a?(Ident) + s( + :procarg0, + [ + s( + :arg, + [required.value.to_sym], + smap_variable( + srange_node(required), + srange_node(required) + ) + ) + ], + smap_collection_bare(srange_node(required)) + ) + else + child = visit(required) + s(:procarg0, child, child.location) + end - type, location = - if node.key.value.start_with?(/[A-Z]/) - [:const, smap_constant(nil, expression, expression)] + [procarg0] else - [:send, smap_send_bare(expression, expression)] + visit(params).children end s( - :pair, - [ - visit(node.key), - s(type, [nil, node.key.value.chomp(":").to_sym], location) - ], - smap_operator( - srange_length(node.key.end_char, -1), - srange_node(node) - ) - ) - else - s( - :pair, - [visit(node.key), visit(node.value)], - smap_operator( - srange_search_between(node.key, node.value, "=>") || - srange_length(node.key.end_char, -1), + :args, + children + shadowargs, + smap_collection( + srange_length(node.start_char, 1), + srange_length(node.end_char, -1), srange_node(node) ) ) end - end - - # Visit an AssocSplat node. - def visit_assoc_splat(node) - s( - :kwsplat, - [visit(node.value)], - smap_operator(srange_length(node.start_char, 2), srange_node(node)) - ) - end - # Visit a Backref node. - def visit_backref(node) - location = smap(srange_node(node)) - - if node.value.match?(/^\$\d+$/) - s(:nth_ref, [node.value[1..].to_i], location) - else - s(:back_ref, [node.value.to_sym], location) - end - end + # Visit a BodyStmt node. + def visit_bodystmt(node) + result = visit(node.statements) + + if node.rescue_clause + rescue_node = visit(node.rescue_clause) + + children = [result] + rescue_node.children + location = rescue_node.location + + if node.else_clause + children.pop + children << visit(node.else_clause) + + location = + smap_condition( + nil, + nil, + srange_length(node.else_clause.start_char - 3, -4), + nil, + srange( + location.expression.begin_pos, + node.else_clause.end_char + ) + ) + end - # Visit a BareAssocHash node. - def visit_bare_assoc_hash(node) - s( - if ::Parser::Builders::Default.emit_kwargs && - !stack[-2].is_a?(ArrayLiteral) - :kwargs - else - :hash - end, - visit_all(node.assocs), - smap_collection_bare(srange_node(node)) - ) - end + result = s(rescue_node.type, children, location) + end - # Visit a BEGINBlock node. - def visit_BEGIN(node) - s( - :preexe, - [visit(node.statements)], - smap_keyword( - srange_length(node.start_char, 5), - srange_find(node.start_char + 5, node.statements.start_char, "{"), - srange_length(node.end_char, -1), - srange_node(node) - ) - ) - end + if node.ensure_clause + ensure_node = visit(node.ensure_clause) - # Visit a Begin node. - def visit_begin(node) - location = - smap_collection( - srange_length(node.start_char, 5), - srange_length(node.end_char, -3), - srange_node(node) - ) + expression = + ( + if result + result.location.expression.join( + ensure_node.location.expression + ) + else + ensure_node.location.expression + end + ) + location = ensure_node.location.with_expression(expression) - if node.bodystmt.empty? - s(:kwbegin, [], location) - elsif node.bodystmt.rescue_clause.nil? && - node.bodystmt.ensure_clause.nil? && node.bodystmt.else_clause.nil? - child = visit(node.bodystmt.statements) + result = + s(ensure_node.type, [result] + ensure_node.children, location) + end - s(:kwbegin, child.type == :begin ? child.children : [child], location) - else - s(:kwbegin, [visit(node.bodystmt)], location) + result end - end - # Visit a Binary node. - def visit_binary(node) - case node.operator - when :| - current = -2 - while stack[current].is_a?(Binary) && stack[current].operator == :| - current -= 1 - end + # Visit a Break node. + def visit_break(node) + s( + :break, + visit_all(node.arguments.parts), + smap_keyword_bare( + srange_length(node.start_char, 5), + srange_node(node) + ) + ) + end - if stack[current].is_a?(In) - s(:match_alt, [visit(node.left), visit(node.right)], nil) - else - visit(canonical_binary(node)) + # Visit a CallNode node. + def visit_call(node) + visit_command_call( + CommandCall.new( + receiver: node.receiver, + operator: node.operator, + message: node.message, + arguments: node.arguments, + block: nil, + location: node.location + ) + ) + end + + # Visit a Case node. + def visit_case(node) + clauses = [node.consequent] + while clauses.last && !clauses.last.is_a?(Else) + clauses << clauses.last.consequent end - when :"=>", :"&&", :and, :"||", :or + + else_token = + if clauses.last.is_a?(Else) + srange_length(clauses.last.start_char, 4) + end + s( - { "=>": :match_as, "&&": :and, "||": :or }.fetch( - node.operator, - node.operator - ), - [visit(node.left), visit(node.right)], - smap_operator( - srange_find_between(node.left, node.right, node.operator.to_s), + node.consequent.is_a?(In) ? :case_match : :case, + [visit(node.value)] + clauses.map { |clause| visit(clause) }, + smap_condition( + srange_length(node.start_char, 4), + nil, + else_token, + srange_length(node.end_char, -3), srange_node(node) ) ) - when :=~ - # When you use a regular expression on the left hand side of a =~ - # operator and it doesn't have interpolatoin, then its named capture - # groups introduce local variables into the scope. In this case the - # parser gem has a different node (match_with_lvasgn) instead of the - # regular send. - if node.left.is_a?(RegexpLiteral) && node.left.parts.length == 1 && - node.left.parts.first.is_a?(TStringContent) - s( - :match_with_lvasgn, - [visit(node.left), visit(node.right)], - smap_operator( - srange_find_between(node.left, node.right, node.operator.to_s), - srange_node(node) - ) + end + + # Visit a CHAR node. + def visit_CHAR(node) + s( + :str, + [node.value[1..]], + smap_collection( + srange_length(node.start_char, 1), + nil, + srange_node(node) ) - else - visit(canonical_binary(node)) - end - else - visit(canonical_binary(node)) + ) end - end - # Visit a BlockArg node. - def visit_blockarg(node) - if node.name.nil? - s(:blockarg, [nil], smap_variable(nil, srange_node(node))) - else + # Visit a ClassDeclaration node. + def visit_class(node) + operator = + if node.superclass + srange_find_between(node.constant, node.superclass, "<") + end + s( - :blockarg, - [node.name.value.to_sym], - smap_variable(srange_node(node.name), srange_node(node)) + :class, + [ + visit(node.constant), + visit(node.superclass), + visit(node.bodystmt) + ], + smap_definition( + srange_length(node.start_char, 5), + operator, + srange_node(node.constant), + srange_length(node.end_char, -3) + ).with_expression(srange_node(node)) ) end - end - # Visit a BlockVar node. - def visit_block_var(node) - shadowargs = - node.locals.map do |local| - s( - :shadowarg, - [local.value.to_sym], - smap_variable(srange_node(local), srange_node(local)) + # Visit a Command node. + def visit_command(node) + visit_command_call( + CommandCall.new( + receiver: nil, + operator: nil, + message: node.message, + arguments: node.arguments, + block: node.block, + location: node.location ) - end + ) + end - params = node.params - children = - if ::Parser::Builders::Default.emit_procarg0 && node.arg0? - # There is a special node type in the parser gem for when a single - # required parameter to a block would potentially be expanded - # automatically. We handle that case here. - required = params.requireds.first - procarg0 = - if ::Parser::Builders::Default.emit_arg_inside_procarg0 && - required.is_a?(Ident) - s( - :procarg0, - [ - s( - :arg, - [required.value.to_sym], - smap_variable( - srange_node(required), - srange_node(required) - ) - ) - ], - smap_collection_bare(srange_node(required)) - ) - else - child = visit(required) - s(:procarg0, child, child.location) - end + # Visit a CommandCall node. + def visit_command_call(node) + children = [ + visit(node.receiver), + node.message == :call ? :call : node.message.value.to_sym + ] + + begin_token = nil + end_token = nil + + case node.arguments + when Args + children += visit_all(node.arguments.parts) + when ArgParen + case node.arguments.arguments + when nil + # skip + when ArgsForward + children << visit(node.arguments.arguments) + else + children += visit_all(node.arguments.arguments.parts) + end - [procarg0] - else - visit(params).children + begin_token = srange_length(node.arguments.start_char, 1) + end_token = srange_length(node.arguments.end_char, -1) end - s( - :args, - children + shadowargs, - smap_collection( - srange_length(node.start_char, 1), - srange_length(node.end_char, -1), - srange_node(node) - ) - ) - end + dot_bound = + if node.arguments + node.arguments.start_char + elsif node.block + node.block.start_char + else + node.end_char + end - # Visit a BodyStmt node. - def visit_bodystmt(node) - result = visit(node.statements) + expression = + if node.arguments.is_a?(ArgParen) + srange(node.start_char, node.arguments.end_char) + elsif node.arguments.is_a?(Args) && node.arguments.parts.any? + last_part = node.arguments.parts.last + end_char = + if last_part.is_a?(Heredoc) + last_part.beginning.end_char + else + last_part.end_char + end - if node.rescue_clause - rescue_node = visit(node.rescue_clause) + srange(node.start_char, end_char) + elsif node.block + srange_node(node.message) + else + srange_node(node) + end - children = [result] + rescue_node.children - location = rescue_node.location + call = + s( + if node.operator.is_a?(Op) && node.operator.value == "&." + :csend + else + :send + end, + children, + smap_send( + if node.operator == :"::" + srange_find( + node.receiver.end_char, + if node.message == :call + dot_bound + else + node.message.start_char + end, + "::" + ) + elsif node.operator + srange_node(node.operator) + end, + node.message == :call ? nil : srange_node(node.message), + begin_token, + end_token, + expression + ) + ) - if node.else_clause - children.pop - children << visit(node.else_clause) + if node.block + type, arguments = block_children(node.block) - location = - smap_condition( - nil, - nil, - srange_length(node.else_clause.start_char - 3, -4), - nil, - srange(location.expression.begin_pos, node.else_clause.end_char) + s( + type, + [call, arguments, visit(node.block.bodystmt)], + smap_collection( + srange_node(node.block.opening), + srange_length( + node.end_char, + node.block.opening.is_a?(Kw) ? -3 : -1 + ), + srange_node(node) ) + ) + else + call end - - result = s(rescue_node.type, children, location) end - if node.ensure_clause - ensure_node = visit(node.ensure_clause) + # Visit a Const node. + def visit_const(node) + s( + :const, + [nil, node.value.to_sym], + smap_constant(nil, srange_node(node), srange_node(node)) + ) + end - expression = - ( - if result - result.location.expression.join(ensure_node.location.expression) - else - ensure_node.location.expression - end + # Visit a ConstPathField node. + def visit_const_path_field(node) + if node.parent.is_a?(VarRef) && node.parent.value.is_a?(Kw) && + node.parent.value.value == "self" && node.constant.is_a?(Ident) + s(:send, [visit(node.parent), :"#{node.constant.value}="], nil) + else + s( + :casgn, + [visit(node.parent), node.constant.value.to_sym], + smap_constant( + srange_find_between(node.parent, node.constant, "::"), + srange_node(node.constant), + srange_node(node) + ) ) - location = ensure_node.location.with_expression(expression) - - result = - s(ensure_node.type, [result] + ensure_node.children, location) + end end - result - end - - # Visit a Break node. - def visit_break(node) - s( - :break, - visit_all(node.arguments.parts), - smap_keyword_bare( - srange_length(node.start_char, 5), - srange_node(node) + # Visit a ConstPathRef node. + def visit_const_path_ref(node) + s( + :const, + [visit(node.parent), node.constant.value.to_sym], + smap_constant( + srange_find_between(node.parent, node.constant, "::"), + srange_node(node.constant), + srange_node(node) + ) ) - ) - end + end - # Visit a CallNode node. - def visit_call(node) - visit_command_call( - CommandCall.new( - receiver: node.receiver, - operator: node.operator, - message: node.message, - arguments: node.arguments, - block: nil, - location: node.location + # Visit a ConstRef node. + def visit_const_ref(node) + s( + :const, + [nil, node.constant.value.to_sym], + smap_constant(nil, srange_node(node.constant), srange_node(node)) ) - ) - end + end - # Visit a Case node. - def visit_case(node) - clauses = [node.consequent] - while clauses.last && !clauses.last.is_a?(Else) - clauses << clauses.last.consequent + # Visit a CVar node. + def visit_cvar(node) + s( + :cvar, + [node.value.to_sym], + smap_variable(srange_node(node), srange_node(node)) + ) end - else_token = - if clauses.last.is_a?(Else) - srange_length(clauses.last.start_char, 4) - end + # Visit a DefNode node. + def visit_def(node) + name = node.name.value.to_sym + args = + case node.params + when Params + child = visit(node.params) - s( - node.consequent.is_a?(In) ? :case_match : :case, - [visit(node.value)] + clauses.map { |clause| visit(clause) }, - smap_condition( - srange_length(node.start_char, 4), - nil, - else_token, - srange_length(node.end_char, -3), - srange_node(node) - ) - ) - end + s( + child.type, + child.children, + smap_collection_bare(child.location&.expression) + ) + when Paren + child = visit(node.params.contents) - # Visit a CHAR node. - def visit_CHAR(node) - s( - :str, - [node.value[1..]], - smap_collection( - srange_length(node.start_char, 1), - nil, - srange_node(node) - ) - ) - end + s( + child.type, + child.children, + smap_collection( + srange_length(node.params.start_char, 1), + srange_length(node.params.end_char, -1), + srange_node(node.params) + ) + ) + else + s(:args, [], smap_collection_bare(nil)) + end - # Visit a ClassDeclaration node. - def visit_class(node) - operator = - if node.superclass - srange_find_between(node.constant, node.superclass, "<") + location = + if node.endless? + smap_method_definition( + srange_length(node.start_char, 3), + nil, + srange_node(node.name), + nil, + srange_find_between( + (node.params || node.name), + node.bodystmt, + "=" + ), + srange_node(node) + ) + else + smap_method_definition( + srange_length(node.start_char, 3), + nil, + srange_node(node.name), + srange_length(node.end_char, -3), + nil, + srange_node(node) + ) + end + + if node.target + target = + node.target.is_a?(Paren) ? node.target.contents : node.target + + s( + :defs, + [visit(target), name, args, visit(node.bodystmt)], + smap_method_definition( + location.keyword, + srange_node(node.operator), + location.name, + location.end, + location.assignment, + location.expression + ) + ) + else + s(:def, [name, args, visit(node.bodystmt)], location) end + end - s( - :class, - [visit(node.constant), visit(node.superclass), visit(node.bodystmt)], - smap_definition( - srange_length(node.start_char, 5), - operator, - srange_node(node.constant), - srange_length(node.end_char, -3) - ).with_expression(srange_node(node)) - ) - end + # Visit a Defined node. + def visit_defined(node) + paren_range = (node.start_char + 8)...node.end_char + begin_token, end_token = + if buffer.source[paren_range].include?("(") + [ + srange_find(paren_range.begin, paren_range.end, "("), + srange_length(node.end_char, -1) + ] + end - # Visit a Command node. - def visit_command(node) - visit_command_call( - CommandCall.new( - receiver: nil, - operator: nil, - message: node.message, - arguments: node.arguments, - block: node.block, - location: node.location + s( + :defined?, + [visit(node.value)], + smap_keyword( + srange_length(node.start_char, 8), + begin_token, + end_token, + srange_node(node) + ) ) - ) - end + end - # Visit a CommandCall node. - def visit_command_call(node) - children = [ - visit(node.receiver), - node.message == :call ? :call : node.message.value.to_sym - ] - - begin_token = nil - end_token = nil - - case node.arguments - when Args - children += visit_all(node.arguments.parts) - when ArgParen - case node.arguments.arguments - when nil - # skip - when ArgsForward - children << visit(node.arguments.arguments) + # Visit a DynaSymbol node. + def visit_dyna_symbol(node) + location = + if node.quote + smap_collection( + srange_length(node.start_char, node.quote.length), + srange_length(node.end_char, -1), + srange_node(node) + ) + else + smap_collection_bare(srange_node(node)) + end + + if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) + s(:sym, ["\"#{node.parts.first.value}\"".undump.to_sym], location) else - children += visit_all(node.arguments.arguments.parts) + s(:dsym, visit_all(node.parts), location) end - - begin_token = srange_length(node.arguments.start_char, 1) - end_token = srange_length(node.arguments.end_char, -1) end - dot_bound = - if node.arguments - node.arguments.start_char - elsif node.block - node.block.start_char + # Visit an Else node. + def visit_else(node) + if node.statements.empty? && stack[-2].is_a?(Case) + s(:empty_else, [], nil) else - node.end_char + visit(node.statements) end + end - expression = - if node.arguments.is_a?(ArgParen) - srange(node.start_char, node.arguments.end_char) - elsif node.arguments.is_a?(Args) && node.arguments.parts.any? - last_part = node.arguments.parts.last - end_char = - if last_part.is_a?(Heredoc) - last_part.beginning.end_char - else - last_part.end_char - end + # Visit an Elsif node. + def visit_elsif(node) + else_token = + case node.consequent + when Elsif + srange_length(node.consequent.start_char, 5) + when Else + srange_length(node.consequent.start_char, 4) + end - srange(node.start_char, end_char) - elsif node.block - srange_node(node.message) - else - srange_node(node) - end + expression = srange(node.start_char, node.statements.end_char - 1) - call = s( - if node.operator.is_a?(Op) && node.operator.value == "&." - :csend - else - :send - end, - children, - smap_send( - if node.operator == :"::" - srange_find( - node.receiver.end_char, - if node.message == :call - dot_bound - else - node.message.start_char - end, - "::" - ) - elsif node.operator - srange_node(node.operator) - end, - node.message == :call ? nil : srange_node(node.message), - begin_token, - end_token, + :if, + [ + visit(node.predicate), + visit(node.statements), + visit(node.consequent) + ], + smap_condition( + srange_length(node.start_char, 5), + nil, + else_token, + nil, expression ) ) + end - if node.block - type, arguments = block_children(node.block) - + # Visit an ENDBlock node. + def visit_END(node) s( - type, - [call, arguments, visit(node.block.bodystmt)], - smap_collection( - srange_node(node.block.opening), - srange_length( - node.end_char, - node.block.opening.is_a?(Kw) ? -3 : -1 - ), + :postexe, + [visit(node.statements)], + smap_keyword( + srange_length(node.start_char, 3), + srange_find(node.start_char + 3, node.statements.start_char, "{"), + srange_length(node.end_char, -1), srange_node(node) ) ) - else - call end - end - # Visit a Const node. - def visit_const(node) - s( - :const, - [nil, node.value.to_sym], - smap_constant(nil, srange_node(node), srange_node(node)) - ) - end + # Visit an Ensure node. + def visit_ensure(node) + start_char = node.start_char + end_char = + if node.statements.empty? + start_char + 6 + else + node.statements.body.last.end_char + end - # Visit a ConstPathField node. - def visit_const_path_field(node) - if node.parent.is_a?(VarRef) && node.parent.value.is_a?(Kw) && - node.parent.value.value == "self" && node.constant.is_a?(Ident) - s(:send, [visit(node.parent), :"#{node.constant.value}="], nil) - else s( - :casgn, - [visit(node.parent), node.constant.value.to_sym], - smap_constant( - srange_find_between(node.parent, node.constant, "::"), - srange_node(node.constant), - srange_node(node) + :ensure, + [visit(node.statements)], + smap_condition( + srange_length(start_char, 6), + nil, + nil, + nil, + srange(start_char, end_char) ) ) end - end - # Visit a ConstPathRef node. - def visit_const_path_ref(node) - s( - :const, - [visit(node.parent), node.constant.value.to_sym], - smap_constant( - srange_find_between(node.parent, node.constant, "::"), - srange_node(node.constant), - srange_node(node) + # Visit a Field node. + def visit_field(node) + message = + case stack[-2] + when Assign, MLHS + Ident.new( + value: "#{node.name.value}=", + location: node.name.location + ) + else + node.name + end + + visit_command_call( + CommandCall.new( + receiver: node.parent, + operator: node.operator, + message: message, + arguments: nil, + block: nil, + location: node.location + ) ) - ) - end + end - # Visit a ConstRef node. - def visit_const_ref(node) - s( - :const, - [nil, node.constant.value.to_sym], - smap_constant(nil, srange_node(node.constant), srange_node(node)) - ) - end + # Visit a FloatLiteral node. + def visit_float(node) + operator = + if %w[+ -].include?(buffer.source[node.start_char]) + srange_length(node.start_char, 1) + end - # Visit a CVar node. - def visit_cvar(node) - s( - :cvar, - [node.value.to_sym], - smap_variable(srange_node(node), srange_node(node)) - ) - end + s( + :float, + [node.value.to_f], + smap_operator(operator, srange_node(node)) + ) + end - # Visit a DefNode node. - def visit_def(node) - name = node.name.value.to_sym - args = - case node.params - when Params - child = visit(node.params) + # Visit a FndPtn node. + def visit_fndptn(node) + left, right = + [node.left, node.right].map do |child| + location = + smap_operator( + srange_length(child.start_char, 1), + srange_node(child) + ) - s( - child.type, - child.children, - smap_collection_bare(child.location&.expression) - ) - when Paren - child = visit(node.params.contents) + if child.is_a?(VarField) && child.value.nil? + s(:match_rest, [], location) + else + s(:match_rest, [visit(child)], location) + end + end + inner = s( - child.type, - child.children, + :find_pattern, + [left, *visit_all(node.values), right], smap_collection( - srange_length(node.params.start_char, 1), - srange_length(node.params.end_char, -1), - srange_node(node.params) + srange_length(node.start_char, 1), + srange_length(node.end_char, -1), + srange_node(node) ) ) + + if node.constant + s(:const_pattern, [visit(node.constant), inner], nil) else - s(:args, [], smap_collection_bare(nil)) + inner end + end - location = - if node.endless? - smap_method_definition( - srange_length(node.start_char, 3), - nil, - srange_node(node.name), - nil, - srange_find_between( - (node.params || node.name), - node.bodystmt, - "=" - ), - srange_node(node) - ) - else - smap_method_definition( + # Visit a For node. + def visit_for(node) + s( + :for, + [visit(node.index), visit(node.collection), visit(node.statements)], + smap_for( srange_length(node.start_char, 3), - nil, - srange_node(node.name), + srange_find_between(node.index, node.collection, "in"), + srange_search_between(node.collection, node.statements, "do") || + srange_search_between(node.collection, node.statements, ";"), srange_length(node.end_char, -3), - nil, srange_node(node) ) - end - - if node.target - target = node.target.is_a?(Paren) ? node.target.contents : node.target - - s( - :defs, - [visit(target), name, args, visit(node.bodystmt)], - smap_method_definition( - location.keyword, - srange_node(node.operator), - location.name, - location.end, - location.assignment, - location.expression - ) ) - else - s(:def, [name, args, visit(node.bodystmt)], location) end - end - # Visit a Defined node. - def visit_defined(node) - paren_range = (node.start_char + 8)...node.end_char - begin_token, end_token = - if buffer.source[paren_range].include?("(") - [ - srange_find(paren_range.begin, paren_range.end, "("), - srange_length(node.end_char, -1) - ] - end - - s( - :defined?, - [visit(node.value)], - smap_keyword( - srange_length(node.start_char, 8), - begin_token, - end_token, - srange_node(node) + # Visit a GVar node. + def visit_gvar(node) + s( + :gvar, + [node.value.to_sym], + smap_variable(srange_node(node), srange_node(node)) ) - ) - end + end - # Visit a DynaSymbol node. - def visit_dyna_symbol(node) - location = - if node.quote + # Visit a HashLiteral node. + def visit_hash(node) + s( + :hash, + visit_all(node.assocs), smap_collection( - srange_length(node.start_char, node.quote.length), + srange_length(node.start_char, 1), srange_length(node.end_char, -1), srange_node(node) ) - else - smap_collection_bare(srange_node(node)) - end - - if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) - s(:sym, ["\"#{node.parts.first.value}\"".undump.to_sym], location) - else - s(:dsym, visit_all(node.parts), location) + ) end - end - # Visit an Else node. - def visit_else(node) - if node.statements.empty? && stack[-2].is_a?(Case) - s(:empty_else, [], nil) - else - visit(node.statements) - end - end + # Visit a Heredoc node. + def visit_heredoc(node) + heredoc = HeredocBuilder.new(node) - # Visit an Elsif node. - def visit_elsif(node) - else_token = - case node.consequent - when Elsif - srange_length(node.consequent.start_char, 5) - when Else - srange_length(node.consequent.start_char, 4) - end + # For each part of the heredoc, if it's a string content node, split + # it into multiple string content nodes, one for each line. Otherwise, + # visit the node as normal. + node.parts.each do |part| + if part.is_a?(TStringContent) && part.value.count("\n") > 1 + index = part.start_char + lines = part.value.split("\n") - expression = srange(node.start_char, node.statements.end_char - 1) - - s( - :if, - [ - visit(node.predicate), - visit(node.statements), - visit(node.consequent) - ], - smap_condition( - srange_length(node.start_char, 5), - nil, - else_token, - nil, - expression - ) - ) - end + lines.each do |line| + length = line.length + 1 + location = smap_collection_bare(srange_length(index, length)) - # Visit an ENDBlock node. - def visit_END(node) - s( - :postexe, - [visit(node.statements)], - smap_keyword( - srange_length(node.start_char, 3), - srange_find(node.start_char + 3, node.statements.start_char, "{"), - srange_length(node.end_char, -1), - srange_node(node) - ) - ) - end + heredoc << s(:str, ["#{line}\n"], location) + index += length + end + else + heredoc << visit(part) + end + end - # Visit an Ensure node. - def visit_ensure(node) - start_char = node.start_char - end_char = - if node.statements.empty? - start_char + 6 + # Now that we have all of the pieces on the heredoc, we can trim it if + # it is a heredoc that supports trimming (i.e., it has a ~ on the + # declaration). + heredoc.trim! + + # Generate the location for the heredoc, which goes from the + # declaration to the ending delimiter. + location = + smap_heredoc( + srange_node(node.beginning), + srange( + if node.parts.empty? + node.beginning.end_char + 1 + else + node.parts.first.start_char + end, + node.ending.start_char + ), + srange(node.ending.start_char, node.ending.end_char - 1) + ) + + # Finally, decide which kind of heredoc node to generate based on its + # declaration and contents. + if node.beginning.value.match?(/`\w+`\z/) + s(:xstr, heredoc.segments, location) + elsif heredoc.segments.length == 1 + segment = heredoc.segments.first + s(segment.type, segment.children, location) else - node.statements.body.last.end_char + s(:dstr, heredoc.segments, location) end + end - s( - :ensure, - [visit(node.statements)], - smap_condition( - srange_length(start_char, 6), - nil, - nil, - nil, - srange(start_char, end_char) - ) - ) - end + # Visit a HshPtn node. + def visit_hshptn(node) + children = + node.keywords.map do |(keyword, value)| + next s(:pair, [visit(keyword), visit(value)], nil) if value + + case keyword + when DynaSymbol + raise if keyword.parts.length > 1 + s(:match_var, [keyword.parts.first.value.to_sym], nil) + when Label + s(:match_var, [keyword.value.chomp(":").to_sym], nil) + end + end - # Visit a Field node. - def visit_field(node) - message = - case stack[-2] - when Assign, MLHS - Ident.new( - value: "#{node.name.value}=", - location: node.name.location - ) + if node.keyword_rest.is_a?(VarField) + children << if node.keyword_rest.value.nil? + s(:match_rest, [], nil) + elsif node.keyword_rest.value == :nil + s(:match_nil_pattern, [], nil) + else + s(:match_rest, [visit(node.keyword_rest)], nil) + end + end + + inner = s(:hash_pattern, children, nil) + if node.constant + s(:const_pattern, [visit(node.constant), inner], nil) else - node.name + inner end + end - visit_command_call( - CommandCall.new( - receiver: node.parent, - operator: node.operator, - message: message, - arguments: nil, - block: nil, - location: node.location + # Visit an Ident node. + def visit_ident(node) + s( + :lvar, + [node.value.to_sym], + smap_variable(srange_node(node), srange_node(node)) ) - ) - end - - # Visit a FloatLiteral node. - def visit_float(node) - operator = - if %w[+ -].include?(buffer.source[node.start_char]) - srange_length(node.start_char, 1) - end + end - s(:float, [node.value.to_f], smap_operator(operator, srange_node(node))) - end + # Visit an IfNode node. + def visit_if(node) + predicate = + case node.predicate + when RangeNode + type = + node.predicate.operator.value == ".." ? :iflipflop : :eflipflop + s(type, visit(node.predicate).children, nil) + when RegexpLiteral + s(:match_current_line, [visit(node.predicate)], nil) + when Unary + if node.predicate.operator.value == "!" && + node.predicate.statement.is_a?(RegexpLiteral) + s( + :send, + [ + s(:match_current_line, [visit(node.predicate.statement)]), + :! + ], + nil + ) + else + visit(node.predicate) + end + else + visit(node.predicate) + end - # Visit a FndPtn node. - def visit_fndptn(node) - left, right = - [node.left, node.right].map do |child| - location = - smap_operator( - srange_length(child.start_char, 1), - srange_node(child) + s( + :if, + [predicate, visit(node.statements), visit(node.consequent)], + if node.modifier? + smap_keyword_bare( + srange_find_between(node.statements, node.predicate, "if"), + srange_node(node) ) - - if child.is_a?(VarField) && child.value.nil? - s(:match_rest, [], location) else - s(:match_rest, [visit(child)], location) + begin_start = node.predicate.end_char + begin_end = + if node.statements.empty? + node.statements.end_char + else + node.statements.body.first.start_char + end + + begin_token = + if buffer.source[begin_start...begin_end].include?("then") + srange_find(begin_start, begin_end, "then") + elsif buffer.source[begin_start...begin_end].include?(";") + srange_find(begin_start, begin_end, ";") + end + + else_token = + case node.consequent + when Elsif + srange_length(node.consequent.start_char, 5) + when Else + srange_length(node.consequent.start_char, 4) + end + + smap_condition( + srange_length(node.start_char, 2), + begin_token, + else_token, + srange_length(node.end_char, -3), + srange_node(node) + ) end - end + ) + end - inner = + # Visit an IfOp node. + def visit_if_op(node) s( - :find_pattern, - [left, *visit_all(node.values), right], - smap_collection( - srange_length(node.start_char, 1), - srange_length(node.end_char, -1), + :if, + [visit(node.predicate), visit(node.truthy), visit(node.falsy)], + smap_ternary( + srange_find_between(node.predicate, node.truthy, "?"), + srange_find_between(node.truthy, node.falsy, ":"), srange_node(node) ) ) - - if node.constant - s(:const_pattern, [visit(node.constant), inner], nil) - else - inner end - end - - # Visit a For node. - def visit_for(node) - s( - :for, - [visit(node.index), visit(node.collection), visit(node.statements)], - smap_for( - srange_length(node.start_char, 3), - srange_find_between(node.index, node.collection, "in"), - srange_search_between(node.collection, node.statements, "do") || - srange_search_between(node.collection, node.statements, ";"), - srange_length(node.end_char, -3), - srange_node(node) - ) - ) - end - - # Visit a GVar node. - def visit_gvar(node) - s( - :gvar, - [node.value.to_sym], - smap_variable(srange_node(node), srange_node(node)) - ) - end - # Visit a HashLiteral node. - def visit_hash(node) - s( - :hash, - visit_all(node.assocs), - smap_collection( - srange_length(node.start_char, 1), - srange_length(node.end_char, -1), - srange_node(node) + # Visit an Imaginary node. + def visit_imaginary(node) + s( + :complex, + [ + # We have to do an eval here in order to get the value in case + # it's something like 42ri. to_c will not give the right value in + # that case. Maybe there's an API for this but I can't find it. + eval(node.value) + ], + smap_operator(nil, srange_node(node)) ) - ) - end - - # Visit a Heredoc node. - def visit_heredoc(node) - heredoc = HeredocBuilder.new(node) - - # For each part of the heredoc, if it's a string content node, split it - # into multiple string content nodes, one for each line. Otherwise, - # visit the node as normal. - node.parts.each do |part| - if part.is_a?(TStringContent) && part.value.count("\n") > 1 - index = part.start_char - lines = part.value.split("\n") - - lines.each do |line| - length = line.length + 1 - location = smap_collection_bare(srange_length(index, length)) - - heredoc << s(:str, ["#{line}\n"], location) - index += length - end - else - heredoc << visit(part) - end end - # Now that we have all of the pieces on the heredoc, we can trim it if - # it is a heredoc that supports trimming (i.e., it has a ~ on the - # declaration). - heredoc.trim! + # Visit an In node. + def visit_in(node) + case node.pattern + when IfNode + s( + :in_pattern, + [ + visit(node.pattern.statements), + s(:if_guard, [visit(node.pattern.predicate)], nil), + visit(node.statements) + ], + nil + ) + when UnlessNode + s( + :in_pattern, + [ + visit(node.pattern.statements), + s(:unless_guard, [visit(node.pattern.predicate)], nil), + visit(node.statements) + ], + nil + ) + else + begin_token = + srange_search_between(node.pattern, node.statements, "then") - # Generate the location for the heredoc, which goes from the declaration - # to the ending delimiter. - location = - smap_heredoc( - srange_node(node.beginning), - srange( - if node.parts.empty? - node.beginning.end_char + 1 + end_char = + if begin_token || node.statements.empty? + node.statements.end_char - 1 else - node.parts.first.start_char - end, - node.ending.start_char - ), - srange(node.ending.start_char, node.ending.end_char - 1) - ) + node.statements.body.last.start_char + end - # Finally, decide which kind of heredoc node to generate based on its - # declaration and contents. - if node.beginning.value.match?(/`\w+`\z/) - s(:xstr, heredoc.segments, location) - elsif heredoc.segments.length == 1 - segment = heredoc.segments.first - s(segment.type, segment.children, location) - else - s(:dstr, heredoc.segments, location) + s( + :in_pattern, + [visit(node.pattern), nil, visit(node.statements)], + smap_keyword( + srange_length(node.start_char, 2), + begin_token, + nil, + srange(node.start_char, end_char) + ) + ) + end end - end - # Visit a HshPtn node. - def visit_hshptn(node) - children = - node.keywords.map do |(keyword, value)| - next s(:pair, [visit(keyword), visit(value)], nil) if value - - case keyword - when DynaSymbol - raise if keyword.parts.length > 1 - s(:match_var, [keyword.parts.first.value.to_sym], nil) - when Label - s(:match_var, [keyword.value.chomp(":").to_sym], nil) + # Visit an Int node. + def visit_int(node) + operator = + if %w[+ -].include?(buffer.source[node.start_char]) + srange_length(node.start_char, 1) end - end - if node.keyword_rest.is_a?(VarField) - children << if node.keyword_rest.value.nil? - s(:match_rest, [], nil) - elsif node.keyword_rest.value == :nil - s(:match_nil_pattern, [], nil) - else - s(:match_rest, [visit(node.keyword_rest)], nil) - end + s(:int, [node.value.to_i], smap_operator(operator, srange_node(node))) end - inner = s(:hash_pattern, children, nil) - if node.constant - s(:const_pattern, [visit(node.constant), inner], nil) - else - inner + # Visit an IVar node. + def visit_ivar(node) + s( + :ivar, + [node.value.to_sym], + smap_variable(srange_node(node), srange_node(node)) + ) end - end - # Visit an Ident node. - def visit_ident(node) - s( - :lvar, - [node.value.to_sym], - smap_variable(srange_node(node), srange_node(node)) - ) - end + # Visit a Kw node. + def visit_kw(node) + location = smap(srange_node(node)) - # Visit an IfNode node. - def visit_if(node) - predicate = - case node.predicate - when RangeNode - type = - node.predicate.operator.value == ".." ? :iflipflop : :eflipflop - s(type, visit(node.predicate).children, nil) - when RegexpLiteral - s(:match_current_line, [visit(node.predicate)], nil) - when Unary - if node.predicate.operator.value == "!" && - node.predicate.statement.is_a?(RegexpLiteral) - s( - :send, - [s(:match_current_line, [visit(node.predicate.statement)]), :!], - nil - ) + case node.value + when "__FILE__" + s(:str, [buffer.name], location) + when "__LINE__" + s( + :int, + [node.location.start_line + buffer.first_line - 1], + location + ) + when "__ENCODING__" + if ::Parser::Builders::Default.emit_encoding + s(:__ENCODING__, [], location) else - visit(node.predicate) + s(:const, [s(:const, [nil, :Encoding], nil), :UTF_8], location) end else - visit(node.predicate) + s(node.value.to_sym, [], location) end + end - s( - :if, - [predicate, visit(node.statements), visit(node.consequent)], - if node.modifier? - smap_keyword_bare( - srange_find_between(node.statements, node.predicate, "if"), - srange_node(node) - ) + # Visit a KwRestParam node. + def visit_kwrest_param(node) + if node.name.nil? + s(:kwrestarg, [], smap_variable(nil, srange_node(node))) else - begin_start = node.predicate.end_char - begin_end = - if node.statements.empty? - node.statements.end_char - else - node.statements.body.first.start_char - end + s( + :kwrestarg, + [node.name.value.to_sym], + smap_variable(srange_node(node.name), srange_node(node)) + ) + end + end - begin_token = - if buffer.source[begin_start...begin_end].include?("then") - srange_find(begin_start, begin_end, "then") - elsif buffer.source[begin_start...begin_end].include?(";") - srange_find(begin_start, begin_end, ";") - end + # Visit a Label node. + def visit_label(node) + s( + :sym, + [node.value.chomp(":").to_sym], + smap_collection_bare(srange(node.start_char, node.end_char - 1)) + ) + end - else_token = - case node.consequent - when Elsif - srange_length(node.consequent.start_char, 5) - when Else - srange_length(node.consequent.start_char, 4) - end + # Visit a Lambda node. + def visit_lambda(node) + args = + node.params.is_a?(LambdaVar) ? node.params : node.params.contents + args_node = visit(args) - smap_condition( - srange_length(node.start_char, 2), - begin_token, - else_token, - srange_length(node.end_char, -3), - srange_node(node) - ) + type = :block + if args.empty? && (maximum = num_block_type(node.statements)) + type = :numblock + args_node = maximum end - ) - end - # Visit an IfOp node. - def visit_if_op(node) - s( - :if, - [visit(node.predicate), visit(node.truthy), visit(node.falsy)], - smap_ternary( - srange_find_between(node.predicate, node.truthy, "?"), - srange_find_between(node.truthy, node.falsy, ":"), - srange_node(node) - ) - ) - end + begin_token, end_token = + if ( + srange = + srange_search_between(node.params, node.statements, "{") + ) + [srange, srange_length(node.end_char, -1)] + else + [ + srange_find_between(node.params, node.statements, "do"), + srange_length(node.end_char, -3) + ] + end - # Visit an Imaginary node. - def visit_imaginary(node) - s( - :complex, - [ - # We have to do an eval here in order to get the value in case it's - # something like 42ri. to_c will not give the right value in that - # case. Maybe there's an API for this but I can't find it. - eval(node.value) - ], - smap_operator(nil, srange_node(node)) - ) - end + selector = srange_length(node.start_char, 2) - # Visit an In node. - def visit_in(node) - case node.pattern - when IfNode - s( - :in_pattern, - [ - visit(node.pattern.statements), - s(:if_guard, [visit(node.pattern.predicate)], nil), - visit(node.statements) - ], - nil - ) - when UnlessNode s( - :in_pattern, + type, [ - visit(node.pattern.statements), - s(:unless_guard, [visit(node.pattern.predicate)], nil), + if ::Parser::Builders::Default.emit_lambda + s(:lambda, [], smap(selector)) + else + s(:send, [nil, :lambda], smap_send_bare(selector, selector)) + end, + args_node, visit(node.statements) ], - nil + smap_collection(begin_token, end_token, srange_node(node)) ) - else - begin_token = - srange_search_between(node.pattern, node.statements, "then") + end - end_char = - if begin_token || node.statements.empty? - node.statements.end_char - 1 + # Visit a LambdaVar node. + def visit_lambda_var(node) + shadowargs = + node.locals.map do |local| + s( + :shadowarg, + [local.value.to_sym], + smap_variable(srange_node(local), srange_node(local)) + ) + end + + location = + if node.start_char == node.end_char + smap_collection_bare(nil) else - node.statements.body.last.start_char + smap_collection( + srange_length(node.start_char, 1), + srange_length(node.end_char, -1), + srange_node(node) + ) end + s(:args, visit(node.params).children + shadowargs, location) + end + + # Visit an MAssign node. + def visit_massign(node) s( - :in_pattern, - [visit(node.pattern), nil, visit(node.statements)], - smap_keyword( - srange_length(node.start_char, 2), - begin_token, - nil, - srange(node.start_char, end_char) + :masgn, + [visit(node.target), visit(node.value)], + smap_operator( + srange_find_between(node.target, node.value, "="), + srange_node(node) ) ) end - end - - # Visit an Int node. - def visit_int(node) - operator = - if %w[+ -].include?(buffer.source[node.start_char]) - srange_length(node.start_char, 1) - end - s(:int, [node.value.to_i], smap_operator(operator, srange_node(node))) - end + # Visit a MethodAddBlock node. + def visit_method_add_block(node) + case node.call + when Break, Next, ReturnNode + type, arguments = block_children(node.block) + call = visit(node.call) - # Visit an IVar node. - def visit_ivar(node) - s( - :ivar, - [node.value.to_sym], - smap_variable(srange_node(node), srange_node(node)) - ) - end + s( + call.type, + [ + s( + type, + [*call.children, arguments, visit(node.block.bodystmt)], + nil + ) + ], + nil + ) + when ARef, Super, ZSuper + type, arguments = block_children(node.block) - # Visit a Kw node. - def visit_kw(node) - location = smap(srange_node(node)) - - case node.value - when "__FILE__" - s(:str, [buffer.name], location) - when "__LINE__" - s(:int, [node.location.start_line + buffer.first_line - 1], location) - when "__ENCODING__" - if ::Parser::Builders::Default.emit_encoding - s(:__ENCODING__, [], location) + s( + type, + [visit(node.call), arguments, visit(node.block.bodystmt)], + nil + ) else - s(:const, [s(:const, [nil, :Encoding], nil), :UTF_8], location) + visit_command_call( + CommandCall.new( + receiver: node.call.receiver, + operator: node.call.operator, + message: node.call.message, + arguments: node.call.arguments, + block: node.block, + location: node.location + ) + ) end - else - s(node.value.to_sym, [], location) end - end - # Visit a KwRestParam node. - def visit_kwrest_param(node) - if node.name.nil? - s(:kwrestarg, [], smap_variable(nil, srange_node(node))) - else + # Visit an MLHS node. + def visit_mlhs(node) s( - :kwrestarg, - [node.name.value.to_sym], - smap_variable(srange_node(node.name), srange_node(node)) + :mlhs, + node.parts.map do |part| + if part.is_a?(Ident) + s( + :arg, + [part.value.to_sym], + smap_variable(srange_node(part), srange_node(part)) + ) + else + visit(part) + end + end, + smap_collection_bare(srange_node(node)) ) end - end - - # Visit a Label node. - def visit_label(node) - s( - :sym, - [node.value.chomp(":").to_sym], - smap_collection_bare(srange(node.start_char, node.end_char - 1)) - ) - end - - # Visit a Lambda node. - def visit_lambda(node) - args = node.params.is_a?(LambdaVar) ? node.params : node.params.contents - args_node = visit(args) - - type = :block - if args.empty? && (maximum = num_block_type(node.statements)) - type = :numblock - args_node = maximum - end - - begin_token, end_token = - if (srange = srange_search_between(node.params, node.statements, "{")) - [srange, srange_length(node.end_char, -1)] - else - [ - srange_find_between(node.params, node.statements, "do"), - srange_length(node.end_char, -3) - ] - end - - selector = srange_length(node.start_char, 2) - - s( - type, - [ - if ::Parser::Builders::Default.emit_lambda - s(:lambda, [], smap(selector)) - else - s(:send, [nil, :lambda], smap_send_bare(selector, selector)) - end, - args_node, - visit(node.statements) - ], - smap_collection(begin_token, end_token, srange_node(node)) - ) - end - # Visit a LambdaVar node. - def visit_lambda_var(node) - shadowargs = - node.locals.map do |local| - s( - :shadowarg, - [local.value.to_sym], - smap_variable(srange_node(local), srange_node(local)) - ) - end + # Visit an MLHSParen node. + def visit_mlhs_paren(node) + child = visit(node.contents) - location = - if node.start_char == node.end_char - smap_collection_bare(nil) - else + s( + child.type, + child.children, smap_collection( srange_length(node.start_char, 1), srange_length(node.end_char, -1), srange_node(node) ) - end - - s(:args, visit(node.params).children + shadowargs, location) - end - - # Visit an MAssign node. - def visit_massign(node) - s( - :masgn, - [visit(node.target), visit(node.value)], - smap_operator( - srange_find_between(node.target, node.value, "="), - srange_node(node) ) - ) - end - - # Visit a MethodAddBlock node. - def visit_method_add_block(node) - case node.call - when Break, Next, ReturnNode - type, arguments = block_children(node.block) - call = visit(node.call) + end + # Visit a ModuleDeclaration node. + def visit_module(node) s( - call.type, - [ - s( - type, - [*call.children, arguments, visit(node.block.bodystmt)], - nil - ) - ], - nil + :module, + [visit(node.constant), visit(node.bodystmt)], + smap_definition( + srange_length(node.start_char, 6), + nil, + srange_node(node.constant), + srange_length(node.end_char, -3) + ).with_expression(srange_node(node)) ) - when ARef, Super, ZSuper - type, arguments = block_children(node.block) + end - s( - type, - [visit(node.call), arguments, visit(node.block.bodystmt)], - nil - ) - else - visit_command_call( - CommandCall.new( - receiver: node.call.receiver, - operator: node.call.operator, - message: node.call.message, - arguments: node.call.arguments, - block: node.block, + # Visit an MRHS node. + def visit_mrhs(node) + visit_array( + ArrayLiteral.new( + lbracket: nil, + contents: Args.new(parts: node.parts, location: node.location), location: node.location ) ) end - end - - # Visit an MLHS node. - def visit_mlhs(node) - s( - :mlhs, - node.parts.map do |part| - if part.is_a?(Ident) - s( - :arg, - [part.value.to_sym], - smap_variable(srange_node(part), srange_node(part)) - ) - else - visit(part) - end - end, - smap_collection_bare(srange_node(node)) - ) - end - - # Visit an MLHSParen node. - def visit_mlhs_paren(node) - child = visit(node.contents) - - s( - child.type, - child.children, - smap_collection( - srange_length(node.start_char, 1), - srange_length(node.end_char, -1), - srange_node(node) - ) - ) - end - - # Visit a ModuleDeclaration node. - def visit_module(node) - s( - :module, - [visit(node.constant), visit(node.bodystmt)], - smap_definition( - srange_length(node.start_char, 6), - nil, - srange_node(node.constant), - srange_length(node.end_char, -3) - ).with_expression(srange_node(node)) - ) - end - - # Visit an MRHS node. - def visit_mrhs(node) - visit_array( - ArrayLiteral.new( - lbracket: nil, - contents: Args.new(parts: node.parts, location: node.location), - location: node.location - ) - ) - end - - # Visit a Next node. - def visit_next(node) - s( - :next, - visit_all(node.arguments.parts), - smap_keyword_bare( - srange_length(node.start_char, 4), - srange_node(node) - ) - ) - end - - # Visit a Not node. - def visit_not(node) - if node.statement.nil? - begin_token = srange_find(node.start_char, nil, "(") - end_token = srange_find(node.start_char, nil, ")") - - s( - :send, - [ - s( - :begin, - [], - smap_collection( - begin_token, - end_token, - begin_token.join(end_token) - ) - ), - :! - ], - smap_send_bare(srange_length(node.start_char, 3), srange_node(node)) - ) - else - begin_token, end_token = - if node.parentheses? - [ - srange_find( - node.start_char + 3, - node.statement.start_char, - "(" - ), - srange_length(node.end_char, -1) - ] - end + # Visit a Next node. + def visit_next(node) s( - :send, - [visit(node.statement), :!], - smap_send( - nil, - srange_length(node.start_char, 3), - begin_token, - end_token, + :next, + visit_all(node.arguments.parts), + smap_keyword_bare( + srange_length(node.start_char, 4), srange_node(node) ) ) end - end - - # Visit an OpAssign node. - def visit_opassign(node) - target = visit(node.target) - location = - target - .location - .with_expression(srange_node(node)) - .with_operator(srange_node(node.operator)) - - case node.operator.value - when "||=" - s(:or_asgn, [target, visit(node.value)], location) - when "&&=" - s(:and_asgn, [target, visit(node.value)], location) - else - s( - :op_asgn, - [target, node.operator.value.chomp("=").to_sym, visit(node.value)], - location - ) - end - end - # Visit a Params node. - def visit_params(node) - children = [] + # Visit a Not node. + def visit_not(node) + if node.statement.nil? + begin_token = srange_find(node.start_char, nil, "(") + end_token = srange_find(node.start_char, nil, ")") - children += - node.requireds.map do |required| - case required - when MLHSParen - visit(required) - else - s( - :arg, - [required.value.to_sym], - smap_variable(srange_node(required), srange_node(required)) + s( + :send, + [ + s( + :begin, + [], + smap_collection( + begin_token, + end_token, + begin_token.join(end_token) + ) + ), + :! + ], + smap_send_bare( + srange_length(node.start_char, 3), + srange_node(node) ) - end - end + ) + else + begin_token, end_token = + if node.parentheses? + [ + srange_find( + node.start_char + 3, + node.statement.start_char, + "(" + ), + srange_length(node.end_char, -1) + ] + end - children += - node.optionals.map do |(name, value)| s( - :optarg, - [name.value.to_sym, visit(value)], - smap_variable( - srange_node(name), - srange_node(name).join(srange_node(value)) - ).with_operator(srange_find_between(name, value, "=")) + :send, + [visit(node.statement), :!], + smap_send( + nil, + srange_length(node.start_char, 3), + begin_token, + end_token, + srange_node(node) + ) ) end - - if node.rest && !node.rest.is_a?(ExcessedComma) - children << visit(node.rest) end - children += - node.posts.map do |post| + # Visit an OpAssign node. + def visit_opassign(node) + target = visit(node.target) + location = + target + .location + .with_expression(srange_node(node)) + .with_operator(srange_node(node.operator)) + + case node.operator.value + when "||=" + s(:or_asgn, [target, visit(node.value)], location) + when "&&=" + s(:and_asgn, [target, visit(node.value)], location) + else s( - :arg, - [post.value.to_sym], - smap_variable(srange_node(post), srange_node(post)) + :op_asgn, + [ + target, + node.operator.value.chomp("=").to_sym, + visit(node.value) + ], + location ) end + end - children += - node.keywords.map do |(name, value)| - key = name.value.chomp(":").to_sym + # Visit a Params node. + def visit_params(node) + children = [] - if value - s( - :kwoptarg, - [key, visit(value)], - smap_variable( - srange(name.start_char, name.end_char - 1), - srange_node(name).join(srange_node(value)) + children += + node.requireds.map do |required| + case required + when MLHSParen + visit(required) + else + s( + :arg, + [required.value.to_sym], + smap_variable(srange_node(required), srange_node(required)) ) - ) - else + end + end + + children += + node.optionals.map do |(name, value)| s( - :kwarg, - [key], + :optarg, + [name.value.to_sym, visit(value)], smap_variable( - srange(name.start_char, name.end_char - 1), - srange_node(name) - ) + srange_node(name), + srange_node(name).join(srange_node(value)) + ).with_operator(srange_find_between(name, value, "=")) ) end + + if node.rest && !node.rest.is_a?(ExcessedComma) + children << visit(node.rest) end - case node.keyword_rest - when nil, ArgsForward - # do nothing - when :nil - children << s( - :kwnilarg, - [], - smap_variable(srange_length(node.end_char, -3), srange_node(node)) - ) - else - children << visit(node.keyword_rest) - end + children += + node.posts.map do |post| + s( + :arg, + [post.value.to_sym], + smap_variable(srange_node(post), srange_node(post)) + ) + end - children << visit(node.block) if node.block + children += + node.keywords.map do |(name, value)| + key = name.value.chomp(":").to_sym - if node.keyword_rest.is_a?(ArgsForward) - location = smap(srange_node(node.keyword_rest)) + if value + s( + :kwoptarg, + [key, visit(value)], + smap_variable( + srange(name.start_char, name.end_char - 1), + srange_node(name).join(srange_node(value)) + ) + ) + else + s( + :kwarg, + [key], + smap_variable( + srange(name.start_char, name.end_char - 1), + srange_node(name) + ) + ) + end + end - # If there are no other arguments and we have the emit_forward_arg - # option enabled, then the entire argument list is represented by a - # single forward_args node. - if children.empty? && !::Parser::Builders::Default.emit_forward_arg - return s(:forward_args, [], location) + case node.keyword_rest + when nil, ArgsForward + # do nothing + when :nil + children << s( + :kwnilarg, + [], + smap_variable(srange_length(node.end_char, -3), srange_node(node)) + ) + else + children << visit(node.keyword_rest) end - # Otherwise, we need to insert a forward_arg node into the list of - # parameters before any keyword rest or block parameters. - index = - node.requireds.length + node.optionals.length + node.keywords.length - children.insert(index, s(:forward_arg, [], location)) - end + children << visit(node.block) if node.block - location = - unless children.empty? - first = children.first.location.expression - last = children.last.location.expression - smap_collection_bare(first.join(last)) + if node.keyword_rest.is_a?(ArgsForward) + location = smap(srange_node(node.keyword_rest)) + + # If there are no other arguments and we have the emit_forward_arg + # option enabled, then the entire argument list is represented by a + # single forward_args node. + if children.empty? && !::Parser::Builders::Default.emit_forward_arg + return s(:forward_args, [], location) + end + + # Otherwise, we need to insert a forward_arg node into the list of + # parameters before any keyword rest or block parameters. + index = + node.requireds.length + node.optionals.length + + node.keywords.length + children.insert(index, s(:forward_arg, [], location)) end - s(:args, children, location) - end + location = + unless children.empty? + first = children.first.location.expression + last = children.last.location.expression + smap_collection_bare(first.join(last)) + end - # Visit a Paren node. - def visit_paren(node) - location = - smap_collection( - srange_length(node.start_char, 1), - srange_length(node.end_char, -1), - srange_node(node) - ) + s(:args, children, location) + end - if node.contents.nil? || - (node.contents.is_a?(Statements) && node.contents.empty?) - s(:begin, [], location) - else - child = visit(node.contents) - child.type == :begin ? child : s(:begin, [child], location) + # Visit a Paren node. + def visit_paren(node) + location = + smap_collection( + srange_length(node.start_char, 1), + srange_length(node.end_char, -1), + srange_node(node) + ) + + if node.contents.nil? || + (node.contents.is_a?(Statements) && node.contents.empty?) + s(:begin, [], location) + else + child = visit(node.contents) + child.type == :begin ? child : s(:begin, [child], location) + end end - end - # Visit a PinnedBegin node. - def visit_pinned_begin(node) - s( - :pin, - [ - s( - :begin, - [visit(node.statement)], - smap_collection( - srange_length(node.start_char + 1, 1), - srange_length(node.end_char, -1), - srange(node.start_char + 1, node.end_char) + # Visit a PinnedBegin node. + def visit_pinned_begin(node) + s( + :pin, + [ + s( + :begin, + [visit(node.statement)], + smap_collection( + srange_length(node.start_char + 1, 1), + srange_length(node.end_char, -1), + srange(node.start_char + 1, node.end_char) + ) ) - ) - ], - smap_send_bare(srange_length(node.start_char, 1), srange_node(node)) - ) - end + ], + smap_send_bare(srange_length(node.start_char, 1), srange_node(node)) + ) + end - # Visit a PinnedVarRef node. - def visit_pinned_var_ref(node) - s( - :pin, - [visit(node.value)], - smap_send_bare(srange_length(node.start_char, 1), srange_node(node)) - ) - end + # Visit a PinnedVarRef node. + def visit_pinned_var_ref(node) + s( + :pin, + [visit(node.value)], + smap_send_bare(srange_length(node.start_char, 1), srange_node(node)) + ) + end - # Visit a Program node. - def visit_program(node) - visit(node.statements) - end + # Visit a Program node. + def visit_program(node) + visit(node.statements) + end - # Visit a QSymbols node. - def visit_qsymbols(node) - parts = - node.elements.map do |element| - SymbolLiteral.new(value: element, location: element.location) - end + # Visit a QSymbols node. + def visit_qsymbols(node) + parts = + node.elements.map do |element| + SymbolLiteral.new(value: element, location: element.location) + end - visit_array( - ArrayLiteral.new( - lbracket: node.beginning, - contents: Args.new(parts: parts, location: node.location), - location: node.location + visit_array( + ArrayLiteral.new( + lbracket: node.beginning, + contents: Args.new(parts: parts, location: node.location), + location: node.location + ) ) - ) - end + end - # Visit a QWords node. - def visit_qwords(node) - visit_array( - ArrayLiteral.new( - lbracket: node.beginning, - contents: Args.new(parts: node.elements, location: node.location), - location: node.location + # Visit a QWords node. + def visit_qwords(node) + visit_array( + ArrayLiteral.new( + lbracket: node.beginning, + contents: Args.new(parts: node.elements, location: node.location), + location: node.location + ) ) - ) - end + end - # Visit a RangeNode node. - def visit_range(node) - s( - node.operator.value == ".." ? :irange : :erange, - [visit(node.left), visit(node.right)], - smap_operator(srange_node(node.operator), srange_node(node)) - ) - end + # Visit a RangeNode node. + def visit_range(node) + s( + node.operator.value == ".." ? :irange : :erange, + [visit(node.left), visit(node.right)], + smap_operator(srange_node(node.operator), srange_node(node)) + ) + end - # Visit an RAssign node. - def visit_rassign(node) - s( - node.operator.value == "=>" ? :match_pattern : :match_pattern_p, - [visit(node.value), visit(node.pattern)], - smap_operator(srange_node(node.operator), srange_node(node)) - ) - end + # Visit an RAssign node. + def visit_rassign(node) + s( + node.operator.value == "=>" ? :match_pattern : :match_pattern_p, + [visit(node.value), visit(node.pattern)], + smap_operator(srange_node(node.operator), srange_node(node)) + ) + end - # Visit a Rational node. - def visit_rational(node) - s(:rational, [node.value.to_r], smap_operator(nil, srange_node(node))) - end + # Visit a Rational node. + def visit_rational(node) + s(:rational, [node.value.to_r], smap_operator(nil, srange_node(node))) + end - # Visit a Redo node. - def visit_redo(node) - s(:redo, [], smap_keyword_bare(srange_node(node), srange_node(node))) - end + # Visit a Redo node. + def visit_redo(node) + s(:redo, [], smap_keyword_bare(srange_node(node), srange_node(node))) + end - # Visit a RegexpLiteral node. - def visit_regexp_literal(node) - s( - :regexp, - visit_all(node.parts).push( - s( - :regopt, - node.ending.scan(/[a-z]/).sort.map(&:to_sym), - smap(srange_length(node.end_char, -(node.ending.length - 1))) + # Visit a RegexpLiteral node. + def visit_regexp_literal(node) + s( + :regexp, + visit_all(node.parts).push( + s( + :regopt, + node.ending.scan(/[a-z]/).sort.map(&:to_sym), + smap(srange_length(node.end_char, -(node.ending.length - 1))) + ) + ), + smap_collection( + srange_length(node.start_char, node.beginning.length), + srange_length(node.end_char - node.ending.length, 1), + srange_node(node) ) - ), - smap_collection( - srange_length(node.start_char, node.beginning.length), - srange_length(node.end_char - node.ending.length, 1), - srange_node(node) ) - ) - end + end - # Visit a Rescue node. - def visit_rescue(node) - # In the parser gem, there is a separation between the rescue node and - # the rescue body. They have different bounds, so we have to calculate - # those here. - start_char = node.start_char + # Visit a Rescue node. + def visit_rescue(node) + # In the parser gem, there is a separation between the rescue node and + # the rescue body. They have different bounds, so we have to calculate + # those here. + start_char = node.start_char - body_end_char = - if node.statements.empty? - start_char + 6 - else - node.statements.body.last.end_char - end + body_end_char = + if node.statements.empty? + start_char + 6 + else + node.statements.body.last.end_char + end - end_char = - if node.consequent - end_node = node.consequent - end_node = end_node.consequent while end_node.consequent + end_char = + if node.consequent + end_node = node.consequent + end_node = end_node.consequent while end_node.consequent - if end_node.statements.empty? - start_char + 6 + if end_node.statements.empty? + start_char + 6 + else + end_node.statements.body.last.end_char + end else - end_node.statements.body.last.end_char + body_end_char end - else - body_end_char - end - # These locations are reused for multiple children. - keyword = srange_length(start_char, 6) - body_expression = srange(start_char, body_end_char) - expression = srange(start_char, end_char) + # These locations are reused for multiple children. + keyword = srange_length(start_char, 6) + body_expression = srange(start_char, body_end_char) + expression = srange(start_char, end_char) - exceptions = - case node.exception&.exceptions - when nil - nil - when MRHS - visit_array( - ArrayLiteral.new( - lbracket: nil, - contents: - Args.new( - parts: node.exception.exceptions.parts, - location: node.exception.exceptions.location - ), - location: node.exception.exceptions.location + exceptions = + case node.exception&.exceptions + when nil + nil + when MRHS + visit_array( + ArrayLiteral.new( + lbracket: nil, + contents: + Args.new( + parts: node.exception.exceptions.parts, + location: node.exception.exceptions.location + ), + location: node.exception.exceptions.location + ) ) - ) - else - visit_array( - ArrayLiteral.new( - lbracket: nil, - contents: - Args.new( - parts: [node.exception.exceptions], - location: node.exception.exceptions.location + else + visit_array( + ArrayLiteral.new( + lbracket: nil, + contents: + Args.new( + parts: [node.exception.exceptions], + location: node.exception.exceptions.location + ), + location: node.exception.exceptions.location + ) + ) + end + + resbody = + if node.exception.nil? + s( + :resbody, + [nil, nil, visit(node.statements)], + smap_rescue_body(keyword, nil, nil, body_expression) + ) + elsif node.exception.variable.nil? + s( + :resbody, + [exceptions, nil, visit(node.statements)], + smap_rescue_body(keyword, nil, nil, body_expression) + ) + else + s( + :resbody, + [ + exceptions, + visit(node.exception.variable), + visit(node.statements) + ], + smap_rescue_body( + keyword, + srange_find( + node.start_char + 6, + node.exception.variable.start_char, + "=>" ), - location: node.exception.exceptions.location + nil, + body_expression + ) ) - ) - end + end - resbody = - if node.exception.nil? - s( - :resbody, - [nil, nil, visit(node.statements)], - smap_rescue_body(keyword, nil, nil, body_expression) - ) - elsif node.exception.variable.nil? - s( - :resbody, - [exceptions, nil, visit(node.statements)], - smap_rescue_body(keyword, nil, nil, body_expression) - ) + children = [resbody] + if node.consequent + children += visit(node.consequent).children else - s( - :resbody, - [ - exceptions, - visit(node.exception.variable), - visit(node.statements) - ], - smap_rescue_body( - keyword, - srange_find( - node.start_char + 6, - node.exception.variable.start_char, - "=>" - ), - nil, - body_expression - ) - ) + children << nil end - children = [resbody] - if node.consequent - children += visit(node.consequent).children - else - children << nil + s(:rescue, children, smap_condition_bare(expression)) end - s(:rescue, children, smap_condition_bare(expression)) - end - - # Visit a RescueMod node. - def visit_rescue_mod(node) - keyword = srange_find_between(node.statement, node.value, "rescue") - - s( - :rescue, - [ - visit(node.statement), - s( - :resbody, - [nil, nil, visit(node.value)], - smap_rescue_body( - keyword, - nil, - nil, - keyword.join(srange_node(node.value)) - ) - ), - nil - ], - smap_condition_bare(srange_node(node)) - ) - end + # Visit a RescueMod node. + def visit_rescue_mod(node) + keyword = srange_find_between(node.statement, node.value, "rescue") - # Visit a RestParam node. - def visit_rest_param(node) - if node.name s( - :restarg, - [node.name.value.to_sym], - smap_variable(srange_node(node.name), srange_node(node)) + :rescue, + [ + visit(node.statement), + s( + :resbody, + [nil, nil, visit(node.value)], + smap_rescue_body( + keyword, + nil, + nil, + keyword.join(srange_node(node.value)) + ) + ), + nil + ], + smap_condition_bare(srange_node(node)) ) - else - s(:restarg, [], smap_variable(nil, srange_node(node))) end - end - - # Visit a Retry node. - def visit_retry(node) - s(:retry, [], smap_keyword_bare(srange_node(node), srange_node(node))) - end - - # Visit a ReturnNode node. - def visit_return(node) - s( - :return, - node.arguments ? visit_all(node.arguments.parts) : [], - smap_keyword_bare( - srange_length(node.start_char, 6), - srange_node(node) - ) - ) - end - # Visit an SClass node. - def visit_sclass(node) - s( - :sclass, - [visit(node.target), visit(node.bodystmt)], - smap_definition( - srange_length(node.start_char, 5), - srange_find(node.start_char + 5, node.target.start_char, "<<"), - nil, - srange_length(node.end_char, -3) - ).with_expression(srange_node(node)) - ) - end - - # Visit a Statements node. - def visit_statements(node) - children = - node.body.reject do |child| - child.is_a?(Comment) || child.is_a?(EmbDoc) || - child.is_a?(EndContent) || child.is_a?(VoidStmt) + # Visit a RestParam node. + def visit_rest_param(node) + if node.name + s( + :restarg, + [node.name.value.to_sym], + smap_variable(srange_node(node.name), srange_node(node)) + ) + else + s(:restarg, [], smap_variable(nil, srange_node(node))) end + end - case children.length - when 0 - nil - when 1 - visit(children.first) - else + # Visit a Retry node. + def visit_retry(node) + s(:retry, [], smap_keyword_bare(srange_node(node), srange_node(node))) + end + + # Visit a ReturnNode node. + def visit_return(node) s( - :begin, - visit_all(children), - smap_collection_bare( - srange(children.first.start_char, children.last.end_char) + :return, + node.arguments ? visit_all(node.arguments.parts) : [], + smap_keyword_bare( + srange_length(node.start_char, 6), + srange_node(node) ) ) end - end - - # Visit a StringConcat node. - def visit_string_concat(node) - s( - :dstr, - [visit(node.left), visit(node.right)], - smap_collection_bare(srange_node(node)) - ) - end - # Visit a StringDVar node. - def visit_string_dvar(node) - visit(node.variable) - end - - # Visit a StringEmbExpr node. - def visit_string_embexpr(node) - s( - :begin, - visit(node.statements).then { |child| child ? [child] : [] }, - smap_collection( - srange_length(node.start_char, 2), - srange_length(node.end_char, -1), - srange_node(node) + # Visit an SClass node. + def visit_sclass(node) + s( + :sclass, + [visit(node.target), visit(node.bodystmt)], + smap_definition( + srange_length(node.start_char, 5), + srange_find(node.start_char + 5, node.target.start_char, "<<"), + nil, + srange_length(node.end_char, -3) + ).with_expression(srange_node(node)) ) - ) - end + end - # Visit a StringLiteral node. - def visit_string_literal(node) - location = - if node.quote - smap_collection( - srange_length(node.start_char, node.quote.length), - srange_length(node.end_char, -1), - srange_node(node) - ) + # Visit a Statements node. + def visit_statements(node) + children = + node.body.reject do |child| + child.is_a?(Comment) || child.is_a?(EmbDoc) || + child.is_a?(EndContent) || child.is_a?(VoidStmt) + end + + case children.length + when 0 + nil + when 1 + visit(children.first) else - smap_collection_bare(srange_node(node)) + s( + :begin, + visit_all(children), + smap_collection_bare( + srange(children.first.start_char, children.last.end_char) + ) + ) end + end - if node.parts.empty? - s(:str, [""], location) - elsif node.parts.length == 1 && node.parts.first.is_a?(TStringContent) - child = visit(node.parts.first) - s(child.type, child.children, location) - else - s(:dstr, visit_all(node.parts), location) + # Visit a StringConcat node. + def visit_string_concat(node) + s( + :dstr, + [visit(node.left), visit(node.right)], + smap_collection_bare(srange_node(node)) + ) end - end - # Visit a Super node. - def visit_super(node) - if node.arguments.is_a?(Args) + # Visit a StringDVar node. + def visit_string_dvar(node) + visit(node.variable) + end + + # Visit a StringEmbExpr node. + def visit_string_embexpr(node) s( - :super, - visit_all(node.arguments.parts), - smap_keyword_bare( - srange_length(node.start_char, 5), + :begin, + visit(node.statements).then { |child| child ? [child] : [] }, + smap_collection( + srange_length(node.start_char, 2), + srange_length(node.end_char, -1), srange_node(node) ) ) - else - case node.arguments.arguments - when nil - s( - :super, - [], - smap_keyword( - srange_length(node.start_char, 5), - srange_find(node.start_char + 5, node.end_char, "("), + end + + # Visit a StringLiteral node. + def visit_string_literal(node) + location = + if node.quote + smap_collection( + srange_length(node.start_char, node.quote.length), srange_length(node.end_char, -1), srange_node(node) ) - ) - when ArgsForward - s(:super, [visit(node.arguments.arguments)], nil) + else + smap_collection_bare(srange_node(node)) + end + + if node.parts.empty? + s(:str, [""], location) + elsif node.parts.length == 1 && node.parts.first.is_a?(TStringContent) + child = visit(node.parts.first) + s(child.type, child.children, location) else + s(:dstr, visit_all(node.parts), location) + end + end + + # Visit a Super node. + def visit_super(node) + if node.arguments.is_a?(Args) s( :super, - visit_all(node.arguments.arguments.parts), - smap_keyword( + visit_all(node.arguments.parts), + smap_keyword_bare( srange_length(node.start_char, 5), - srange_find(node.start_char + 5, node.end_char, "("), - srange_length(node.end_char, -1), srange_node(node) ) ) + else + case node.arguments.arguments + when nil + s( + :super, + [], + smap_keyword( + srange_length(node.start_char, 5), + srange_find(node.start_char + 5, node.end_char, "("), + srange_length(node.end_char, -1), + srange_node(node) + ) + ) + when ArgsForward + s(:super, [visit(node.arguments.arguments)], nil) + else + s( + :super, + visit_all(node.arguments.arguments.parts), + smap_keyword( + srange_length(node.start_char, 5), + srange_find(node.start_char + 5, node.end_char, "("), + srange_length(node.end_char, -1), + srange_node(node) + ) + ) + end end end - end - # Visit a SymbolLiteral node. - def visit_symbol_literal(node) - begin_token = - if buffer.source[node.start_char] == ":" - srange_length(node.start_char, 1) - end + # Visit a SymbolLiteral node. + def visit_symbol_literal(node) + begin_token = + if buffer.source[node.start_char] == ":" + srange_length(node.start_char, 1) + end - s( - :sym, - [node.value.value.to_sym], - smap_collection(begin_token, nil, srange_node(node)) - ) - end + s( + :sym, + [node.value.value.to_sym], + smap_collection(begin_token, nil, srange_node(node)) + ) + end - # Visit a Symbols node. - def visit_symbols(node) - parts = - node.elements.map do |element| - part = element.parts.first + # Visit a Symbols node. + def visit_symbols(node) + parts = + node.elements.map do |element| + part = element.parts.first - if element.parts.length == 1 && part.is_a?(TStringContent) - SymbolLiteral.new(value: part, location: part.location) - else - DynaSymbol.new( - parts: element.parts, - quote: nil, - location: element.location - ) + if element.parts.length == 1 && part.is_a?(TStringContent) + SymbolLiteral.new(value: part, location: part.location) + else + DynaSymbol.new( + parts: element.parts, + quote: nil, + location: element.location + ) + end end - end - - visit_array( - ArrayLiteral.new( - lbracket: node.beginning, - contents: Args.new(parts: parts, location: node.location), - location: node.location - ) - ) - end - # Visit a TopConstField node. - def visit_top_const_field(node) - s( - :casgn, - [ - s(:cbase, [], smap(srange_length(node.start_char, 2))), - node.constant.value.to_sym - ], - smap_constant( - srange_length(node.start_char, 2), - srange_node(node.constant), - srange_node(node) + visit_array( + ArrayLiteral.new( + lbracket: node.beginning, + contents: Args.new(parts: parts, location: node.location), + location: node.location + ) ) - ) - end + end - # Visit a TopConstRef node. - def visit_top_const_ref(node) - s( - :const, - [ - s(:cbase, [], smap(srange_length(node.start_char, 2))), - node.constant.value.to_sym - ], - smap_constant( - srange_length(node.start_char, 2), - srange_node(node.constant), - srange_node(node) + # Visit a TopConstField node. + def visit_top_const_field(node) + s( + :casgn, + [ + s(:cbase, [], smap(srange_length(node.start_char, 2))), + node.constant.value.to_sym + ], + smap_constant( + srange_length(node.start_char, 2), + srange_node(node.constant), + srange_node(node) + ) ) - ) - end - - # Visit a TStringContent node. - def visit_tstring_content(node) - dumped = node.value.gsub(/([^[:ascii:]])/) { $1.dump[1...-1] } - - s( - :str, - ["\"#{dumped}\"".undump], - smap_collection_bare(srange_node(node)) - ) - end + end - # Visit a Unary node. - def visit_unary(node) - # Special handling here for flipflops - if node.statement.is_a?(Paren) && - node.statement.contents.is_a?(Statements) && - node.statement.contents.body.length == 1 && - (range = node.statement.contents.body.first).is_a?(RangeNode) && - node.operator == "!" - type = range.operator.value == ".." ? :iflipflop : :eflipflop - return( - s( - :send, - [s(:begin, [s(type, visit(range).children, nil)], nil), :!], - nil + # Visit a TopConstRef node. + def visit_top_const_ref(node) + s( + :const, + [ + s(:cbase, [], smap(srange_length(node.start_char, 2))), + node.constant.value.to_sym + ], + smap_constant( + srange_length(node.start_char, 2), + srange_node(node.constant), + srange_node(node) ) ) end - visit(canonical_unary(node)) - end + # Visit a TStringContent node. + def visit_tstring_content(node) + dumped = node.value.gsub(/([^[:ascii:]])/) { $1.dump[1...-1] } - # Visit an Undef node. - def visit_undef(node) - s( - :undef, - visit_all(node.symbols), - smap_keyword_bare( - srange_length(node.start_char, 5), - srange_node(node) + s( + :str, + ["\"#{dumped}\"".undump], + smap_collection_bare(srange_node(node)) ) - ) - end + end - # Visit an UnlessNode node. - def visit_unless(node) - predicate = - case node.predicate - when RegexpLiteral - s(:match_current_line, [visit(node.predicate)], nil) - when Unary - if node.predicate.operator.value == "!" && - node.predicate.statement.is_a?(RegexpLiteral) + # Visit a Unary node. + def visit_unary(node) + # Special handling here for flipflops + if node.statement.is_a?(Paren) && + node.statement.contents.is_a?(Statements) && + node.statement.contents.body.length == 1 && + (range = node.statement.contents.body.first).is_a?(RangeNode) && + node.operator == "!" + type = range.operator.value == ".." ? :iflipflop : :eflipflop + return( s( :send, - [s(:match_current_line, [visit(node.predicate.statement)]), :!], + [s(:begin, [s(type, visit(range).children, nil)], nil), :!], nil ) - else - visit(node.predicate) - end - else - visit(node.predicate) - end - - s( - :if, - [predicate, visit(node.consequent), visit(node.statements)], - if node.modifier? - smap_keyword_bare( - srange_find_between(node.statements, node.predicate, "unless"), - srange_node(node) - ) - else - smap_condition( - srange_length(node.start_char, 6), - srange_search_between(node.predicate, node.statements, "then"), - nil, - srange_length(node.end_char, -3), - srange_node(node) ) end - ) - end - # Visit an UntilNode node. - def visit_until(node) - s( - loop_post?(node) ? :until_post : :until, - [visit(node.predicate), visit(node.statements)], - if node.modifier? + visit(canonical_unary(node)) + end + + # Visit an Undef node. + def visit_undef(node) + s( + :undef, + visit_all(node.symbols), smap_keyword_bare( - srange_find_between(node.statements, node.predicate, "until"), - srange_node(node) - ) - else - smap_keyword( srange_length(node.start_char, 5), - srange_search_between(node.predicate, node.statements, "do") || - srange_search_between(node.predicate, node.statements, ";"), - srange_length(node.end_char, -3), srange_node(node) ) - end - ) - end + ) + end - # Visit a VarField node. - def visit_var_field(node) - name = node.value.value.to_sym - match_var = - [stack[-3], stack[-2]].any? do |parent| - case parent - when AryPtn, FndPtn, HshPtn, In, RAssign - true - when Binary - parent.operator == :"=>" + # Visit an UnlessNode node. + def visit_unless(node) + predicate = + case node.predicate + when RegexpLiteral + s(:match_current_line, [visit(node.predicate)], nil) + when Unary + if node.predicate.operator.value == "!" && + node.predicate.statement.is_a?(RegexpLiteral) + s( + :send, + [ + s(:match_current_line, [visit(node.predicate.statement)]), + :! + ], + nil + ) + else + visit(node.predicate) + end else - false + visit(node.predicate) end - end - if match_var s( - :match_var, - [name], - smap_variable(srange_node(node.value), srange_node(node.value)) + :if, + [predicate, visit(node.consequent), visit(node.statements)], + if node.modifier? + smap_keyword_bare( + srange_find_between(node.statements, node.predicate, "unless"), + srange_node(node) + ) + else + smap_condition( + srange_length(node.start_char, 6), + srange_search_between(node.predicate, node.statements, "then"), + nil, + srange_length(node.end_char, -3), + srange_node(node) + ) + end ) - elsif node.value.is_a?(Const) + end + + # Visit an UntilNode node. + def visit_until(node) s( - :casgn, - [nil, name], - smap_constant(nil, srange_node(node.value), srange_node(node)) + loop_post?(node) ? :until_post : :until, + [visit(node.predicate), visit(node.statements)], + if node.modifier? + smap_keyword_bare( + srange_find_between(node.statements, node.predicate, "until"), + srange_node(node) + ) + else + smap_keyword( + srange_length(node.start_char, 5), + srange_search_between(node.predicate, node.statements, "do") || + srange_search_between(node.predicate, node.statements, ";"), + srange_length(node.end_char, -3), + srange_node(node) + ) + end ) - else - location = smap_variable(srange_node(node), srange_node(node)) + end - case node.value - when CVar - s(:cvasgn, [name], location) - when GVar - s(:gvasgn, [name], location) - when Ident - s(:lvasgn, [name], location) - when IVar - s(:ivasgn, [name], location) - when VarRef - s(:lvasgn, [name], location) + # Visit a VarField node. + def visit_var_field(node) + name = node.value.value.to_sym + match_var = + [stack[-3], stack[-2]].any? do |parent| + case parent + when AryPtn, FndPtn, HshPtn, In, RAssign + true + when Binary + parent.operator == :"=>" + else + false + end + end + + if match_var + s( + :match_var, + [name], + smap_variable(srange_node(node.value), srange_node(node.value)) + ) + elsif node.value.is_a?(Const) + s( + :casgn, + [nil, name], + smap_constant(nil, srange_node(node.value), srange_node(node)) + ) else - s(:match_rest, [], nil) + location = smap_variable(srange_node(node), srange_node(node)) + + case node.value + when CVar + s(:cvasgn, [name], location) + when GVar + s(:gvasgn, [name], location) + when Ident + s(:lvasgn, [name], location) + when IVar + s(:ivasgn, [name], location) + when VarRef + s(:lvasgn, [name], location) + else + s(:match_rest, [], nil) + end end end - end - # Visit a VarRef node. - def visit_var_ref(node) - visit(node.value) - end + # Visit a VarRef node. + def visit_var_ref(node) + visit(node.value) + end - # Visit a VCall node. - def visit_vcall(node) - visit_command_call( - CommandCall.new( - receiver: nil, - operator: nil, - message: node.value, - arguments: nil, - block: nil, - location: node.location + # Visit a VCall node. + def visit_vcall(node) + visit_command_call( + CommandCall.new( + receiver: nil, + operator: nil, + message: node.value, + arguments: nil, + block: nil, + location: node.location + ) ) - ) - end - - # Visit a When node. - def visit_when(node) - keyword = srange_length(node.start_char, 4) - begin_token = - if buffer.source[node.statements.start_char] == ";" - srange_length(node.statements.start_char, 1) - end + end - end_char = - if node.statements.body.empty? - node.statements.end_char - else - node.statements.body.last.end_char - end + # Visit a When node. + def visit_when(node) + keyword = srange_length(node.start_char, 4) + begin_token = + if buffer.source[node.statements.start_char] == ";" + srange_length(node.statements.start_char, 1) + end - s( - :when, - visit_all(node.arguments.parts) + [visit(node.statements)], - smap_keyword( - keyword, - begin_token, - nil, - srange(keyword.begin_pos, end_char) - ) - ) - end + end_char = + if node.statements.body.empty? + node.statements.end_char + else + node.statements.body.last.end_char + end - # Visit a WhileNode node. - def visit_while(node) - s( - loop_post?(node) ? :while_post : :while, - [visit(node.predicate), visit(node.statements)], - if node.modifier? - smap_keyword_bare( - srange_find_between(node.statements, node.predicate, "while"), - srange_node(node) - ) - else + s( + :when, + visit_all(node.arguments.parts) + [visit(node.statements)], smap_keyword( - srange_length(node.start_char, 5), - srange_search_between(node.predicate, node.statements, "do") || - srange_search_between(node.predicate, node.statements, ";"), - srange_length(node.end_char, -3), - srange_node(node) + keyword, + begin_token, + nil, + srange(keyword.begin_pos, end_char) ) - end - ) - end - - # Visit a Word node. - def visit_word(node) - visit_string_literal( - StringLiteral.new( - parts: node.parts, - quote: nil, - location: node.location ) - ) - end + end - # Visit a Words node. - def visit_words(node) - visit_array( - ArrayLiteral.new( - lbracket: node.beginning, - contents: Args.new(parts: node.elements, location: node.location), - location: node.location + # Visit a WhileNode node. + def visit_while(node) + s( + loop_post?(node) ? :while_post : :while, + [visit(node.predicate), visit(node.statements)], + if node.modifier? + smap_keyword_bare( + srange_find_between(node.statements, node.predicate, "while"), + srange_node(node) + ) + else + smap_keyword( + srange_length(node.start_char, 5), + srange_search_between(node.predicate, node.statements, "do") || + srange_search_between(node.predicate, node.statements, ";"), + srange_length(node.end_char, -3), + srange_node(node) + ) + end ) - ) - end + end - # Visit an XStringLiteral node. - def visit_xstring_literal(node) - s( - :xstr, - visit_all(node.parts), - smap_collection( - srange_length( - node.start_char, - buffer.source[node.start_char] == "%" ? 3 : 1 - ), - srange_length(node.end_char, -1), - srange_node(node) + # Visit a Word node. + def visit_word(node) + visit_string_literal( + StringLiteral.new( + parts: node.parts, + quote: nil, + location: node.location + ) ) - ) - end + end - def visit_yield(node) - case node.arguments - when nil - s( - :yield, - [], - smap_keyword_bare( - srange_length(node.start_char, 5), - srange_node(node) + # Visit a Words node. + def visit_words(node) + visit_array( + ArrayLiteral.new( + lbracket: node.beginning, + contents: Args.new(parts: node.elements, location: node.location), + location: node.location ) ) - when Args + end + + # Visit an XStringLiteral node. + def visit_xstring_literal(node) s( - :yield, - visit_all(node.arguments.parts), - smap_keyword_bare( - srange_length(node.start_char, 5), + :xstr, + visit_all(node.parts), + smap_collection( + srange_length( + node.start_char, + buffer.source[node.start_char] == "%" ? 3 : 1 + ), + srange_length(node.end_char, -1), srange_node(node) ) ) - else + end + + def visit_yield(node) + case node.arguments + when nil + s( + :yield, + [], + smap_keyword_bare( + srange_length(node.start_char, 5), + srange_node(node) + ) + ) + when Args + s( + :yield, + visit_all(node.arguments.parts), + smap_keyword_bare( + srange_length(node.start_char, 5), + srange_node(node) + ) + ) + else + s( + :yield, + visit_all(node.arguments.contents.parts), + smap_keyword( + srange_length(node.start_char, 5), + srange_length(node.arguments.start_char, 1), + srange_length(node.end_char, -1), + srange_node(node) + ) + ) + end + end + + # Visit a ZSuper node. + def visit_zsuper(node) s( - :yield, - visit_all(node.arguments.contents.parts), - smap_keyword( + :zsuper, + [], + smap_keyword_bare( srange_length(node.start_char, 5), - srange_length(node.arguments.start_char, 1), - srange_length(node.end_char, -1), srange_node(node) ) ) end end - # Visit a ZSuper node. - def visit_zsuper(node) - s( - :zsuper, - [], - smap_keyword_bare( - srange_length(node.start_char, 5), - srange_node(node) - ) - ) - end - private def block_children(node) diff --git a/lib/syntax_tree/with_environment.rb b/lib/syntax_tree/with_environment.rb index 13f5e080..da300dc0 100644 --- a/lib/syntax_tree/with_environment.rb +++ b/lib/syntax_tree/with_environment.rb @@ -121,9 +121,9 @@ def visit_module(node) with_new_environment { super } end - # When we find a method invocation with a block, only the code that happens - # inside of the block needs a fresh environment. The method invocation - # itself happens in the same environment. + # When we find a method invocation with a block, only the code that + # happens inside of the block needs a fresh environment. The method + # invocation itself happens in the same environment. def visit_method_add_block(node) visit(node.call) with_new_environment { visit(node.block) } diff --git a/lib/syntax_tree/yarv/compiler.rb b/lib/syntax_tree/yarv/compiler.rb index a8044faf..bd20bc19 100644 --- a/lib/syntax_tree/yarv/compiler.rb +++ b/lib/syntax_tree/yarv/compiler.rb @@ -124,76 +124,122 @@ def self.compile(node) rescue CompilationError end - def visit_array(node) - node.contents ? visit_all(node.contents.parts) : [] - end + visit_methods do + def visit_array(node) + node.contents ? visit_all(node.contents.parts) : [] + end - def visit_bare_assoc_hash(node) - node.assocs.to_h do |assoc| - # We can only convert regular key-value pairs. A double splat ** - # operator means it has to be converted at run-time. - raise CompilationError unless assoc.is_a?(Assoc) - [visit(assoc.key), visit(assoc.value)] + def visit_bare_assoc_hash(node) + node.assocs.to_h do |assoc| + # We can only convert regular key-value pairs. A double splat ** + # operator means it has to be converted at run-time. + raise CompilationError unless assoc.is_a?(Assoc) + [visit(assoc.key), visit(assoc.value)] + end end - end - def visit_float(node) - node.value.to_f - end + def visit_float(node) + node.value.to_f + end - alias visit_hash visit_bare_assoc_hash + alias visit_hash visit_bare_assoc_hash - def visit_imaginary(node) - node.value.to_c - end + def visit_imaginary(node) + node.value.to_c + end - def visit_int(node) - case (value = node.value) - when /^0b/ - value[2..].to_i(2) - when /^0o/ - value[2..].to_i(8) - when /^0d/ - value[2..].to_i - when /^0x/ - value[2..].to_i(16) - else - value.to_i + def visit_int(node) + case (value = node.value) + when /^0b/ + value[2..].to_i(2) + when /^0o/ + value[2..].to_i(8) + when /^0d/ + value[2..].to_i + when /^0x/ + value[2..].to_i(16) + else + value.to_i + end end - end - def visit_label(node) - node.value.chomp(":").to_sym - end + def visit_label(node) + node.value.chomp(":").to_sym + end - def visit_mrhs(node) - visit_all(node.parts) - end + def visit_mrhs(node) + visit_all(node.parts) + end - def visit_qsymbols(node) - node.elements.map { |element| visit(element).to_sym } - end + def visit_qsymbols(node) + node.elements.map { |element| visit(element).to_sym } + end - def visit_qwords(node) - visit_all(node.elements) - end + def visit_qwords(node) + visit_all(node.elements) + end - def visit_range(node) - left, right = [visit(node.left), visit(node.right)] - node.operator.value === ".." ? left..right : left...right - end + def visit_range(node) + left, right = [visit(node.left), visit(node.right)] + node.operator.value === ".." ? left..right : left...right + end - def visit_rational(node) - node.value.to_r - end + def visit_rational(node) + node.value.to_r + end - def visit_regexp_literal(node) - if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) - Regexp.new(node.parts.first.value, visit_regexp_literal_flags(node)) - else - # Any interpolation of expressions or variables will result in the - # regular expression being constructed at run-time. - raise CompilationError + def visit_regexp_literal(node) + if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) + Regexp.new( + node.parts.first.value, + visit_regexp_literal_flags(node) + ) + else + # Any interpolation of expressions or variables will result in the + # regular expression being constructed at run-time. + raise CompilationError + end + end + + def visit_symbol_literal(node) + node.value.value.to_sym + end + + def visit_symbols(node) + node.elements.map { |element| visit(element).to_sym } + end + + def visit_tstring_content(node) + node.value + end + + def visit_var_ref(node) + raise CompilationError unless node.value.is_a?(Kw) + + case node.value.value + when "nil" + nil + when "true" + true + when "false" + false + else + raise CompilationError + end + end + + def visit_word(node) + if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) + node.parts.first.value + else + # Any interpolation of expressions or variables will result in the + # string being constructed at run-time. + raise CompilationError + end + end + + def visit_words(node) + visit_all(node.elements) end end @@ -219,47 +265,6 @@ def visit_regexp_literal_flags(node) end end - def visit_symbol_literal(node) - node.value.value.to_sym - end - - def visit_symbols(node) - node.elements.map { |element| visit(element).to_sym } - end - - def visit_tstring_content(node) - node.value - end - - def visit_var_ref(node) - raise CompilationError unless node.value.is_a?(Kw) - - case node.value.value - when "nil" - nil - when "true" - true - when "false" - false - else - raise CompilationError - end - end - - def visit_word(node) - if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) - node.parts.first.value - else - # Any interpolation of expressions or variables will result in the - # string being constructed at run-time. - raise CompilationError - end - end - - def visit_words(node) - visit_all(node.elements) - end - def visit_unsupported(_node) raise CompilationError end diff --git a/test/visitor_test.rb b/test/visitor_test.rb index 86ff1b01..d9637df0 100644 --- a/test/visitor_test.rb +++ b/test/visitor_test.rb @@ -30,13 +30,15 @@ def initialize @visited_nodes = [] end - visit_method def visit_class(node) - @visited_nodes << node.constant.constant.value - super - end + visit_methods do + def visit_class(node) + @visited_nodes << node.constant.constant.value + super + end - visit_method def visit_def(node) - @visited_nodes << node.name.value + def visit_def(node) + @visited_nodes << node.name.value + end end end diff --git a/test/visitor_with_environment_test.rb b/test/visitor_with_environment_test.rb index cc4007fe..278ae361 100644 --- a/test/visitor_with_environment_test.rb +++ b/test/visitor_with_environment_test.rb @@ -14,26 +14,28 @@ def initialize @arguments = {} end - def visit_ident(node) - local = current_environment.find_local(node.value) - return unless local - - value = node.value.delete_suffix(":") - - case local.type - when :argument - @arguments[value] = local - when :variable - @variables[value] = local + visit_methods do + def visit_ident(node) + local = current_environment.find_local(node.value) + return unless local + + value = node.value.delete_suffix(":") + + case local.type + when :argument + @arguments[value] = local + when :variable + @variables[value] = local + end end - end - def visit_label(node) - value = node.value.delete_suffix(":") - local = current_environment.find_local(value) - return unless local + def visit_label(node) + value = node.value.delete_suffix(":") + local = current_environment.find_local(value) + return unless local - @arguments[value] = node if local.type == :argument + @arguments[value] = node if local.type == :argument + end end end @@ -625,13 +627,15 @@ def initialize @locals = [] end - def visit_assign(node) - level = 0 - environment = current_environment - level += 1 until (environment = environment.parent).nil? + visit_methods do + def visit_assign(node) + level = 0 + environment = current_environment + level += 1 until (environment = environment.parent).nil? - locals << [node.target.value.value, level] - super + locals << [node.target.value.value, level] + super + end end end From 174cc6bae01dc6825858906fa46a9f3213608c24 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 10 Feb 2023 12:50:59 -0500 Subject: [PATCH 393/536] Make environment break at boundaries --- lib/syntax_tree/with_environment.rb | 56 ++- test/visitor_with_environment_test.rb | 663 -------------------------- test/with_environment_test.rb | 457 ++++++++++++++++++ 3 files changed, 499 insertions(+), 677 deletions(-) delete mode 100644 test/visitor_with_environment_test.rb create mode 100644 test/with_environment_test.rb diff --git a/lib/syntax_tree/with_environment.rb b/lib/syntax_tree/with_environment.rb index da300dc0..3a6f04b9 100644 --- a/lib/syntax_tree/with_environment.rb +++ b/lib/syntax_tree/with_environment.rb @@ -55,14 +55,18 @@ def add_usage(location) end end - # [Array[Local]] The local variables and arguments defined in this + # [Integer] a unique identifier for this environment + attr_reader :id + + # [Hash[String, Local]] The local variables and arguments defined in this # environment attr_reader :locals # [Environment | nil] The parent environment attr_reader :parent - def initialize(parent = nil) + def initialize(id, parent = nil) + @id = id @locals = {} @parent = parent end @@ -74,8 +78,14 @@ def initialize(parent = nil) def add_local_definition(identifier, type) name = identifier.value.delete_suffix(":") - @locals[name] ||= Local.new(type) - @locals[name].add_definition(identifier.location) + local = + if type == :argument + locals[name] ||= Local.new(type) + else + resolve_local(name, type) + end + + local.add_definition(identifier.location) end # Adding a local usage will either insert a new entry in the locals @@ -84,28 +94,42 @@ def add_local_definition(identifier, type) # registered. def add_local_usage(identifier, type) name = identifier.value.delete_suffix(":") - - @locals[name] ||= Local.new(type) - @locals[name].add_usage(identifier.location) + resolve_local(name, type).add_usage(identifier.location) end # Try to find the local given its name in this environment or any of its # parents. def find_local(name) - local = @locals[name] - return local unless local.nil? + locals[name] || parent&.find_local(name) + end - @parent&.find_local(name) + private + + def resolve_local(name, type) + local = find_local(name) + + unless local + local = Local.new(type) + locals[name] = local + end + + local end end + def initialize(*args, **kwargs, &block) + super + @environment_id = 0 + end + def current_environment - @current_environment ||= Environment.new + @current_environment ||= Environment.new(next_environment_id) end - def with_new_environment + def with_new_environment(parent_environment = nil) previous_environment = @current_environment - @current_environment = Environment.new(previous_environment) + @current_environment = + Environment.new(next_environment_id, parent_environment) yield ensure @current_environment = previous_environment @@ -126,7 +150,7 @@ def visit_module(node) # invocation itself happens in the same environment. def visit_method_add_block(node) visit(node.call) - with_new_environment { visit(node.block) } + with_new_environment(current_environment) { visit(node.block) } end def visit_def(node) @@ -213,5 +237,9 @@ def add_argument_definitions(list) end end end + + def next_environment_id + @environment_id += 1 + end end end diff --git a/test/visitor_with_environment_test.rb b/test/visitor_with_environment_test.rb deleted file mode 100644 index 278ae361..00000000 --- a/test/visitor_with_environment_test.rb +++ /dev/null @@ -1,663 +0,0 @@ -# frozen_string_literal: true - -require_relative "test_helper" - -module SyntaxTree - class VisitorWithEnvironmentTest < Minitest::Test - class Collector < Visitor - include WithEnvironment - - attr_reader :variables, :arguments - - def initialize - @variables = {} - @arguments = {} - end - - visit_methods do - def visit_ident(node) - local = current_environment.find_local(node.value) - return unless local - - value = node.value.delete_suffix(":") - - case local.type - when :argument - @arguments[value] = local - when :variable - @variables[value] = local - end - end - - def visit_label(node) - value = node.value.delete_suffix(":") - local = current_environment.find_local(value) - return unless local - - @arguments[value] = node if local.type == :argument - end - end - end - - def test_collecting_simple_variables - tree = SyntaxTree.parse(<<~RUBY) - def foo - a = 1 - a - end - RUBY - - visitor = Collector.new - visitor.visit(tree) - - assert_equal(1, visitor.variables.length) - - variable = visitor.variables["a"] - assert_equal(1, variable.definitions.length) - assert_equal(1, variable.usages.length) - - assert_equal(2, variable.definitions[0].start_line) - assert_equal(3, variable.usages[0].start_line) - end - - def test_collecting_aref_variables - tree = SyntaxTree.parse(<<~RUBY) - def foo - a = [] - a[1] - end - RUBY - - visitor = Collector.new - visitor.visit(tree) - - assert_equal(1, visitor.variables.length) - - variable = visitor.variables["a"] - assert_equal(1, variable.definitions.length) - assert_equal(1, variable.usages.length) - - assert_equal(2, variable.definitions[0].start_line) - assert_equal(3, variable.usages[0].start_line) - end - - def test_collecting_multi_assign_variables - tree = SyntaxTree.parse(<<~RUBY) - def foo - a, b = [1, 2] - puts a - puts b - end - RUBY - - visitor = Collector.new - visitor.visit(tree) - - assert_equal(2, visitor.variables.length) - - variable_a = visitor.variables["a"] - assert_equal(1, variable_a.definitions.length) - assert_equal(1, variable_a.usages.length) - - assert_equal(2, variable_a.definitions[0].start_line) - assert_equal(3, variable_a.usages[0].start_line) - - variable_b = visitor.variables["b"] - assert_equal(1, variable_b.definitions.length) - assert_equal(1, variable_b.usages.length) - - assert_equal(2, variable_b.definitions[0].start_line) - assert_equal(4, variable_b.usages[0].start_line) - end - - def test_collecting_pattern_matching_variables - tree = SyntaxTree.parse(<<~RUBY) - def foo - case [1, 2] - in Integer => a, Integer - puts a - end - end - RUBY - - visitor = Collector.new - visitor.visit(tree) - - # There are two occurrences, one on line 3 for pinning and one on line 4 - # for reference - assert_equal(1, visitor.variables.length) - - variable = visitor.variables["a"] - - # Assignment a - assert_equal(3, variable.definitions[0].start_line) - assert_equal(4, variable.usages[0].start_line) - end - - def test_collecting_pinned_variables - tree = SyntaxTree.parse(<<~RUBY) - def foo - a = 18 - case [1, 2] - in ^a, *rest - puts a - puts rest - end - end - RUBY - - visitor = Collector.new - visitor.visit(tree) - - assert_equal(2, visitor.variables.length) - - variable_a = visitor.variables["a"] - assert_equal(2, variable_a.definitions.length) - assert_equal(1, variable_a.usages.length) - - assert_equal(2, variable_a.definitions[0].start_line) - assert_equal(4, variable_a.definitions[1].start_line) - assert_equal(5, variable_a.usages[0].start_line) - - variable_rest = visitor.variables["rest"] - assert_equal(1, variable_rest.definitions.length) - assert_equal(4, variable_rest.definitions[0].start_line) - - # Rest is considered a vcall by the parser instead of a var_ref - # assert_equal(1, variable_rest.usages.length) - # assert_equal(6, variable_rest.usages[0].start_line) - end - - if RUBY_VERSION >= "3.1" - def test_collecting_one_line_pattern_matching_variables - tree = SyntaxTree.parse(<<~RUBY) - def foo - [1] => a - puts a - end - RUBY - - visitor = Collector.new - visitor.visit(tree) - - assert_equal(1, visitor.variables.length) - - variable = visitor.variables["a"] - assert_equal(1, variable.definitions.length) - assert_equal(1, variable.usages.length) - - assert_equal(2, variable.definitions[0].start_line) - assert_equal(3, variable.usages[0].start_line) - end - - def test_collecting_endless_method_arguments - tree = SyntaxTree.parse(<<~RUBY) - def foo(a) = puts a - RUBY - - visitor = Collector.new - visitor.visit(tree) - - assert_equal(1, visitor.arguments.length) - - argument = visitor.arguments["a"] - assert_equal(1, argument.definitions.length) - assert_equal(1, argument.usages.length) - - assert_equal(1, argument.definitions[0].start_line) - assert_equal(1, argument.usages[0].start_line) - end - end - - def test_collecting_method_arguments - tree = SyntaxTree.parse(<<~RUBY) - def foo(a) - puts a - end - RUBY - - visitor = Collector.new - visitor.visit(tree) - - assert_equal(1, visitor.arguments.length) - - argument = visitor.arguments["a"] - assert_equal(1, argument.definitions.length) - assert_equal(1, argument.usages.length) - - assert_equal(1, argument.definitions[0].start_line) - assert_equal(2, argument.usages[0].start_line) - end - - def test_collecting_singleton_method_arguments - tree = SyntaxTree.parse(<<~RUBY) - def self.foo(a) - puts a - end - RUBY - - visitor = Collector.new - visitor.visit(tree) - - assert_equal(1, visitor.arguments.length) - - argument = visitor.arguments["a"] - assert_equal(1, argument.definitions.length) - assert_equal(1, argument.usages.length) - - assert_equal(1, argument.definitions[0].start_line) - assert_equal(2, argument.usages[0].start_line) - end - - def test_collecting_method_arguments_all_types - tree = SyntaxTree.parse(<<~RUBY) - def foo(a, b = 1, *c, d, e: 1, **f, &block) - puts a - puts b - puts c - puts d - puts e - puts f - block.call - end - RUBY - - visitor = Collector.new - visitor.visit(tree) - - assert_equal(7, visitor.arguments.length) - - argument_a = visitor.arguments["a"] - assert_equal(1, argument_a.definitions.length) - assert_equal(1, argument_a.usages.length) - assert_equal(1, argument_a.definitions[0].start_line) - assert_equal(2, argument_a.usages[0].start_line) - - argument_b = visitor.arguments["b"] - assert_equal(1, argument_b.definitions.length) - assert_equal(1, argument_b.usages.length) - assert_equal(1, argument_b.definitions[0].start_line) - assert_equal(3, argument_b.usages[0].start_line) - - argument_c = visitor.arguments["c"] - assert_equal(1, argument_c.definitions.length) - assert_equal(1, argument_c.usages.length) - assert_equal(1, argument_c.definitions[0].start_line) - assert_equal(4, argument_c.usages[0].start_line) - - argument_d = visitor.arguments["d"] - assert_equal(1, argument_d.definitions.length) - assert_equal(1, argument_d.usages.length) - assert_equal(1, argument_d.definitions[0].start_line) - assert_equal(5, argument_d.usages[0].start_line) - - argument_e = visitor.arguments["e"] - assert_equal(1, argument_e.definitions.length) - assert_equal(1, argument_e.usages.length) - assert_equal(1, argument_e.definitions[0].start_line) - assert_equal(6, argument_e.usages[0].start_line) - - argument_f = visitor.arguments["f"] - assert_equal(1, argument_f.definitions.length) - assert_equal(1, argument_f.usages.length) - assert_equal(1, argument_f.definitions[0].start_line) - assert_equal(7, argument_f.usages[0].start_line) - - argument_block = visitor.arguments["block"] - assert_equal(1, argument_block.definitions.length) - assert_equal(1, argument_block.usages.length) - assert_equal(1, argument_block.definitions[0].start_line) - assert_equal(8, argument_block.usages[0].start_line) - end - - def test_collecting_block_arguments - tree = SyntaxTree.parse(<<~RUBY) - def foo - [].each do |i| - puts i - end - end - RUBY - - visitor = Collector.new - visitor.visit(tree) - - assert_equal(1, visitor.arguments.length) - - argument = visitor.arguments["i"] - assert_equal(1, argument.definitions.length) - assert_equal(1, argument.usages.length) - assert_equal(2, argument.definitions[0].start_line) - assert_equal(3, argument.usages[0].start_line) - end - - def test_collecting_one_line_block_arguments - tree = SyntaxTree.parse(<<~RUBY) - def foo - [].each { |i| puts i } - end - RUBY - - visitor = Collector.new - visitor.visit(tree) - - assert_equal(1, visitor.arguments.length) - - argument = visitor.arguments["i"] - assert_equal(1, argument.definitions.length) - assert_equal(1, argument.usages.length) - assert_equal(2, argument.definitions[0].start_line) - assert_equal(2, argument.usages[0].start_line) - end - - def test_collecting_shadowed_block_arguments - tree = SyntaxTree.parse(<<~RUBY) - def foo - i = "something" - - [].each do |i| - puts i - end - - i - end - RUBY - - visitor = Collector.new - visitor.visit(tree) - - assert_equal(1, visitor.arguments.length) - assert_equal(1, visitor.variables.length) - - argument = visitor.arguments["i"] - assert_equal(1, argument.definitions.length) - assert_equal(1, argument.usages.length) - assert_equal(4, argument.definitions[0].start_line) - assert_equal(5, argument.usages[0].start_line) - - variable = visitor.variables["i"] - assert_equal(1, variable.definitions.length) - assert_equal(1, variable.usages.length) - assert_equal(2, variable.definitions[0].start_line) - assert_equal(8, variable.usages[0].start_line) - end - - def test_collecting_shadowed_local_variables - tree = SyntaxTree.parse(<<~RUBY) - def foo(a) - puts a - a = 123 - a - end - RUBY - - visitor = Collector.new - visitor.visit(tree) - - # All occurrences are considered arguments, despite overriding the - # argument value - assert_equal(1, visitor.arguments.length) - assert_equal(0, visitor.variables.length) - - argument = visitor.arguments["a"] - assert_equal(2, argument.definitions.length) - assert_equal(2, argument.usages.length) - - assert_equal(1, argument.definitions[0].start_line) - assert_equal(3, argument.definitions[1].start_line) - assert_equal(2, argument.usages[0].start_line) - assert_equal(4, argument.usages[1].start_line) - end - - def test_variables_in_the_top_level - tree = SyntaxTree.parse(<<~RUBY) - a = 123 - a - RUBY - - visitor = Collector.new - visitor.visit(tree) - - assert_equal(0, visitor.arguments.length) - assert_equal(1, visitor.variables.length) - - variable = visitor.variables["a"] - assert_equal(1, variable.definitions.length) - assert_equal(1, variable.usages.length) - - assert_equal(1, variable.definitions[0].start_line) - assert_equal(2, variable.usages[0].start_line) - end - - def test_aref_field - tree = SyntaxTree.parse(<<~RUBY) - object = {} - object["name"] = "something" - RUBY - - visitor = Collector.new - visitor.visit(tree) - - assert_equal(0, visitor.arguments.length) - assert_equal(1, visitor.variables.length) - - variable = visitor.variables["object"] - assert_equal(1, variable.definitions.length) - assert_equal(1, variable.usages.length) - - assert_equal(1, variable.definitions[0].start_line) - assert_equal(2, variable.usages[0].start_line) - end - - def test_aref_on_a_method_call - tree = SyntaxTree.parse(<<~RUBY) - object = MyObject.new - object.attributes["name"] = "something" - RUBY - - visitor = Collector.new - visitor.visit(tree) - - assert_equal(0, visitor.arguments.length) - assert_equal(1, visitor.variables.length) - - variable = visitor.variables["object"] - assert_equal(1, variable.definitions.length) - assert_equal(1, variable.usages.length) - - assert_equal(1, variable.definitions[0].start_line) - assert_equal(2, variable.usages[0].start_line) - end - - def test_aref_with_two_accesses - tree = SyntaxTree.parse(<<~RUBY) - object = MyObject.new - object["first"]["second"] ||= [] - RUBY - - visitor = Collector.new - visitor.visit(tree) - - assert_equal(0, visitor.arguments.length) - assert_equal(1, visitor.variables.length) - - variable = visitor.variables["object"] - assert_equal(1, variable.definitions.length) - assert_equal(1, variable.usages.length) - - assert_equal(1, variable.definitions[0].start_line) - assert_equal(2, variable.usages[0].start_line) - end - - def test_aref_on_a_method_call_with_arguments - tree = SyntaxTree.parse(<<~RUBY) - object = MyObject.new - object.instance_variable_get(:@attributes)[:something] = :other_thing - RUBY - - visitor = Collector.new - visitor.visit(tree) - - assert_equal(0, visitor.arguments.length) - assert_equal(1, visitor.variables.length) - - variable = visitor.variables["object"] - assert_equal(1, variable.definitions.length) - assert_equal(1, variable.usages.length) - - assert_equal(1, variable.definitions[0].start_line) - assert_equal(2, variable.usages[0].start_line) - end - - def test_double_aref_on_method_call - tree = SyntaxTree.parse(<<~RUBY) - object = MyObject.new - object["attributes"].find { |a| a["field"] == "expected" }["value"] = "changed" - RUBY - - visitor = Collector.new - visitor.visit(tree) - - assert_equal(1, visitor.arguments.length) - assert_equal(1, visitor.variables.length) - - variable = visitor.variables["object"] - assert_equal(1, variable.definitions.length) - assert_equal(1, variable.usages.length) - - assert_equal(1, variable.definitions[0].start_line) - assert_equal(2, variable.usages[0].start_line) - - argument = visitor.arguments["a"] - assert_equal(1, argument.definitions.length) - assert_equal(1, argument.usages.length) - - assert_equal(2, argument.definitions[0].start_line) - assert_equal(2, argument.usages[0].start_line) - end - - def test_nested_arguments - tree = SyntaxTree.parse(<<~RUBY) - [[1, [2, 3]]].each do |one, (two, three)| - one - two - three - end - RUBY - - visitor = Collector.new - visitor.visit(tree) - - assert_equal(3, visitor.arguments.length) - assert_equal(0, visitor.variables.length) - - argument = visitor.arguments["one"] - assert_equal(1, argument.definitions.length) - assert_equal(1, argument.usages.length) - - assert_equal(1, argument.definitions[0].start_line) - assert_equal(2, argument.usages[0].start_line) - - argument = visitor.arguments["two"] - assert_equal(1, argument.definitions.length) - assert_equal(1, argument.usages.length) - - assert_equal(1, argument.definitions[0].start_line) - assert_equal(3, argument.usages[0].start_line) - - argument = visitor.arguments["three"] - assert_equal(1, argument.definitions.length) - assert_equal(1, argument.usages.length) - - assert_equal(1, argument.definitions[0].start_line) - assert_equal(4, argument.usages[0].start_line) - end - - def test_double_nested_arguments - tree = SyntaxTree.parse(<<~RUBY) - [[1, [2, 3]]].each do |one, (two, (three, four))| - one - two - three - four - end - RUBY - - visitor = Collector.new - visitor.visit(tree) - - assert_equal(4, visitor.arguments.length) - assert_equal(0, visitor.variables.length) - - argument = visitor.arguments["one"] - assert_equal(1, argument.definitions.length) - assert_equal(1, argument.usages.length) - - assert_equal(1, argument.definitions[0].start_line) - assert_equal(2, argument.usages[0].start_line) - - argument = visitor.arguments["two"] - assert_equal(1, argument.definitions.length) - assert_equal(1, argument.usages.length) - - assert_equal(1, argument.definitions[0].start_line) - assert_equal(3, argument.usages[0].start_line) - - argument = visitor.arguments["three"] - assert_equal(1, argument.definitions.length) - assert_equal(1, argument.usages.length) - - assert_equal(1, argument.definitions[0].start_line) - assert_equal(4, argument.usages[0].start_line) - - argument = visitor.arguments["four"] - assert_equal(1, argument.definitions.length) - assert_equal(1, argument.usages.length) - - assert_equal(1, argument.definitions[0].start_line) - assert_equal(5, argument.usages[0].start_line) - end - - class Resolver < Visitor - include WithEnvironment - - attr_reader :locals - - def initialize - @locals = [] - end - - visit_methods do - def visit_assign(node) - level = 0 - environment = current_environment - level += 1 until (environment = environment.parent).nil? - - locals << [node.target.value.value, level] - super - end - end - end - - def test_class - source = <<~RUBY - module Level0 - level0 = 0 - - module Level1 - level1 = 1 - - class Level2 - level2 = 2 - end - end - end - RUBY - - visitor = Resolver.new - SyntaxTree.parse(source).accept(visitor) - - assert_equal [["level0", 0], ["level1", 1], ["level2", 2]], visitor.locals - end - end -end diff --git a/test/with_environment_test.rb b/test/with_environment_test.rb new file mode 100644 index 00000000..b6f79c14 --- /dev/null +++ b/test/with_environment_test.rb @@ -0,0 +1,457 @@ +# frozen_string_literal: true + +require_relative "test_helper" + +module SyntaxTree + class WithEnvironmentTest < Minitest::Test + class Collector < Visitor + prepend WithEnvironment + + attr_reader :arguments, :variables + + def initialize + @arguments = {} + @variables = {} + end + + def self.collect(source) + new.tap { SyntaxTree.parse(source).accept(_1) } + end + + visit_methods do + def visit_ident(node) + value = node.value.delete_suffix(":") + local = current_environment.find_local(node.value) + + case local&.type + when :argument + arguments[[current_environment.id, value]] = local + when :variable + variables[[current_environment.id, value]] = local + end + end + + def visit_label(node) + value = node.value.delete_suffix(":") + local = current_environment.find_local(value) + + if local&.type == :argument + arguments[[current_environment.id, value]] = node + end + end + end + end + + def test_collecting_simple_variables + collector = Collector.collect(<<~RUBY) + def foo + a = 1 + a + end + RUBY + + assert_equal(1, collector.variables.length) + assert_variable(collector, "a", definitions: [2], usages: [3]) + end + + def test_collecting_aref_variables + collector = Collector.collect(<<~RUBY) + def foo + a = [] + a[1] + end + RUBY + + assert_equal(1, collector.variables.length) + assert_variable(collector, "a", definitions: [2], usages: [3]) + end + + def test_collecting_multi_assign_variables + collector = Collector.collect(<<~RUBY) + def foo + a, b = [1, 2] + puts a + puts b + end + RUBY + + assert_equal(2, collector.variables.length) + assert_variable(collector, "a", definitions: [2], usages: [3]) + assert_variable(collector, "b", definitions: [2], usages: [4]) + end + + def test_collecting_pattern_matching_variables + collector = Collector.collect(<<~RUBY) + def foo + case [1, 2] + in Integer => a, Integer + puts a + end + end + RUBY + + # There are two occurrences, one on line 3 for pinning and one on line 4 + # for reference + assert_equal(1, collector.variables.length) + assert_variable(collector, "a", definitions: [3], usages: [4]) + end + + def test_collecting_pinned_variables + collector = Collector.collect(<<~RUBY) + def foo + a = 18 + case [1, 2] + in ^a, *rest + puts a + puts rest + end + end + RUBY + + assert_equal(2, collector.variables.length) + assert_variable(collector, "a", definitions: [2, 4], usages: [5]) + assert_variable(collector, "rest", definitions: [4]) + + # Rest is considered a vcall by the parser instead of a var_ref + # assert_equal(1, variable_rest.usages.length) + # assert_equal(6, variable_rest.usages[0].start_line) + end + + if RUBY_VERSION >= "3.1" + def test_collecting_one_line_pattern_matching_variables + collector = Collector.collect(<<~RUBY) + def foo + [1] => a + puts a + end + RUBY + + assert_equal(1, collector.variables.length) + assert_variable(collector, "a", definitions: [2], usages: [3]) + end + + def test_collecting_endless_method_arguments + collector = Collector.collect(<<~RUBY) + def foo(a) = puts a + RUBY + + assert_equal(1, collector.arguments.length) + assert_argument(collector, "a", definitions: [1], usages: [1]) + end + end + + def test_collecting_method_arguments + collector = Collector.collect(<<~RUBY) + def foo(a) + puts a + end + RUBY + + assert_equal(1, collector.arguments.length) + assert_argument(collector, "a", definitions: [1], usages: [2]) + end + + def test_collecting_singleton_method_arguments + collector = Collector.collect(<<~RUBY) + def self.foo(a) + puts a + end + RUBY + + assert_equal(1, collector.arguments.length) + assert_argument(collector, "a", definitions: [1], usages: [2]) + end + + def test_collecting_method_arguments_all_types + collector = Collector.collect(<<~RUBY) + def foo(a, b = 1, *c, d, e: 1, **f, &block) + puts a + puts b + puts c + puts d + puts e + puts f + block.call + end + RUBY + + assert_equal(7, collector.arguments.length) + assert_argument(collector, "a", definitions: [1], usages: [2]) + assert_argument(collector, "b", definitions: [1], usages: [3]) + assert_argument(collector, "c", definitions: [1], usages: [4]) + assert_argument(collector, "d", definitions: [1], usages: [5]) + assert_argument(collector, "e", definitions: [1], usages: [6]) + assert_argument(collector, "f", definitions: [1], usages: [7]) + assert_argument(collector, "block", definitions: [1], usages: [8]) + end + + def test_collecting_block_arguments + collector = Collector.collect(<<~RUBY) + def foo + [].each do |i| + puts i + end + end + RUBY + + assert_equal(1, collector.arguments.length) + assert_argument(collector, "i", definitions: [2], usages: [3]) + end + + def test_collecting_one_line_block_arguments + collector = Collector.collect(<<~RUBY) + def foo + [].each { |i| puts i } + end + RUBY + + assert_equal(1, collector.arguments.length) + assert_argument(collector, "i", definitions: [2], usages: [2]) + end + + def test_collecting_shadowed_block_arguments + collector = Collector.collect(<<~RUBY) + def foo + i = "something" + + [].each do |i| + puts i + end + + i + end + RUBY + + assert_equal(1, collector.arguments.length) + assert_argument(collector, "i", definitions: [4], usages: [5]) + + assert_equal(1, collector.variables.length) + assert_variable(collector, "i", definitions: [2], usages: [8]) + end + + def test_collecting_shadowed_local_variables + collector = Collector.collect(<<~RUBY) + def foo(a) + puts a + a = 123 + a + end + RUBY + + # All occurrences are considered arguments, despite overriding the + # argument value + assert_equal(1, collector.arguments.length) + assert_equal(0, collector.variables.length) + assert_argument(collector, "a", definitions: [1, 3], usages: [2, 4]) + end + + def test_variables_in_the_top_level + collector = Collector.collect(<<~RUBY) + a = 123 + a + RUBY + + assert_equal(0, collector.arguments.length) + assert_equal(1, collector.variables.length) + assert_variable(collector, "a", definitions: [1], usages: [2]) + end + + def test_aref_field + collector = Collector.collect(<<~RUBY) + object = {} + object["name"] = "something" + RUBY + + assert_equal(0, collector.arguments.length) + assert_equal(1, collector.variables.length) + assert_variable(collector, "object", definitions: [1], usages: [2]) + end + + def test_aref_on_a_method_call + collector = Collector.collect(<<~RUBY) + object = MyObject.new + object.attributes["name"] = "something" + RUBY + + assert_equal(0, collector.arguments.length) + assert_equal(1, collector.variables.length) + assert_variable(collector, "object", definitions: [1], usages: [2]) + end + + def test_aref_with_two_accesses + collector = Collector.collect(<<~RUBY) + object = MyObject.new + object["first"]["second"] ||= [] + RUBY + + assert_equal(0, collector.arguments.length) + assert_equal(1, collector.variables.length) + assert_variable(collector, "object", definitions: [1], usages: [2]) + end + + def test_aref_on_a_method_call_with_arguments + collector = Collector.collect(<<~RUBY) + object = MyObject.new + object.instance_variable_get(:@attributes)[:something] = :other_thing + RUBY + + assert_equal(0, collector.arguments.length) + assert_equal(1, collector.variables.length) + assert_variable(collector, "object", definitions: [1], usages: [2]) + end + + def test_double_aref_on_method_call + collector = Collector.collect(<<~RUBY) + object = MyObject.new + object["attributes"].find { |a| a["field"] == "expected" }["value"] = "changed" + RUBY + + assert_equal(1, collector.arguments.length) + assert_argument(collector, "a", definitions: [2], usages: [2]) + + assert_equal(1, collector.variables.length) + assert_variable(collector, "object", definitions: [1], usages: [2]) + end + + def test_nested_arguments + collector = Collector.collect(<<~RUBY) + [[1, [2, 3]]].each do |one, (two, three)| + one + two + three + end + RUBY + + assert_equal(3, collector.arguments.length) + assert_equal(0, collector.variables.length) + + assert_argument(collector, "one", definitions: [1], usages: [2]) + assert_argument(collector, "two", definitions: [1], usages: [3]) + assert_argument(collector, "three", definitions: [1], usages: [4]) + end + + def test_double_nested_arguments + collector = Collector.collect(<<~RUBY) + [[1, [2, 3]]].each do |one, (two, (three, four))| + one + two + three + four + end + RUBY + + assert_equal(4, collector.arguments.length) + assert_equal(0, collector.variables.length) + + assert_argument(collector, "one", definitions: [1], usages: [2]) + assert_argument(collector, "two", definitions: [1], usages: [3]) + assert_argument(collector, "three", definitions: [1], usages: [4]) + assert_argument(collector, "four", definitions: [1], usages: [5]) + end + + class Resolver < Visitor + prepend WithEnvironment + + attr_reader :locals + + def initialize + @locals = [] + end + + visit_methods do + def visit_assign(node) + super.tap do + level = 0 + name = node.target.value.value + + environment = current_environment + while !environment.locals.key?(name) && !environment.parent.nil? + level += 1 + environment = environment.parent + end + + locals << [name, level] + end + end + end + end + + def test_resolver + source = <<~RUBY + module Level0 + level0 = 0 + + class Level1 + level1 = 1 + + def level2 + level2 = 2 + + tap do |level3| + level2 = 2 + level3 = 3 + + tap do |level4| + level2 = 2 + level4 = 4 + end + end + end + end + end + RUBY + + resolver = Resolver.new + SyntaxTree.parse(source).accept(resolver) + + expected = [ + ["level0", 0], + ["level1", 0], + ["level2", 0], + ["level2", 1], + ["level3", 0], + ["level2", 2], + ["level4", 0] + ] + + assert_equal expected, resolver.locals + end + + private + + def assert_collected(field, name, definitions: [], usages: []) + keys = field.keys.select { |key| key[1] == name } + assert_equal(1, keys.length) + + variable = field[keys.first] + + assert_equal(definitions.length, variable.definitions.length) + definitions.each_with_index do |definition, index| + assert_equal(definition, variable.definitions[index].start_line) + end + + assert_equal(usages.length, variable.usages.length) + usages.each_with_index do |usage, index| + assert_equal(usage, variable.usages[index].start_line) + end + end + + def assert_argument(collector, name, definitions: [], usages: []) + assert_collected( + collector.arguments, + name, + definitions: definitions, + usages: usages + ) + end + + def assert_variable(collector, name, definitions: [], usages: []) + assert_collected( + collector.variables, + name, + definitions: definitions, + usages: usages + ) + end + end +end From 4a6fc77abd4c696b3d38498250ab37e571f27d9a Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 10 Feb 2023 13:00:40 -0500 Subject: [PATCH 394/536] WithEnvironment -> WithScope --- README.md | 12 +- lib/syntax_tree.rb | 2 +- .../{with_environment.rb => with_scope.rb} | 107 ++++++++---------- ...environment_test.rb => with_scope_test.rb} | 22 ++-- 4 files changed, 67 insertions(+), 76 deletions(-) rename lib/syntax_tree/{with_environment.rb => with_scope.rb} (65%) rename test/{with_environment_test.rb => with_scope_test.rb} (95%) diff --git a/README.md b/README.md index 5f447ad8..500d5fad 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ It is built with only standard library dependencies. It additionally ships with - [visit_methods](#visit_methods) - [BasicVisitor](#basicvisitor) - [MutationVisitor](#mutationvisitor) - - [WithEnvironment](#withenvironment) + - [WithScope](#withscope) - [Language server](#language-server) - [textDocument/formatting](#textdocumentformatting) - [textDocument/inlayHint](#textdocumentinlayhint) @@ -588,20 +588,18 @@ SyntaxTree::Formatter.format(source, program.accept(visitor)) # => "if (a = 1)\nend\n" ``` -### WithEnvironment +### WithScope -The `WithEnvironment` module can be included in visitors to automatically keep track of local variables and arguments -defined inside each environment. A `current_environment` accessor is made available to the request, allowing it to find -all usages and definitions of a local. +The `WithScope` module can be included in visitors to automatically keep track of local variables and arguments defined inside each scope. A `current_scope` accessor is made available to the request, allowing it to find all usages and definitions of a local. ```ruby class MyVisitor < Visitor - include WithEnvironment + prepend WithScope def visit_ident(node) # find_local will return a Local for any local variables or arguments # present in the current environment or nil if the identifier is not a local - local = current_environment.find_local(node) + local = current_scope.find_local(node) puts local.type # the type of the local (:variable or :argument) puts local.definitions # the array of locations where this local is defined diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index 70126b14..4e183383 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -33,7 +33,7 @@ module SyntaxTree autoload :PrettyPrintVisitor, "syntax_tree/pretty_print_visitor" autoload :Search, "syntax_tree/search" autoload :Translation, "syntax_tree/translation" - autoload :WithEnvironment, "syntax_tree/with_environment" + autoload :WithScope, "syntax_tree/with_scope" autoload :YARV, "syntax_tree/yarv" # This holds references to objects that respond to both #parse and #format diff --git a/lib/syntax_tree/with_environment.rb b/lib/syntax_tree/with_scope.rb similarity index 65% rename from lib/syntax_tree/with_environment.rb rename to lib/syntax_tree/with_scope.rb index 3a6f04b9..efa8d075 100644 --- a/lib/syntax_tree/with_environment.rb +++ b/lib/syntax_tree/with_scope.rb @@ -1,18 +1,18 @@ # frozen_string_literal: true module SyntaxTree - # WithEnvironment is a module intended to be included in classes inheriting - # from Visitor. The module overrides a few visit methods to automatically keep - # track of local variables and arguments defined in the current environment. + # WithScope is a module intended to be included in classes inheriting from + # Visitor. The module overrides a few visit methods to automatically keep + # track of local variables and arguments defined in the current scope. # Example usage: # # class MyVisitor < Visitor - # include WithEnvironment + # include WithScope # # def visit_ident(node) # # Check if we're visiting an identifier for an argument, a local # # variable or something else - # local = current_environment.find_local(node) + # local = current_scope.find_local(node) # # if local.type == :argument # # handle identifiers for arguments @@ -24,11 +24,11 @@ module SyntaxTree # end # end # - module WithEnvironment - # The environment class is used to keep track of local variables and - # arguments inside a particular scope - class Environment - # This class tracks the occurrences of a local variable or argument + module WithScope + # The scope class is used to keep track of local variables and arguments + # inside a particular scope. + class Scope + # This class tracks the occurrences of a local variable or argument. class Local # [Symbol] The type of the local (e.g. :argument, :variable) attr_reader :type @@ -55,20 +55,20 @@ def add_usage(location) end end - # [Integer] a unique identifier for this environment + # [Integer] a unique identifier for this scope attr_reader :id + # [scope | nil] The parent scope + attr_reader :parent + # [Hash[String, Local]] The local variables and arguments defined in this - # environment + # scope attr_reader :locals - # [Environment | nil] The parent environment - attr_reader :parent - def initialize(id, parent = nil) @id = id - @locals = {} @parent = parent + @locals = {} end # Adding a local definition will either insert a new entry in the locals @@ -97,7 +97,7 @@ def add_local_usage(identifier, type) resolve_local(name, type).add_usage(identifier.location) end - # Try to find the local given its name in this environment or any of its + # Try to find the local given its name in this scope or any of its # parents. def find_local(name) locals[name] || parent&.find_local(name) @@ -117,44 +117,35 @@ def resolve_local(name, type) end end + attr_reader :current_scope + def initialize(*args, **kwargs, &block) super - @environment_id = 0 - end - - def current_environment - @current_environment ||= Environment.new(next_environment_id) - end - def with_new_environment(parent_environment = nil) - previous_environment = @current_environment - @current_environment = - Environment.new(next_environment_id, parent_environment) - yield - ensure - @current_environment = previous_environment + @current_scope = Scope.new(0) + @next_scope_id = 0 end - # Visits for nodes that create new environments, such as classes, modules + # Visits for nodes that create new scopes, such as classes, modules # and method definitions. def visit_class(node) - with_new_environment { super } + with_scope { super } end def visit_module(node) - with_new_environment { super } + with_scope { super } end - # When we find a method invocation with a block, only the code that - # happens inside of the block needs a fresh environment. The method - # invocation itself happens in the same environment. + # When we find a method invocation with a block, only the code that happens + # inside of the block needs a fresh scope. The method invocation + # itself happens in the same scope. def visit_method_add_block(node) visit(node.call) - with_new_environment(current_environment) { visit(node.block) } + with_scope(current_scope) { visit(node.block) } end def visit_def(node) - with_new_environment { super } + with_scope { super } end # Visit for keeping track of local arguments, such as method and block @@ -163,15 +154,15 @@ def visit_params(node) add_argument_definitions(node.requireds) node.posts.each do |param| - current_environment.add_local_definition(param, :argument) + current_scope.add_local_definition(param, :argument) end node.keywords.each do |param| - current_environment.add_local_definition(param.first, :argument) + current_scope.add_local_definition(param.first, :argument) end node.optionals.each do |param| - current_environment.add_local_definition(param.first, :argument) + current_scope.add_local_definition(param.first, :argument) end super @@ -179,21 +170,21 @@ def visit_params(node) def visit_rest_param(node) name = node.name - current_environment.add_local_definition(name, :argument) if name + current_scope.add_local_definition(name, :argument) if name super end def visit_kwrest_param(node) name = node.name - current_environment.add_local_definition(name, :argument) if name + current_scope.add_local_definition(name, :argument) if name super end def visit_blockarg(node) name = node.name - current_environment.add_local_definition(name, :argument) if name + current_scope.add_local_definition(name, :argument) if name super end @@ -201,10 +192,7 @@ def visit_blockarg(node) # Visit for keeping track of local variable definitions def visit_var_field(node) value = node.value - - if value.is_a?(SyntaxTree::Ident) - current_environment.add_local_definition(value, :variable) - end + current_scope.add_local_definition(value, :variable) if value.is_a?(Ident) super end @@ -215,12 +203,9 @@ def visit_var_field(node) def visit_var_ref(node) value = node.value - if value.is_a?(SyntaxTree::Ident) - definition = current_environment.find_local(value.value) - - if definition - current_environment.add_local_usage(value, definition.type) - end + if value.is_a?(Ident) + definition = current_scope.find_local(value.value) + current_scope.add_local_usage(value, definition.type) if definition end super @@ -233,13 +218,21 @@ def add_argument_definitions(list) if param.is_a?(SyntaxTree::MLHSParen) add_argument_definitions(param.contents.parts) else - current_environment.add_local_definition(param, :argument) + current_scope.add_local_definition(param, :argument) end end end - def next_environment_id - @environment_id += 1 + def next_scope_id + @next_scope_id += 1 + end + + def with_scope(parent_scope = nil) + previous_scope = @current_scope + @current_scope = Scope.new(next_scope_id, parent_scope) + yield + ensure + @current_scope = previous_scope end end end diff --git a/test/with_environment_test.rb b/test/with_scope_test.rb similarity index 95% rename from test/with_environment_test.rb rename to test/with_scope_test.rb index b6f79c14..1a4c5468 100644 --- a/test/with_environment_test.rb +++ b/test/with_scope_test.rb @@ -3,9 +3,9 @@ require_relative "test_helper" module SyntaxTree - class WithEnvironmentTest < Minitest::Test + class WithScopeTest < Minitest::Test class Collector < Visitor - prepend WithEnvironment + prepend WithScope attr_reader :arguments, :variables @@ -21,22 +21,22 @@ def self.collect(source) visit_methods do def visit_ident(node) value = node.value.delete_suffix(":") - local = current_environment.find_local(node.value) + local = current_scope.find_local(node.value) case local&.type when :argument - arguments[[current_environment.id, value]] = local + arguments[[current_scope.id, value]] = local when :variable - variables[[current_environment.id, value]] = local + variables[[current_scope.id, value]] = local end end def visit_label(node) value = node.value.delete_suffix(":") - local = current_environment.find_local(value) + local = current_scope.find_local(value) if local&.type == :argument - arguments[[current_environment.id, value]] = node + arguments[[current_scope.id, value]] = node end end end @@ -350,7 +350,7 @@ def test_double_nested_arguments end class Resolver < Visitor - prepend WithEnvironment + prepend WithScope attr_reader :locals @@ -364,10 +364,10 @@ def visit_assign(node) level = 0 name = node.target.value.value - environment = current_environment - while !environment.locals.key?(name) && !environment.parent.nil? + scope = current_scope + while !scope.locals.key?(name) && !scope.parent.nil? level += 1 - environment = environment.parent + scope = scope.parent end locals << [name, level] From 0068978479bb18b581aa745a12bb104f52ebe82f Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 10 Feb 2023 14:24:02 -0500 Subject: [PATCH 395/536] Pinned variables should be treated as usages, not definitions --- lib/syntax_tree/with_scope.rb | 8 +++++++- test/with_scope_test.rb | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/syntax_tree/with_scope.rb b/lib/syntax_tree/with_scope.rb index efa8d075..7fcef067 100644 --- a/lib/syntax_tree/with_scope.rb +++ b/lib/syntax_tree/with_scope.rb @@ -197,7 +197,13 @@ def visit_var_field(node) super end - alias visit_pinned_var_ref visit_var_field + # Visit for keeping track of local variable definitions + def visit_pinned_var_ref(node) + value = node.value + current_scope.add_local_usage(value, :variable) if value.is_a?(Ident) + + super + end # Visits for keeping track of variable and argument usages def visit_var_ref(node) diff --git a/test/with_scope_test.rb b/test/with_scope_test.rb index 1a4c5468..9675e811 100644 --- a/test/with_scope_test.rb +++ b/test/with_scope_test.rb @@ -109,7 +109,7 @@ def foo RUBY assert_equal(2, collector.variables.length) - assert_variable(collector, "a", definitions: [2, 4], usages: [5]) + assert_variable(collector, "a", definitions: [2], usages: [4, 5]) assert_variable(collector, "rest", definitions: [4]) # Rest is considered a vcall by the parser instead of a var_ref From 575ae3ea24a66a74b254090e421c6cd439e63fee Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 10 Feb 2023 14:38:26 -0500 Subject: [PATCH 396/536] No submodules needed --- .github/workflows/main.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8bca2fc4..3f811317 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,8 +23,6 @@ jobs: # TESTOPTS: --verbose steps: - uses: actions/checkout@master - with: - submodules: true - uses: ruby/setup-ruby@v1 with: bundler-cache: true From cfc297925a056201825f76c1aea67ce72a65dcfc Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 10 Feb 2023 14:39:04 -0500 Subject: [PATCH 397/536] Remove unused sections of rubocop config --- .rubocop.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index e5a3fe96..e74cdc1b 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -8,8 +8,6 @@ AllCops: TargetRubyVersion: 2.7 Exclude: - '{.git,.github,bin,coverage,pkg,spec,test/fixtures,vendor,tmp}/**/*' - - test/ruby-syntax-fixtures/**/* - - test/suites/parser/**/* - test.rb Gemspec/DevelopmentDependencies: From 4dac90b53df388f726dce50ce638a1ba71cc59f8 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 10 Feb 2023 15:19:00 -0500 Subject: [PATCH 398/536] Bump to version 6.0.0 --- CHANGELOG.md | 65 +++++++++++++++++++++++++++++++++++++- Gemfile.lock | 2 +- lib/syntax_tree/version.rb | 2 +- 3 files changed, 66 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c39bed36..34c40e40 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,68 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +## [6.0.0] - 2023-02-10 + +### Added + +- `SyntaxTree::BasicVisitor::visit_methods` has been added to allow you to check multiple visit methods inside of a block. There _was_ a method called `visit_methods` previously, but it was undocumented because it was meant as a private API. That method has been renamed to `valid_visit_methods`. +- `rake sorbet:rbi` has been added as a task within the repository to generate an RBI file corresponding to the nodes in the tree. This can be used to help aid consumers of Syntax Tree that are using Sorbet. +- `SyntaxTree::Reflection` has been added to allow you to get information about the nodes in the tree. It is not required by default, since it takes a small amount of time to parse `node.rb` and get all of the information. +- `SyntaxTree::Node#to_mermaid` has been added to allow you to generate a Mermaid diagram of the node and its children. This is useful for debugging and understanding the structure of the tree. +- `SyntaxTree::Translation` has been added as an experimental API to transform the Syntax Tree syntax tree into the syntax trees represented by the whitequark/parser and rubocop/rubocop-ast gems. + - `SyntaxTree::Translation.to_parser(node, buffer)` will return a `Parser::AST::Node` object. + - `SyntaxTree::Translation.to_rubocop_ast(node, buffer)` will return a `RuboCop::AST::Node` object. +- `SyntaxTree::index` and `SyntaxTree::index_file` have been added to allow you to get a list of all of the classes, modules, and methods defined in a given source string or file. +- Various convenience methods have been added: + - `SyntaxTree::format_file` - which calls format with the result of reading the file + - `SyntaxTree::format_node` - which formats the node directly + - `SyntaxTree::parse_file` - which calls parse with the result of reading the file + - `SyntaxTree::search_file` - which calls search with the result of reading the file + - `SyntaxTree::Node#start_char` - which is the same as calling `node.location.start_char` + - `SyntaxTree::Node#end_char` - which is the same as calling `node.location.end_char` +- `SyntaxTree::Assoc` nodes can now be formatted on their own without a parent hash node. +- `SyntaxTree::BlockVar#arg0?` has been added to check if a single required block parameter is present and would potentially be expanded. +- More experimental APIs have been added to the `SyntaxTree::YARV` module, including: + - `SyntaxTree::YARV::ControlFlowGraph` + - `SyntaxTree::YARV::DataFlowGraph` + - `SyntaxTree::YARV::SeaOfNodes` + +### Changed + +#### Major changes + +- *BREAKING* Updates to `WithEnvironment`: + - The `WithEnvironment` module has been renamed to `WithScope`. + - The `current_environment` method has been renamed to `current_scope`. + - The `with_current_environment` method has been removed. + - Previously scopes were always able to look up the tree, as in: `a = 1; def foo; a = 2; end` would see only a single `a` variable. That has been corrected. + - Previously accessing variables from inside of blocks that were not shadowed would mark them as being local to the block only. This has been correct. +- *BREAKING* Lots of constants moved out of `SyntaxTree::Visitor` to just `SyntaxTree`: + * `SyntaxTree::Visitor::FieldVisitor` is now `SyntaxTree::FieldVisitor` + * `SyntaxTree::Visitor::JSONVisitor` is now `SyntaxTree::JSONVisitor` + * `SyntaxTree::Visitor::MatchVisitor` is now `SyntaxTree::MatchVisitor` + * `SyntaxTree::Visitor::MutationVisitor` is now `SyntaxTree::MutationVisitor` + * `SyntaxTree::Visitor::PrettyPrintVisitor` is now `SyntaxTree::PrettyPrintVisitor` +- *BREAKING* Lots of constants are now autoloaded instead of required by default. This is only particularly relevant if you are in a forking environment and want to preload constants before forking for better memory usage with copy-on-write. +- *BREAKING* The `SyntaxTree::Statements#initialize` method no longer accepts a parser as the first argument. It now mirrors the other nodes in that it accepts its children and location. As a result, Syntax Tree nodes are now marshalable (and therefore can be sent over DRb). Previously the `Statements` node was not able to be marshaled because it held a reference to the parser. + +#### Minor changes + +- Many places where embedded documents (`=begin` to `=end`) were being treated as real comments have been fixed for formatting. +- Dynamic symbols in keyword pattern matching now have better formatting. +- Endless method definitions used to have a `SyntaxTree::BodyStmt` node that had any kind of node as its `statements` field. That has been corrected to be more consistent such that now going from `def_node.bodystmt.statements` always returns a `SyntaxTree::Statements` node, which is more consistent. +- We no longer assume that `fiddle` is able to be required, and only require it when it is actually needed. + +#### Tiny changes + +- Empty parameter nodes within blocks now have more accurate location information. +- Pinned variables have more correct location information now. (Previously the location was just around the variable itself, but it now includes the pin.) +- Array patterns in pattern matching now have more accurate location information when they are using parentheses with a constant present. +- Find patterns in pattern matching now have more correct location information for their `left` and `right` fields. +- Lots of nodes have more correct types in the comments on their attributes. +- The expressions `break foo.bar :baz do |qux| qux end` and `next fun foo do end` now correctly parses as a control-flow statement with a method call that has a block attached, as opposed to a control-flow statement with a block attached. +- The expression `self::a, b = 1, 2` would previously yield a `SyntaxTree::ConstPathField` node for the first element of the left-hand-side of the multiple assignment. Semantically this is incorrect, and we have fixed this to now be a `SyntaxTree::Field` node instead. + ## [5.3.0] - 2023-01-26 ### Added @@ -497,7 +559,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - 🎉 Initial release! 🎉 -[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v5.3.0...HEAD +[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v6.0.0...HEAD +[6.0.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v5.3.0...v6.0.0 [5.3.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v5.2.0...v5.3.0 [5.2.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v5.1.0...v5.2.0 [5.1.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v5.0.1...v5.1.0 diff --git a/Gemfile.lock b/Gemfile.lock index 46111ea4..325d89b3 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - syntax_tree (5.3.0) + syntax_tree (6.0.0) prettier_print (>= 1.2.0) GEM diff --git a/lib/syntax_tree/version.rb b/lib/syntax_tree/version.rb index 6cb1fccf..1f028f89 100644 --- a/lib/syntax_tree/version.rb +++ b/lib/syntax_tree/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SyntaxTree - VERSION = "5.3.0" + VERSION = "6.0.0" end From f5f8b6a8dcbf499db95d2c3f8c13ff57a4782bcc Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Sat, 18 Feb 2023 10:14:58 -0500 Subject: [PATCH 399/536] Even more parser gem locations --- lib/syntax_tree/translation/parser.rb | 129 +++++++++++++------------- 1 file changed, 67 insertions(+), 62 deletions(-) diff --git a/lib/syntax_tree/translation/parser.rb b/lib/syntax_tree/translation/parser.rb index ad889478..0ed2c61f 100644 --- a/lib/syntax_tree/translation/parser.rb +++ b/lib/syntax_tree/translation/parser.rb @@ -1287,35 +1287,13 @@ def visit_ident(node) # Visit an IfNode node. def visit_if(node) - predicate = - case node.predicate - when RangeNode - type = - node.predicate.operator.value == ".." ? :iflipflop : :eflipflop - s(type, visit(node.predicate).children, nil) - when RegexpLiteral - s(:match_current_line, [visit(node.predicate)], nil) - when Unary - if node.predicate.operator.value == "!" && - node.predicate.statement.is_a?(RegexpLiteral) - s( - :send, - [ - s(:match_current_line, [visit(node.predicate.statement)]), - :! - ], - nil - ) - else - visit(node.predicate) - end - else - visit(node.predicate) - end - s( :if, - [predicate, visit(node.statements), visit(node.consequent)], + [ + visit_predicate(node.predicate), + visit(node.statements), + visit(node.consequent) + ], if node.modifier? smap_keyword_bare( srange_find_between(node.statements, node.predicate, "if"), @@ -2376,22 +2354,42 @@ def visit_tstring_content(node) # Visit a Unary node. def visit_unary(node) # Special handling here for flipflops - if node.statement.is_a?(Paren) && - node.statement.contents.is_a?(Statements) && - node.statement.contents.body.length == 1 && - (range = node.statement.contents.body.first).is_a?(RangeNode) && + if (paren = node.statement).is_a?(Paren) && + paren.contents.is_a?(Statements) && + paren.contents.body.length == 1 && + (range = paren.contents.body.first).is_a?(RangeNode) && node.operator == "!" - type = range.operator.value == ".." ? :iflipflop : :eflipflop - return( - s( - :send, - [s(:begin, [s(type, visit(range).children, nil)], nil), :!], - nil + s( + :send, + [ + s( + :begin, + [ + s( + range.operator.value == ".." ? :iflipflop : :eflipflop, + visit(range).children, + smap_operator( + srange_node(range.operator), + srange_node(range) + ) + ) + ], + smap_collection( + srange_length(paren.start_char, 1), + srange_length(paren.end_char, -1), + srange_node(paren) + ) + ), + :! + ], + smap_send_bare( + srange_length(node.start_char, 1), + srange_node(node) ) ) + else + visit(canonical_unary(node)) end - - visit(canonical_unary(node)) end # Visit an Undef node. @@ -2408,31 +2406,13 @@ def visit_undef(node) # Visit an UnlessNode node. def visit_unless(node) - predicate = - case node.predicate - when RegexpLiteral - s(:match_current_line, [visit(node.predicate)], nil) - when Unary - if node.predicate.operator.value == "!" && - node.predicate.statement.is_a?(RegexpLiteral) - s( - :send, - [ - s(:match_current_line, [visit(node.predicate.statement)]), - :! - ], - nil - ) - else - visit(node.predicate) - end - else - visit(node.predicate) - end - s( :if, - [predicate, visit(node.consequent), visit(node.statements)], + [ + visit_predicate(node.predicate), + visit(node.consequent), + visit(node.statements) + ], if node.modifier? smap_keyword_bare( srange_find_between(node.statements, node.predicate, "unless"), @@ -3014,6 +2994,31 @@ def srange_node(node) location = node.location srange(location.start_char, location.end_char) end + + def visit_predicate(node) + case node + when RangeNode + s( + node.operator.value == ".." ? :iflipflop : :eflipflop, + visit(node).children, + smap_operator(srange_node(node.operator), srange_node(node)) + ) + when RegexpLiteral + s(:match_current_line, [visit(node)], smap(srange_node(node))) + when Unary + if node.operator.value == "!" && node.statement.is_a?(RegexpLiteral) + s( + :send, + [s(:match_current_line, [visit(node.statement)]), :!], + smap_send_bare(srange_node(node.operator), srange_node(node)) + ) + else + visit(node) + end + else + visit(node) + end + end end end end From 4057dfa17c3fc80ed8b4b11722e97fd53de50cf2 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Sat, 18 Feb 2023 18:19:04 -0500 Subject: [PATCH 400/536] Handle matching current line --- lib/syntax_tree/translation/parser.rb | 16 ++++++++++++++++ test/translation/parser_test.rb | 1 - 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/lib/syntax_tree/translation/parser.rb b/lib/syntax_tree/translation/parser.rb index 0ed2c61f..9c53ad14 100644 --- a/lib/syntax_tree/translation/parser.rb +++ b/lib/syntax_tree/translation/parser.rb @@ -2387,6 +2387,22 @@ def visit_unary(node) srange_node(node) ) ) + elsif node.operator == "!" && node.statement.is_a?(RegexpLiteral) + s( + :send, + [ + s( + :match_current_line, + [visit(node.statement)], + smap(srange_node(node.statement)) + ), + :! + ], + smap_send_bare( + srange_length(node.start_char, 1), + srange_node(node) + ) + ) else visit(canonical_unary(node)) end diff --git a/test/translation/parser_test.rb b/test/translation/parser_test.rb index ad87d8c6..1df98f47 100644 --- a/test/translation/parser_test.rb +++ b/test/translation/parser_test.rb @@ -55,7 +55,6 @@ class ParserTest < Minitest::Test "test_dedenting_heredoc:399", "test_slash_newline_in_heredocs:7194", "test_parser_slash_slash_n_escaping_in_literals:*", - "test_cond_match_current_line:4801", "test_forwarded_restarg:*", "test_forwarded_kwrestarg:*", "test_forwarded_argument_with_restarg:*", From 6f135be2dbcd002afb67da194759190f752c59fc Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Sat, 18 Feb 2023 18:22:50 -0500 Subject: [PATCH 401/536] Block on super location --- lib/syntax_tree/translation/parser.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/syntax_tree/translation/parser.rb b/lib/syntax_tree/translation/parser.rb index 9c53ad14..243b460b 100644 --- a/lib/syntax_tree/translation/parser.rb +++ b/lib/syntax_tree/translation/parser.rb @@ -1576,7 +1576,11 @@ def visit_method_add_block(node) s( type, [visit(node.call), arguments, visit(node.block.bodystmt)], - nil + smap_collection( + srange_node(node.block.opening), + srange_length(node.block.end_char, node.block.opening.is_a?(Kw) ? -3 : -1), + srange_node(node) + ) ) else visit_command_call( From 305ee004c932718ca39af8815a4debc1aa72e745 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Sat, 18 Feb 2023 19:48:15 -0500 Subject: [PATCH 402/536] ; delimiting unless nodes --- lib/syntax_tree/translation/parser.rb | 37 +++++++++++++++------------ 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/lib/syntax_tree/translation/parser.rb b/lib/syntax_tree/translation/parser.rb index 243b460b..4f32c933 100644 --- a/lib/syntax_tree/translation/parser.rb +++ b/lib/syntax_tree/translation/parser.rb @@ -1555,21 +1555,6 @@ def visit_massign(node) # Visit a MethodAddBlock node. def visit_method_add_block(node) case node.call - when Break, Next, ReturnNode - type, arguments = block_children(node.block) - call = visit(node.call) - - s( - call.type, - [ - s( - type, - [*call.children, arguments, visit(node.block.bodystmt)], - nil - ) - ], - nil - ) when ARef, Super, ZSuper type, arguments = block_children(node.block) @@ -1578,7 +1563,10 @@ def visit_method_add_block(node) [visit(node.call), arguments, visit(node.block.bodystmt)], smap_collection( srange_node(node.block.opening), - srange_length(node.block.end_char, node.block.opening.is_a?(Kw) ? -3 : -1), + srange_length( + node.block.end_char, + node.block.opening.is_a?(Kw) ? -3 : -1 + ), srange_node(node) ) ) @@ -2439,9 +2427,24 @@ def visit_unless(node) srange_node(node) ) else + begin_start = node.predicate.end_char + begin_end = + if node.statements.empty? + node.statements.end_char + else + node.statements.body.first.start_char + end + + begin_token = + if buffer.source[begin_start...begin_end].include?("then") + srange_find(begin_start, begin_end, "then") + elsif buffer.source[begin_start...begin_end].include?(";") + srange_find(begin_start, begin_end, ";") + end + smap_condition( srange_length(node.start_char, 6), - srange_search_between(node.predicate, node.statements, "then"), + begin_token, nil, srange_length(node.end_char, -3), srange_node(node) From 1eec9e708387c13766e7fa48d1447b408049df27 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 13 Feb 2023 10:28:09 -0500 Subject: [PATCH 403/536] More whitequark/parser translation --- lib/syntax_tree/parser.rb | 11 +++- lib/syntax_tree/translation/parser.rb | 80 +++++++++++++++++++++++---- 2 files changed, 80 insertions(+), 11 deletions(-) diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index 426bd945..d0a5bf67 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -1559,7 +1559,14 @@ def on_elsif(predicate, statements, consequent) beginning = consume_keyword(:elsif) ending = consequent || consume_keyword(:end) - start_char = find_next_statement_start(predicate.location.end_char) + delimiter = + find_keyword_between(:then, predicate, statements) || + find_token_between(Semicolon, predicate, statements) + + tokens.delete(delimiter) if delimiter + start_char = + find_next_statement_start((delimiter || predicate).location.end_char) + statements.bind( self, start_char, @@ -2045,6 +2052,7 @@ def on_if(predicate, statements, consequent) start_char = find_next_statement_start((keyword || predicate).location.end_char) + statements.bind( self, start_char, @@ -3805,6 +3813,7 @@ def on_unless(predicate, statements, consequent) start_char = find_next_statement_start((keyword || predicate).location.end_char) + statements.bind( self, start_char, diff --git a/lib/syntax_tree/translation/parser.rb b/lib/syntax_tree/translation/parser.rb index 4f32c933..8be4fc79 100644 --- a/lib/syntax_tree/translation/parser.rb +++ b/lib/syntax_tree/translation/parser.rb @@ -336,8 +336,8 @@ def visit_assign(node) # Visit an Assoc node. def visit_assoc(node) if node.value.nil? + # { foo: } expression = srange(node.start_char, node.end_char - 1) - type, location = if node.key.value.start_with?(/[A-Z]/) [:const, smap_constant(nil, expression, expression)] @@ -356,13 +356,38 @@ def visit_assoc(node) srange_node(node) ) ) - else + elsif node.key.is_a?(Label) + # { foo: 1 } s( :pair, [visit(node.key), visit(node.value)], smap_operator( - srange_search_between(node.key, node.value, "=>") || - srange_length(node.key.end_char, -1), + srange_length(node.key.end_char, -1), + srange_node(node) + ) + ) + elsif (operator = srange_search_between(node.key, node.value, "=>")) + # { :foo => 1 } + s( + :pair, + [visit(node.key), visit(node.value)], + smap_operator(operator, srange_node(node)) + ) + else + # { "foo": 1 } + key = visit(node.key) + key_location = + smap_collection( + key.location.begin, + srange_length(node.key.end_char - 2, 1), + srange(node.key.start_char, node.key.end_char - 1) + ) + + s( + :pair, + [s(key.type, key.children, key_location), visit(node.value)], + smap_operator( + srange_length(node.key.end_char, -1), srange_node(node) ) ) @@ -769,7 +794,11 @@ def visit_command_call(node) srange(node.start_char, end_char) elsif node.block - srange_node(node.message) + if node.receiver + srange(node.receiver.start_char, node.message.end_char) + else + srange_node(node.message) + end else srange_node(node) end @@ -1010,6 +1039,21 @@ def visit_else(node) # Visit an Elsif node. def visit_elsif(node) + begin_start = node.predicate.end_char + begin_end = + if node.statements.empty? + node.statements.end_char + else + node.statements.body.first.start_char + end + + begin_token = + if buffer.source[begin_start...begin_end].include?("then") + srange_find(begin_start, begin_end, "then") + elsif buffer.source[begin_start...begin_end].include?(";") + srange_find(begin_start, begin_end, ";") + end + else_token = case node.consequent when Elsif @@ -1029,7 +1073,7 @@ def visit_elsif(node) ], smap_condition( srange_length(node.start_char, 5), - nil, + begin_token, else_token, nil, expression @@ -1529,12 +1573,14 @@ def visit_lambda_var(node) location = if node.start_char == node.end_char smap_collection_bare(nil) - else + elsif buffer.source[node.start_char - 1] == "(" smap_collection( srange_length(node.start_char, 1), srange_length(node.end_char, -1), srange_node(node) ) + else + smap_collection_bare(srange_node(node)) end s(:args, visit(node.params).children + shadowargs, location) @@ -1565,7 +1611,7 @@ def visit_method_add_block(node) srange_node(node.block.opening), srange_length( node.block.end_char, - node.block.opening.is_a?(Kw) ? -3 : -1 + node.block.keywords? ? -3 : -1 ), srange_node(node) ) @@ -2244,7 +2290,16 @@ def visit_super(node) ) ) when ArgsForward - s(:super, [visit(node.arguments.arguments)], nil) + s( + :super, + [visit(node.arguments.arguments)], + smap_keyword( + srange_length(node.start_char, 5), + srange_find(node.start_char + 5, node.end_char, "("), + srange_length(node.end_char, -1), + srange_node(node) + ) + ) else s( :super, @@ -2442,10 +2497,15 @@ def visit_unless(node) srange_find(begin_start, begin_end, ";") end + else_token = + if node.consequent + srange_length(node.consequent.start_char, 4) + end + smap_condition( srange_length(node.start_char, 6), begin_token, - nil, + else_token, srange_length(node.end_char, -3), srange_node(node) ) From ce9de3114c537de85cc86f90bf603d56d7eba653 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 21 Feb 2023 10:03:42 -0500 Subject: [PATCH 404/536] Better handle nested constant names --- lib/syntax_tree/index.rb | 75 +++++++++++++++++++++++++++++++++------- test/index_test.rb | 33 ++++++++++++++---- 2 files changed, 90 insertions(+), 18 deletions(-) diff --git a/lib/syntax_tree/index.rb b/lib/syntax_tree/index.rb index ab2460dd..c067606f 100644 --- a/lib/syntax_tree/index.rb +++ b/lib/syntax_tree/index.rb @@ -176,30 +176,64 @@ def location_for(iseq) Location.new(code_location[0], code_location[1]) end + def find_constant_path(insns, index) + insn = insns[index] + + if insn.is_a?(Array) && insn[0] == :opt_getconstant_path + # In this case we're on Ruby 3.2+ and we have an opt_getconstant_path + # instruction, so we already know all of the symbols in the nesting. + insn[1] + elsif insn.is_a?(Symbol) && insn.match?(/\Alabel_\d+/) + # Otherwise, if we have a label then this is very likely the + # destination of an opt_getinlinecache instruction, in which case + # we'll walk backwards to grab up all of the constants. + names = [] + + index -= 1 + until insns[index][0] == :opt_getinlinecache + names.unshift(insns[index][1]) if insns[index][0] == :getconstant + index -= 1 + end + + names + end + end + def index_iseq(iseq, file_comments) results = [] queue = [[iseq, []]] while (current_iseq, current_nesting = queue.shift) - current_iseq[13].each_with_index do |insn, index| + insns = current_iseq[13] + insns.each_with_index do |insn, index| next unless insn.is_a?(Array) case insn[0] when :defineclass _, name, class_iseq, flags = insn + next_nesting = current_nesting.dup + + if (nesting = find_constant_path(insns, index - 2)) + # If there is a constant path in the class name, then we need to + # handle that by updating the nesting. + next_nesting << (nesting << name) + else + # Otherwise we'll add the class name to the nesting. + next_nesting << [name] + end if flags == VM_DEFINECLASS_TYPE_SINGLETON_CLASS # At the moment, we don't support singletons that aren't # defined on self. We could, but it would require more # emulation. - if current_iseq[13][index - 2] != [:putself] + if insns[index - 2] != [:putself] raise NotImplementedError, "singleton class with non-self receiver" end elsif flags & VM_DEFINECLASS_TYPE_MODULE > 0 location = location_for(class_iseq) results << ModuleDefinition.new( - current_nesting, + next_nesting, name, location, EntryComments.new(file_comments, location) @@ -207,14 +241,14 @@ def index_iseq(iseq, file_comments) else location = location_for(class_iseq) results << ClassDefinition.new( - current_nesting, + next_nesting, name, location, EntryComments.new(file_comments, location) ) end - queue << [class_iseq, current_nesting + [name]] + queue << [class_iseq, next_nesting] when :definemethod location = location_for(insn[2]) results << MethodDefinition.new( @@ -259,24 +293,36 @@ def initialize visit_methods do def visit_class(node) - name = visit(node.constant).to_sym + names = visit(node.constant) + nesting << names + location = Location.new(node.location.start_line, node.location.start_column) results << ClassDefinition.new( nesting.dup, - name, + names.last, location, comments_for(node) ) - nesting << name super nesting.pop end def visit_const_ref(node) - node.constant.value + [node.constant.value.to_sym] + end + + def visit_const_path_ref(node) + names = + if node.parent.is_a?(ConstPathRef) + visit(node.parent) + else + [visit(node.parent)] + end + + names << node.constant.value.to_sym end def visit_def(node) @@ -302,18 +348,19 @@ def visit_def(node) end def visit_module(node) - name = visit(node.constant).to_sym + names = visit(node.constant) + nesting << names + location = Location.new(node.location.start_line, node.location.start_column) results << ModuleDefinition.new( nesting.dup, - name, + names.last, location, comments_for(node) ) - nesting << name super nesting.pop end @@ -327,6 +374,10 @@ def visit_statements(node) @statements = node super end + + def visit_var_ref(node) + node.value.value.to_sym + end end private diff --git a/test/index_test.rb b/test/index_test.rb index 6bb83881..b00b4bc6 100644 --- a/test/index_test.rb +++ b/test/index_test.rb @@ -7,14 +7,14 @@ class IndexTest < Minitest::Test def test_module index_each("module Foo; end") do |entry| assert_equal :Foo, entry.name - assert_empty entry.nesting + assert_equal [[:Foo]], entry.nesting end end def test_module_nested index_each("module Foo; module Bar; end; end") do |entry| assert_equal :Bar, entry.name - assert_equal [:Foo], entry.nesting + assert_equal [[:Foo], [:Bar]], entry.nesting end end @@ -28,14 +28,35 @@ def test_module_comments def test_class index_each("class Foo; end") do |entry| assert_equal :Foo, entry.name - assert_empty entry.nesting + assert_equal [[:Foo]], entry.nesting + end + end + + def test_class_paths_2 + index_each("class Foo::Bar; end") do |entry| + assert_equal :Bar, entry.name + assert_equal [[:Foo, :Bar]], entry.nesting + end + end + + def test_class_paths_3 + index_each("class Foo::Bar::Baz; end") do |entry| + assert_equal :Baz, entry.name + assert_equal [[:Foo, :Bar, :Baz]], entry.nesting end end def test_class_nested index_each("class Foo; class Bar; end; end") do |entry| assert_equal :Bar, entry.name - assert_equal [:Foo], entry.nesting + assert_equal [[:Foo], [:Bar]], entry.nesting + end + end + + def test_class_paths_nested + index_each("class Foo; class Bar::Baz::Qux; end; end") do |entry| + assert_equal :Qux, entry.name + assert_equal [[:Foo], [:Bar, :Baz, :Qux]], entry.nesting end end @@ -56,7 +77,7 @@ def test_method def test_method_nested index_each("class Foo; def foo; end; end") do |entry| assert_equal :foo, entry.name - assert_equal [:Foo], entry.nesting + assert_equal [[:Foo]], entry.nesting end end @@ -77,7 +98,7 @@ def test_singleton_method def test_singleton_method_nested index_each("class Foo; def self.foo; end; end") do |entry| assert_equal :foo, entry.name - assert_equal [:Foo], entry.nesting + assert_equal [[:Foo]], entry.nesting end end From 2d5f9fc2d4af804662b470c64fe0479277a4b88c Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 21 Feb 2023 10:16:03 -0500 Subject: [PATCH 405/536] Handle superclasses --- lib/syntax_tree/index.rb | 56 ++++++++++++++++++++++++++++++---------- test/index_test.rb | 30 +++++++++++++++++++++ 2 files changed, 72 insertions(+), 14 deletions(-) diff --git a/lib/syntax_tree/index.rb b/lib/syntax_tree/index.rb index c067606f..c2850f6a 100644 --- a/lib/syntax_tree/index.rb +++ b/lib/syntax_tree/index.rb @@ -20,11 +20,12 @@ def initialize(line, column) # This entry represents a class definition using the class keyword. class ClassDefinition - attr_reader :nesting, :name, :location, :comments + attr_reader :nesting, :name, :superclass, :location, :comments - def initialize(nesting, name, location, comments) + def initialize(nesting, name, superclass, location, comments) @nesting = nesting @name = name + @superclass = superclass @location = location @comments = comments end @@ -182,7 +183,7 @@ def find_constant_path(insns, index) if insn.is_a?(Array) && insn[0] == :opt_getconstant_path # In this case we're on Ruby 3.2+ and we have an opt_getconstant_path # instruction, so we already know all of the symbols in the nesting. - insn[1] + [index - 1, insn[1]] elsif insn.is_a?(Symbol) && insn.match?(/\Alabel_\d+/) # Otherwise, if we have a label then this is very likely the # destination of an opt_getinlinecache instruction, in which case @@ -195,7 +196,9 @@ def find_constant_path(insns, index) index -= 1 end - names + [index - 1, names] + else + [index, []] end end @@ -213,7 +216,24 @@ def index_iseq(iseq, file_comments) _, name, class_iseq, flags = insn next_nesting = current_nesting.dup - if (nesting = find_constant_path(insns, index - 2)) + # This is the index we're going to search for the nested constant + # path within the declaration name. + constant_index = index - 2 + + # This is the superclass of the class being defined. + superclass = [] + + # If there is a superclass, then we're going to find it here and + # then update the constant_index as necessary. + if flags & VM_DEFINECLASS_FLAG_HAS_SUPERCLASS > 0 + constant_index, superclass = find_constant_path(insns, index - 1) + + if superclass.empty? + raise NotImplementedError, "superclass with non constant path" + end + end + + if (_, nesting = find_constant_path(insns, constant_index)) # If there is a constant path in the class name, then we need to # handle that by updating the nesting. next_nesting << (nesting << name) @@ -243,6 +263,7 @@ def index_iseq(iseq, file_comments) results << ClassDefinition.new( next_nesting, name, + superclass, location, EntryComments.new(file_comments, location) ) @@ -299,9 +320,23 @@ def visit_class(node) location = Location.new(node.location.start_line, node.location.start_column) + superclass = + if node.superclass + visited = visit(node.superclass) + + if visited == [[]] + raise NotImplementedError, "superclass with non constant path" + end + + visited + else + [] + end + results << ClassDefinition.new( nesting.dup, names.last, + superclass, location, comments_for(node) ) @@ -315,14 +350,7 @@ def visit_const_ref(node) end def visit_const_path_ref(node) - names = - if node.parent.is_a?(ConstPathRef) - visit(node.parent) - else - [visit(node.parent)] - end - - names << node.constant.value.to_sym + visit(node.parent) << node.constant.value.to_sym end def visit_def(node) @@ -376,7 +404,7 @@ def visit_statements(node) end def visit_var_ref(node) - node.value.value.to_sym + [node.value.value.to_sym] end end diff --git a/test/index_test.rb b/test/index_test.rb index b00b4bc6..9101870b 100644 --- a/test/index_test.rb +++ b/test/index_test.rb @@ -60,6 +60,36 @@ def test_class_paths_nested end end + def test_class_superclass + index_each("class Foo < Bar; end") do |entry| + assert_equal :Foo, entry.name + assert_equal [[:Foo]], entry.nesting + assert_equal [:Bar], entry.superclass + end + end + + def test_class_path_superclass + index_each("class Foo::Bar < Baz::Qux; end") do |entry| + assert_equal :Bar, entry.name + assert_equal [[:Foo, :Bar]], entry.nesting + assert_equal [:Baz, :Qux], entry.superclass + end + end + + def test_class_path_superclass_unknown + source = "class Foo < bar; end" + + assert_raises NotImplementedError do + Index.index(source, backend: Index::ParserBackend.new) + end + + if defined?(RubyVM::InstructionSequence) + assert_raises NotImplementedError do + Index.index(source, backend: Index::ISeqBackend.new) + end + end + end + def test_class_comments index_each("# comment1\n# comment2\nclass Foo; end") do |entry| assert_equal :Foo, entry.name From a886179e15831e22f958c859fec4456a48eddcc8 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 21 Feb 2023 10:43:08 -0500 Subject: [PATCH 406/536] Handle line numbers in constant searching --- lib/syntax_tree/index.rb | 28 +++++++++++++++++++++++----- test/index_test.rb | 10 +++++----- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/lib/syntax_tree/index.rb b/lib/syntax_tree/index.rb index c2850f6a..c6973847 100644 --- a/lib/syntax_tree/index.rb +++ b/lib/syntax_tree/index.rb @@ -178,6 +178,7 @@ def location_for(iseq) end def find_constant_path(insns, index) + index -= 1 while insns[index].is_a?(Integer) insn = insns[index] if insn.is_a?(Array) && insn[0] == :opt_getconstant_path @@ -191,8 +192,12 @@ def find_constant_path(insns, index) names = [] index -= 1 - until insns[index][0] == :opt_getinlinecache - names.unshift(insns[index][1]) if insns[index][0] == :getconstant + until insns[index].is_a?(Array) && + insns[index][0] == :opt_getinlinecache + if insns[index].is_a?(Array) && insns[index][0] == :getconstant + names.unshift(insns[index][1]) + end + index -= 1 end @@ -207,9 +212,20 @@ def index_iseq(iseq, file_comments) queue = [[iseq, []]] while (current_iseq, current_nesting = queue.shift) + line = current_iseq[8] insns = current_iseq[13] + insns.each_with_index do |insn, index| - next unless insn.is_a?(Array) + case insn + when Integer + line = insn + next + when Array + # continue on + else + # skip everything else + next + end case insn[0] when :defineclass @@ -226,10 +242,12 @@ def index_iseq(iseq, file_comments) # If there is a superclass, then we're going to find it here and # then update the constant_index as necessary. if flags & VM_DEFINECLASS_FLAG_HAS_SUPERCLASS > 0 - constant_index, superclass = find_constant_path(insns, index - 1) + constant_index, superclass = + find_constant_path(insns, index - 1) if superclass.empty? - raise NotImplementedError, "superclass with non constant path" + raise NotImplementedError, + "superclass with non constant path on line #{line}" end end diff --git a/test/index_test.rb b/test/index_test.rb index 9101870b..60c51d9d 100644 --- a/test/index_test.rb +++ b/test/index_test.rb @@ -35,14 +35,14 @@ def test_class def test_class_paths_2 index_each("class Foo::Bar; end") do |entry| assert_equal :Bar, entry.name - assert_equal [[:Foo, :Bar]], entry.nesting + assert_equal [%i[Foo Bar]], entry.nesting end end def test_class_paths_3 index_each("class Foo::Bar::Baz; end") do |entry| assert_equal :Baz, entry.name - assert_equal [[:Foo, :Bar, :Baz]], entry.nesting + assert_equal [%i[Foo Bar Baz]], entry.nesting end end @@ -56,7 +56,7 @@ def test_class_nested def test_class_paths_nested index_each("class Foo; class Bar::Baz::Qux; end; end") do |entry| assert_equal :Qux, entry.name - assert_equal [[:Foo], [:Bar, :Baz, :Qux]], entry.nesting + assert_equal [[:Foo], %i[Bar Baz Qux]], entry.nesting end end @@ -71,8 +71,8 @@ def test_class_superclass def test_class_path_superclass index_each("class Foo::Bar < Baz::Qux; end") do |entry| assert_equal :Bar, entry.name - assert_equal [[:Foo, :Bar]], entry.nesting - assert_equal [:Baz, :Qux], entry.superclass + assert_equal [%i[Foo Bar]], entry.nesting + assert_equal %i[Baz Qux], entry.superclass end end From e68e3f6e34c8ff7cde4ec69bd45d8a5af72b418f Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 21 Feb 2023 10:45:42 -0500 Subject: [PATCH 407/536] Document indexing --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 500d5fad..03942d46 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ It is built with only standard library dependencies. It additionally ships with - [SyntaxTree.format(source)](#syntaxtreeformatsource) - [SyntaxTree.mutation(&block)](#syntaxtreemutationblock) - [SyntaxTree.search(source, query, &block)](#syntaxtreesearchsource-query-block) + - [SyntaxTree.index(source)](#syntaxtreeindexsource) - [Nodes](#nodes) - [child_nodes](#child_nodes) - [copy(**attrs)](#copyattrs) @@ -347,6 +348,10 @@ This function yields a new mutation visitor to the block, and then returns the i This function takes an input string containing Ruby code, an input string containing a valid Ruby `in` clause expression that can be used to match against nodes in the tree (can be generated using `stree expr`, `stree match`, or `Node#construct_keys`), and a block. Each node that matches the given query will be yielded to the block. The block will receive the node as its only argument. +### SyntaxTree.index(source) + +This function takes an input string containing Ruby code and returns a list of all of the class declarations, module declarations, and method definitions within a file. Each of the entries also has access to its associated comments. This is useful for generating documentation or index information for a file to support something like go-to-definition. + ## Nodes There are many different node types in the syntax tree. They are meant to be treated as immutable structs containing links to child nodes with minimal logic contained within their implementation. However, for the most part they all respond to a certain set of APIs, listed below. From 4cb8b9bb6745c6512bc34f12dd13c57d08b8a1d0 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 21 Feb 2023 10:50:24 -0500 Subject: [PATCH 408/536] Changelog for indexing --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 34c40e40..3548fa6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,14 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +### Added + +- The class declarations returned as the result of the indexing operation now have their superclass as a field. It is returned as an array of constants. If the superclass is anything other than a constant lookup, then it raises an error. + +### Changed + +- The `nesting` field on the results of the indexing operation is no longer a single flat array. Instead it is an array of arrays, where each array is a single nesting level. This more accurately reflects the nesting of the nodes in the tree. For example, `class Foo::Bar::Baz; end` would result in `[Foo, Bar, Baz]`, but that incorrectly implies that you can see constants at each of those levels. Now this would result in `[[Foo, Bar, Baz]]` to indicate that it can see either the top level or constants within the scope of `Foo::Bar::Baz` only. + ## [6.0.0] - 2023-02-10 ### Added From 3742be00e332b9910c6c0ebcf693c589e5c5da54 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Feb 2023 18:00:40 +0000 Subject: [PATCH 409/536] Bump rubocop from 1.45.1 to 1.46.0 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.45.1 to 1.46.0. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.45.1...v1.46.0) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 325d89b3..1995351b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -12,25 +12,25 @@ GEM json (2.6.3) minitest (5.17.0) parallel (1.22.1) - parser (3.2.0.0) + parser (3.2.1.0) ast (~> 2.4.1) prettier_print (1.2.0) rainbow (3.1.1) rake (13.0.6) - regexp_parser (2.6.2) + regexp_parser (2.7.0) rexml (3.2.5) - rubocop (1.45.1) + rubocop (1.46.0) json (~> 2.3) parallel (~> 1.10) parser (>= 3.2.0.0) rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 1.8, < 3.0) rexml (>= 3.2.5, < 4.0) - rubocop-ast (>= 1.24.1, < 2.0) + rubocop-ast (>= 1.26.0, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.24.1) - parser (>= 3.1.1.0) + rubocop-ast (1.26.0) + parser (>= 3.2.1.0) ruby-progressbar (1.11.0) simplecov (0.22.0) docile (~> 1.1) From 2993e27af7a87e369a7b6df0de6bd2fa646acafb Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Sun, 26 Feb 2023 12:38:22 -0500 Subject: [PATCH 410/536] Handle assoc value omission with mixed delimiters --- CHANGELOG.md | 1 + lib/syntax_tree/node.rb | 37 ++++++++++++++++++++++++++----------- test/fixtures/hash.rb | 2 ++ 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3548fa6e..27b1813f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ### Changed - The `nesting` field on the results of the indexing operation is no longer a single flat array. Instead it is an array of arrays, where each array is a single nesting level. This more accurately reflects the nesting of the nodes in the tree. For example, `class Foo::Bar::Baz; end` would result in `[Foo, Bar, Baz]`, but that incorrectly implies that you can see constants at each of those levels. Now this would result in `[[Foo, Bar, Baz]]` to indicate that it can see either the top level or constants within the scope of `Foo::Bar::Baz` only. +- When formatting hashes that have omitted values and mixed hash rockets with labels, the formatting now maintains whichever delimiter was used in the source. This is because forcing the use of hash rockets with omitted values results in a syntax error. ## [6.0.0] - 2023-02-10 diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 567ec0c8..dd4eb3ab 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -1780,13 +1780,25 @@ def format_key(q, key) end def self.for(container) - labels = - container.assocs.all? do |assoc| - next true if assoc.is_a?(AssocSplat) - + container.assocs.each do |assoc| + if assoc.is_a?(AssocSplat) + # Splat nodes do not impact the formatting choice. + elsif assoc.value.nil? + # If the value is nil, then it has been omitted. In this case we have + # to match the existing formatting because standardizing would + # potentially break the code. For example: + # + # { first:, "second" => "value" } + # + return Identity.new + else + # Otherwise, we need to check the type of the key. If it's a label or + # dynamic symbol, we can use labels. If it's a symbol literal then it + # needs to match a certain pattern to be used as a label. If it's + # anything else, then we need to use hash rockets. case assoc.key - when Label - true + when Label, DynaSymbol + # Here labels can be used. when SymbolLiteral # When attempting to convert a hash rocket into a hash label, # you need to take care because only certain patterns are @@ -1794,15 +1806,18 @@ def self.for(container) # arguments to methods, but don't specify what that is. After # some experimentation, it looks like it's: value = assoc.key.value.value - value.match?(/^[_A-Za-z]/) && !value.end_with?("=") - when DynaSymbol - true + + if !value.match?(/^[_A-Za-z]/) || value.end_with?("=") + return Rockets.new + end else - false + # If the value is anything else, we have to use hash rockets. + return Rockets.new end end + end - (labels ? Labels : Rockets).new + Labels.new end end diff --git a/test/fixtures/hash.rb b/test/fixtures/hash.rb index 9c43a4fe..70e89f69 100644 --- a/test/fixtures/hash.rb +++ b/test/fixtures/hash.rb @@ -29,3 +29,5 @@ { # comment } +% # >= 3.1.0 +{ foo:, "bar" => "baz" } From b0ba92edf5fc371243cb1875c892387515816532 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Sun, 26 Feb 2023 12:57:03 -0500 Subject: [PATCH 411/536] Handle keywords with bare hashes --- CHANGELOG.md | 1 + lib/syntax_tree/node.rb | 10 +++++++++- test/fixtures/break.rb | 2 ++ test/fixtures/next.rb | 2 ++ test/fixtures/return.rb | 2 ++ 5 files changed, 16 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 27b1813f..d3fd9964 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - The `nesting` field on the results of the indexing operation is no longer a single flat array. Instead it is an array of arrays, where each array is a single nesting level. This more accurately reflects the nesting of the nodes in the tree. For example, `class Foo::Bar::Baz; end` would result in `[Foo, Bar, Baz]`, but that incorrectly implies that you can see constants at each of those levels. Now this would result in `[[Foo, Bar, Baz]]` to indicate that it can see either the top level or constants within the scope of `Foo::Bar::Baz` only. - When formatting hashes that have omitted values and mixed hash rockets with labels, the formatting now maintains whichever delimiter was used in the source. This is because forcing the use of hash rockets with omitted values results in a syntax error. +- Handle the case where a bare hash is used after the `break`, `next`, or `return` keywords. Previously this would result in hash labels which is not valid syntax. Now it maintains the delimiters used in the source. ## [6.0.0] - 2023-02-10 diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index dd4eb3ab..2dbe3a79 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -1874,7 +1874,15 @@ def ===(other) end def format_key(q, key) - (@key_formatter ||= HashKeyFormatter.for(self)).format_key(q, key) + @key_formatter ||= + case q.parents.take(3).last + when Break, Next, ReturnNode + HashKeyFormatter::Identity.new + else + HashKeyFormatter.for(self) + end + + @key_formatter.format_key(q, key) end end diff --git a/test/fixtures/break.rb b/test/fixtures/break.rb index a608a6b2..519becda 100644 --- a/test/fixtures/break.rb +++ b/test/fixtures/break.rb @@ -33,3 +33,5 @@ qux end ) +% +break :foo => "bar" diff --git a/test/fixtures/next.rb b/test/fixtures/next.rb index 79a8c62e..66e90028 100644 --- a/test/fixtures/next.rb +++ b/test/fixtures/next.rb @@ -72,3 +72,5 @@ fun foo do end ) +% +next :foo => "bar" diff --git a/test/fixtures/return.rb b/test/fixtures/return.rb index 8f7d0aa3..7092464f 100644 --- a/test/fixtures/return.rb +++ b/test/fixtures/return.rb @@ -37,3 +37,5 @@ return [] % return [1] +% +return :foo => "bar" From 7dcc84396bf196b24b37165b9d38e6cde46265be Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Sun, 26 Feb 2023 13:11:52 -0500 Subject: [PATCH 412/536] Split up chained << expressions --- CHANGELOG.md | 1 + lib/syntax_tree/node.rb | 16 ++++++++++++---- test/fixtures/binary.rb | 5 +++++ 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d3fd9964..bb8425bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - The `nesting` field on the results of the indexing operation is no longer a single flat array. Instead it is an array of arrays, where each array is a single nesting level. This more accurately reflects the nesting of the nodes in the tree. For example, `class Foo::Bar::Baz; end` would result in `[Foo, Bar, Baz]`, but that incorrectly implies that you can see constants at each of those levels. Now this would result in `[[Foo, Bar, Baz]]` to indicate that it can see either the top level or constants within the scope of `Foo::Bar::Baz` only. - When formatting hashes that have omitted values and mixed hash rockets with labels, the formatting now maintains whichever delimiter was used in the source. This is because forcing the use of hash rockets with omitted values results in a syntax error. - Handle the case where a bare hash is used after the `break`, `next`, or `return` keywords. Previously this would result in hash labels which is not valid syntax. Now it maintains the delimiters used in the source. +- The `<<` operator will now break on chained `<<` expressions. Previously it would always stay flat. ## [6.0.0] - 2023-02-10 diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 2dbe3a79..c4bc1495 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -2097,10 +2097,7 @@ def format(q) q.group { q.format(left) } q.text(" ") unless power - if operator == :<< - q.text("<< ") - q.format(right) - else + if operator != :<< q.group do q.text(operator.name) q.indent do @@ -2108,6 +2105,17 @@ def format(q) q.format(right) end end + elsif left.is_a?(Binary) && left.operator == :<< + q.group do + q.text(operator.name) + q.indent do + power ? q.breakable_empty : q.breakable_space + q.format(right) + end + end + else + q.text("<< ") + q.format(right) end end end diff --git a/test/fixtures/binary.rb b/test/fixtures/binary.rb index f8833cdc..4cb56cbf 100644 --- a/test/fixtures/binary.rb +++ b/test/fixtures/binary.rb @@ -3,6 +3,11 @@ % foo << bar % +foo << barrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr << barrrrrrrrrrrrr << barrrrrrrrrrrrrrrrrr +- +foo << barrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr << barrrrrrrrrrrrr << + barrrrrrrrrrrrrrrrrr +% foo**bar % foo * barrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr From ff9094ac1364e78041872b38b642e4e1b5e21a1e Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Sun, 26 Feb 2023 14:15:34 -0500 Subject: [PATCH 413/536] Bump to v6.0.1 --- CHANGELOG.md | 5 ++++- Gemfile.lock | 2 +- lib/syntax_tree/version.rb | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb8425bd..018d5b25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +## [6.0.1] - 2023-02-26 + ### Added - The class declarations returned as the result of the indexing operation now have their superclass as a field. It is returned as an array of constants. If the superclass is anything other than a constant lookup, then it raises an error. @@ -570,7 +572,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - 🎉 Initial release! 🎉 -[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v6.0.0...HEAD +[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v6.0.1...HEAD +[6.0.1]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v6.0.0...v6.0.1 [6.0.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v5.3.0...v6.0.0 [5.3.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v5.2.0...v5.3.0 [5.2.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v5.1.0...v5.2.0 diff --git a/Gemfile.lock b/Gemfile.lock index 1995351b..c7ffc7d0 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - syntax_tree (6.0.0) + syntax_tree (6.0.1) prettier_print (>= 1.2.0) GEM diff --git a/lib/syntax_tree/version.rb b/lib/syntax_tree/version.rb index 1f028f89..0b3502d1 100644 --- a/lib/syntax_tree/version.rb +++ b/lib/syntax_tree/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SyntaxTree - VERSION = "6.0.0" + VERSION = "6.0.1" end From ed6e20624293cccd64d6a3f84a7c9f6071970da7 Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Mon, 27 Feb 2023 14:06:16 +0100 Subject: [PATCH 414/536] Disable SimpleCov on truffleruby as it adds around 5 seconds to tests for little value --- test/test_helper.rb | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/test/test_helper.rb b/test/test_helper.rb index 2c8f6466..f7f8be61 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -1,10 +1,12 @@ # frozen_string_literal: true -require "simplecov" -SimpleCov.start do - add_filter("idempotency_test.rb") unless ENV["CI"] - add_group("lib", "lib") - add_group("test", "test") +unless RUBY_ENGINE == "truffleruby" + require "simplecov" + SimpleCov.start do + add_filter("idempotency_test.rb") unless ENV["CI"] + add_group("lib", "lib") + add_group("test", "test") + end end $LOAD_PATH.unshift(File.expand_path("../lib", __dir__)) From d3410ffad8ac4f9fbc53eecc111a3dc84b1c6e52 Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Mon, 27 Feb 2023 14:06:27 +0100 Subject: [PATCH 415/536] Disable reflection type verification on truffleruby * It fails transiently and Ripper does not seem to provide any way to investigate the actual error/exception. --- lib/syntax_tree/reflection.rb | 3 +- test/test_helper.rb | 58 ++++++++++++++++++----------------- 2 files changed, 32 insertions(+), 29 deletions(-) diff --git a/lib/syntax_tree/reflection.rb b/lib/syntax_tree/reflection.rb index bf4b95f3..b2ffec6d 100644 --- a/lib/syntax_tree/reflection.rb +++ b/lib/syntax_tree/reflection.rb @@ -176,7 +176,8 @@ def parse_comments(statements, index) program = SyntaxTree.parse(SyntaxTree.read(File.expand_path("node.rb", __dir__))) - main_statements = program.statements.body.last.bodystmt.statements.body + program_statements = program.statements + main_statements = program_statements.body.last.bodystmt.statements.body main_statements.each_with_index do |main_statement, main_statement_index| # Ensure we are only looking at class declarations. next unless main_statement.is_a?(SyntaxTree::ClassDeclaration) diff --git a/test/test_helper.rb b/test/test_helper.rb index f7f8be61..8015be14 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -13,36 +13,38 @@ require "syntax_tree" require "syntax_tree/cli" -# Here we are going to establish type verification whenever a new node is -# created. We do this through the reflection module, which in turn parses the -# source code of the node classes. -require "syntax_tree/reflection" -SyntaxTree::Reflection.nodes.each do |name, node| - next if name == :Statements - - clazz = SyntaxTree.const_get(name) - parameters = clazz.instance_method(:initialize).parameters - - # First, verify that all of the parameters listed in the list of attributes. - # If there are any parameters that aren't listed in the attributes, then - # something went wrong with the parsing in the reflection module. - raise unless (parameters.map(&:last) - node.attributes.keys).empty? - - # Now we're going to use an alias chain to redefine the initialize method to - # include type checking. - clazz.alias_method(:initialize_without_verify, :initialize) - clazz.define_method(:initialize) do |**kwargs| - kwargs.each do |kwarg, value| - attribute = node.attributes.fetch(kwarg) - - unless attribute.type === value - raise TypeError, - "invalid type for #{name}##{kwarg}, expected " \ - "#{attribute.type.inspect}, got #{value.inspect}" +unless RUBY_ENGINE == "truffleruby" + # Here we are going to establish type verification whenever a new node is + # created. We do this through the reflection module, which in turn parses the + # source code of the node classes. + require "syntax_tree/reflection" + SyntaxTree::Reflection.nodes.each do |name, node| + next if name == :Statements + + clazz = SyntaxTree.const_get(name) + parameters = clazz.instance_method(:initialize).parameters + + # First, verify that all of the parameters listed in the list of attributes. + # If there are any parameters that aren't listed in the attributes, then + # something went wrong with the parsing in the reflection module. + raise unless (parameters.map(&:last) - node.attributes.keys).empty? + + # Now we're going to use an alias chain to redefine the initialize method to + # include type checking. + clazz.alias_method(:initialize_without_verify, :initialize) + clazz.define_method(:initialize) do |**kwargs| + kwargs.each do |kwarg, value| + attribute = node.attributes.fetch(kwarg) + + unless attribute.type === value + raise TypeError, + "invalid type for #{name}##{kwarg}, expected " \ + "#{attribute.type.inspect}, got #{value.inspect}" + end end - end - initialize_without_verify(**kwargs) + initialize_without_verify(**kwargs) + end end end From 58d2ab4718f197ae39010329a636e3d6cc47ef0b Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 27 Feb 2023 15:27:05 -0500 Subject: [PATCH 416/536] Fix AST output for Command and CommandCall nodes --- CHANGELOG.md | 4 ++++ lib/syntax_tree/field_visitor.rb | 2 ++ 2 files changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 018d5b25..b06032f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +### Changed + +- Fixed the AST output by adding blocks to `Command` and `CommandCall` nodes in the `FieldVisitor`. + ## [6.0.1] - 2023-02-26 ### Added diff --git a/lib/syntax_tree/field_visitor.rb b/lib/syntax_tree/field_visitor.rb index ca1df55b..f5607c67 100644 --- a/lib/syntax_tree/field_visitor.rb +++ b/lib/syntax_tree/field_visitor.rb @@ -263,6 +263,7 @@ def visit_command(node) node(node, "command") do field("message", node.message) field("arguments", node.arguments) + field("block", node.block) if node.block comments(node) end end @@ -273,6 +274,7 @@ def visit_command_call(node) field("operator", node.operator) field("message", node.message) field("arguments", node.arguments) if node.arguments + field("block", node.block) if node.block comments(node) end end From ead887486315177a79d808a163fb986b618f1041 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Mar 2023 18:02:43 +0000 Subject: [PATCH 417/536] Bump rubocop from 1.46.0 to 1.47.0 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.46.0 to 1.47.0. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.46.0...v1.47.0) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index c7ffc7d0..208850fe 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -19,7 +19,7 @@ GEM rake (13.0.6) regexp_parser (2.7.0) rexml (3.2.5) - rubocop (1.46.0) + rubocop (1.47.0) json (~> 2.3) parallel (~> 1.10) parser (>= 3.2.0.0) @@ -29,9 +29,9 @@ GEM rubocop-ast (>= 1.26.0, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.26.0) + rubocop-ast (1.27.0) parser (>= 3.2.1.0) - ruby-progressbar (1.11.0) + ruby-progressbar (1.12.0) simplecov (0.22.0) docile (~> 1.1) simplecov-html (~> 0.11) From 2367cb49884d076ec6d137930c72a3f89dec524e Mon Sep 17 00:00:00 2001 From: Vinicius Stock Date: Thu, 2 Mar 2023 17:05:13 -0500 Subject: [PATCH 418/536] Fix lambda local locations Co-authored-by: Kevin Newton --- lib/syntax_tree/parser.rb | 6 +++++ test/parser_test.rb | 48 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index d0a5bf67..ed0de408 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -2391,8 +2391,14 @@ def lambda_locals(source) } } + parent_line = lineno - 1 + parent_column = + consume_token(Semicolon).location.start_column - tokens[index][0][1] + tokens[(index + 1)..].each_with_object([]) do |token, locals| (lineno, column), type, value, = token + column += parent_column if lineno == 1 + lineno += parent_line # Make the state transition for the parser. If there isn't a transition # from the current state to a new state for this type, then we're in a diff --git a/test/parser_test.rb b/test/parser_test.rb index 8d6c0a16..7ac07381 100644 --- a/test/parser_test.rb +++ b/test/parser_test.rb @@ -74,5 +74,53 @@ def test_does_not_choke_on_invalid_characters_in_source_string \xC5 RUBY end + + def test_lambda_vars_with_parameters_location + tree = SyntaxTree.parse(<<~RUBY) + # comment + # comment + ->(_i; a) { a } + RUBY + + local_location = + tree.statements.body.last.params.contents.locals.first.location + + assert_equal(3, local_location.start_line) + assert_equal(3, local_location.end_line) + assert_equal(7, local_location.start_column) + assert_equal(8, local_location.end_column) + end + + def test_lambda_vars_location + tree = SyntaxTree.parse(<<~RUBY) + # comment + # comment + ->(; a) { a } + RUBY + + local_location = + tree.statements.body.last.params.contents.locals.first.location + + assert_equal(3, local_location.start_line) + assert_equal(3, local_location.end_line) + assert_equal(5, local_location.start_column) + assert_equal(6, local_location.end_column) + end + + def test_multiple_lambda_vars_location + tree = SyntaxTree.parse(<<~RUBY) + # comment + # comment + ->(; a, b, c) { a } + RUBY + + local_location = + tree.statements.body.last.params.contents.locals.last.location + + assert_equal(3, local_location.start_line) + assert_equal(3, local_location.end_line) + assert_equal(11, local_location.start_column) + assert_equal(12, local_location.end_column) + end end end From ac63bef6bd59dbe93632b5acff5acc57bca6db26 Mon Sep 17 00:00:00 2001 From: Vinicius Stock Date: Thu, 2 Mar 2023 16:06:30 -0500 Subject: [PATCH 419/536] Add support for regexp locals Co-authored-by: Kevin Newton --- lib/syntax_tree/with_scope.rb | 57 +++++++++++++++++++++++++++++++++++ test/with_scope_test.rb | 38 +++++++++++++++++++++++ 2 files changed, 95 insertions(+) diff --git a/lib/syntax_tree/with_scope.rb b/lib/syntax_tree/with_scope.rb index 7fcef067..2f691927 100644 --- a/lib/syntax_tree/with_scope.rb +++ b/lib/syntax_tree/with_scope.rb @@ -217,6 +217,63 @@ def visit_var_ref(node) super end + # When using regex named capture groups, vcalls might actually be a variable + def visit_vcall(node) + value = node.value + definition = current_scope.find_local(value.value) + current_scope.add_local_usage(value, definition.type) if definition + + super + end + + # Visit for capturing local variables defined in regex named capture groups + def visit_binary(node) + if node.operator == :=~ + left = node.left + + if left.is_a?(RegexpLiteral) && left.parts.length == 1 && + left.parts.first.is_a?(TStringContent) + content = left.parts.first + + value = content.value + location = content.location + start_line = location.start_line + + Regexp + .new(value, Regexp::FIXEDENCODING) + .names + .each do |name| + offset = value.index(/\(\?<#{Regexp.escape(name)}>/) + line = start_line + value[0...offset].count("\n") + + # We need to add 3 to account for these three characters + # prefixing a named capture (?< + column = location.start_column + offset + 3 + if value[0...offset].include?("\n") + column = + value[0...offset].length - value[0...offset].rindex("\n") + + 3 - 1 + end + + ident_location = + Location.new( + start_line: line, + start_char: location.start_char + offset, + start_column: column, + end_line: line, + end_char: location.start_char + offset + name.length, + end_column: column + name.length + ) + + identifier = Ident.new(value: name, location: ident_location) + current_scope.add_local_definition(identifier, :variable) + end + end + end + + super + end + private def add_argument_definitions(list) diff --git a/test/with_scope_test.rb b/test/with_scope_test.rb index 9675e811..bb804df5 100644 --- a/test/with_scope_test.rb +++ b/test/with_scope_test.rb @@ -39,6 +39,13 @@ def visit_label(node) arguments[[current_scope.id, value]] = node end end + + def visit_vcall(node) + local = current_scope.find_local(node.value) + variables[[current_scope.id, value]] = local if local + + super + end end end @@ -349,6 +356,37 @@ def test_double_nested_arguments assert_argument(collector, "four", definitions: [1], usages: [5]) end + def test_regex_named_capture_groups + collector = Collector.collect(<<~RUBY) + if /(?\\w+)-(?\\w+)/ =~ "something-else" + one + two + end + RUBY + + assert_equal(2, collector.variables.length) + + assert_variable(collector, "one", definitions: [1], usages: [2]) + assert_variable(collector, "two", definitions: [1], usages: [3]) + end + + def test_multiline_regex_named_capture_groups + collector = Collector.collect(<<~RUBY) + if %r{ + (?\\w+)- + (?\\w+) + } =~ "something-else" + one + two + end + RUBY + + assert_equal(2, collector.variables.length) + + assert_variable(collector, "one", definitions: [2], usages: [5]) + assert_variable(collector, "two", definitions: [3], usages: [6]) + end + class Resolver < Visitor prepend WithScope From 039c0874e08316e3f9a9c80f7838177ccad1cd6c Mon Sep 17 00:00:00 2001 From: Vinicius Stock Date: Thu, 2 Mar 2023 17:04:42 -0500 Subject: [PATCH 420/536] Add support for lambda and block locals Co-authored-by: Kevin Newton --- lib/syntax_tree/with_scope.rb | 9 +++++++++ test/with_scope_test.rb | 21 +++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/lib/syntax_tree/with_scope.rb b/lib/syntax_tree/with_scope.rb index 2f691927..f7c3a203 100644 --- a/lib/syntax_tree/with_scope.rb +++ b/lib/syntax_tree/with_scope.rb @@ -189,6 +189,15 @@ def visit_blockarg(node) super end + def visit_block_var(node) + node.locals.each do |local| + current_scope.add_local_definition(local, :variable) + end + + super + end + alias visit_lambda_var visit_block_var + # Visit for keeping track of local variable definitions def visit_var_field(node) value = node.value diff --git a/test/with_scope_test.rb b/test/with_scope_test.rb index bb804df5..6928a322 100644 --- a/test/with_scope_test.rb +++ b/test/with_scope_test.rb @@ -356,6 +356,27 @@ def test_double_nested_arguments assert_argument(collector, "four", definitions: [1], usages: [5]) end + def test_block_locals + collector = Collector.collect(<<~RUBY) + [].each do |; a| + end + RUBY + + assert_equal(1, collector.variables.length) + + assert_variable(collector, "a", definitions: [1]) + end + + def test_lambda_locals + collector = Collector.collect(<<~RUBY) + ->(;a) { } + RUBY + + assert_equal(1, collector.variables.length) + + assert_variable(collector, "a", definitions: [1]) + end + def test_regex_named_capture_groups collector = Collector.collect(<<~RUBY) if /(?\\w+)-(?\\w+)/ =~ "something-else" From 48a630ac84cb5faa3734a5a2560d0a562d54951d Mon Sep 17 00:00:00 2001 From: Vinicius Stock Date: Thu, 2 Mar 2023 17:14:00 -0500 Subject: [PATCH 421/536] Fix test for pinned variable Co-authored-by: Kevin Newton --- test/with_scope_test.rb | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/test/with_scope_test.rb b/test/with_scope_test.rb index 6928a322..eb88576a 100644 --- a/test/with_scope_test.rb +++ b/test/with_scope_test.rb @@ -117,11 +117,7 @@ def foo assert_equal(2, collector.variables.length) assert_variable(collector, "a", definitions: [2], usages: [4, 5]) - assert_variable(collector, "rest", definitions: [4]) - - # Rest is considered a vcall by the parser instead of a var_ref - # assert_equal(1, variable_rest.usages.length) - # assert_equal(6, variable_rest.usages[0].start_line) + assert_variable(collector, "rest", definitions: [4], usages: [6]) end if RUBY_VERSION >= "3.1" From 77bdc1275514ad225d6157c70c1d9a465e9d2549 Mon Sep 17 00:00:00 2001 From: Vinicius Stock Date: Thu, 2 Mar 2023 15:13:26 -0500 Subject: [PATCH 422/536] Fix WithScope visits for deconstructed block params Co-authored-by: Kevin Newton --- lib/syntax_tree/with_scope.rb | 6 +++++- test/with_scope_test.rb | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/lib/syntax_tree/with_scope.rb b/lib/syntax_tree/with_scope.rb index 7fcef067..1a3d74fa 100644 --- a/lib/syntax_tree/with_scope.rb +++ b/lib/syntax_tree/with_scope.rb @@ -221,7 +221,11 @@ def visit_var_ref(node) def add_argument_definitions(list) list.each do |param| - if param.is_a?(SyntaxTree::MLHSParen) + case param + when ArgStar + value = param.value + current_scope.add_local_definition(value, :argument) if value + when MLHSParen add_argument_definitions(param.contents.parts) else current_scope.add_local_definition(param, :argument) diff --git a/test/with_scope_test.rb b/test/with_scope_test.rb index 9675e811..9a4189a5 100644 --- a/test/with_scope_test.rb +++ b/test/with_scope_test.rb @@ -198,6 +198,25 @@ def foo assert_argument(collector, "i", definitions: [2], usages: [3]) end + def test_collecting_destructured_block_arguments + collector = Collector.collect(<<~RUBY) + [].each do |(a, *b)| + end + RUBY + + assert_equal(2, collector.arguments.length) + assert_argument(collector, "b", definitions: [1]) + end + + def test_collecting_anonymous_destructured_block_arguments + collector = Collector.collect(<<~RUBY) + [].each do |(a, *)| + end + RUBY + + assert_equal(1, collector.arguments.length) + end + def test_collecting_one_line_block_arguments collector = Collector.collect(<<~RUBY) def foo From 4864692d4a27d1a39fb2fc0bbd5500ab4a2c85d3 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 3 Mar 2023 09:10:44 -0500 Subject: [PATCH 423/536] Bump to version 6.0.2 --- CHANGELOG.md | 11 ++++++++++- Gemfile.lock | 2 +- lib/syntax_tree/version.rb | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b06032f7..960bb0e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,9 +6,17 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +## [6.0.2] - 2023-03-03 + +### Added + +- The `WithScope` visitor mixin will now additionally report local variables defined through regular expression named captures. +- The `WithScope` visitor mixin now properly handles destructured splat arguments in required positions. + ### Changed - Fixed the AST output by adding blocks to `Command` and `CommandCall` nodes in the `FieldVisitor`. +- Fixed the location of lambda local variables (e.g., `->(; a) {}`). ## [6.0.1] - 2023-02-26 @@ -576,7 +584,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - 🎉 Initial release! 🎉 -[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v6.0.1...HEAD +[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v6.0.2...HEAD +[6.0.2]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v6.0.1...v6.0.2 [6.0.1]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v6.0.0...v6.0.1 [6.0.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v5.3.0...v6.0.0 [5.3.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v5.2.0...v5.3.0 diff --git a/Gemfile.lock b/Gemfile.lock index 208850fe..735a5025 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - syntax_tree (6.0.1) + syntax_tree (6.0.2) prettier_print (>= 1.2.0) GEM diff --git a/lib/syntax_tree/version.rb b/lib/syntax_tree/version.rb index 0b3502d1..ff3db370 100644 --- a/lib/syntax_tree/version.rb +++ b/lib/syntax_tree/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SyntaxTree - VERSION = "6.0.1" + VERSION = "6.0.2" end From 528662658ec016c41fd4e510d3f180cf22b81783 Mon Sep 17 00:00:00 2001 From: Nolan Date: Sat, 4 Mar 2023 21:21:49 -0800 Subject: [PATCH 424/536] List editor support for Emacs --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 03942d46..aa57eabf 100644 --- a/README.md +++ b/README.md @@ -788,6 +788,7 @@ inherit_gem: * [Neovim](https://neovim.io/) - [neovim/nvim-lspconfig](https://github.com/neovim/nvim-lspconfig). * [Vim](https://www.vim.org/) - [dense-analysis/ale](https://github.com/dense-analysis/ale). * [VSCode](https://code.visualstudio.com/) - [ruby-syntax-tree/vscode-syntax-tree](https://github.com/ruby-syntax-tree/vscode-syntax-tree). +* [Emacs](https://www.gnu.org/software/emacs/) - [emacs-format-all-the-code](https://github.com/lassik/emacs-format-all-the-code). ## Contributing From a5a071091ccfd7cbd045b4998d1321fc6389d996 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Sun, 5 Mar 2023 13:06:38 -0500 Subject: [PATCH 425/536] Capture alias methods in index --- lib/syntax_tree/index.rb | 47 +++++++++++++++++++++++++++++++++++++++- test/index_test.rb | 7 ++++++ 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/lib/syntax_tree/index.rb b/lib/syntax_tree/index.rb index c6973847..4e84ab2a 100644 --- a/lib/syntax_tree/index.rb +++ b/lib/syntax_tree/index.rb @@ -68,6 +68,19 @@ def initialize(nesting, name, location, comments) end end + # This entry represents a method definition that was created using the alias + # keyword. + class AliasMethodDefinition + attr_reader :nesting, :name, :location, :comments + + def initialize(nesting, name, location, comments) + @nesting = nesting + @name = name + @location = location + @comments = comments + end + end + # When you're using the instruction sequence backend, this class is used to # lazily parse comments out of the source code. class FileComments @@ -297,7 +310,7 @@ def index_iseq(iseq, file_comments) EntryComments.new(file_comments, location) ) when :definesmethod - if current_iseq[13][index - 1] != [:putself] + if insns[index - 1] != [:putself] raise NotImplementedError, "singleton method with non-self receiver" end @@ -309,6 +322,24 @@ def index_iseq(iseq, file_comments) location, EntryComments.new(file_comments, location) ) + when :opt_send_without_block, :send + if insn[1][:mid] == :"core#set_method_alias" + # Now we have to validate that the alias is happening with a + # non-interpolated value. To do this we'll match the specific + # pattern we're expecting. + values = insns[(index - 4)...index].map { |insn| insn.is_a?(Array) ? insn[0] : insn } + next if values != %i[putspecialobject putspecialobject putobject putobject] + + # Now that we know it's in the structure we want it, we can use + # the values of the putobject to determine the alias. + location = Location.new(line, 0) + results << AliasMethodDefinition.new( + current_nesting, + insns[index - 2][1], + location, + EntryComments.new(file_comments, location) + ) + end end end end @@ -331,6 +362,20 @@ def initialize end visit_methods do + def visit_alias(node) + if node.left.is_a?(SymbolLiteral) && node.right.is_a?(SymbolLiteral) + location = + Location.new(node.location.start_line, node.location.start_column) + + results << AliasMethodDefinition.new( + nesting.dup, + node.left.value.value.to_sym, + location, + comments_for(node) + ) + end + end + def visit_class(node) names = visit(node.constant) nesting << names diff --git a/test/index_test.rb b/test/index_test.rb index 60c51d9d..0813dc02 100644 --- a/test/index_test.rb +++ b/test/index_test.rb @@ -139,6 +139,13 @@ def test_singleton_method_comments end end + def test_alias_method + index_each("alias foo bar") do |entry| + assert_equal :foo, entry.name + assert_empty entry.nesting + end + end + def test_this_file entries = Index.index_file(__FILE__, backend: Index::ParserBackend.new) From 31e4a4724c495017f65d674a5211ddb9cb7349e9 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Sun, 5 Mar 2023 16:35:03 -0500 Subject: [PATCH 426/536] Index attr_readers --- lib/syntax_tree/index.rb | 83 +++++++++++++++++++++++++++++++++------- test/index_test.rb | 7 ++++ 2 files changed, 76 insertions(+), 14 deletions(-) diff --git a/lib/syntax_tree/index.rb b/lib/syntax_tree/index.rb index 4e84ab2a..f0788619 100644 --- a/lib/syntax_tree/index.rb +++ b/lib/syntax_tree/index.rb @@ -323,7 +323,34 @@ def index_iseq(iseq, file_comments) EntryComments.new(file_comments, location) ) when :opt_send_without_block, :send - if insn[1][:mid] == :"core#set_method_alias" + case insn[1][:mid] + when :attr_reader + # We're going to scan backward finding symbols until we hit a + # different instruction. We'll then use that to determine the + # receiver. It needs to be self if we're going to understand it. + names = [] + current = index - 1 + + while current >= 0 && names.length < insn[1][:orig_argc] + if insns[current].is_a?(Array) && insns[current][0] == :putobject + names.unshift(insns[current][1]) + end + + current -= 1 + end + + next if insns[current] != [:putself] + + location = Location.new(line, 0) + names.each do |name| + results << MethodDefinition.new( + current_nesting, + name, + location, + EntryComments.new(file_comments, location) + ) + end + when :"core#set_method_alias" # Now we have to validate that the alias is happening with a # non-interpolated value. To do this we'll match the specific # pattern we're expecting. @@ -352,6 +379,20 @@ def index_iseq(iseq, file_comments) # It is not as fast as using the instruction sequences directly, but is # supported on all runtimes. class ParserBackend + class ConstantNameVisitor < Visitor + def visit_const_ref(node) + [node.constant.value.to_sym] + end + + def visit_const_path_ref(node) + visit(node.parent) << node.constant.value.to_sym + end + + def visit_var_ref(node) + [node.value.value.to_sym] + end + end + class IndexVisitor < Visitor attr_reader :results, :nesting, :statements @@ -374,10 +415,12 @@ def visit_alias(node) comments_for(node) ) end + + super end def visit_class(node) - names = visit(node.constant) + names = node.constant.accept(ConstantNameVisitor.new) nesting << names location = @@ -385,7 +428,7 @@ def visit_class(node) superclass = if node.superclass - visited = visit(node.superclass) + visited = node.superclass.accept(ConstantNameVisitor.new) if visited == [[]] raise NotImplementedError, "superclass with non constant path" @@ -408,12 +451,24 @@ def visit_class(node) nesting.pop end - def visit_const_ref(node) - [node.constant.value.to_sym] - end + def visit_command(node) + if node.message.value == "attr_reader" + location = + Location.new(node.location.start_line, node.location.start_column) + + node.arguments.parts.each do |argument| + next unless argument.is_a?(SymbolLiteral) + + results << MethodDefinition.new( + nesting.dup, + argument.value.value.to_sym, + location, + comments_for(node) + ) + end + end - def visit_const_path_ref(node) - visit(node.parent) << node.constant.value.to_sym + super end def visit_def(node) @@ -436,10 +491,12 @@ def visit_def(node) comments_for(node) ) end + + super end def visit_module(node) - names = visit(node.constant) + names = node.constant.accept(ConstantNameVisitor.new) nesting << names location = @@ -465,10 +522,6 @@ def visit_statements(node) @statements = node super end - - def visit_var_ref(node) - [node.value.value.to_sym] - end end private @@ -478,8 +531,10 @@ def comments_for(node) body = statements.body line = node.location.start_line - 1 - index = body.index(node) - 1 + index = body.index(node) + return comments if index.nil? + index -= 1 while index >= 0 && body[index].is_a?(Comment) && (line - body[index].location.start_line < 2) comments.unshift(body[index].value) diff --git a/test/index_test.rb b/test/index_test.rb index 0813dc02..41c9495f 100644 --- a/test/index_test.rb +++ b/test/index_test.rb @@ -146,6 +146,13 @@ def test_alias_method end end + def test_attr_reader + index_each("attr_reader :foo") do |entry| + assert_equal :foo, entry.name + assert_empty entry.nesting + end + end + def test_this_file entries = Index.index_file(__FILE__, backend: Index::ParserBackend.new) From ee2db3ff99a68756d10fc7eb522a11a7c7dfe5bf Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Sun, 5 Mar 2023 17:07:37 -0500 Subject: [PATCH 427/536] attr_writer and attr_accessor --- lib/syntax_tree/index.rb | 76 +++++++++++++++++++++++++--------------- test/index_test.rb | 14 ++++++++ 2 files changed, 61 insertions(+), 29 deletions(-) diff --git a/lib/syntax_tree/index.rb b/lib/syntax_tree/index.rb index f0788619..ad090f95 100644 --- a/lib/syntax_tree/index.rb +++ b/lib/syntax_tree/index.rb @@ -220,6 +220,22 @@ def find_constant_path(insns, index) end end + def find_attr_arguments(insns, index) + orig_argc = insns[index][1][:orig_argc] + names = [] + + current = index - 1 + while current >= 0 && names.length < orig_argc + if insns[current].is_a?(Array) && insns[current][0] == :putobject + names.unshift(insns[current][1]) + end + + current -= 1 + end + + names if insns[current] == [:putself] && names.length == orig_argc + end + def index_iseq(iseq, file_comments) results = [] queue = [[iseq, []]] @@ -324,31 +340,29 @@ def index_iseq(iseq, file_comments) ) when :opt_send_without_block, :send case insn[1][:mid] - when :attr_reader - # We're going to scan backward finding symbols until we hit a - # different instruction. We'll then use that to determine the - # receiver. It needs to be self if we're going to understand it. - names = [] - current = index - 1 - - while current >= 0 && names.length < insn[1][:orig_argc] - if insns[current].is_a?(Array) && insns[current][0] == :putobject - names.unshift(insns[current][1]) - end - - current -= 1 - end - - next if insns[current] != [:putself] + when :attr_reader, :attr_writer, :attr_accessor + names = find_attr_arguments(insns, index) + next unless names location = Location.new(line, 0) names.each do |name| - results << MethodDefinition.new( - current_nesting, - name, - location, - EntryComments.new(file_comments, location) - ) + if insn[1][:mid] != :attr_writer + results << MethodDefinition.new( + current_nesting, + name, + location, + EntryComments.new(file_comments, location) + ) + end + + if insn[1][:mid] != :attr_reader + results << MethodDefinition.new( + current_nesting, + :"#{name}=", + location, + EntryComments.new(file_comments, location) + ) + end end when :"core#set_method_alias" # Now we have to validate that the alias is happening with a @@ -452,19 +466,23 @@ def visit_class(node) end def visit_command(node) - if node.message.value == "attr_reader" + case node.message.value + when "attr_reader", "attr_writer", "attr_accessor" + comments = comments_for(node) location = Location.new(node.location.start_line, node.location.start_column) node.arguments.parts.each do |argument| next unless argument.is_a?(SymbolLiteral) + name = argument.value.value.to_sym - results << MethodDefinition.new( - nesting.dup, - argument.value.value.to_sym, - location, - comments_for(node) - ) + if node.message.value != "attr_writer" + results << MethodDefinition.new(nesting.dup, name, location, comments) + end + + if node.message.value != "attr_reader" + results << MethodDefinition.new(nesting.dup, :"#{name}=", location, comments) + end end end diff --git a/test/index_test.rb b/test/index_test.rb index 41c9495f..42da9704 100644 --- a/test/index_test.rb +++ b/test/index_test.rb @@ -153,6 +153,20 @@ def test_attr_reader end end + def test_attr_writer + index_each("attr_writer :foo") do |entry| + assert_equal :foo=, entry.name + assert_empty entry.nesting + end + end + + def test_attr_accessor + index_each("attr_accessor :foo") do |entry| + assert_equal :foo=, entry.name + assert_empty entry.nesting + end + end + def test_this_file entries = Index.index_file(__FILE__, backend: Index::ParserBackend.new) From f712366084241bd7c0ab38da768f3d56f5705399 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Sun, 5 Mar 2023 17:30:51 -0500 Subject: [PATCH 428/536] Constant definitions --- lib/syntax_tree/index.rb | 44 +++++++++++++++++++++++++++++++++++++++- test/index_test.rb | 7 +++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/lib/syntax_tree/index.rb b/lib/syntax_tree/index.rb index ad090f95..35dbb898 100644 --- a/lib/syntax_tree/index.rb +++ b/lib/syntax_tree/index.rb @@ -31,6 +31,18 @@ def initialize(nesting, name, superclass, location, comments) end end + # This entry represents a constant assignment. + class ConstantDefinition + attr_reader :nesting, :name, :location, :comments + + def initialize(nesting, name, location, comments) + @nesting = nesting + @name = name + @location = location + @comments = comments + end + end + # This entry represents a module definition using the module keyword. class ModuleDefinition attr_reader :nesting, :name, :location, :comments @@ -191,7 +203,7 @@ def location_for(iseq) end def find_constant_path(insns, index) - index -= 1 while insns[index].is_a?(Integer) + index -= 1 while index >= 0 && (insns[index].is_a?(Integer) || (insns[index].is_a?(Array) && %i[swap topn].include?(insns[index][0]))) insn = insns[index] if insn.is_a?(Array) && insn[0] == :opt_getconstant_path @@ -338,6 +350,20 @@ def index_iseq(iseq, file_comments) location, EntryComments.new(file_comments, location) ) + when :setconstant + next_nesting = current_nesting.dup + name = insn[1] + + _, nesting = find_constant_path(insns, index - 1) + next_nesting << nesting if nesting.any? + + location = Location.new(line, 0) + results << ConstantDefinition.new( + next_nesting, + name, + location, + EntryComments.new(file_comments, location) + ) when :opt_send_without_block, :send case insn[1][:mid] when :attr_reader, :attr_writer, :attr_accessor @@ -433,6 +459,22 @@ def visit_alias(node) super end + def visit_assign(node) + if node.target.is_a?(VarField) && node.target.value.is_a?(Const) + location = + Location.new(node.location.start_line, node.location.start_column) + + results << ConstantDefinition.new( + nesting.dup, + node.target.value.value.to_sym, + location, + comments_for(node) + ) + end + + super + end + def visit_class(node) names = node.constant.accept(ConstantNameVisitor.new) nesting << names diff --git a/test/index_test.rb b/test/index_test.rb index 42da9704..855e36ec 100644 --- a/test/index_test.rb +++ b/test/index_test.rb @@ -167,6 +167,13 @@ def test_attr_accessor end end + def test_constant + index_each("FOO = 1") do |entry| + assert_equal :FOO, entry.name + assert_empty entry.nesting + end + end + def test_this_file entries = Index.index_file(__FILE__, backend: Index::ParserBackend.new) From 474931b89d847f17b40f9df8e942c26fb927b539 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Sun, 5 Mar 2023 19:39:20 -0500 Subject: [PATCH 429/536] Correctly set singleton method status --- lib/syntax_tree/index.rb | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/lib/syntax_tree/index.rb b/lib/syntax_tree/index.rb index 35dbb898..c5945470 100644 --- a/lib/syntax_tree/index.rb +++ b/lib/syntax_tree/index.rb @@ -248,6 +248,16 @@ def find_attr_arguments(insns, index) names if insns[current] == [:putself] && names.length == orig_argc end + def method_definition(nesting, name, location, file_comments) + comments = EntryComments.new(file_comments, location) + + if nesting.last == [:singletonclass] + SingletonMethodDefinition.new(nesting[0...-1], name, location, comments) + else + MethodDefinition.new(nesting, name, location, comments) + end + end + def index_iseq(iseq, file_comments) results = [] queue = [[iseq, []]] @@ -331,12 +341,7 @@ def index_iseq(iseq, file_comments) queue << [class_iseq, next_nesting] when :definemethod location = location_for(insn[2]) - results << MethodDefinition.new( - current_nesting, - insn[1], - location, - EntryComments.new(file_comments, location) - ) + results << method_definition(current_nesting, insn[1], location, file_comments) when :definesmethod if insns[index - 1] != [:putself] raise NotImplementedError, @@ -373,21 +378,11 @@ def index_iseq(iseq, file_comments) location = Location.new(line, 0) names.each do |name| if insn[1][:mid] != :attr_writer - results << MethodDefinition.new( - current_nesting, - name, - location, - EntryComments.new(file_comments, location) - ) + results << method_definition(current_nesting, name, location, file_comments) end if insn[1][:mid] != :attr_reader - results << MethodDefinition.new( - current_nesting, - :"#{name}=", - location, - EntryComments.new(file_comments, location) - ) + results << method_definition(current_nesting, :"#{name}=", location, file_comments) end end when :"core#set_method_alias" From eeea72003fd36ad4e72f3fc6339995b1186ffb86 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Sun, 5 Mar 2023 19:46:47 -0500 Subject: [PATCH 430/536] Explicitly specify that locations can have :unknown columns --- lib/syntax_tree/index.rb | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/syntax_tree/index.rb b/lib/syntax_tree/index.rb index c5945470..7865a949 100644 --- a/lib/syntax_tree/index.rb +++ b/lib/syntax_tree/index.rb @@ -297,8 +297,8 @@ def index_iseq(iseq, file_comments) find_constant_path(insns, index - 1) if superclass.empty? - raise NotImplementedError, - "superclass with non constant path on line #{line}" + warn("superclass with non constant path on line #{line}") + next end end @@ -316,8 +316,8 @@ def index_iseq(iseq, file_comments) # defined on self. We could, but it would require more # emulation. if insns[index - 2] != [:putself] - raise NotImplementedError, - "singleton class with non-self receiver" + warn("singleton class with non-self receiver") + next end elsif flags & VM_DEFINECLASS_TYPE_MODULE > 0 location = location_for(class_iseq) @@ -344,8 +344,8 @@ def index_iseq(iseq, file_comments) results << method_definition(current_nesting, insn[1], location, file_comments) when :definesmethod if insns[index - 1] != [:putself] - raise NotImplementedError, - "singleton method with non-self receiver" + warn("singleton method with non-self receiver") + next end location = location_for(insn[2]) @@ -362,7 +362,7 @@ def index_iseq(iseq, file_comments) _, nesting = find_constant_path(insns, index - 1) next_nesting << nesting if nesting.any? - location = Location.new(line, 0) + location = Location.new(line, :unknown) results << ConstantDefinition.new( next_nesting, name, @@ -375,7 +375,7 @@ def index_iseq(iseq, file_comments) names = find_attr_arguments(insns, index) next unless names - location = Location.new(line, 0) + location = Location.new(line, :unknown) names.each do |name| if insn[1][:mid] != :attr_writer results << method_definition(current_nesting, name, location, file_comments) @@ -394,7 +394,7 @@ def index_iseq(iseq, file_comments) # Now that we know it's in the structure we want it, we can use # the values of the putobject to determine the alias. - location = Location.new(line, 0) + location = Location.new(line, :unknown) results << AliasMethodDefinition.new( current_nesting, insns[index - 2][1], From c187683d70e70c8ea4a5377b4d8e407690f5dc9f Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 6 Mar 2023 10:52:47 -0500 Subject: [PATCH 431/536] CTags CLI action --- lib/syntax_tree/cli.rb | 84 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/lib/syntax_tree/cli.rb b/lib/syntax_tree/cli.rb index cbe10446..9243d3bf 100644 --- a/lib/syntax_tree/cli.rb +++ b/lib/syntax_tree/cli.rb @@ -154,6 +154,88 @@ def failure end end + # An action of the CLI that generates ctags for the given source. + class CTags < Action + attr_reader :entries + + def initialize(options) + super(options) + @entries = [] + end + + def run(item) + lines = item.source.lines(chomp: true) + + SyntaxTree.index(item.source).each do |entry| + line = lines[entry.location.line - 1] + pattern = "/^#{line.gsub("\\", "\\\\\\\\").gsub("/", "\\/")}$/;\"" + + entries << + case entry + when SyntaxTree::Index::ModuleDefinition + parts = [entry.name, item.filepath, pattern, "m"] + + if entry.nesting != [[entry.name]] + parts << "class:#{entry.nesting.flatten.tap(&:pop).join(".")}" + end + + parts.join("\t") + when SyntaxTree::Index::ClassDefinition + parts = [entry.name, item.filepath, pattern, "c"] + + if entry.nesting != [[entry.name]] + parts << "class:#{entry.nesting.flatten.tap(&:pop).join(".")}" + end + + unless entry.superclass.empty? + inherits = entry.superclass.join(".").delete_prefix(".") + parts << "inherits:#{inherits}" + end + + parts.join("\t") + when SyntaxTree::Index::MethodDefinition + parts = [entry.name, item.filepath, pattern, "f"] + + unless entry.nesting.empty? + parts << "class:#{entry.nesting.flatten.join(".")}" + end + + parts.join("\t") + when SyntaxTree::Index::SingletonMethodDefinition + parts = [entry.name, item.filepath, pattern, "F"] + + unless entry.nesting.empty? + parts << "class:#{entry.nesting.flatten.join(".")}" + end + + parts.join("\t") + when SyntaxTree::Index::AliasMethodDefinition + parts = [entry.name, item.filepath, pattern, "a"] + + unless entry.nesting.empty? + parts << "class:#{entry.nesting.flatten.join(".")}" + end + + parts.join("\t") + when SyntaxTree::Index::ConstantDefinition + parts = [entry.name, item.filepath, pattern, "C"] + + unless entry.nesting.empty? + parts << "class:#{entry.nesting.flatten.join(".")}" + end + + parts.join("\t") + end + end + end + + def success + puts("!_TAG_FILE_FORMAT 2 /extended format; --format=1 will not append ;\" to lines/") + puts("!_TAG_FILE_SORTED 1 /0=unsorted, 1=sorted, 2=foldcase/") + entries.sort.each { |entry| puts(entry) } + end + end + # An action of the CLI that formats the source twice to check if the first # format is not idempotent. class Debug < Action @@ -488,6 +570,8 @@ def run(argv) AST.new(options) when "c", "check" Check.new(options) + when "ctags" + CTags.new(options) when "debug" Debug.new(options) when "doc" From dea5da2527bc8d23500ee517cb9700226cdf7c60 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 6 Mar 2023 10:53:45 -0500 Subject: [PATCH 432/536] Reformat --- lib/syntax_tree/cli.rb | 15 ++++---- lib/syntax_tree/index.rb | 76 +++++++++++++++++++++++++++++++++------- 2 files changed, 73 insertions(+), 18 deletions(-) diff --git a/lib/syntax_tree/cli.rb b/lib/syntax_tree/cli.rb index 9243d3bf..02f8f55d 100644 --- a/lib/syntax_tree/cli.rb +++ b/lib/syntax_tree/cli.rb @@ -166,12 +166,13 @@ def initialize(options) def run(item) lines = item.source.lines(chomp: true) - SyntaxTree.index(item.source).each do |entry| - line = lines[entry.location.line - 1] - pattern = "/^#{line.gsub("\\", "\\\\\\\\").gsub("/", "\\/")}$/;\"" + SyntaxTree + .index(item.source) + .each do |entry| + line = lines[entry.location.line - 1] + pattern = "/^#{line.gsub("\\", "\\\\\\\\").gsub("/", "\\/")}$/;\"" - entries << - case entry + entries << case entry when SyntaxTree::Index::ModuleDefinition parts = [entry.name, item.filepath, pattern, "m"] @@ -230,7 +231,9 @@ def run(item) end def success - puts("!_TAG_FILE_FORMAT 2 /extended format; --format=1 will not append ;\" to lines/") + puts( + "!_TAG_FILE_FORMAT 2 /extended format; --format=1 will not append ;\" to lines/" + ) puts("!_TAG_FILE_SORTED 1 /0=unsorted, 1=sorted, 2=foldcase/") entries.sort.each { |entry| puts(entry) } end diff --git a/lib/syntax_tree/index.rb b/lib/syntax_tree/index.rb index 7865a949..fef97be4 100644 --- a/lib/syntax_tree/index.rb +++ b/lib/syntax_tree/index.rb @@ -203,7 +203,14 @@ def location_for(iseq) end def find_constant_path(insns, index) - index -= 1 while index >= 0 && (insns[index].is_a?(Integer) || (insns[index].is_a?(Array) && %i[swap topn].include?(insns[index][0]))) + index -= 1 while index >= 0 && + ( + insns[index].is_a?(Integer) || + ( + insns[index].is_a?(Array) && + %i[swap topn].include?(insns[index][0]) + ) + ) insn = insns[index] if insn.is_a?(Array) && insn[0] == :opt_getconstant_path @@ -252,7 +259,12 @@ def method_definition(nesting, name, location, file_comments) comments = EntryComments.new(file_comments, location) if nesting.last == [:singletonclass] - SingletonMethodDefinition.new(nesting[0...-1], name, location, comments) + SingletonMethodDefinition.new( + nesting[0...-1], + name, + location, + comments + ) else MethodDefinition.new(nesting, name, location, comments) end @@ -341,7 +353,12 @@ def index_iseq(iseq, file_comments) queue << [class_iseq, next_nesting] when :definemethod location = location_for(insn[2]) - results << method_definition(current_nesting, insn[1], location, file_comments) + results << method_definition( + current_nesting, + insn[1], + location, + file_comments + ) when :definesmethod if insns[index - 1] != [:putself] warn("singleton method with non-self receiver") @@ -378,19 +395,35 @@ def index_iseq(iseq, file_comments) location = Location.new(line, :unknown) names.each do |name| if insn[1][:mid] != :attr_writer - results << method_definition(current_nesting, name, location, file_comments) + results << method_definition( + current_nesting, + name, + location, + file_comments + ) end if insn[1][:mid] != :attr_reader - results << method_definition(current_nesting, :"#{name}=", location, file_comments) + results << method_definition( + current_nesting, + :"#{name}=", + location, + file_comments + ) end end when :"core#set_method_alias" # Now we have to validate that the alias is happening with a # non-interpolated value. To do this we'll match the specific # pattern we're expecting. - values = insns[(index - 4)...index].map { |insn| insn.is_a?(Array) ? insn[0] : insn } - next if values != %i[putspecialobject putspecialobject putobject putobject] + values = + insns[(index - 4)...index].map do |insn| + insn.is_a?(Array) ? insn[0] : insn + end + if values != + %i[putspecialobject putspecialobject putobject putobject] + next + end # Now that we know it's in the structure we want it, we can use # the values of the putobject to determine the alias. @@ -441,7 +474,10 @@ def initialize def visit_alias(node) if node.left.is_a?(SymbolLiteral) && node.right.is_a?(SymbolLiteral) location = - Location.new(node.location.start_line, node.location.start_column) + Location.new( + node.location.start_line, + node.location.start_column + ) results << AliasMethodDefinition.new( nesting.dup, @@ -457,7 +493,10 @@ def visit_alias(node) def visit_assign(node) if node.target.is_a?(VarField) && node.target.value.is_a?(Const) location = - Location.new(node.location.start_line, node.location.start_column) + Location.new( + node.location.start_line, + node.location.start_column + ) results << ConstantDefinition.new( nesting.dup, @@ -507,18 +546,31 @@ def visit_command(node) when "attr_reader", "attr_writer", "attr_accessor" comments = comments_for(node) location = - Location.new(node.location.start_line, node.location.start_column) + Location.new( + node.location.start_line, + node.location.start_column + ) node.arguments.parts.each do |argument| next unless argument.is_a?(SymbolLiteral) name = argument.value.value.to_sym if node.message.value != "attr_writer" - results << MethodDefinition.new(nesting.dup, name, location, comments) + results << MethodDefinition.new( + nesting.dup, + name, + location, + comments + ) end if node.message.value != "attr_reader" - results << MethodDefinition.new(nesting.dup, :"#{name}=", location, comments) + results << MethodDefinition.new( + nesting.dup, + :"#{name}=", + location, + comments + ) end end end From 600d94c262cb951e1fa212c18d2fa01c46e8801e Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 6 Mar 2023 11:05:01 -0500 Subject: [PATCH 433/536] Fix up index test --- lib/syntax_tree/cli.rb | 9 +++++---- lib/syntax_tree/index.rb | 23 +++++++++++++---------- test/index_test.rb | 14 -------------- 3 files changed, 18 insertions(+), 28 deletions(-) diff --git a/lib/syntax_tree/cli.rb b/lib/syntax_tree/cli.rb index 02f8f55d..43265c2b 100644 --- a/lib/syntax_tree/cli.rb +++ b/lib/syntax_tree/cli.rb @@ -231,10 +231,11 @@ def run(item) end def success - puts( - "!_TAG_FILE_FORMAT 2 /extended format; --format=1 will not append ;\" to lines/" - ) - puts("!_TAG_FILE_SORTED 1 /0=unsorted, 1=sorted, 2=foldcase/") + puts(<<~HEADER) + !_TAG_FILE_FORMAT 2 /extended format; --format=1 will not append ;" to lines/ + !_TAG_FILE_SORTED 1 /0=unsorted, 1=sorted, 2=foldcase/ + HEADER + entries.sort.each { |entry| puts(entry) } end end diff --git a/lib/syntax_tree/index.rb b/lib/syntax_tree/index.rb index fef97be4..0280749f 100644 --- a/lib/syntax_tree/index.rb +++ b/lib/syntax_tree/index.rb @@ -275,6 +275,7 @@ def index_iseq(iseq, file_comments) queue = [[iseq, []]] while (current_iseq, current_nesting = queue.shift) + file = current_iseq[5] line = current_iseq[8] insns = current_iseq[13] @@ -309,7 +310,7 @@ def index_iseq(iseq, file_comments) find_constant_path(insns, index - 1) if superclass.empty? - warn("superclass with non constant path on line #{line}") + warn("#{file}:#{line}: superclass with non constant path") next end end @@ -328,7 +329,9 @@ def index_iseq(iseq, file_comments) # defined on self. We could, but it would require more # emulation. if insns[index - 2] != [:putself] - warn("singleton class with non-self receiver") + warn( + "#{file}:#{line}: singleton class with non-self receiver" + ) next end elsif flags & VM_DEFINECLASS_TYPE_MODULE > 0 @@ -361,7 +364,7 @@ def index_iseq(iseq, file_comments) ) when :definesmethod if insns[index - 1] != [:putself] - warn("singleton method with non-self receiver") + warn("#{file}:#{line}: singleton method with non-self receiver") next end @@ -389,15 +392,15 @@ def index_iseq(iseq, file_comments) when :opt_send_without_block, :send case insn[1][:mid] when :attr_reader, :attr_writer, :attr_accessor - names = find_attr_arguments(insns, index) - next unless names + attr_names = find_attr_arguments(insns, index) + next unless attr_names location = Location.new(line, :unknown) - names.each do |name| + attr_names.each do |attr_name| if insn[1][:mid] != :attr_writer results << method_definition( current_nesting, - name, + attr_name, location, file_comments ) @@ -406,7 +409,7 @@ def index_iseq(iseq, file_comments) if insn[1][:mid] != :attr_reader results << method_definition( current_nesting, - :"#{name}=", + :"#{attr_name}=", location, file_comments ) @@ -417,8 +420,8 @@ def index_iseq(iseq, file_comments) # non-interpolated value. To do this we'll match the specific # pattern we're expecting. values = - insns[(index - 4)...index].map do |insn| - insn.is_a?(Array) ? insn[0] : insn + insns[(index - 4)...index].map do |previous| + previous.is_a?(Array) ? previous[0] : previous end if values != %i[putspecialobject putspecialobject putobject putobject] diff --git a/test/index_test.rb b/test/index_test.rb index 855e36ec..1e2a7fc7 100644 --- a/test/index_test.rb +++ b/test/index_test.rb @@ -76,20 +76,6 @@ def test_class_path_superclass end end - def test_class_path_superclass_unknown - source = "class Foo < bar; end" - - assert_raises NotImplementedError do - Index.index(source, backend: Index::ParserBackend.new) - end - - if defined?(RubyVM::InstructionSequence) - assert_raises NotImplementedError do - Index.index(source, backend: Index::ISeqBackend.new) - end - end - end - def test_class_comments index_each("# comment1\n# comment2\nclass Foo; end") do |entry| assert_equal :Foo, entry.name From d9b21ee7393cc02647ce9f29c75f45924f336c03 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 6 Mar 2023 11:25:26 -0500 Subject: [PATCH 434/536] Document CTags --- README.md | 28 ++++++++++++++++++++++++++++ lib/syntax_tree/cli.rb | 3 +++ 2 files changed, 31 insertions(+) diff --git a/README.md b/README.md index 03942d46..d15bb5f1 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ It is built with only standard library dependencies. It additionally ships with - [CLI](#cli) - [ast](#ast) - [check](#check) + - [ctags](#ctags) - [expr](#expr) - [format](#format) - [json](#json) @@ -139,6 +140,33 @@ To change the print width that you are checking against, specify the `--print-wi stree check --print-width=100 path/to/file.rb ``` +### ctags + +This command will output to stdout a set of tags suitable for usage with [ctags](https://github.com/universal-ctags/ctags). + +```sh +stree ctags path/to/file.rb +``` + +For a file containing the following Ruby code: + +```ruby +class Foo +end + +class Bar < Foo +end +``` + +you will receive: + +``` +!_TAG_FILE_FORMAT 2 /extended format; --format=1 will not append ;" to lines/ +!_TAG_FILE_SORTED 1 /0=unsorted, 1=sorted, 2=foldcase/ +Bar test.rb /^class Bar < Foo$/;" c inherits:Foo +Foo test.rb /^class Foo$/;" c +``` + ### expr This command will output a Ruby case-match expression that would match correctly against the first expression of the input. diff --git a/lib/syntax_tree/cli.rb b/lib/syntax_tree/cli.rb index 43265c2b..f2616c87 100644 --- a/lib/syntax_tree/cli.rb +++ b/lib/syntax_tree/cli.rb @@ -413,6 +413,9 @@ def run(item) #{Color.bold("stree check [--plugins=...] [--print-width=NUMBER] [-e SCRIPT] FILE")} Check that the given files are formatted as syntax tree would format them + #{Color.bold("stree ctags [-e SCRIPT] FILE")} + Print out a ctags-compatible index of the given files + #{Color.bold("stree debug [--plugins=...] [--print-width=NUMBER] [-e SCRIPT] FILE")} Check that the given files can be formatted idempotently From 9a10b4e84d0314afbf3abb364d363c5cda12e850 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Mar 2023 18:10:26 +0000 Subject: [PATCH 435/536] Bump rubocop from 1.47.0 to 1.48.0 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.47.0 to 1.48.0. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.47.0...v1.48.0) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 735a5025..5bb2d3bb 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -19,7 +19,7 @@ GEM rake (13.0.6) regexp_parser (2.7.0) rexml (3.2.5) - rubocop (1.47.0) + rubocop (1.48.0) json (~> 2.3) parallel (~> 1.10) parser (>= 3.2.0.0) @@ -31,7 +31,7 @@ GEM unicode-display_width (>= 2.4.0, < 3.0) rubocop-ast (1.27.0) parser (>= 3.2.1.0) - ruby-progressbar (1.12.0) + ruby-progressbar (1.13.0) simplecov (0.22.0) docile (~> 1.1) simplecov-html (~> 0.11) From 989c5d83a579f102b52864f7e2deda60b7ab9f9f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Mar 2023 18:10:27 +0000 Subject: [PATCH 436/536] Bump minitest from 5.17.0 to 5.18.0 Bumps [minitest](https://github.com/seattlerb/minitest) from 5.17.0 to 5.18.0. - [Release notes](https://github.com/seattlerb/minitest/releases) - [Changelog](https://github.com/minitest/minitest/blob/master/History.rdoc) - [Commits](https://github.com/seattlerb/minitest/compare/v5.17.0...v5.18.0) --- updated-dependencies: - dependency-name: minitest dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 735a5025..f14f4912 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -10,7 +10,7 @@ GEM ast (2.4.2) docile (1.4.0) json (2.6.3) - minitest (5.17.0) + minitest (5.18.0) parallel (1.22.1) parser (3.2.1.0) ast (~> 2.4.1) From 22bea0cbe3d3d19f321c1ffe5c256f35ae74f2d9 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 8 Mar 2023 10:51:31 -0500 Subject: [PATCH 437/536] Fix Ruby build on HEAD This fixes two bugs. The first is that in an `in` clause, you need to use the `then` keyword or parentheses if the pattern you are matching is an endless range. The second is that we are associating the `then` keyword with the wrong `in` clauses because they come in in reverse order and we're deleting them from the parent clauses incorrectly. --- lib/syntax_tree/node.rb | 1 + lib/syntax_tree/parser.rb | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index c4bc1495..63a5d466 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -6767,6 +6767,7 @@ def format(q) q.group do q.text(keyword) q.nest(keyword.length) { q.format(pattern) } + q.text(" then") if pattern.is_a?(RangeNode) && pattern.right.nil? unless statements.empty? q.indent do diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index ed0de408..825cd90e 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -2132,13 +2132,20 @@ def on_in(pattern, statements, consequent) ending = consequent || consume_keyword(:end) statements_start = pattern - if (token = find_keyword(:then)) + if (token = find_keyword_between(:then, pattern, statements)) tokens.delete(token) statements_start = token end start_char = find_next_statement_start((token || statements_start).location.end_char) + + # Ripper ignores parentheses on patterns, so we need to do the same in + # order to attach comments correctly to the pattern. + if source[start_char] == ")" + start_char = find_next_statement_start(start_char + 1) + end + statements.bind( self, start_char, From dcae7057a46ad62df742d38ab6eff621847ab6c0 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 8 Mar 2023 11:38:46 -0500 Subject: [PATCH 438/536] A little bit of Sorbet --- .gitignore | 1 + .rubocop.yml | 2 +- .ruby-version | 1 + lib/syntax_tree/node.rb | 96 +++++++++++++------- lib/syntax_tree/reflection.rb | 5 +- lib/syntax_tree/yarv/instruction_sequence.rb | 2 +- tasks/sorbet.rake | 33 +++++++ test/language_server_test.rb | 69 ++++++++++++-- 8 files changed, 165 insertions(+), 44 deletions(-) create mode 100644 .ruby-version diff --git a/.gitignore b/.gitignore index 69755243..3ce1e327 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ /coverage/ /pkg/ /rdocs/ +/sorbet/ /spec/reports/ /tmp/ /vendor/ diff --git a/.rubocop.yml b/.rubocop.yml index e74cdc1b..c1c17001 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -7,7 +7,7 @@ AllCops: SuggestExtensions: false TargetRubyVersion: 2.7 Exclude: - - '{.git,.github,bin,coverage,pkg,spec,test/fixtures,vendor,tmp}/**/*' + - '{.git,.github,bin,coverage,pkg,sorbet,spec,test/fixtures,vendor,tmp}/**/*' - test.rb Gemspec/DevelopmentDependencies: diff --git a/.ruby-version b/.ruby-version new file mode 100644 index 00000000..944880fa --- /dev/null +++ b/.ruby-version @@ -0,0 +1 @@ +3.2.0 diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 63a5d466..3f013b31 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -792,9 +792,10 @@ def arity private def trailing_comma? + arguments = self.arguments return false unless arguments.is_a?(Args) - parts = arguments.parts + parts = arguments.parts if parts.last.is_a?(ArgBlock) # If the last argument is a block, then we can't put a trailing comma # after it without resulting in a syntax error. @@ -1188,8 +1189,11 @@ def deconstruct_keys(_keys) end def format(q) - if lbracket.comments.empty? && contents && contents.comments.empty? && - contents.parts.length > 1 + lbracket = self.lbracket + contents = self.contents + + if lbracket.is_a?(LBracket) && lbracket.comments.empty? && contents && + contents.comments.empty? && contents.parts.length > 1 if qwords? QWordsFormatter.new(contents).format(q) return @@ -2091,6 +2095,7 @@ def deconstruct_keys(_keys) end def format(q) + left = self.left power = operator == :** q.group do @@ -2307,6 +2312,8 @@ def initialize( end def bind(parser, start_char, start_column, end_char, end_column) + rescue_clause = self.rescue_clause + @location = Location.new( start_line: location.start_line, @@ -2330,6 +2337,7 @@ def bind(parser, start_char, start_column, end_char, end_column) # Next we're going to determine the rescue clause if there is one if rescue_clause consequent = else_clause || ensure_clause + rescue_clause.bind_end( consequent ? consequent.location.start_char : end_char, consequent ? consequent.location.start_column : end_column @@ -2735,7 +2743,7 @@ def format(q) children << receiver end when MethodAddBlock - if receiver.call.is_a?(CallNode) && !receiver.call.receiver.nil? + if (call = receiver.call).is_a?(CallNode) && !call.receiver.nil? children << receiver else break @@ -2744,8 +2752,8 @@ def format(q) break end when MethodAddBlock - if child.call.is_a?(CallNode) && !child.call.receiver.nil? - children << child.call + if (call = child.call).is_a?(CallNode) && !call.receiver.nil? + children << call else break end @@ -2767,8 +2775,8 @@ def format(q) # of just Statements nodes. parent = parents[3] if parent.is_a?(BlockNode) && parent.keywords? - if parent.is_a?(MethodAddBlock) && parent.call.is_a?(CallNode) && - parent.call.message.value == "sig" + if parent.is_a?(MethodAddBlock) && + (call = parent.call).is_a?(CallNode) && call.message.value == "sig" threshold = 2 end end @@ -2813,10 +2821,10 @@ def format_chain(q, children) while (child = children.pop) if child.is_a?(CallNode) - if child.receiver.is_a?(CallNode) && - (child.receiver.message != :call) && - (child.receiver.message.value == "where") && - (child.message.value == "not") + if (receiver = child.receiver).is_a?(CallNode) && + (receiver.message != :call) && + (receiver.message.value == "where") && + (message.value == "not") # This is very specialized behavior wherein we group # .where.not calls together because it looks better. For more # information, see @@ -2872,7 +2880,8 @@ def self.chained?(node) when CallNode !node.receiver.nil? when MethodAddBlock - node.call.is_a?(CallNode) && !node.call.receiver.nil? + call = node.call + call.is_a?(CallNode) && !call.receiver.nil? else false end @@ -3629,6 +3638,10 @@ def deconstruct_keys(_keys) end def format(q) + message = self.message + arguments = self.arguments + block = self.block + q.group do doc = q.nest(0) do @@ -3637,7 +3650,7 @@ def format(q) # If there are leading comments on the message then we know we have # a newline in the source that is forcing these things apart. In # this case we will have to use a trailing operator. - if message.comments.any?(&:leading?) + if message != :call && message.comments.any?(&:leading?) q.format(CallOperatorFormatter.new(operator), stackable: false) q.indent do q.breakable_empty @@ -4153,6 +4166,9 @@ def deconstruct_keys(_keys) end def format(q) + params = self.params + bodystmt = self.bodystmt + q.group do q.group do q.text("def") @@ -4209,6 +4225,8 @@ def endless? end def arity + params = self.params + case params when Params params.arity @@ -5293,6 +5311,7 @@ def accept(visitor) end def child_nodes + operator = self.operator [parent, (operator if operator != :"::"), name] end @@ -5674,7 +5693,7 @@ def accept(visitor) end def child_nodes - [lbrace] + assocs + [lbrace].concat(assocs) end def copy(lbrace: nil, assocs: nil, location: nil) @@ -5766,7 +5785,7 @@ class Heredoc < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(beginning:, ending: nil, dedent: 0, parts: [], location:) + def initialize(beginning:, location:, ending: nil, dedent: 0, parts: []) @beginning = beginning @ending = ending @dedent = dedent @@ -6134,6 +6153,8 @@ def ===(other) private def format_contents(q, parts, nested) + keyword_rest = self.keyword_rest + q.group { q.seplist(parts) { |part| q.format(part, stackable: false) } } # If there isn't a constant, and there's a blank keyword_rest, then we @@ -6763,6 +6784,8 @@ def deconstruct_keys(_keys) def format(q) keyword = "in " + pattern = self.pattern + consequent = self.consequent q.group do q.text(keyword) @@ -7165,6 +7188,8 @@ def deconstruct_keys(_keys) end def format(q) + params = self.params + q.text("->") q.group do if params.is_a?(Paren) @@ -7643,7 +7668,7 @@ class MLHS < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(parts:, comma: false, location:) + def initialize(parts:, location:, comma: false) @parts = parts @comma = comma @location = location @@ -7704,7 +7729,7 @@ class MLHSParen < Node # [Array[ Comment | EmbDoc ]] the comments attached to this node attr_reader :comments - def initialize(contents:, comma: false, location:) + def initialize(contents:, location:, comma: false) @contents = contents @comma = comma @location = location @@ -8287,14 +8312,14 @@ def format(q) attr_reader :comments def initialize( + location:, requireds: [], optionals: [], rest: nil, posts: [], keywords: [], keyword_rest: nil, - block: nil, - location: + block: nil ) @requireds = requireds @optionals = optionals @@ -8321,6 +8346,8 @@ def accept(visitor) end def child_nodes + keyword_rest = self.keyword_rest + [ *requireds, *optionals.flatten(1), @@ -8375,16 +8402,19 @@ def deconstruct_keys(_keys) end def format(q) + rest = self.rest + keyword_rest = self.keyword_rest + parts = [ *requireds, *optionals.map { |(name, value)| OptionalFormatter.new(name, value) } ] parts << rest if rest && !rest.is_a?(ExcessedComma) - parts += [ - *posts, - *keywords.map { |(name, value)| KeywordFormatter.new(name, value) } - ] + parts.concat(posts) + parts.concat( + keywords.map { |(name, value)| KeywordFormatter.new(name, value) } + ) parts << KeywordRestFormatter.new(keyword_rest) if keyword_rest parts << block if block @@ -8511,6 +8541,8 @@ def deconstruct_keys(_keys) end def format(q) + contents = self.contents + q.group do q.format(lparen) @@ -9425,11 +9457,11 @@ def bind_end(end_char, end_column) end_column: end_column ) - if consequent - consequent.bind_end(end_char, end_column) + if (next_node = consequent) + next_node.bind_end(end_char, end_column) statements.bind_end( - consequent.location.start_char, - consequent.location.start_column + next_node.location.start_char, + next_node.location.start_column ) else statements.bind_end(end_char, end_column) @@ -9872,8 +9904,8 @@ def bind(parser, start_char, start_column, end_char, end_column) end_column: end_column ) - if body[0].is_a?(VoidStmt) - location = body[0].location + if (void_stmt = body[0]).is_a?(VoidStmt) + location = void_stmt.location location = Location.new( start_line: location.start_line, @@ -10352,7 +10384,7 @@ def format(q) opening_quote, closing_quote = if !Quotes.locked?(self, q.quote) [q.quote, q.quote] - elsif quote.start_with?("%") + elsif quote&.start_with?("%") [quote, Quotes.matching(quote[/%[qQ]?(.)/, 1])] else [quote, quote] @@ -11521,7 +11553,7 @@ def accept(visitor) end def child_nodes - [value] + value == :nil ? [] : [value] end def copy(value: nil, location: nil) diff --git a/lib/syntax_tree/reflection.rb b/lib/syntax_tree/reflection.rb index b2ffec6d..a27593ee 100644 --- a/lib/syntax_tree/reflection.rb +++ b/lib/syntax_tree/reflection.rb @@ -183,10 +183,11 @@ def parse_comments(statements, index) next unless main_statement.is_a?(SyntaxTree::ClassDeclaration) # Ensure we're looking at class declarations with superclasses. - next unless main_statement.superclass.is_a?(SyntaxTree::VarRef) + superclass = main_statement.superclass + next unless superclass.is_a?(SyntaxTree::VarRef) # Ensure we're looking at class declarations that inherit from Node. - next unless main_statement.superclass.value.value == "Node" + next unless superclass.value.value == "Node" # All child nodes inherit the location attr_reader from Node, so we'll add # that to the list of attributes first. diff --git a/lib/syntax_tree/yarv/instruction_sequence.rb b/lib/syntax_tree/yarv/instruction_sequence.rb index 45b543e6..5aaaef44 100644 --- a/lib/syntax_tree/yarv/instruction_sequence.rb +++ b/lib/syntax_tree/yarv/instruction_sequence.rb @@ -50,7 +50,7 @@ def initialize @tail_node = nil end - def each + def each(&_blk) return to_enum(__method__) unless block_given? each_node { |node| yield node.value } end diff --git a/tasks/sorbet.rake b/tasks/sorbet.rake index e4152664..c80ec91d 100644 --- a/tasks/sorbet.rake +++ b/tasks/sorbet.rake @@ -122,8 +122,41 @@ module SyntaxTree @line += 1 node_body << generate_def_node("child_nodes", nil) + @line += 2 + + node_body << sig_block do + CallNode( + sig_params do + BareAssocHash( + [ + Assoc( + Label("other:"), + CallNode( + VarRef(Const("T")), + Period("."), + Ident("untyped"), + nil + ) + ) + ] + ) + end, + Period("."), + sig_returns { ConstPathRef(VarRef(Const("T")), Const("Boolean")) }, + nil + ) + end @line += 1 + node_body << generate_def_node( + "==", + Paren( + LParen("("), + Params.new(location: location, requireds: [Ident("other")]) + ) + ) + @line += 2 + node_body end diff --git a/test/language_server_test.rb b/test/language_server_test.rb index 2fe4e60a..f5a6ca57 100644 --- a/test/language_server_test.rb +++ b/test/language_server_test.rb @@ -6,19 +6,38 @@ module SyntaxTree # stree-ignore class LanguageServerTest < Minitest::Test - class Initialize < Struct.new(:id) + class Initialize + attr_reader :id + + def initialize(id) + @id = id + end + def to_hash { method: "initialize", id: id } end end - class Shutdown < Struct.new(:id) + class Shutdown + attr_reader :id + + def initialize(id) + @id = id + end + def to_hash { method: "shutdown", id: id } end end - class TextDocumentDidOpen < Struct.new(:uri, :text) + class TextDocumentDidOpen + attr_reader :uri, :text + + def initialize(uri, text) + @uri = uri + @text = text + end + def to_hash { method: "textDocument/didOpen", @@ -27,7 +46,14 @@ def to_hash end end - class TextDocumentDidChange < Struct.new(:uri, :text) + class TextDocumentDidChange + attr_reader :uri, :text + + def initialize(uri, text) + @uri = uri + @text = text + end + def to_hash { method: "textDocument/didChange", @@ -39,7 +65,13 @@ def to_hash end end - class TextDocumentDidClose < Struct.new(:uri) + class TextDocumentDidClose + attr_reader :uri + + def initialize(uri) + @uri = uri + end + def to_hash { method: "textDocument/didClose", @@ -48,7 +80,14 @@ def to_hash end end - class TextDocumentFormatting < Struct.new(:id, :uri) + class TextDocumentFormatting + attr_reader :id, :uri + + def initialize(id, uri) + @id = id + @uri = uri + end + def to_hash { method: "textDocument/formatting", @@ -58,7 +97,14 @@ def to_hash end end - class TextDocumentInlayHint < Struct.new(:id, :uri) + class TextDocumentInlayHint + attr_reader :id, :uri + + def initialize(id, uri) + @id = id + @uri = uri + end + def to_hash { method: "textDocument/inlayHint", @@ -68,7 +114,14 @@ def to_hash end end - class SyntaxTreeVisualizing < Struct.new(:id, :uri) + class SyntaxTreeVisualizing + attr_reader :id, :uri + + def initialize(id, uri) + @id = id + @uri = uri + end + def to_hash { method: "syntaxTree/visualizing", From ec0396d235ac5a9a0408707c40dd7713eec624d8 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 8 Mar 2023 14:16:47 -0500 Subject: [PATCH 439/536] Require the pp gem, necessary for pretty printing --- lib/syntax_tree.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index 4e183383..24d8426f 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require "prettier_print" +require "pp" require "ripper" require_relative "syntax_tree/node" From acd8238b47ee2f831554645c181f609c7be90ceb Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 9 Mar 2023 09:36:48 -0500 Subject: [PATCH 440/536] Add the visitor method on reflection --- lib/syntax_tree/reflection.rb | 20 ++++++++++++-- tasks/sorbet.rake | 52 +++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 3 deletions(-) diff --git a/lib/syntax_tree/reflection.rb b/lib/syntax_tree/reflection.rb index a27593ee..aa7b85b6 100644 --- a/lib/syntax_tree/reflection.rb +++ b/lib/syntax_tree/reflection.rb @@ -138,12 +138,13 @@ def initialize(name, comment) # as a placeholder for collecting all of the various places that nodes are # used. class Node - attr_reader :name, :comment, :attributes + attr_reader :name, :comment, :attributes, :visitor_method - def initialize(name, comment, attributes) + def initialize(name, comment, attributes, visitor_method) @name = name @comment = comment @attributes = attributes + @visitor_method = visitor_method end end @@ -196,6 +197,10 @@ def parse_comments(statements, index) Attribute.new(:location, "[Location] the location of this node") } + # This is the name of the method tha gets called on the given visitor when + # the accept method is called on this node. + visitor_method = nil + statements = main_statement.bodystmt.statements.body statements.each_with_index do |statement, statement_index| case statement @@ -225,16 +230,25 @@ def parse_comments(statements, index) end attributes[attribute.name] = attribute + when SyntaxTree::DefNode + if statement.name.value == "accept" + call_node = statement.bodystmt.statements.body.first + visitor_method = call_node.message.value.to_sym + end end end + # If we never found a visitor method, then we have an error. + raise if visitor_method.nil? + # Finally, set it up in the hash of nodes so that we can use it later. comments = parse_comments(main_statements, main_statement_index) node = Node.new( main_statement.constant.constant.value.to_sym, "#{comments.join("\n")}\n", - attributes + attributes, + visitor_method ) @nodes[node.name] = node diff --git a/tasks/sorbet.rake b/tasks/sorbet.rake index c80ec91d..134b6011 100644 --- a/tasks/sorbet.rake +++ b/tasks/sorbet.rake @@ -20,6 +20,22 @@ module SyntaxTree generate_parent Reflection.nodes.sort.each { |(_, node)| generate_node(node) } + body << ClassDeclaration( + ConstPathRef(VarRef(Const("SyntaxTree")), Const("BasicVisitor")), + nil, + BodyStmt( + Statements(generate_visitor("overridable")), nil, nil, nil, nil + ), + location + ) + + body << ClassDeclaration( + ConstPathRef(VarRef(Const("SyntaxTree")), Const("Visitor")), + ConstPathRef(VarRef(Const("SyntaxTree")), Const("BasicVisitor")), + BodyStmt(Statements(generate_visitor("override")), nil, nil, nil, nil), + location + ) + Formatter.format(nil, Program(Statements(body))) end @@ -228,6 +244,42 @@ module SyntaxTree ) end + def generate_visitor(override) + body = [] + + Reflection.nodes.each do |name, node| + body << sig_block do + CallNode( + CallNode( + Ident(override), + Period("."), + sig_params do + BareAssocHash([ + Assoc(Label("node:"), + sig_type_for(SyntaxTree.const_get(name))) + ]) + end, + nil + ), + Period("."), + sig_returns do + CallNode(VarRef(Const("T")), Period("."), Ident("untyped"), nil) + end, + nil + ) + end + + body << generate_def_node(node.visitor_method, Paren( + LParen("("), + Params.new(requireds: [Ident("node")], location: location) + )) + + @line += 2 + end + + body + end + def sig_block MethodAddBlock( CallNode(nil, nil, Ident("sig"), nil), From 57caa25b945a48b20dafe7d454bdf9f99ff2caae Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 9 Mar 2023 10:03:25 -0500 Subject: [PATCH 441/536] Fix up formatting --- tasks/sorbet.rake | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/tasks/sorbet.rake b/tasks/sorbet.rake index 134b6011..05f48874 100644 --- a/tasks/sorbet.rake +++ b/tasks/sorbet.rake @@ -24,7 +24,11 @@ module SyntaxTree ConstPathRef(VarRef(Const("SyntaxTree")), Const("BasicVisitor")), nil, BodyStmt( - Statements(generate_visitor("overridable")), nil, nil, nil, nil + Statements(generate_visitor("overridable")), + nil, + nil, + nil, + nil ), location ) @@ -254,10 +258,14 @@ module SyntaxTree Ident(override), Period("."), sig_params do - BareAssocHash([ - Assoc(Label("node:"), - sig_type_for(SyntaxTree.const_get(name))) - ]) + BareAssocHash( + [ + Assoc( + Label("node:"), + sig_type_for(SyntaxTree.const_get(name)) + ) + ] + ) end, nil ), @@ -269,10 +277,13 @@ module SyntaxTree ) end - body << generate_def_node(node.visitor_method, Paren( - LParen("("), - Params.new(requireds: [Ident("node")], location: location) - )) + body << generate_def_node( + node.visitor_method, + Paren( + LParen("("), + Params.new(requireds: [Ident("node")], location: location) + ) + ) @line += 2 end From e348f8fb75b2c532b35311f43703ad1d646eb9f7 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 9 Mar 2023 10:18:19 -0500 Subject: [PATCH 442/536] defined_ivar instruction --- lib/syntax_tree/yarv/compiler.rb | 3 +- lib/syntax_tree/yarv/instruction_sequence.rb | 15 ++++- lib/syntax_tree/yarv/instructions.rb | 58 ++++++++++++++++++++ 3 files changed, 72 insertions(+), 4 deletions(-) diff --git a/lib/syntax_tree/yarv/compiler.rb b/lib/syntax_tree/yarv/compiler.rb index bd20bc19..b0afcc99 100644 --- a/lib/syntax_tree/yarv/compiler.rb +++ b/lib/syntax_tree/yarv/compiler.rb @@ -875,8 +875,7 @@ def visit_defined(node) when Ident iseq.putobject("local-variable") when IVar - iseq.putnil - iseq.defined(Defined::TYPE_IVAR, name, "instance-variable") + iseq.defined_ivar(name, iseq.inline_storage, "instance-variable") when Kw case name when :false diff --git a/lib/syntax_tree/yarv/instruction_sequence.rb b/lib/syntax_tree/yarv/instruction_sequence.rb index 5aaaef44..2d89e052 100644 --- a/lib/syntax_tree/yarv/instruction_sequence.rb +++ b/lib/syntax_tree/yarv/instruction_sequence.rb @@ -673,12 +673,21 @@ def concatstrings(number) push(ConcatStrings.new(number)) end + def defineclass(name, class_iseq, flags) + push(DefineClass.new(name, class_iseq, flags)) + end + def defined(type, name, message) push(Defined.new(type, name, message)) end - def defineclass(name, class_iseq, flags) - push(DefineClass.new(name, class_iseq, flags)) + def defined_ivar(name, cache, message) + if RUBY_VERSION < "3.3" + push(PutNil.new) + push(Defined.new(Defined::TYPE_IVAR, name, message)) + else + push(DefinedIVar.new(name, cache, message)) + end end def definemethod(name, method_iseq) @@ -1058,6 +1067,8 @@ def self.from(source, options = Compiler::Options.new, parent_iseq = nil) iseq.defineclass(opnds[0], from(opnds[1], options, iseq), opnds[2]) when :defined iseq.defined(opnds[0], opnds[1], opnds[2]) + when :defined_ivar + iseq.defined_ivar(opnds[0], opnds[1], opnds[2]) when :definemethod iseq.definemethod(opnds[0], from(opnds[1], options, iseq)) when :definesmethod diff --git a/lib/syntax_tree/yarv/instructions.rb b/lib/syntax_tree/yarv/instructions.rb index 38c80fde..cf83ddeb 100644 --- a/lib/syntax_tree/yarv/instructions.rb +++ b/lib/syntax_tree/yarv/instructions.rb @@ -994,6 +994,64 @@ def call(vm) end end + # ### Summary + # + # `defined_ivar` checks if an instance variable is defined. It is a + # specialization of the `defined` instruction. It accepts three arguments: + # the name of the instance variable, an inline cache, and the string that + # should be pushed onto the stack in the event that the instance variable + # is defined. + # + # ### Usage + # + # ~~~ruby + # defined?(@value) + # ~~~ + # + class DefinedIVar < Instruction + attr_reader :name, :cache, :message + + def initialize(name, cache, message) + @name = name + @cache = cache + @message = message + end + + def disasm(fmt) + fmt.instruction( + "defined_ivar", + [fmt.object(name), fmt.inline_storage(cache), fmt.object(message)] + ) + end + + def to_a(_iseq) + [:defined_ivar, name, cache, message] + end + + def deconstruct_keys(_keys) + { name: name, cache: cache, message: message } + end + + def ==(other) + other.is_a?(DefinedIVar) && other.name == name && + other.cache == cache && other.message == message + end + + def length + 4 + end + + def pushes + 1 + end + + def call(vm) + result = (message if vm.frame._self.instance_variable_defined?(name)) + + vm.push(result) + end + end + # ### Summary # # `definemethod` defines a method on the class of the current value of From 3e4fcd533ab983645da555ce1ad02e673ab80ab9 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 9 Mar 2023 10:42:33 -0500 Subject: [PATCH 443/536] definedivar --- lib/syntax_tree/yarv/compiler.rb | 2 +- lib/syntax_tree/yarv/instruction_sequence.rb | 6 +++--- lib/syntax_tree/yarv/instructions.rb | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/syntax_tree/yarv/compiler.rb b/lib/syntax_tree/yarv/compiler.rb index b0afcc99..0f7e7372 100644 --- a/lib/syntax_tree/yarv/compiler.rb +++ b/lib/syntax_tree/yarv/compiler.rb @@ -875,7 +875,7 @@ def visit_defined(node) when Ident iseq.putobject("local-variable") when IVar - iseq.defined_ivar(name, iseq.inline_storage, "instance-variable") + iseq.definedivar(name, iseq.inline_storage, "instance-variable") when Kw case name when :false diff --git a/lib/syntax_tree/yarv/instruction_sequence.rb b/lib/syntax_tree/yarv/instruction_sequence.rb index 2d89e052..7ce7bcdd 100644 --- a/lib/syntax_tree/yarv/instruction_sequence.rb +++ b/lib/syntax_tree/yarv/instruction_sequence.rb @@ -681,7 +681,7 @@ def defined(type, name, message) push(Defined.new(type, name, message)) end - def defined_ivar(name, cache, message) + def definedivar(name, cache, message) if RUBY_VERSION < "3.3" push(PutNil.new) push(Defined.new(Defined::TYPE_IVAR, name, message)) @@ -1067,8 +1067,8 @@ def self.from(source, options = Compiler::Options.new, parent_iseq = nil) iseq.defineclass(opnds[0], from(opnds[1], options, iseq), opnds[2]) when :defined iseq.defined(opnds[0], opnds[1], opnds[2]) - when :defined_ivar - iseq.defined_ivar(opnds[0], opnds[1], opnds[2]) + when :definedivar + iseq.definedivar(opnds[0], opnds[1], opnds[2]) when :definemethod iseq.definemethod(opnds[0], from(opnds[1], options, iseq)) when :definesmethod diff --git a/lib/syntax_tree/yarv/instructions.rb b/lib/syntax_tree/yarv/instructions.rb index cf83ddeb..ceb237dc 100644 --- a/lib/syntax_tree/yarv/instructions.rb +++ b/lib/syntax_tree/yarv/instructions.rb @@ -996,7 +996,7 @@ def call(vm) # ### Summary # - # `defined_ivar` checks if an instance variable is defined. It is a + # `definedivar` checks if an instance variable is defined. It is a # specialization of the `defined` instruction. It accepts three arguments: # the name of the instance variable, an inline cache, and the string that # should be pushed onto the stack in the event that the instance variable @@ -1019,13 +1019,13 @@ def initialize(name, cache, message) def disasm(fmt) fmt.instruction( - "defined_ivar", + "definedivar", [fmt.object(name), fmt.inline_storage(cache), fmt.object(message)] ) end def to_a(_iseq) - [:defined_ivar, name, cache, message] + [:definedivar, name, cache, message] end def deconstruct_keys(_keys) From ed033e49603be8fb1a1b9a523aa4669e384c6df1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Mar 2023 17:15:01 +0000 Subject: [PATCH 444/536] Bump prettier_print from 1.2.0 to 1.2.1 Bumps [prettier_print](https://github.com/ruby-syntax-tree/prettier_print) from 1.2.0 to 1.2.1. - [Release notes](https://github.com/ruby-syntax-tree/prettier_print/releases) - [Changelog](https://github.com/ruby-syntax-tree/prettier_print/blob/main/CHANGELOG.md) - [Commits](https://github.com/ruby-syntax-tree/prettier_print/compare/v1.2.0...v1.2.1) --- updated-dependencies: - dependency-name: prettier_print dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 9ba5adf8..cd726fb4 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -14,7 +14,7 @@ GEM parallel (1.22.1) parser (3.2.1.0) ast (~> 2.4.1) - prettier_print (1.2.0) + prettier_print (1.2.1) rainbow (3.1.1) rake (13.0.6) regexp_parser (2.7.0) From 662b9f273b3c8e37749c07b0dd0033d36d8c9ddc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Mar 2023 18:08:50 +0000 Subject: [PATCH 445/536] Bump rubocop from 1.48.0 to 1.48.1 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.48.0 to 1.48.1. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.48.0...v1.48.1) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index cd726fb4..565fb7ad 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -12,14 +12,14 @@ GEM json (2.6.3) minitest (5.18.0) parallel (1.22.1) - parser (3.2.1.0) + parser (3.2.1.1) ast (~> 2.4.1) prettier_print (1.2.1) rainbow (3.1.1) rake (13.0.6) regexp_parser (2.7.0) rexml (3.2.5) - rubocop (1.48.0) + rubocop (1.48.1) json (~> 2.3) parallel (~> 1.10) parser (>= 3.2.0.0) From 1e13c69fb65f0ddbc3818dceb3845fcb00430c41 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 20 Mar 2023 09:53:02 -0400 Subject: [PATCH 446/536] Bump to version 6.1.0 --- CHANGELOG.md | 13 +++++++++++++ Gemfile.lock | 2 +- lib/syntax_tree/version.rb | 2 +- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 960bb0e9..2d3daa58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,19 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +## [6.1.0] - 2023-03-20 + +### Added + +- The `stree ctags` command for generating ctags like `universal-ctags` or `ripper-tags` would. +- The `definedivar` YARV instruction has been added to match CRuby's implementation. +- We now generate better Sorbet RBI files for the nodes in the tree and the visitors. +- `SyntaxTree::Reflection.nodes` now includes the visitor method. + +### Changed + +- We now explicitly require `pp` in environments that need it. + ## [6.0.2] - 2023-03-03 ### Added diff --git a/Gemfile.lock b/Gemfile.lock index 565fb7ad..f69c40d1 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - syntax_tree (6.0.2) + syntax_tree (6.1.0) prettier_print (>= 1.2.0) GEM diff --git a/lib/syntax_tree/version.rb b/lib/syntax_tree/version.rb index ff3db370..3ed889e4 100644 --- a/lib/syntax_tree/version.rb +++ b/lib/syntax_tree/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SyntaxTree - VERSION = "6.0.2" + VERSION = "6.1.0" end From 82bab149162b86f8f333fa312198c33eeabb7a5a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Mar 2023 18:08:04 +0000 Subject: [PATCH 447/536] Bump actions/deploy-pages from 1 to 2 Bumps [actions/deploy-pages](https://github.com/actions/deploy-pages) from 1 to 2. - [Release notes](https://github.com/actions/deploy-pages/releases) - [Commits](https://github.com/actions/deploy-pages/compare/v1...v2) --- updated-dependencies: - dependency-name: actions/deploy-pages dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/gh-pages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 6c64676d..5b662631 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -51,4 +51,4 @@ jobs: steps: - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@v1 + uses: actions/deploy-pages@v2 From 491b86de38462ddcf0c6a34e51f007c6d9dfcf27 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 21 Mar 2023 09:46:05 -0400 Subject: [PATCH 448/536] Bump to version 6.1.1 --- CHANGELOG.md | 6 ++++++ Gemfile.lock | 2 +- lib/syntax_tree/node.rb | 6 +++--- lib/syntax_tree/version.rb | 2 +- test/fixtures/call.rb | 7 +++++++ 5 files changed, 18 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d3daa58..273d4003 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +## [6.1.1] - 2023-03-21 + +### Changed + +- Fixed a bug where the call chain formatter was incorrectly looking at call messages. + ## [6.1.0] - 2023-03-20 ### Added diff --git a/Gemfile.lock b/Gemfile.lock index f69c40d1..ad2aeaa5 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - syntax_tree (6.1.0) + syntax_tree (6.1.1) prettier_print (>= 1.2.0) GEM diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 3f013b31..54d132e6 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -2824,7 +2824,7 @@ def format_chain(q, children) if (receiver = child.receiver).is_a?(CallNode) && (receiver.message != :call) && (receiver.message.value == "where") && - (message.value == "not") + (child.message != :call && child.message.value == "not") # This is very specialized behavior wherein we group # .where.not calls together because it looks better. For more # information, see @@ -2848,8 +2848,8 @@ def format_chain(q, children) # If the parent call node has a comment on the message then we need # to print the operator trailing in order to keep it working. last_child = children.last - if last_child.is_a?(CallNode) && last_child.message.comments.any? && - last_child.operator + if last_child.is_a?(CallNode) && last_child.message != :call && + last_child.message.comments.any? && last_child.operator q.format(CallOperatorFormatter.new(last_child.operator)) skip_operator = true else diff --git a/lib/syntax_tree/version.rb b/lib/syntax_tree/version.rb index 3ed889e4..ad87d3e1 100644 --- a/lib/syntax_tree/version.rb +++ b/lib/syntax_tree/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SyntaxTree - VERSION = "6.1.0" + VERSION = "6.1.1" end diff --git a/test/fixtures/call.rb b/test/fixtures/call.rb index d35c6036..eec717f0 100644 --- a/test/fixtures/call.rb +++ b/test/fixtures/call.rb @@ -65,3 +65,10 @@ =begin =end to_s +% +fooooooooooooooooooooooooooooooooooo.barrrrrrrrrrrrrrrrrrrrrrrrrrrrrr.where.not(:id).order(:id) +- +fooooooooooooooooooooooooooooooooooo + .barrrrrrrrrrrrrrrrrrrrrrrrrrrrrr + .where.not(:id) + .order(:id) From ed215c4592417d53bc7f37a94066af02395b08d8 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 24 Mar 2023 12:17:04 -0400 Subject: [PATCH 449/536] Experimental database API --- lib/syntax_tree.rb | 1 + lib/syntax_tree/database.rb | 331 ++++++++++++++++++++++++++++++++++++ 2 files changed, 332 insertions(+) create mode 100644 lib/syntax_tree/database.rb diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index 24d8426f..6c595db5 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -21,6 +21,7 @@ module SyntaxTree # CLI. Requiring those features takes time, so we autoload as many constants # as possible in order to keep the CLI as fast as possible. + autoload :Database, "syntax_tree/database" autoload :DSL, "syntax_tree/dsl" autoload :FieldVisitor, "syntax_tree/field_visitor" autoload :Index, "syntax_tree/index" diff --git a/lib/syntax_tree/database.rb b/lib/syntax_tree/database.rb new file mode 100644 index 00000000..c9981f35 --- /dev/null +++ b/lib/syntax_tree/database.rb @@ -0,0 +1,331 @@ +# frozen_string_literal: true + +module SyntaxTree + # Provides the ability to index source files into a database, then query for + # the nodes. + module Database + class IndexingVisitor < SyntaxTree::FieldVisitor + attr_reader :database, :filepath, :node_id + + def initialize(database, filepath) + @database = database + @filepath = filepath + @node_id = nil + end + + private + + def comments(node) + end + + def field(name, value) + return unless value.is_a?(SyntaxTree::Node) + + binds = [node_id, visit(value), name] + database.execute(<<~SQL, binds) + INSERT INTO edges (from_id, to_id, name) + VALUES (?, ?, ?) + SQL + end + + def list(name, values) + values.each_with_index do |value, index| + binds = [node_id, visit(value), name, index] + database.execute(<<~SQL, binds) + INSERT INTO edges (from_id, to_id, name, list_index) + VALUES (?, ?, ?, ?) + SQL + end + end + + def node(node, _name) + previous = node_id + binds = [ + node.class.name.delete_prefix("SyntaxTree::"), + filepath, + node.location.start_line, + node.location.start_column + ] + + database.execute(<<~SQL, binds) + INSERT INTO nodes (type, path, line, column) + VALUES (?, ?, ?, ?) + SQL + + begin + @node_id = database.last_insert_row_id + yield + @node_id + ensure + @node_id = previous + end + end + + def text(name, value) + end + + def pairs(name, values) + values.each_with_index do |(key, value), index| + binds = [node_id, visit(key), "#{name}[0]", index] + database.execute(<<~SQL, binds) + INSERT INTO edges (from_id, to_id, name, list_index) + VALUES (?, ?, ?, ?) + SQL + + binds = [node_id, visit(value), "#{name}[1]", index] + database.execute(<<~SQL, binds) + INSERT INTO edges (from_id, to_id, name, list_index) + VALUES (?, ?, ?, ?) + SQL + end + end + end + + # Query for a specific type of node. + class TypeQuery + attr_reader :type + + def initialize(type) + @type = type + end + + def each(database, &block) + sql = "SELECT * FROM nodes WHERE type = ?" + database.execute(sql, type).each(&block) + end + end + + # Query for the attributes of a node, optionally also filtering by type. + class AttrQuery + attr_reader :type, :attrs + + def initialize(type, attrs) + @type = type + @attrs = attrs + end + + def each(database, &block) + joins = [] + binds = [] + + attrs.each do |name, query| + ids = query.each(database).map { |row| row[0] } + joins << <<~SQL + JOIN edges AS #{name} + ON #{name}.from_id = nodes.id + AND #{name}.name = ? + AND #{name}.to_id IN (#{(["?"] * ids.size).join(", ")}) + SQL + + binds.push(name).concat(ids) + end + + sql = +"SELECT nodes.* FROM nodes, edges #{joins.join(" ")}" + + if type + sql << " WHERE nodes.type = ?" + binds << type + end + + sql << " GROUP BY nodes.id" + database.execute(sql, binds).each(&block) + end + end + + # Query for the results of either query. + class OrQuery + attr_reader :left, :right + + def initialize(left, right) + @left = left + @right = right + end + + def each(database, &block) + left.each(database, &block) + right.each(database, &block) + end + end + + # A lazy query result. + class QueryResult + attr_reader :database, :query + + def initialize(database, query) + @database = database + @query = query + end + + def each(&block) + return enum_for(__method__) unless block_given? + query.each(database, &block) + end + end + + # A pattern matching expression that will be compiled into a query. + class Pattern + class CompilationError < StandardError + end + + attr_reader :query + + def initialize(query) + @query = query + end + + def compile + program = + begin + SyntaxTree.parse("case nil\nin #{query}\nend") + rescue Parser::ParseError + raise CompilationError, query + end + + compile_node(program.statements.body.first.consequent.pattern) + end + + private + + def compile_error(node) + raise CompilationError, PP.pp(node, +"").chomp + end + + # Shortcut for combining two queries into one that returns the results of + # if either query matches. + def combine_or(left, right) + OrQuery.new(left, right) + end + + # in foo | bar + def compile_binary(node) + compile_error(node) if node.operator != :| + + combine_or(compile_node(node.left), compile_node(node.right)) + end + + # in Ident + def compile_const(node) + value = node.value + + if SyntaxTree.const_defined?(value, false) + clazz = SyntaxTree.const_get(value) + TypeQuery.new(clazz.name.delete_prefix("SyntaxTree::")) + else + compile_error(node) + end + end + + # in SyntaxTree::Ident + def compile_const_path_ref(node) + parent = node.parent + if !parent.is_a?(SyntaxTree::VarRef) || + !parent.value.is_a?(SyntaxTree::Const) + compile_error(node) + end + + if parent.value.value == "SyntaxTree" + compile_node(node.constant) + else + compile_error(node) + end + end + + # in Ident[value: String] + def compile_hshptn(node) + compile_error(node) unless node.keyword_rest.nil? + + attrs = {} + node.keywords.each do |keyword, value| + compile_error(node) unless keyword.is_a?(SyntaxTree::Label) + attrs[keyword.value.chomp(":")] = compile_node(value) + end + + type = node.constant ? compile_node(node.constant).type : nil + AttrQuery.new(type, attrs) + end + + # in Foo + def compile_var_ref(node) + value = node.value + + if value.is_a?(SyntaxTree::Const) + compile_node(value) + else + compile_error(node) + end + end + + def compile_node(node) + case node + when SyntaxTree::Binary + compile_binary(node) + when SyntaxTree::Const + compile_const(node) + when SyntaxTree::ConstPathRef + compile_const_path_ref(node) + when SyntaxTree::HshPtn + compile_hshptn(node) + when SyntaxTree::VarRef + compile_var_ref(node) + else + compile_error(node) + end + end + end + + class Connection + attr_reader :raw_connection + + def initialize(raw_connection) + @raw_connection = raw_connection + end + + def execute(query, binds = []) + raw_connection.execute(query, binds) + end + + def index_file(filepath) + program = SyntaxTree.parse(SyntaxTree.read(filepath)) + program.accept(IndexingVisitor.new(self, filepath)) + end + + def last_insert_row_id + raw_connection.last_insert_row_id + end + + def prepare + raw_connection.execute(<<~SQL) + CREATE TABLE nodes ( + id integer primary key, + type varchar(20), + path varchar(200), + line integer, + column integer + ); + SQL + + raw_connection.execute(<<~SQL) + CREATE INDEX nodes_type ON nodes (type); + SQL + + raw_connection.execute(<<~SQL) + CREATE TABLE edges ( + id integer primary key, + from_id integer, + to_id integer, + name varchar(20), + list_index integer + ); + SQL + + raw_connection.execute(<<~SQL) + CREATE INDEX edges_name ON edges (name); + SQL + end + + def search(query) + QueryResult.new(self, Pattern.new(query).compile) + end + end + end +end From 5c379c05e9a62503276ed3b7921860ff55e76478 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Sat, 1 Apr 2023 14:08:31 -0400 Subject: [PATCH 450/536] Retrigger HEAD build From 66d70ea618994fe20b61efa0deacc0090302d710 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 3 Apr 2023 10:00:40 -0400 Subject: [PATCH 451/536] Remove obsolete blockiseq flag --- lib/syntax_tree/yarv/assembler.rb | 1 - lib/syntax_tree/yarv/calldata.rb | 34 ++++++++++++++++++------------- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/lib/syntax_tree/yarv/assembler.rb b/lib/syntax_tree/yarv/assembler.rb index ac400506..b29c252a 100644 --- a/lib/syntax_tree/yarv/assembler.rb +++ b/lib/syntax_tree/yarv/assembler.rb @@ -31,7 +31,6 @@ def visit_string_literal(node) "FCALL" => CallData::CALL_FCALL, "VCALL" => CallData::CALL_VCALL, "ARGS_SIMPLE" => CallData::CALL_ARGS_SIMPLE, - "BLOCKISEQ" => CallData::CALL_BLOCKISEQ, "KWARG" => CallData::CALL_KWARG, "KW_SPLAT" => CallData::CALL_KW_SPLAT, "TAILCALL" => CallData::CALL_TAILCALL, diff --git a/lib/syntax_tree/yarv/calldata.rb b/lib/syntax_tree/yarv/calldata.rb index fadea61b..e35992f5 100644 --- a/lib/syntax_tree/yarv/calldata.rb +++ b/lib/syntax_tree/yarv/calldata.rb @@ -5,19 +5,26 @@ module YARV # This is an operand to various YARV instructions that represents the # information about a specific call site. class CallData - CALL_ARGS_SPLAT = 1 << 0 - CALL_ARGS_BLOCKARG = 1 << 1 - CALL_FCALL = 1 << 2 - CALL_VCALL = 1 << 3 - CALL_ARGS_SIMPLE = 1 << 4 - CALL_BLOCKISEQ = 1 << 5 - CALL_KWARG = 1 << 6 - CALL_KW_SPLAT = 1 << 7 - CALL_TAILCALL = 1 << 8 - CALL_SUPER = 1 << 9 - CALL_ZSUPER = 1 << 10 - CALL_OPT_SEND = 1 << 11 - CALL_KW_SPLAT_MUT = 1 << 12 + flags = %i[ + CALL_ARGS_SPLAT + CALL_ARGS_BLOCKARG + CALL_FCALL + CALL_VCALL + CALL_ARGS_SIMPLE + CALL_KWARG + CALL_KW_SPLAT + CALL_TAILCALL + CALL_SUPER + CALL_ZSUPER + CALL_OPT_SEND + CALL_KW_SPLAT_MUT + ] + + # Insert the legacy CALL_BLOCKISEQ flag for Ruby 3.2 and earlier. + flags.insert(5, :CALL_BLOCKISEQ) if RUBY_VERSION < "3.3" + + # Set the flags as constants on the class. + flags.each_with_index { |name, index| const_set(name, 1 << index) } attr_reader :method, :argc, :flags, :kw_arg @@ -50,7 +57,6 @@ def inspect names << :FCALL if flag?(CALL_FCALL) names << :VCALL if flag?(CALL_VCALL) names << :ARGS_SIMPLE if flag?(CALL_ARGS_SIMPLE) - names << :BLOCKISEQ if flag?(CALL_BLOCKISEQ) names << :KWARG if flag?(CALL_KWARG) names << :KW_SPLAT if flag?(CALL_KW_SPLAT) names << :TAILCALL if flag?(CALL_TAILCALL) From 1af7a77b56b584f27429e77dfde372620178bf11 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Apr 2023 18:00:17 +0000 Subject: [PATCH 452/536] Bump rubocop from 1.48.1 to 1.49.0 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.48.1 to 1.49.0. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.48.1...v1.49.0) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index ad2aeaa5..2f03cb6e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -12,24 +12,24 @@ GEM json (2.6.3) minitest (5.18.0) parallel (1.22.1) - parser (3.2.1.1) + parser (3.2.2.0) ast (~> 2.4.1) prettier_print (1.2.1) rainbow (3.1.1) rake (13.0.6) regexp_parser (2.7.0) rexml (3.2.5) - rubocop (1.48.1) + rubocop (1.49.0) json (~> 2.3) parallel (~> 1.10) parser (>= 3.2.0.0) rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 1.8, < 3.0) rexml (>= 3.2.5, < 4.0) - rubocop-ast (>= 1.26.0, < 2.0) + rubocop-ast (>= 1.28.0, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.27.0) + rubocop-ast (1.28.0) parser (>= 3.2.1.0) ruby-progressbar (1.13.0) simplecov (0.22.0) From c583a18d49bd5d60df9629d9a04869c115be9011 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 11 Apr 2023 17:59:11 +0000 Subject: [PATCH 453/536] Bump rubocop from 1.49.0 to 1.50.0 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.49.0 to 1.50.0. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.49.0...v1.50.0) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 2f03cb6e..e0ce76bd 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -19,7 +19,7 @@ GEM rake (13.0.6) regexp_parser (2.7.0) rexml (3.2.5) - rubocop (1.49.0) + rubocop (1.50.0) json (~> 2.3) parallel (~> 1.10) parser (>= 3.2.0.0) From f3f307de586caa5017994d4139248721d961444e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 12 Apr 2023 17:58:56 +0000 Subject: [PATCH 454/536] Bump rubocop from 1.50.0 to 1.50.1 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.50.0 to 1.50.1. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.50.0...v1.50.1) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index e0ce76bd..f705736f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -19,7 +19,7 @@ GEM rake (13.0.6) regexp_parser (2.7.0) rexml (3.2.5) - rubocop (1.50.0) + rubocop (1.50.1) json (~> 2.3) parallel (~> 1.10) parser (>= 3.2.0.0) From ada73b3afe1269762ea75972bdf1a544f8d8c519 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Apr 2023 18:00:28 +0000 Subject: [PATCH 455/536] Bump rubocop from 1.50.1 to 1.50.2 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.50.1 to 1.50.2. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.50.1...v1.50.2) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index f705736f..8ee39a70 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -17,9 +17,9 @@ GEM prettier_print (1.2.1) rainbow (3.1.1) rake (13.0.6) - regexp_parser (2.7.0) + regexp_parser (2.8.0) rexml (3.2.5) - rubocop (1.50.1) + rubocop (1.50.2) json (~> 2.3) parallel (~> 1.10) parser (>= 3.2.0.0) From fefc783422e7b26a63510446faa426ac1e169b85 Mon Sep 17 00:00:00 2001 From: Felipe Vicente Date: Tue, 18 Apr 2023 23:14:34 -0300 Subject: [PATCH 456/536] Fix typo on README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3a3f7d2d..113c0f29 100644 --- a/README.md +++ b/README.md @@ -297,7 +297,7 @@ Note that the output of the `match` CLI command creates a valid pattern that can ### write -This command will format the listed files and write that formatted version back to the source files. Note that this overwrites the original content, to be sure to be using a version control system. +This command will format the listed files and write that formatted version back to the source files. Note that this overwrites the original content, so be sure to be using a version control system. ```sh stree write path/to/file.rb From 5529448bd303d2fb7684c469dc20dc0031badec6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 19 Apr 2023 18:02:16 +0000 Subject: [PATCH 457/536] Bump dependabot/fetch-metadata from 1.3.6 to 1.4.0 Bumps [dependabot/fetch-metadata](https://github.com/dependabot/fetch-metadata) from 1.3.6 to 1.4.0. - [Release notes](https://github.com/dependabot/fetch-metadata/releases) - [Commits](https://github.com/dependabot/fetch-metadata/compare/v1.3.6...v1.4.0) --- updated-dependencies: - dependency-name: dependabot/fetch-metadata dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/auto-merge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/auto-merge.yml b/.github/workflows/auto-merge.yml index e54c9100..85e9fdb7 100644 --- a/.github/workflows/auto-merge.yml +++ b/.github/workflows/auto-merge.yml @@ -12,7 +12,7 @@ jobs: steps: - name: Dependabot metadata id: metadata - uses: dependabot/fetch-metadata@v1.3.6 + uses: dependabot/fetch-metadata@v1.4.0 with: github-token: "${{ secrets.GITHUB_TOKEN }}" - name: Enable auto-merge for Dependabot PRs From 117c6b5f8e23070355420c57647b5b254abf7097 Mon Sep 17 00:00:00 2001 From: Andy Waite <13400+andyw8@users.noreply.github.com> Date: Wed, 19 Apr 2023 22:10:38 -0400 Subject: [PATCH 458/536] Update README.md - fix link for visitors --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3a3f7d2d..3fdf0e6e 100644 --- a/README.md +++ b/README.md @@ -525,7 +525,7 @@ With visitors, you only define handlers for the nodes that you need. You can fin * call `visit(child)` with each child that you want to visit * call nothing if you're sure you don't want to descend further -There are a couple of visitors that ship with Syntax Tree that can be used as examples. They live in the [lib/syntax_tree/visitor](lib/syntax_tree/visitor) directory. +There are a couple of visitors that ship with Syntax Tree that can be used as examples. They live in the [lib/syntax_tree](lib/syntax_tree) directory. ### visit_method From c921b551f7caf51eb25c86a8b9e8a4e076ccd0a1 Mon Sep 17 00:00:00 2001 From: t-mario-y Date: Wed, 3 May 2023 19:23:15 +0900 Subject: [PATCH 459/536] rename left variables and files for plugin/disable_auto_ternary --- lib/syntax_tree/formatter.rb | 2 +- .../plugin/{disable_ternary.rb => disable_auto_ternary.rb} | 2 +- .../{disable_ternary_test.rb => disable_auto_ternary_test.rb} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename lib/syntax_tree/plugin/{disable_ternary.rb => disable_auto_ternary.rb} (70%) rename test/plugin/{disable_ternary_test.rb => disable_auto_ternary_test.rb} (100%) diff --git a/lib/syntax_tree/formatter.rb b/lib/syntax_tree/formatter.rb index 60858bf2..2b229885 100644 --- a/lib/syntax_tree/formatter.rb +++ b/lib/syntax_tree/formatter.rb @@ -60,7 +60,7 @@ def initialize( # constant. That constant is responsible for determining the default # disable ternary value. If it's defined, then we default to true. # Otherwise we default to false. - defined?(DISABLE_TERNARY) + defined?(DISABLE_AUTO_TERNARY) else disable_auto_ternary end diff --git a/lib/syntax_tree/plugin/disable_ternary.rb b/lib/syntax_tree/plugin/disable_auto_ternary.rb similarity index 70% rename from lib/syntax_tree/plugin/disable_ternary.rb rename to lib/syntax_tree/plugin/disable_auto_ternary.rb index 0cb48d84..dd38c783 100644 --- a/lib/syntax_tree/plugin/disable_ternary.rb +++ b/lib/syntax_tree/plugin/disable_auto_ternary.rb @@ -2,6 +2,6 @@ module SyntaxTree class Formatter - DISABLE_TERNARY = true + DISABLE_AUTO_TERNARY = true end end diff --git a/test/plugin/disable_ternary_test.rb b/test/plugin/disable_auto_ternary_test.rb similarity index 100% rename from test/plugin/disable_ternary_test.rb rename to test/plugin/disable_auto_ternary_test.rb From c91f495270168e9a6df1cb1029a29d6f60636c64 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 May 2023 18:00:15 +0000 Subject: [PATCH 460/536] Bump rubocop from 1.50.2 to 1.51.0 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.50.2 to 1.51.0. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.50.2...v1.51.0) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 8ee39a70..0a60c87e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -11,15 +11,15 @@ GEM docile (1.4.0) json (2.6.3) minitest (5.18.0) - parallel (1.22.1) - parser (3.2.2.0) + parallel (1.23.0) + parser (3.2.2.1) ast (~> 2.4.1) prettier_print (1.2.1) rainbow (3.1.1) rake (13.0.6) regexp_parser (2.8.0) rexml (3.2.5) - rubocop (1.50.2) + rubocop (1.51.0) json (~> 2.3) parallel (~> 1.10) parser (>= 3.2.0.0) @@ -29,7 +29,7 @@ GEM rubocop-ast (>= 1.28.0, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.28.0) + rubocop-ast (1.28.1) parser (>= 3.2.1.0) ruby-progressbar (1.13.0) simplecov (0.22.0) From 692d23f169d203176e6a13f891b637f93bf1f33e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 May 2023 18:07:01 +0000 Subject: [PATCH 461/536] Bump dependabot/fetch-metadata from 1.4.0 to 1.5.0 Bumps [dependabot/fetch-metadata](https://github.com/dependabot/fetch-metadata) from 1.4.0 to 1.5.0. - [Release notes](https://github.com/dependabot/fetch-metadata/releases) - [Commits](https://github.com/dependabot/fetch-metadata/compare/v1.4.0...v1.5.0) --- updated-dependencies: - dependency-name: dependabot/fetch-metadata dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/auto-merge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/auto-merge.yml b/.github/workflows/auto-merge.yml index 85e9fdb7..d56afdfb 100644 --- a/.github/workflows/auto-merge.yml +++ b/.github/workflows/auto-merge.yml @@ -12,7 +12,7 @@ jobs: steps: - name: Dependabot metadata id: metadata - uses: dependabot/fetch-metadata@v1.4.0 + uses: dependabot/fetch-metadata@v1.5.0 with: github-token: "${{ secrets.GITHUB_TOKEN }}" - name: Enable auto-merge for Dependabot PRs From ab0be75a1162cdcd525852f8d2bc0036ede40f3a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 24 May 2023 18:01:45 +0000 Subject: [PATCH 462/536] Bump dependabot/fetch-metadata from 1.5.0 to 1.5.1 Bumps [dependabot/fetch-metadata](https://github.com/dependabot/fetch-metadata) from 1.5.0 to 1.5.1. - [Release notes](https://github.com/dependabot/fetch-metadata/releases) - [Commits](https://github.com/dependabot/fetch-metadata/compare/v1.5.0...v1.5.1) --- updated-dependencies: - dependency-name: dependabot/fetch-metadata dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/auto-merge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/auto-merge.yml b/.github/workflows/auto-merge.yml index d56afdfb..57830be5 100644 --- a/.github/workflows/auto-merge.yml +++ b/.github/workflows/auto-merge.yml @@ -12,7 +12,7 @@ jobs: steps: - name: Dependabot metadata id: metadata - uses: dependabot/fetch-metadata@v1.5.0 + uses: dependabot/fetch-metadata@v1.5.1 with: github-token: "${{ secrets.GITHUB_TOKEN }}" - name: Enable auto-merge for Dependabot PRs From b3457654ee196f752c7c936b426a99f6888b1608 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 31 May 2023 11:01:20 -0400 Subject: [PATCH 463/536] Support opt_newarray_send --- lib/syntax_tree/node.rb | 11 +- lib/syntax_tree/yarv/instruction_sequence.rb | 25 ++++- lib/syntax_tree/yarv/instructions.rb | 82 ++++----------- lib/syntax_tree/yarv/legacy.rb | 104 +++++++++++++++++++ test/compiler_test.rb | 6 ++ 5 files changed, 157 insertions(+), 71 deletions(-) diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 54d132e6..ac6092ca 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -1299,7 +1299,7 @@ def format(q) end end - # [nil | VarRef] the optional constant wrapper + # [nil | VarRef | ConstPathRef] the optional constant wrapper attr_reader :constant # [Array[ Node ]] the regular positional arguments that this array @@ -2849,7 +2849,10 @@ def format_chain(q, children) # to print the operator trailing in order to keep it working. last_child = children.last if last_child.is_a?(CallNode) && last_child.message != :call && - last_child.message.comments.any? && last_child.operator + ( + (last_child.message.comments.any? && last_child.operator) || + (last_child.operator && last_child.operator.comments.any?) + ) q.format(CallOperatorFormatter.new(last_child.operator)) skip_operator = true else @@ -5413,7 +5416,7 @@ def ===(other) # end # class FndPtn < Node - # [nil | Node] the optional constant wrapper + # [nil | VarRef | ConstPathRef] the optional constant wrapper attr_reader :constant # [VarField] the splat on the left-hand side @@ -6035,7 +6038,7 @@ def format(q) end end - # [nil | Node] the optional constant wrapper + # [nil | VarRef | ConstPathRef] the optional constant wrapper attr_reader :constant # [Array[ [DynaSymbol | Label, nil | Node] ]] the set of tuples diff --git a/lib/syntax_tree/yarv/instruction_sequence.rb b/lib/syntax_tree/yarv/instruction_sequence.rb index 7ce7bcdd..df92799b 100644 --- a/lib/syntax_tree/yarv/instruction_sequence.rb +++ b/lib/syntax_tree/yarv/instruction_sequence.rb @@ -353,11 +353,27 @@ def specialize_instructions! next unless calldata.argc == 0 case calldata.method + when :min + node.value = + if RUBY_VERSION < "3.3" + Legacy::OptNewArrayMin.new(value.number) + else + OptNewArraySend.new(value.number, :min) + end + + node.next_node = next_node.next_node when :max - node.value = OptNewArrayMax.new(value.number) + node.value = + if RUBY_VERSION < "3.3" + Legacy::OptNewArrayMax.new(value.number) + else + OptNewArraySend.new(value.number, :max) + end + node.next_node = next_node.next_node - when :min - node.value = OptNewArrayMin.new(value.number) + when :hash + next if RUBY_VERSION < "3.3" + node.value = OptNewArraySend.new(value.number, :hash) node.next_node = next_node.next_node end when PutObject, PutString @@ -1174,6 +1190,9 @@ def self.from(source, options = Compiler::Options.new, parent_iseq = nil) when :opt_newarray_min iseq.newarray(opnds[0]) iseq.send(YARV.calldata(:min)) + when :opt_newarray_send + iseq.newarray(opnds[0]) + iseq.send(CallData.new(opnds[1])) when :opt_neq iseq.push( OptNEq.new(CallData.from(opnds[0]), CallData.from(opnds[1])) diff --git a/lib/syntax_tree/yarv/instructions.rb b/lib/syntax_tree/yarv/instructions.rb index ceb237dc..ffeebe65 100644 --- a/lib/syntax_tree/yarv/instructions.rb +++ b/lib/syntax_tree/yarv/instructions.rb @@ -3818,9 +3818,10 @@ def call(vm) # ### Summary # - # `opt_newarray_max` is a specialization that occurs when the `max` method - # is called on an array literal. It pops the values of the array off the - # stack and pushes on the result. + # `opt_newarray_send` is a specialization that occurs when a dynamic array + # literal is created and immediately sent the `min`, `max`, or `hash` + # methods. It pops the values of the array off the stack and pushes on the + # result of the method call. # # ### Usage # @@ -3828,83 +3829,36 @@ def call(vm) # [a, b, c].max # ~~~ # - class OptNewArrayMax < Instruction - attr_reader :number - - def initialize(number) - @number = number - end - - def disasm(fmt) - fmt.instruction("opt_newarray_max", [fmt.object(number)]) - end - - def to_a(_iseq) - [:opt_newarray_max, number] - end - - def deconstruct_keys(_keys) - { number: number } - end - - def ==(other) - other.is_a?(OptNewArrayMax) && other.number == number - end - - def length - 2 - end - - def pops - number - end + class OptNewArraySend < Instruction + attr_reader :number, :method - def pushes - 1 - end - - def call(vm) - vm.push(vm.pop(number).max) - end - end - - # ### Summary - # - # `opt_newarray_min` is a specialization that occurs when the `min` method - # is called on an array literal. It pops the values of the array off the - # stack and pushes on the result. - # - # ### Usage - # - # ~~~ruby - # [a, b, c].min - # ~~~ - # - class OptNewArrayMin < Instruction - attr_reader :number - - def initialize(number) + def initialize(number, method) @number = number + @method = method end def disasm(fmt) - fmt.instruction("opt_newarray_min", [fmt.object(number)]) + fmt.instruction( + "opt_newarray_send", + [fmt.object(number), fmt.object(method)] + ) end def to_a(_iseq) - [:opt_newarray_min, number] + [:opt_newarray_send, number, method] end def deconstruct_keys(_keys) - { number: number } + { number: number, method: method } end def ==(other) - other.is_a?(OptNewArrayMin) && other.number == number + other.is_a?(OptNewArraySend) && other.number == number && + other.method == method end def length - 2 + 3 end def pops @@ -3916,7 +3870,7 @@ def pushes end def call(vm) - vm.push(vm.pop(number).min) + vm.push(vm.pop(number).__send__(method)) end end diff --git a/lib/syntax_tree/yarv/legacy.rb b/lib/syntax_tree/yarv/legacy.rb index e20729d9..8715993a 100644 --- a/lib/syntax_tree/yarv/legacy.rb +++ b/lib/syntax_tree/yarv/legacy.rb @@ -124,6 +124,110 @@ def falls_through? end end + # ### Summary + # + # `opt_newarray_max` is a specialization that occurs when the `max` method + # is called on an array literal. It pops the values of the array off the + # stack and pushes on the result. + # + # ### Usage + # + # ~~~ruby + # [a, b, c].max + # ~~~ + # + class OptNewArrayMax < Instruction + attr_reader :number + + def initialize(number) + @number = number + end + + def disasm(fmt) + fmt.instruction("opt_newarray_max", [fmt.object(number)]) + end + + def to_a(_iseq) + [:opt_newarray_max, number] + end + + def deconstruct_keys(_keys) + { number: number } + end + + def ==(other) + other.is_a?(OptNewArrayMax) && other.number == number + end + + def length + 2 + end + + def pops + number + end + + def pushes + 1 + end + + def call(vm) + vm.push(vm.pop(number).max) + end + end + + # ### Summary + # + # `opt_newarray_min` is a specialization that occurs when the `min` method + # is called on an array literal. It pops the values of the array off the + # stack and pushes on the result. + # + # ### Usage + # + # ~~~ruby + # [a, b, c].min + # ~~~ + # + class OptNewArrayMin < Instruction + attr_reader :number + + def initialize(number) + @number = number + end + + def disasm(fmt) + fmt.instruction("opt_newarray_min", [fmt.object(number)]) + end + + def to_a(_iseq) + [:opt_newarray_min, number] + end + + def deconstruct_keys(_keys) + { number: number } + end + + def ==(other) + other.is_a?(OptNewArrayMin) && other.number == number + end + + def length + 2 + end + + def pops + number + end + + def pushes + 1 + end + + def call(vm) + vm.push(vm.pop(number).min) + end + end + # ### Summary # # `opt_setinlinecache` sets an inline cache for a constant lookup. It pops diff --git a/test/compiler_test.rb b/test/compiler_test.rb index 1922f8c6..ca3e8dde 100644 --- a/test/compiler_test.rb +++ b/test/compiler_test.rb @@ -311,6 +311,12 @@ class CompilerTest < Minitest::Test "[1, 2, 3].min", "[foo, bar, baz].min", "[foo, bar, baz].min(1)", + "[1, 2, 3].hash", + "[foo, bar, baz].hash", + "[foo, bar, baz].hash(1)", + "[1, 2, 3].foo", + "[foo, bar, baz].foo", + "[foo, bar, baz].foo(1)", "[**{ x: true }][0][:x]", # Core method calls "alias foo bar", From 32d6d8a62cacfdd220e47defe5a2d1a445541087 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Barri=C3=A9?= Date: Thu, 1 Jun 2023 14:52:20 +0200 Subject: [PATCH 464/536] Always use do/end for multiline lambdas Previously lambda blocks inside a Command/CommandCall were always using braces, even when multiline. --- lib/syntax_tree/node.rb | 33 +++++++-------------------------- test/fixtures/lambda.rb | 28 ++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 26 deletions(-) diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index ac6092ca..a2c78677 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -7210,36 +7210,17 @@ def format(q) q.text(" ") q .if_break do - force_parens = - q.parents.any? do |node| - node.is_a?(Command) || node.is_a?(CommandCall) - end - - if force_parens - q.text("{") + q.text("do") - unless statements.empty? - q.indent do - q.breakable_space - q.format(statements) - end + unless statements.empty? + q.indent do q.breakable_space + q.format(statements) end - - q.text("}") - else - q.text("do") - - unless statements.empty? - q.indent do - q.breakable_space - q.format(statements) - end - end - - q.breakable_space - q.text("end") end + + q.breakable_space + q.text("end") end .if_flat do q.text("{") diff --git a/test/fixtures/lambda.rb b/test/fixtures/lambda.rb index 5dba3be3..8b922ef0 100644 --- a/test/fixtures/lambda.rb +++ b/test/fixtures/lambda.rb @@ -80,3 +80,31 @@ -> do # comment1 # comment2 end +% # multiline lambda in a command +command "arg" do + -> { + multi + line + } +end +- +command "arg" do + -> do + multi + line + end +end +% # multiline lambda in a command call +command.call "arg" do + -> { + multi + line + } +end +- +command.call "arg" do + -> do + multi + line + end +end From c7de3f653cec55ac5aa9d71b46ccca19430e00d8 Mon Sep 17 00:00:00 2001 From: nobuyo Date: Mon, 12 Jun 2023 13:52:21 +0900 Subject: [PATCH 465/536] Fix formatting for CHAR node when single_quotes plugin is enabled --- lib/syntax_tree/node.rb | 2 +- test/plugin/single_quotes_test.rb | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index a2c78677..3c5f93da 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -288,7 +288,7 @@ def format(q) q.text(value) else q.text(q.quote) - q.text(value[1] == "\"" ? "\\\"" : value[1]) + q.text(value[1] == q.quote ? "\\#{q.quote}" : value[1]) q.text(q.quote) end end diff --git a/test/plugin/single_quotes_test.rb b/test/plugin/single_quotes_test.rb index 6ce10448..b1359ac7 100644 --- a/test/plugin/single_quotes_test.rb +++ b/test/plugin/single_quotes_test.rb @@ -8,6 +8,14 @@ def test_empty_string_literal assert_format("''\n", "\"\"") end + def test_character_literal_with_double_quote + assert_format("'\"'\n", "?\"") + end + + def test_character_literal_with_singlee_quote + assert_format("'\\''\n", "?'") + end + def test_string_literal assert_format("'string'\n", "\"string\"") end From 4e94775e6d27d9407f554cb6728e71a330511004 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Jun 2023 18:00:06 +0000 Subject: [PATCH 466/536] Bump minitest from 5.18.0 to 5.18.1 Bumps [minitest](https://github.com/minitest/minitest) from 5.18.0 to 5.18.1. - [Changelog](https://github.com/minitest/minitest/blob/master/History.rdoc) - [Commits](https://github.com/minitest/minitest/compare/v5.18.0...v5.18.1) --- updated-dependencies: - dependency-name: minitest dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 0a60c87e..2bd42028 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -10,7 +10,7 @@ GEM ast (2.4.2) docile (1.4.0) json (2.6.3) - minitest (5.18.0) + minitest (5.18.1) parallel (1.23.0) parser (3.2.2.1) ast (~> 2.4.1) From 7fc6be2cc612f8f56416204454621ca47f2036f7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 28 Jun 2023 17:10:53 +0000 Subject: [PATCH 467/536] Bump dependabot/fetch-metadata from 1.5.1 to 1.6.0 Bumps [dependabot/fetch-metadata](https://github.com/dependabot/fetch-metadata) from 1.5.1 to 1.6.0. - [Release notes](https://github.com/dependabot/fetch-metadata/releases) - [Commits](https://github.com/dependabot/fetch-metadata/compare/v1.5.1...v1.6.0) --- updated-dependencies: - dependency-name: dependabot/fetch-metadata dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/auto-merge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/auto-merge.yml b/.github/workflows/auto-merge.yml index 57830be5..8ca265e0 100644 --- a/.github/workflows/auto-merge.yml +++ b/.github/workflows/auto-merge.yml @@ -12,7 +12,7 @@ jobs: steps: - name: Dependabot metadata id: metadata - uses: dependabot/fetch-metadata@v1.5.1 + uses: dependabot/fetch-metadata@v1.6.0 with: github-token: "${{ secrets.GITHUB_TOKEN }}" - name: Enable auto-merge for Dependabot PRs From 1d4e6a5b07110e34346eeaa5d6db51656722666f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 11 Jul 2023 17:38:22 +0000 Subject: [PATCH 468/536] Bump actions/upload-pages-artifact from 1 to 2 Bumps [actions/upload-pages-artifact](https://github.com/actions/upload-pages-artifact) from 1 to 2. - [Release notes](https://github.com/actions/upload-pages-artifact/releases) - [Commits](https://github.com/actions/upload-pages-artifact/compare/v1...v2) --- updated-dependencies: - dependency-name: actions/upload-pages-artifact dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/gh-pages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 5b662631..f2419d00 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -39,7 +39,7 @@ jobs: rdoc --main README.md --op _site --exclude={Gemfile,Rakefile,"coverage/*","vendor/*","bin/*","test/*","tmp/*"} cp -r doc _site/doc - name: Upload artifact - uses: actions/upload-pages-artifact@v1 + uses: actions/upload-pages-artifact@v2 # Deployment job deploy: From 9e8a2c3db7f6bb4f862c3c87c8d77198db613589 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 14 Jul 2023 11:46:48 -0400 Subject: [PATCH 469/536] Fix rubocop build --- .rubocop.yml | 5 ++++- Gemfile.lock | 14 +++++++++----- lib/syntax_tree/reflection.rb | 2 +- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index c1c17001..2142296f 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -7,7 +7,7 @@ AllCops: SuggestExtensions: false TargetRubyVersion: 2.7 Exclude: - - '{.git,.github,bin,coverage,pkg,sorbet,spec,test/fixtures,vendor,tmp}/**/*' + - '{.git,.github,.ruby-lsp,bin,coverage,doc,pkg,sorbet,spec,test/fixtures,vendor,tmp}/**/*' - test.rb Gemspec/DevelopmentDependencies: @@ -154,6 +154,9 @@ Style/ParallelAssignment: Style/PerlBackrefs: Enabled: false +Style/RedundantArrayConstructor: + Enabled: false + Style/SafeNavigation: Enabled: false diff --git a/Gemfile.lock b/Gemfile.lock index 2bd42028..a26c0e17 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -10,26 +10,30 @@ GEM ast (2.4.2) docile (1.4.0) json (2.6.3) + language_server-protocol (3.17.0.3) minitest (5.18.1) parallel (1.23.0) - parser (3.2.2.1) + parser (3.2.2.3) ast (~> 2.4.1) + racc prettier_print (1.2.1) + racc (1.7.1) rainbow (3.1.1) rake (13.0.6) - regexp_parser (2.8.0) + regexp_parser (2.8.1) rexml (3.2.5) - rubocop (1.51.0) + rubocop (1.54.2) json (~> 2.3) + language_server-protocol (>= 3.17.0) parallel (~> 1.10) - parser (>= 3.2.0.0) + parser (>= 3.2.2.3) rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 1.8, < 3.0) rexml (>= 3.2.5, < 4.0) rubocop-ast (>= 1.28.0, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.28.1) + rubocop-ast (1.29.0) parser (>= 3.2.1.0) ruby-progressbar (1.13.0) simplecov (0.22.0) diff --git a/lib/syntax_tree/reflection.rb b/lib/syntax_tree/reflection.rb index aa7b85b6..6955aa21 100644 --- a/lib/syntax_tree/reflection.rb +++ b/lib/syntax_tree/reflection.rb @@ -64,7 +64,7 @@ def inspect class << self def parse(comment) - comment = comment.gsub(/\n/, " ") + comment = comment.gsub("\n", " ") unless comment.start_with?("[") raise "Comment does not start with a bracket: #{comment.inspect}" From fcc974a2e91f183de79d1d038aba9725e327c604 Mon Sep 17 00:00:00 2001 From: Vinicius Stock Date: Wed, 12 Jul 2023 13:48:15 -0600 Subject: [PATCH 470/536] Fix WithScope for destructured post arguments --- lib/syntax_tree/node.rb | 4 ++-- lib/syntax_tree/with_scope.rb | 5 +---- test/with_scope_test.rb | 36 +++++++++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index a2c78677..3b676552 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -8277,8 +8277,8 @@ def format(q) # parameter attr_reader :rest - # [Array[ Ident ]] any positional parameters that exist after a rest - # parameter + # [Array[ Ident | MLHSParen ]] any positional parameters that exist after a + # rest parameter attr_reader :posts # [Array[ [ Label, nil | Node ] ]] any keyword parameters and their diff --git a/lib/syntax_tree/with_scope.rb b/lib/syntax_tree/with_scope.rb index c479fd3e..8c4908f3 100644 --- a/lib/syntax_tree/with_scope.rb +++ b/lib/syntax_tree/with_scope.rb @@ -152,10 +152,7 @@ def visit_def(node) # arguments. def visit_params(node) add_argument_definitions(node.requireds) - - node.posts.each do |param| - current_scope.add_local_definition(param, :argument) - end + add_argument_definitions(node.posts) node.keywords.each do |param| current_scope.add_local_definition(param.first, :argument) diff --git a/test/with_scope_test.rb b/test/with_scope_test.rb index 5bf276be..6b48d17d 100644 --- a/test/with_scope_test.rb +++ b/test/with_scope_test.rb @@ -154,6 +154,42 @@ def foo(a) assert_argument(collector, "a", definitions: [1], usages: [2]) end + def test_collecting_methods_with_destructured_post_arguments + collector = Collector.collect(<<~RUBY) + def foo(optional = 1, (bin, bag)) + end + RUBY + + assert_equal(3, collector.arguments.length) + assert_argument(collector, "optional", definitions: [1], usages: []) + assert_argument(collector, "bin", definitions: [1], usages: []) + assert_argument(collector, "bag", definitions: [1], usages: []) + end + + def test_collecting_methods_with_desctructured_post_using_splat + collector = Collector.collect(<<~RUBY) + def foo(optional = 1, (bin, bag, *)) + end + RUBY + + assert_equal(3, collector.arguments.length) + assert_argument(collector, "optional", definitions: [1], usages: []) + assert_argument(collector, "bin", definitions: [1], usages: []) + assert_argument(collector, "bag", definitions: [1], usages: []) + end + + def test_collecting_methods_with_nested_desctructured + collector = Collector.collect(<<~RUBY) + def foo(optional = 1, (bin, (bag))) + end + RUBY + + assert_equal(3, collector.arguments.length) + assert_argument(collector, "optional", definitions: [1], usages: []) + assert_argument(collector, "bin", definitions: [1], usages: []) + assert_argument(collector, "bag", definitions: [1], usages: []) + end + def test_collecting_singleton_method_arguments collector = Collector.collect(<<~RUBY) def self.foo(a) From c8bff434e03bfb1ae24ad344e47bf712f8f83846 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 25 Jul 2023 17:06:01 +0000 Subject: [PATCH 471/536] Bump rubocop from 1.54.2 to 1.55.0 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.54.2 to 1.55.0. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.54.2...v1.55.0) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index a26c0e17..b99cf938 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -22,7 +22,7 @@ GEM rake (13.0.6) regexp_parser (2.8.1) rexml (3.2.5) - rubocop (1.54.2) + rubocop (1.55.0) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10) @@ -30,7 +30,7 @@ GEM rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 1.8, < 3.0) rexml (>= 3.2.5, < 4.0) - rubocop-ast (>= 1.28.0, < 2.0) + rubocop-ast (>= 1.28.1, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 3.0) rubocop-ast (1.29.0) From 54aced14aa95c26b3810e4b6db11ac40eb87ce1d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 27 Jul 2023 17:14:46 +0000 Subject: [PATCH 472/536] Bump minitest from 5.18.1 to 5.19.0 Bumps [minitest](https://github.com/minitest/minitest) from 5.18.1 to 5.19.0. - [Changelog](https://github.com/minitest/minitest/blob/master/History.rdoc) - [Commits](https://github.com/minitest/minitest/compare/v5.18.1...v5.19.0) --- updated-dependencies: - dependency-name: minitest dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index b99cf938..ea447d12 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -11,7 +11,7 @@ GEM docile (1.4.0) json (2.6.3) language_server-protocol (3.17.0.3) - minitest (5.18.1) + minitest (5.19.0) parallel (1.23.0) parser (3.2.2.3) ast (~> 2.4.1) From 4b554b38cebf1e1ac33e93e0bb0e8e2d27f168cb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Jul 2023 17:22:03 +0000 Subject: [PATCH 473/536] Bump rubocop from 1.55.0 to 1.55.1 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.55.0 to 1.55.1. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.55.0...v1.55.1) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index ea447d12..5ac7c4a5 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -21,8 +21,8 @@ GEM rainbow (3.1.1) rake (13.0.6) regexp_parser (2.8.1) - rexml (3.2.5) - rubocop (1.55.0) + rexml (3.2.6) + rubocop (1.55.1) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10) From 16125ec48cc4bbac72515f283e0cb2387351aaec Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 9 Aug 2023 17:27:44 +0000 Subject: [PATCH 474/536] Bump rubocop from 1.55.1 to 1.56.0 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.55.1 to 1.56.0. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.55.1...v1.56.0) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 5ac7c4a5..d8d12e74 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -8,6 +8,7 @@ GEM remote: https://rubygems.org/ specs: ast (2.4.2) + base64 (0.1.1) docile (1.4.0) json (2.6.3) language_server-protocol (3.17.0.3) @@ -22,7 +23,8 @@ GEM rake (13.0.6) regexp_parser (2.8.1) rexml (3.2.6) - rubocop (1.55.1) + rubocop (1.56.0) + base64 (~> 0.1.1) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10) From b3faf1070b888aad1d53132e0121b966442893d2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Aug 2023 17:43:26 +0000 Subject: [PATCH 475/536] Bump rubocop from 1.56.0 to 1.56.1 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.56.0 to 1.56.1. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.56.0...v1.56.1) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index d8d12e74..cb658916 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -23,7 +23,7 @@ GEM rake (13.0.6) regexp_parser (2.8.1) rexml (3.2.6) - rubocop (1.56.0) + rubocop (1.56.1) base64 (~> 0.1.1) json (~> 2.3) language_server-protocol (>= 3.17.0) From 385b07bf7128e280224f5633e83681f7901f1448 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Aug 2023 17:55:06 +0000 Subject: [PATCH 476/536] Bump rubocop from 1.56.1 to 1.56.2 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.56.1 to 1.56.2. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.56.1...v1.56.2) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index cb658916..939a6e22 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -23,7 +23,7 @@ GEM rake (13.0.6) regexp_parser (2.8.1) rexml (3.2.6) - rubocop (1.56.1) + rubocop (1.56.2) base64 (~> 0.1.1) json (~> 2.3) language_server-protocol (>= 3.17.0) From 4629899aaae5ab5114fdea2c86ae45da5bafe807 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Sep 2023 17:46:15 +0000 Subject: [PATCH 477/536] Bump actions/checkout from 3 to 4 Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/gh-pages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index f2419d00..4bbfc0a2 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Pages uses: actions/configure-pages@v3 - name: Set up Ruby From b92a2cff094cb8f360b0dea02365b7ed53878809 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 6 Sep 2023 17:55:45 +0000 Subject: [PATCH 478/536] Bump minitest from 5.19.0 to 5.20.0 Bumps [minitest](https://github.com/minitest/minitest) from 5.19.0 to 5.20.0. - [Changelog](https://github.com/minitest/minitest/blob/master/History.rdoc) - [Commits](https://github.com/minitest/minitest/compare/v5.19.0...v5.20.0) --- updated-dependencies: - dependency-name: minitest dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 939a6e22..53021ebb 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -12,7 +12,7 @@ GEM docile (1.4.0) json (2.6.3) language_server-protocol (3.17.0.3) - minitest (5.19.0) + minitest (5.20.0) parallel (1.23.0) parser (3.2.2.3) ast (~> 2.4.1) From 55ecb2b29ea8405f9674424238ff74b7ba1f2267 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Sep 2023 17:14:36 +0000 Subject: [PATCH 479/536] Bump rubocop from 1.56.2 to 1.56.3 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.56.2 to 1.56.3. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.56.2...v1.56.3) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 53021ebb..1e97c593 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -23,7 +23,7 @@ GEM rake (13.0.6) regexp_parser (2.8.1) rexml (3.2.6) - rubocop (1.56.2) + rubocop (1.56.3) base64 (~> 0.1.1) json (~> 2.3) language_server-protocol (>= 3.17.0) From 68d34667370d55953a6fb53e8f5077917357d6c6 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 20 Sep 2023 10:58:33 -0400 Subject: [PATCH 480/536] Bump to version 6.2.0 --- CHANGELOG.md | 15 ++++++++++++++- Gemfile.lock | 2 +- lib/syntax_tree/version.rb | 2 +- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 273d4003..1beac42f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,16 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +## [6.2.0] - 2023-09-20 + +### Added + +- Fix `WithScope` for destructured post arguments. + +### Changed + +- Always use `do`/`end` for multi-line lambdas. + ## [6.1.1] - 2023-03-21 ### Changed @@ -603,7 +613,10 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - 🎉 Initial release! 🎉 -[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v6.0.2...HEAD +[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v6.2.0...HEAD +[6.2.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v6.1.1...v6.2.0 +[6.1.1]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v6.1.0...v6.1.1 +[6.1.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v6.0.2...v6.1.0 [6.0.2]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v6.0.1...v6.0.2 [6.0.1]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v6.0.0...v6.0.1 [6.0.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v5.3.0...v6.0.0 diff --git a/Gemfile.lock b/Gemfile.lock index 1e97c593..a9fc60b1 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - syntax_tree (6.1.1) + syntax_tree (6.2.0) prettier_print (>= 1.2.0) GEM diff --git a/lib/syntax_tree/version.rb b/lib/syntax_tree/version.rb index ad87d3e1..51599f77 100644 --- a/lib/syntax_tree/version.rb +++ b/lib/syntax_tree/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SyntaxTree - VERSION = "6.1.1" + VERSION = "6.2.0" end From 687b3d2232eb8fd663bf59c2675345077f830b0a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 28 Sep 2023 17:45:47 +0000 Subject: [PATCH 481/536] Bump rubocop from 1.56.3 to 1.56.4 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.56.3 to 1.56.4. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.56.3...v1.56.4) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index a9fc60b1..0b40c733 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -23,7 +23,7 @@ GEM rake (13.0.6) regexp_parser (2.8.1) rexml (3.2.6) - rubocop (1.56.3) + rubocop (1.56.4) base64 (~> 0.1.1) json (~> 2.3) language_server-protocol (>= 3.17.0) From 2ad0552d7081a12087e044c64869c0502496635a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 11 Oct 2023 17:08:55 +0000 Subject: [PATCH 482/536] Bump rubocop from 1.56.4 to 1.57.0 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.56.4 to 1.57.0. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.56.4...v1.57.0) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 0b40c733..f4eaa649 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -14,21 +14,21 @@ GEM language_server-protocol (3.17.0.3) minitest (5.20.0) parallel (1.23.0) - parser (3.2.2.3) + parser (3.2.2.4) ast (~> 2.4.1) racc prettier_print (1.2.1) racc (1.7.1) rainbow (3.1.1) rake (13.0.6) - regexp_parser (2.8.1) + regexp_parser (2.8.2) rexml (3.2.6) - rubocop (1.56.4) + rubocop (1.57.0) base64 (~> 0.1.1) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10) - parser (>= 3.2.2.3) + parser (>= 3.2.2.4) rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 1.8, < 3.0) rexml (>= 3.2.5, < 4.0) @@ -44,7 +44,7 @@ GEM simplecov_json_formatter (~> 0.1) simplecov-html (0.12.3) simplecov_json_formatter (0.1.4) - unicode-display_width (2.4.2) + unicode-display_width (2.5.0) PLATFORMS arm64-darwin-21 From 8a30c348feacaf2fea50f67f13f5219a79f740d8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 13 Oct 2023 17:10:21 +0000 Subject: [PATCH 483/536] Bump rubocop from 1.57.0 to 1.57.1 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.57.0 to 1.57.1. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.57.0...v1.57.1) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index f4eaa649..62a11d31 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -23,7 +23,7 @@ GEM rake (13.0.6) regexp_parser (2.8.2) rexml (3.2.6) - rubocop (1.57.0) + rubocop (1.57.1) base64 (~> 0.1.1) json (~> 2.3) language_server-protocol (>= 3.17.0) From 171d0020b2d5123a74eb7decc4b6336e8687afd8 Mon Sep 17 00:00:00 2001 From: Alex Rattray Date: Thu, 19 Oct 2023 18:19:21 -0400 Subject: [PATCH 484/536] Mention `write` from the `format` command When trying to figure out how to format a new Ruby project, we were choosing between `@prettier/plugin-ruby`, this project, and Rufo. At first, we thought this project didn't have built-in support for writing to disk at all, so we almost switched to Rufo (or prettier-ruby, despite that library's admonition to use this gem). Only when I said, "no, Kevin wouldn't ship a library like this without a `--write` facility!" did we keep scrolling and find the `write` command. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6e1119df..c238620e 100644 --- a/README.md +++ b/README.md @@ -187,7 +187,7 @@ SyntaxTree::Binary[ ### format -This command will output the formatted version of each of the listed files. Importantly, it will not write that content back to the source files. It is meant to display the formatted version only. +This command will output the formatted version of each of the listed files to stdout. Importantly, it will not write that content back to the source files – for that, you want [`write`](#write). ```sh stree format path/to/file.rb From d0bf2197631157f832804aced66aff98d9346d4b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 Oct 2023 17:42:55 +0000 Subject: [PATCH 485/536] Bump rubocop from 1.57.1 to 1.57.2 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.57.1 to 1.57.2. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.57.1...v1.57.2) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 62a11d31..c2f3665e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -8,7 +8,6 @@ GEM remote: https://rubygems.org/ specs: ast (2.4.2) - base64 (0.1.1) docile (1.4.0) json (2.6.3) language_server-protocol (3.17.0.3) @@ -23,8 +22,7 @@ GEM rake (13.0.6) regexp_parser (2.8.2) rexml (3.2.6) - rubocop (1.57.1) - base64 (~> 0.1.1) + rubocop (1.57.2) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10) @@ -35,7 +33,7 @@ GEM rubocop-ast (>= 1.28.1, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.29.0) + rubocop-ast (1.30.0) parser (>= 3.2.1.0) ruby-progressbar (1.13.0) simplecov (0.22.0) From 47a8a95b76c30dc012b0077b0685ea199abc117f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Oct 2023 17:23:15 +0000 Subject: [PATCH 486/536] Bump rake from 13.0.6 to 13.1.0 Bumps [rake](https://github.com/ruby/rake) from 13.0.6 to 13.1.0. - [Release notes](https://github.com/ruby/rake/releases) - [Changelog](https://github.com/ruby/rake/blob/master/History.rdoc) - [Commits](https://github.com/ruby/rake/compare/v13.0.6...v13.1.0) --- updated-dependencies: - dependency-name: rake dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index c2f3665e..02679ac8 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -19,7 +19,7 @@ GEM prettier_print (1.2.1) racc (1.7.1) rainbow (3.1.1) - rake (13.0.6) + rake (13.1.0) regexp_parser (2.8.2) rexml (3.2.6) rubocop (1.57.2) From a7cdb6042df68c85b8be34936d8e35df30cd81c5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Dec 2023 17:18:05 +0000 Subject: [PATCH 487/536] Bump actions/configure-pages from 3 to 4 Bumps [actions/configure-pages](https://github.com/actions/configure-pages) from 3 to 4. - [Release notes](https://github.com/actions/configure-pages/releases) - [Commits](https://github.com/actions/configure-pages/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/configure-pages dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/gh-pages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 4bbfc0a2..60f229d0 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -27,7 +27,7 @@ jobs: - name: Checkout uses: actions/checkout@v4 - name: Setup Pages - uses: actions/configure-pages@v3 + uses: actions/configure-pages@v4 - name: Set up Ruby uses: ruby/setup-ruby@v1 with: From bb92b915cfc8c4f770b97331661efbee086f5949 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Dec 2023 17:28:14 +0000 Subject: [PATCH 488/536] Bump actions/deploy-pages from 2 to 4 Bumps [actions/deploy-pages](https://github.com/actions/deploy-pages) from 2 to 4. - [Release notes](https://github.com/actions/deploy-pages/releases) - [Commits](https://github.com/actions/deploy-pages/compare/v2...v4) --- updated-dependencies: - dependency-name: actions/deploy-pages dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/gh-pages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 60f229d0..94bee969 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -51,4 +51,4 @@ jobs: steps: - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@v2 + uses: actions/deploy-pages@v4 From adaf133643ef7a7bdeff8503b904c02537a9ceb7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 12 Jan 2024 17:02:43 +0000 Subject: [PATCH 489/536] Bump minitest from 5.20.0 to 5.21.1 Bumps [minitest](https://github.com/minitest/minitest) from 5.20.0 to 5.21.1. - [Changelog](https://github.com/minitest/minitest/blob/master/History.rdoc) - [Commits](https://github.com/minitest/minitest/compare/v5.20.0...v5.21.1) --- updated-dependencies: - dependency-name: minitest dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 02679ac8..2d6c2174 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -11,7 +11,7 @@ GEM docile (1.4.0) json (2.6.3) language_server-protocol (3.17.0.3) - minitest (5.20.0) + minitest (5.21.1) parallel (1.23.0) parser (3.2.2.4) ast (~> 2.4.1) From c6624dee611d75084ffb8217ef8ec2e8aa5df15c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 18 Jan 2024 17:38:24 +0000 Subject: [PATCH 490/536] Bump minitest from 5.21.1 to 5.21.2 Bumps [minitest](https://github.com/minitest/minitest) from 5.21.1 to 5.21.2. - [Changelog](https://github.com/minitest/minitest/blob/master/History.rdoc) - [Commits](https://github.com/minitest/minitest/compare/v5.21.1...v5.21.2) --- updated-dependencies: - dependency-name: minitest dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 2d6c2174..d58aae0c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -11,7 +11,7 @@ GEM docile (1.4.0) json (2.6.3) language_server-protocol (3.17.0.3) - minitest (5.21.1) + minitest (5.21.2) parallel (1.23.0) parser (3.2.2.4) ast (~> 2.4.1) From ce3e996379885951fae3805f67e8979c396137e2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 Feb 2024 17:17:28 +0000 Subject: [PATCH 491/536] Bump minitest from 5.21.2 to 5.22.0 Bumps [minitest](https://github.com/minitest/minitest) from 5.21.2 to 5.22.0. - [Changelog](https://github.com/minitest/minitest/blob/master/History.rdoc) - [Commits](https://github.com/minitest/minitest/compare/v5.21.2...v5.22.0) --- updated-dependencies: - dependency-name: minitest dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index d58aae0c..475cd0b1 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -11,7 +11,7 @@ GEM docile (1.4.0) json (2.6.3) language_server-protocol (3.17.0.3) - minitest (5.21.2) + minitest (5.22.0) parallel (1.23.0) parser (3.2.2.4) ast (~> 2.4.1) From 24fba6fdc77fe1ce7cec0674d98f8f17d36f1aff Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 7 Feb 2024 17:08:32 +0000 Subject: [PATCH 492/536] Bump minitest from 5.22.0 to 5.22.1 Bumps [minitest](https://github.com/minitest/minitest) from 5.22.0 to 5.22.1. - [Changelog](https://github.com/minitest/minitest/blob/master/History.rdoc) - [Commits](https://github.com/minitest/minitest/compare/v5.22.0...v5.22.1) --- updated-dependencies: - dependency-name: minitest dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 475cd0b1..9a8dd7d7 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -11,7 +11,7 @@ GEM docile (1.4.0) json (2.6.3) language_server-protocol (3.17.0.3) - minitest (5.22.0) + minitest (5.22.1) parallel (1.23.0) parser (3.2.2.4) ast (~> 2.4.1) From b1e8fd91a130550f9385045294261b18cd3765ed Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 8 Feb 2024 17:50:17 +0000 Subject: [PATCH 493/536] Bump minitest from 5.22.1 to 5.22.2 Bumps [minitest](https://github.com/minitest/minitest) from 5.22.1 to 5.22.2. - [Changelog](https://github.com/minitest/minitest/blob/master/History.rdoc) - [Commits](https://github.com/minitest/minitest/compare/v5.22.1...v5.22.2) --- updated-dependencies: - dependency-name: minitest dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 9a8dd7d7..8fa50a1e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -11,7 +11,7 @@ GEM docile (1.4.0) json (2.6.3) language_server-protocol (3.17.0.3) - minitest (5.22.1) + minitest (5.22.2) parallel (1.23.0) parser (3.2.2.4) ast (~> 2.4.1) From b9d5fd3215a4f63162c5fa790f053033b80eea62 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 14 Mar 2024 17:43:58 +0000 Subject: [PATCH 494/536] Bump minitest from 5.22.2 to 5.22.3 Bumps [minitest](https://github.com/minitest/minitest) from 5.22.2 to 5.22.3. - [Changelog](https://github.com/minitest/minitest/blob/master/History.rdoc) - [Commits](https://github.com/minitest/minitest/commits) --- updated-dependencies: - dependency-name: minitest dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 8fa50a1e..ca33d6ee 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -11,7 +11,7 @@ GEM docile (1.4.0) json (2.6.3) language_server-protocol (3.17.0.3) - minitest (5.22.2) + minitest (5.22.3) parallel (1.23.0) parser (3.2.2.4) ast (~> 2.4.1) From 8a13b44bca2a92d8b526a8f8a117de4f1af41d38 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Mar 2024 18:00:21 +0000 Subject: [PATCH 495/536] Bump dependabot/fetch-metadata from 1.6.0 to 1.7.0 Bumps [dependabot/fetch-metadata](https://github.com/dependabot/fetch-metadata) from 1.6.0 to 1.7.0. - [Release notes](https://github.com/dependabot/fetch-metadata/releases) - [Commits](https://github.com/dependabot/fetch-metadata/compare/v1.6.0...v1.7.0) --- updated-dependencies: - dependency-name: dependabot/fetch-metadata dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/auto-merge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/auto-merge.yml b/.github/workflows/auto-merge.yml index 8ca265e0..67fbe586 100644 --- a/.github/workflows/auto-merge.yml +++ b/.github/workflows/auto-merge.yml @@ -12,7 +12,7 @@ jobs: steps: - name: Dependabot metadata id: metadata - uses: dependabot/fetch-metadata@v1.6.0 + uses: dependabot/fetch-metadata@v1.7.0 with: github-token: "${{ secrets.GITHUB_TOKEN }}" - name: Enable auto-merge for Dependabot PRs From ebf1f6d5c9d934c5538132eed708126a4e785846 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 22 Mar 2024 17:47:53 +0000 Subject: [PATCH 496/536] Bump dependabot/fetch-metadata from 1.7.0 to 2.0.0 Bumps [dependabot/fetch-metadata](https://github.com/dependabot/fetch-metadata) from 1.7.0 to 2.0.0. - [Release notes](https://github.com/dependabot/fetch-metadata/releases) - [Commits](https://github.com/dependabot/fetch-metadata/compare/v1.7.0...v2.0.0) --- updated-dependencies: - dependency-name: dependabot/fetch-metadata dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/auto-merge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/auto-merge.yml b/.github/workflows/auto-merge.yml index 67fbe586..44908aae 100644 --- a/.github/workflows/auto-merge.yml +++ b/.github/workflows/auto-merge.yml @@ -12,7 +12,7 @@ jobs: steps: - name: Dependabot metadata id: metadata - uses: dependabot/fetch-metadata@v1.7.0 + uses: dependabot/fetch-metadata@v2.0.0 with: github-token: "${{ secrets.GITHUB_TOKEN }}" - name: Enable auto-merge for Dependabot PRs From c2e233cc3f9b2e2c71d31ea32e5f795b2f955029 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 17:30:23 +0000 Subject: [PATCH 497/536] Bump actions/configure-pages from 4 to 5 Bumps [actions/configure-pages](https://github.com/actions/configure-pages) from 4 to 5. - [Release notes](https://github.com/actions/configure-pages/releases) - [Commits](https://github.com/actions/configure-pages/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/configure-pages dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/gh-pages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 94bee969..7ff5f5f1 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -27,7 +27,7 @@ jobs: - name: Checkout uses: actions/checkout@v4 - name: Setup Pages - uses: actions/configure-pages@v4 + uses: actions/configure-pages@v5 - name: Set up Ruby uses: ruby/setup-ruby@v1 with: From b084dc8ebaec41b13271fd39f82ed6533f0fc347 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 Apr 2024 18:00:36 +0000 Subject: [PATCH 498/536] Bump rake from 13.1.0 to 13.2.0 Bumps [rake](https://github.com/ruby/rake) from 13.1.0 to 13.2.0. - [Release notes](https://github.com/ruby/rake/releases) - [Changelog](https://github.com/ruby/rake/blob/master/History.rdoc) - [Commits](https://github.com/ruby/rake/compare/v13.1.0...v13.2.0) --- updated-dependencies: - dependency-name: rake dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index ca33d6ee..e66562e2 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -19,7 +19,7 @@ GEM prettier_print (1.2.1) racc (1.7.1) rainbow (3.1.1) - rake (13.1.0) + rake (13.2.0) regexp_parser (2.8.2) rexml (3.2.6) rubocop (1.57.2) From 28cc432d942e8edc208b35ea83d22a45e8992059 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 5 Apr 2024 17:22:15 +0000 Subject: [PATCH 499/536] Bump rake from 13.2.0 to 13.2.1 Bumps [rake](https://github.com/ruby/rake) from 13.2.0 to 13.2.1. - [Release notes](https://github.com/ruby/rake/releases) - [Changelog](https://github.com/ruby/rake/blob/master/History.rdoc) - [Commits](https://github.com/ruby/rake/compare/v13.2.0...v13.2.1) --- updated-dependencies: - dependency-name: rake dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index e66562e2..794e25d7 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -19,7 +19,7 @@ GEM prettier_print (1.2.1) racc (1.7.1) rainbow (3.1.1) - rake (13.2.0) + rake (13.2.1) regexp_parser (2.8.2) rexml (3.2.6) rubocop (1.57.2) From 07c1f38ddb43c5de8ee969c1d8355628cc2b5592 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 24 Apr 2024 17:07:55 +0000 Subject: [PATCH 500/536] Bump dependabot/fetch-metadata from 2.0.0 to 2.1.0 Bumps [dependabot/fetch-metadata](https://github.com/dependabot/fetch-metadata) from 2.0.0 to 2.1.0. - [Release notes](https://github.com/dependabot/fetch-metadata/releases) - [Commits](https://github.com/dependabot/fetch-metadata/compare/v2.0.0...v2.1.0) --- updated-dependencies: - dependency-name: dependabot/fetch-metadata dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/auto-merge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/auto-merge.yml b/.github/workflows/auto-merge.yml index 44908aae..6efe37ff 100644 --- a/.github/workflows/auto-merge.yml +++ b/.github/workflows/auto-merge.yml @@ -12,7 +12,7 @@ jobs: steps: - name: Dependabot metadata id: metadata - uses: dependabot/fetch-metadata@v2.0.0 + uses: dependabot/fetch-metadata@v2.1.0 with: github-token: "${{ secrets.GITHUB_TOKEN }}" - name: Enable auto-merge for Dependabot PRs From 2279e3373a18fe56a72c3801429f4a53768302db Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 May 2024 17:10:10 +0000 Subject: [PATCH 501/536] Bump minitest from 5.22.3 to 5.23.0 Bumps [minitest](https://github.com/minitest/minitest) from 5.22.3 to 5.23.0. - [Changelog](https://github.com/minitest/minitest/blob/master/History.rdoc) - [Commits](https://github.com/minitest/minitest/compare/v5.22.3...v5.23.0) --- updated-dependencies: - dependency-name: minitest dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 794e25d7..baf469d8 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -11,7 +11,7 @@ GEM docile (1.4.0) json (2.6.3) language_server-protocol (3.17.0.3) - minitest (5.22.3) + minitest (5.23.0) parallel (1.23.0) parser (3.2.2.4) ast (~> 2.4.1) From e507a537f46daa91fd6b32551d9029ad3d2f0ead Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 22 May 2024 17:37:05 +0000 Subject: [PATCH 502/536] --- updated-dependencies: - dependency-name: minitest dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index baf469d8..0931995d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -11,7 +11,7 @@ GEM docile (1.4.0) json (2.6.3) language_server-protocol (3.17.0.3) - minitest (5.23.0) + minitest (5.23.1) parallel (1.23.0) parser (3.2.2.4) ast (~> 2.4.1) From 56cd8acdf461966f68a75af90f6c9e2ac55b83a2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 19 Jun 2024 17:41:50 +0000 Subject: [PATCH 503/536] Bump minitest from 5.23.1 to 5.24.0 Bumps [minitest](https://github.com/minitest/minitest) from 5.23.1 to 5.24.0. - [Changelog](https://github.com/minitest/minitest/blob/master/History.rdoc) - [Commits](https://github.com/minitest/minitest/compare/v5.23.1...v5.24.0) --- updated-dependencies: - dependency-name: minitest dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 0931995d..3326d05b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -11,7 +11,7 @@ GEM docile (1.4.0) json (2.6.3) language_server-protocol (3.17.0.3) - minitest (5.23.1) + minitest (5.24.0) parallel (1.23.0) parser (3.2.2.4) ast (~> 2.4.1) From 13fd57eb9abc530c62c50f66d679736bff368059 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Jul 2024 17:03:00 +0000 Subject: [PATCH 504/536] Bump minitest from 5.24.0 to 5.24.1 Bumps [minitest](https://github.com/minitest/minitest) from 5.24.0 to 5.24.1. - [Changelog](https://github.com/minitest/minitest/blob/master/History.rdoc) - [Commits](https://github.com/minitest/minitest/compare/v5.24.0...v5.24.1) --- updated-dependencies: - dependency-name: minitest dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 3326d05b..94791460 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -11,7 +11,7 @@ GEM docile (1.4.0) json (2.6.3) language_server-protocol (3.17.0.3) - minitest (5.24.0) + minitest (5.24.1) parallel (1.23.0) parser (3.2.2.4) ast (~> 2.4.1) From 3f96ef12bc2c08e9c2deb34164544a9874d030bc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 5 Jul 2024 17:54:00 +0000 Subject: [PATCH 505/536] Bump dependabot/fetch-metadata from 2.1.0 to 2.2.0 Bumps [dependabot/fetch-metadata](https://github.com/dependabot/fetch-metadata) from 2.1.0 to 2.2.0. - [Release notes](https://github.com/dependabot/fetch-metadata/releases) - [Commits](https://github.com/dependabot/fetch-metadata/compare/v2.1.0...v2.2.0) --- updated-dependencies: - dependency-name: dependabot/fetch-metadata dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/auto-merge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/auto-merge.yml b/.github/workflows/auto-merge.yml index 6efe37ff..977b53e4 100644 --- a/.github/workflows/auto-merge.yml +++ b/.github/workflows/auto-merge.yml @@ -12,7 +12,7 @@ jobs: steps: - name: Dependabot metadata id: metadata - uses: dependabot/fetch-metadata@v2.1.0 + uses: dependabot/fetch-metadata@v2.2.0 with: github-token: "${{ secrets.GITHUB_TOKEN }}" - name: Enable auto-merge for Dependabot PRs From 9084b7a5e7ee9678b6ffd7c03b1d543b07c4632f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 14 Aug 2024 17:30:51 +0000 Subject: [PATCH 506/536] Bump minitest from 5.24.1 to 5.25.0 Bumps [minitest](https://github.com/minitest/minitest) from 5.24.1 to 5.25.0. - [Changelog](https://github.com/minitest/minitest/blob/master/History.rdoc) - [Commits](https://github.com/minitest/minitest/compare/v5.24.1...v5.25.0) --- updated-dependencies: - dependency-name: minitest dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 94791460..cac4d61c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -11,7 +11,7 @@ GEM docile (1.4.0) json (2.6.3) language_server-protocol (3.17.0.3) - minitest (5.24.1) + minitest (5.25.0) parallel (1.23.0) parser (3.2.2.4) ast (~> 2.4.1) From 882ac94123dd1ae5303ad3043d32abaabc9fe2ed Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 17:56:49 +0000 Subject: [PATCH 507/536] Bump minitest from 5.25.0 to 5.25.1 Bumps [minitest](https://github.com/minitest/minitest) from 5.25.0 to 5.25.1. - [Changelog](https://github.com/minitest/minitest/blob/master/History.rdoc) - [Commits](https://github.com/minitest/minitest/compare/v5.25.0...v5.25.1) --- updated-dependencies: - dependency-name: minitest dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index cac4d61c..def0ba89 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -11,7 +11,7 @@ GEM docile (1.4.0) json (2.6.3) language_server-protocol (3.17.0.3) - minitest (5.25.0) + minitest (5.25.1) parallel (1.23.0) parser (3.2.2.4) ast (~> 2.4.1) From 57af9682334f44164b23189efe2de085002570a0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 22 Nov 2024 17:24:27 +0000 Subject: [PATCH 508/536] Bump minitest from 5.25.1 to 5.25.2 Bumps [minitest](https://github.com/minitest/minitest) from 5.25.1 to 5.25.2. - [Changelog](https://github.com/minitest/minitest/blob/master/History.rdoc) - [Commits](https://github.com/minitest/minitest/compare/v5.25.1...v5.25.2) --- updated-dependencies: - dependency-name: minitest dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index def0ba89..fa5cdfe2 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -11,7 +11,7 @@ GEM docile (1.4.0) json (2.6.3) language_server-protocol (3.17.0.3) - minitest (5.25.1) + minitest (5.25.2) parallel (1.23.0) parser (3.2.2.4) ast (~> 2.4.1) From c23fcbec4f99d8943842e604a73e2a7152119224 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 4 Dec 2024 17:57:17 +0000 Subject: [PATCH 509/536] Bump minitest from 5.25.2 to 5.25.4 Bumps [minitest](https://github.com/minitest/minitest) from 5.25.2 to 5.25.4. - [Changelog](https://github.com/minitest/minitest/blob/master/History.rdoc) - [Commits](https://github.com/minitest/minitest/compare/v5.25.2...v5.25.4) --- updated-dependencies: - dependency-name: minitest dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index fa5cdfe2..d14291a4 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -11,7 +11,7 @@ GEM docile (1.4.0) json (2.6.3) language_server-protocol (3.17.0.3) - minitest (5.25.2) + minitest (5.25.4) parallel (1.23.0) parser (3.2.2.4) ast (~> 2.4.1) From 258e52a5a73d6ab563028f863f611028382f42e4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jan 2025 17:57:50 +0000 Subject: [PATCH 510/536] Bump dependabot/fetch-metadata from 2.2.0 to 2.3.0 Bumps [dependabot/fetch-metadata](https://github.com/dependabot/fetch-metadata) from 2.2.0 to 2.3.0. - [Release notes](https://github.com/dependabot/fetch-metadata/releases) - [Commits](https://github.com/dependabot/fetch-metadata/compare/v2.2.0...v2.3.0) --- updated-dependencies: - dependency-name: dependabot/fetch-metadata dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/auto-merge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/auto-merge.yml b/.github/workflows/auto-merge.yml index 977b53e4..295e8b18 100644 --- a/.github/workflows/auto-merge.yml +++ b/.github/workflows/auto-merge.yml @@ -12,7 +12,7 @@ jobs: steps: - name: Dependabot metadata id: metadata - uses: dependabot/fetch-metadata@v2.2.0 + uses: dependabot/fetch-metadata@v2.3.0 with: github-token: "${{ secrets.GITHUB_TOKEN }}" - name: Enable auto-merge for Dependabot PRs From 46476ce3cb45b48bfec0801ecddb3ecb7433b387 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 13 Mar 2025 17:20:26 +0000 Subject: [PATCH 511/536] Bump minitest from 5.25.4 to 5.25.5 Bumps [minitest](https://github.com/minitest/minitest) from 5.25.4 to 5.25.5. - [Changelog](https://github.com/minitest/minitest/blob/master/History.rdoc) - [Commits](https://github.com/minitest/minitest/compare/v5.25.4...v5.25.5) --- updated-dependencies: - dependency-name: minitest dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index d14291a4..22453392 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -11,7 +11,7 @@ GEM docile (1.4.0) json (2.6.3) language_server-protocol (3.17.0.3) - minitest (5.25.4) + minitest (5.25.5) parallel (1.23.0) parser (3.2.2.4) ast (~> 2.4.1) From af1a8325fbb77b73f295a01d9691e5ee43c6726e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 May 2025 17:31:27 +0000 Subject: [PATCH 512/536] Bump dependabot/fetch-metadata from 2.3.0 to 2.4.0 Bumps [dependabot/fetch-metadata](https://github.com/dependabot/fetch-metadata) from 2.3.0 to 2.4.0. - [Release notes](https://github.com/dependabot/fetch-metadata/releases) - [Commits](https://github.com/dependabot/fetch-metadata/compare/v2.3.0...v2.4.0) --- updated-dependencies: - dependency-name: dependabot/fetch-metadata dependency-version: 2.4.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/auto-merge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/auto-merge.yml b/.github/workflows/auto-merge.yml index 295e8b18..5468e6d0 100644 --- a/.github/workflows/auto-merge.yml +++ b/.github/workflows/auto-merge.yml @@ -12,7 +12,7 @@ jobs: steps: - name: Dependabot metadata id: metadata - uses: dependabot/fetch-metadata@v2.3.0 + uses: dependabot/fetch-metadata@v2.4.0 with: github-token: "${{ secrets.GITHUB_TOKEN }}" - name: Enable auto-merge for Dependabot PRs From f32655bbb61050d47bc71135fc60f1a9b9833f26 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 30 May 2025 17:21:22 +0000 Subject: [PATCH 513/536] Bump rake from 13.2.1 to 13.3.0 Bumps [rake](https://github.com/ruby/rake) from 13.2.1 to 13.3.0. - [Release notes](https://github.com/ruby/rake/releases) - [Changelog](https://github.com/ruby/rake/blob/master/History.rdoc) - [Commits](https://github.com/ruby/rake/compare/v13.2.1...v13.3.0) --- updated-dependencies: - dependency-name: rake dependency-version: 13.3.0 dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 22453392..1bf158a2 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -19,7 +19,7 @@ GEM prettier_print (1.2.1) racc (1.7.1) rainbow (3.1.1) - rake (13.2.1) + rake (13.3.0) regexp_parser (2.8.2) rexml (3.2.6) rubocop (1.57.2) From d0b3eafd8248b15b28c98c3c0cf3b976a26aa5e9 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 10 Jul 2025 12:07:06 -0400 Subject: [PATCH 514/536] Updates * Update fixtures to handle invalid jumps * Update YARV to handle newer instructions * Update whitequark fixtures * Update dependencies and fix rubocop violations --- .github/workflows/main.yml | 3 +- .rubocop.yml | 6 + .ruby-version | 1 - Gemfile | 2 + Gemfile.lock | 46 +- lib/syntax_tree/cli.rb | 2 +- lib/syntax_tree/node.rb | 4 + lib/syntax_tree/parser.rb | 10 +- lib/syntax_tree/pattern.rb | 1 + lib/syntax_tree/yarv/assembler.rb | 4 +- lib/syntax_tree/yarv/calldata.rb | 2 +- lib/syntax_tree/yarv/decompiler.rb | 12 +- lib/syntax_tree/yarv/instruction_sequence.rb | 43 +- lib/syntax_tree/yarv/instructions.rb | 147 ++ tasks/whitequark.rake | 9 +- test/cli_test.rb | 4 +- test/compiler_test.rb | 4 +- test/fixtures/bodystmt.rb | 1 + test/fixtures/break.rb | 64 +- test/fixtures/ifop.rb | 10 +- test/fixtures/next.rb | 100 +- test/fixtures/redo.rb | 6 +- test/fixtures/retry.rb | 10 +- test/fixtures/var_field_rassign.rb | 1 + test/fixtures/yield.rb | 32 +- test/fixtures/yield0.rb | 8 +- test/node_test.rb | 33 +- test/test_helper.rb | 20 +- test/translation/parser.txt | 2146 +++++++++--------- test/translation/parser_test.rb | 158 +- 30 files changed, 1577 insertions(+), 1312 deletions(-) delete mode 100644 .ruby-version diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3f811317..468591bd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -14,7 +14,8 @@ jobs: - '3.0' - '3.1' - '3.2' - - head + - '3.3' + - '3.4' - truffleruby-head name: CI runs-on: ubuntu-latest diff --git a/.rubocop.yml b/.rubocop.yml index 2142296f..1b81a535 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -136,6 +136,9 @@ Style/KeywordParametersOrder: Style/MissingRespondToMissing: Enabled: false +Style/MultipleComparison: + Enabled: false + Style/MutableConstant: Enabled: false @@ -157,6 +160,9 @@ Style/PerlBackrefs: Style/RedundantArrayConstructor: Enabled: false +Style/RedundantParentheses: + Enabled: false + Style/SafeNavigation: Enabled: false diff --git a/.ruby-version b/.ruby-version deleted file mode 100644 index 944880fa..00000000 --- a/.ruby-version +++ /dev/null @@ -1 +0,0 @@ -3.2.0 diff --git a/Gemfile b/Gemfile index be173b20..b4252fb5 100644 --- a/Gemfile +++ b/Gemfile @@ -3,3 +3,5 @@ source "https://rubygems.org" gemspec + +gem "fiddle" diff --git a/Gemfile.lock b/Gemfile.lock index 1bf158a2..b855c712 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -7,42 +7,47 @@ PATH GEM remote: https://rubygems.org/ specs: - ast (2.4.2) - docile (1.4.0) - json (2.6.3) - language_server-protocol (3.17.0.3) + ast (2.4.3) + docile (1.4.1) + fiddle (1.1.8) + json (2.12.2) + language_server-protocol (3.17.0.5) + lint_roller (1.1.0) minitest (5.25.5) - parallel (1.23.0) - parser (3.2.2.4) + parallel (1.27.0) + parser (3.3.8.0) ast (~> 2.4.1) racc prettier_print (1.2.1) - racc (1.7.1) + prism (1.4.0) + racc (1.8.1) rainbow (3.1.1) rake (13.3.0) - regexp_parser (2.8.2) - rexml (3.2.6) - rubocop (1.57.2) + regexp_parser (2.10.0) + rubocop (1.78.0) json (~> 2.3) - language_server-protocol (>= 3.17.0) + language_server-protocol (~> 3.17.0.2) + lint_roller (~> 1.1.0) parallel (~> 1.10) - parser (>= 3.2.2.4) + parser (>= 3.3.0.2) rainbow (>= 2.2.2, < 4.0) - regexp_parser (>= 1.8, < 3.0) - rexml (>= 3.2.5, < 4.0) - rubocop-ast (>= 1.28.1, < 2.0) + regexp_parser (>= 2.9.3, < 3.0) + rubocop-ast (>= 1.45.1, < 2.0) ruby-progressbar (~> 1.7) - unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.30.0) - parser (>= 3.2.1.0) + unicode-display_width (>= 2.4.0, < 4.0) + rubocop-ast (1.45.1) + parser (>= 3.3.7.2) + prism (~> 1.4) ruby-progressbar (1.13.0) simplecov (0.22.0) docile (~> 1.1) simplecov-html (~> 0.11) simplecov_json_formatter (~> 0.1) - simplecov-html (0.12.3) + simplecov-html (0.13.1) simplecov_json_formatter (0.1.4) - unicode-display_width (2.5.0) + unicode-display_width (3.1.4) + unicode-emoji (~> 4.0, >= 4.0.4) + unicode-emoji (4.0.4) PLATFORMS arm64-darwin-21 @@ -53,6 +58,7 @@ PLATFORMS DEPENDENCIES bundler + fiddle minitest rake rubocop diff --git a/lib/syntax_tree/cli.rb b/lib/syntax_tree/cli.rb index f2616c87..e0bafce9 100644 --- a/lib/syntax_tree/cli.rb +++ b/lib/syntax_tree/cli.rb @@ -159,7 +159,7 @@ class CTags < Action attr_reader :entries def initialize(options) - super(options) + super @entries = [] end diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 3b676552..5a92a5a7 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -11644,6 +11644,10 @@ def pin(parent, pin) elsif value.is_a?(Array) && (index = value.index(self)) parent.public_send(key)[index] = replace break + elsif value.is_a?(Array) && + (index = value.index { |(_k, v)| v == self }) + parent.public_send(key)[index][1] = replace + break end end end diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index 825cd90e..326d3ec7 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -670,7 +670,11 @@ def visit(node) visit_methods do def visit_var_ref(node) - node.pin(stack[-2], pins.shift) + if node.start_char > pins.first.start_char + node.pin(stack[-2], pins.shift) + else + super + end end end @@ -1732,13 +1736,13 @@ def on_fcall(value) # :call-seq: # on_field: ( # untyped parent, - # (:"::" | Op | Period) operator + # (:"::" | Op | Period | 73) operator # (Const | Ident) name # ) -> Field def on_field(parent, operator, name) Field.new( parent: parent, - operator: operator, + operator: operator == 73 ? :"::" : operator, name: name, location: parent.location.to(name.location) ) diff --git a/lib/syntax_tree/pattern.rb b/lib/syntax_tree/pattern.rb index ca49c6bf..a5e88bfa 100644 --- a/lib/syntax_tree/pattern.rb +++ b/lib/syntax_tree/pattern.rb @@ -70,6 +70,7 @@ def compile raise CompilationError, query end + raise CompilationError, query if program.nil? compile_node(program.statements.body.first.consequent.pattern) end diff --git a/lib/syntax_tree/yarv/assembler.rb b/lib/syntax_tree/yarv/assembler.rb index b29c252a..a48c58fd 100644 --- a/lib/syntax_tree/yarv/assembler.rb +++ b/lib/syntax_tree/yarv/assembler.rb @@ -408,7 +408,7 @@ def assemble_iseq(iseq, lines) def find_local(iseq, operands) name_string, level_string = operands.split(/,\s*/) name = name_string.to_sym - level = level_string&.to_i || 0 + level = level_string.to_i iseq.local_table.plain(name) iseq.local_table.find(name, level) @@ -455,7 +455,7 @@ def parse_calldata(value) CallData::CALL_ARGS_SIMPLE end - YARV.calldata(message.to_sym, argc_value&.to_i || 0, flags) + YARV.calldata(message.to_sym, argc_value.to_i, flags) end end end diff --git a/lib/syntax_tree/yarv/calldata.rb b/lib/syntax_tree/yarv/calldata.rb index e35992f5..278a3dd9 100644 --- a/lib/syntax_tree/yarv/calldata.rb +++ b/lib/syntax_tree/yarv/calldata.rb @@ -41,7 +41,7 @@ def initialize( end def flag?(mask) - (flags & mask) > 0 + flags.anybits?(mask) end def to_h diff --git a/lib/syntax_tree/yarv/decompiler.rb b/lib/syntax_tree/yarv/decompiler.rb index 4ea99e3a..6a2cddbd 100644 --- a/lib/syntax_tree/yarv/decompiler.rb +++ b/lib/syntax_tree/yarv/decompiler.rb @@ -45,7 +45,7 @@ def node_for(value) when Integer Int(value.to_s) when Symbol - SymbolLiteral(Ident(value.to_s)) + SymbolLiteral(Ident(value.name)) end end @@ -88,10 +88,10 @@ def decompile(iseq) clause << HashLiteral(LBrace("{"), assocs) when GetGlobal - clause << VarRef(GVar(insn.name.to_s)) + clause << VarRef(GVar(insn.name.name)) when GetLocalWC0 local = iseq.local_table.locals[insn.index] - clause << VarRef(Ident(local.name.to_s)) + clause << VarRef(Ident(local.name.name)) when Jump clause << Assign(block_label.field, node_for(insn.label.name)) clause << Next(Args([])) @@ -123,7 +123,7 @@ def decompile(iseq) left, right = clause.pop(2) clause << Binary(left, :"!=", right) when OptSendWithoutBlock - method = insn.calldata.method.to_s + method = insn.calldata.method.name argc = insn.calldata.argc if insn.calldata.flag?(CallData::CALL_FCALL) @@ -182,7 +182,7 @@ def decompile(iseq) when PutSelf clause << VarRef(Kw("self")) when SetGlobal - target = GVar(insn.name.to_s) + target = GVar(insn.name.name) value = clause.pop clause << if value.is_a?(Binary) && VarRef(target) === value.left @@ -256,7 +256,7 @@ def decompile(iseq) def local_name(index, level) current = iseq level.times { current = current.parent_iseq } - current.local_table.locals[index].name.to_s + current.local_table.locals[index].name.name end end end diff --git a/lib/syntax_tree/yarv/instruction_sequence.rb b/lib/syntax_tree/yarv/instruction_sequence.rb index df92799b..4f2e0d9a 100644 --- a/lib/syntax_tree/yarv/instruction_sequence.rb +++ b/lib/syntax_tree/yarv/instruction_sequence.rb @@ -252,19 +252,23 @@ def to_a dumped_options = argument_options.dup dumped_options[:opt].map!(&:name) if dumped_options[:opt] + metadata = { + arg_size: argument_size, + local_size: local_table.size, + stack_max: stack.maximum_size, + node_id: -1, + node_ids: [-1] * insns.length + } + + metadata[:parser] = :prism if RUBY_VERSION >= "3.3" + # Next, return the instruction sequence as an array. [ MAGIC, versions[0], versions[1], 1, - { - arg_size: argument_size, - local_size: local_table.size, - stack_max: stack.maximum_size, - node_id: -1, - node_ids: [-1] * insns.length - }, + metadata, name, file, "", @@ -689,6 +693,10 @@ def concatstrings(number) push(ConcatStrings.new(number)) end + def concattoarray(object) + push(ConcatToArray.new(object)) + end + def defineclass(name, class_iseq, flags) push(DefineClass.new(name, class_iseq, flags)) end @@ -897,6 +905,14 @@ def pop push(Pop.new) end + def pushtoarraykwsplat + push(PushToArrayKwSplat.new) + end + + def putchilledstring(object) + push(PutChilledString.new(object)) + end + def putnil push(PutNil.new) end @@ -1079,6 +1095,8 @@ def self.from(source, options = Compiler::Options.new, parent_iseq = nil) iseq.concatarray when :concatstrings iseq.concatstrings(opnds[0]) + when :concattoarray + iseq.concattoarray(opnds[0]) when :defineclass iseq.defineclass(opnds[0], from(opnds[1], options, iseq), opnds[2]) when :defined @@ -1191,8 +1209,13 @@ def self.from(source, options = Compiler::Options.new, parent_iseq = nil) iseq.newarray(opnds[0]) iseq.send(YARV.calldata(:min)) when :opt_newarray_send + mid = opnds[1] + if RUBY_VERSION >= "3.4" + mid = %i[max min hash pack pack_buffer include?][mid - 1] + end + iseq.newarray(opnds[0]) - iseq.send(CallData.new(opnds[1])) + iseq.send(CallData.new(mid)) when :opt_neq iseq.push( OptNEq.new(CallData.from(opnds[0]), CallData.from(opnds[1])) @@ -1207,6 +1230,10 @@ def self.from(source, options = Compiler::Options.new, parent_iseq = nil) iseq.send(YARV.calldata(:-@)) when :pop iseq.pop + when :pushtoarraykwsplat + iseq.pushtoarraykwsplat + when :putchilledstring + iseq.putchilledstring(opnds[0]) when :putnil iseq.putnil when :putobject diff --git a/lib/syntax_tree/yarv/instructions.rb b/lib/syntax_tree/yarv/instructions.rb index ffeebe65..02188dfe 100644 --- a/lib/syntax_tree/yarv/instructions.rb +++ b/lib/syntax_tree/yarv/instructions.rb @@ -757,6 +757,59 @@ def call(vm) end end + # ### Summary + # + # `concattoarray` pops a single value off the stack and attempts to concat + # it to the Array on top of the stack. If the value is not an Array, it + # will be coerced into one. + # + # ### Usage + # + # ~~~ruby + # [1, *2] + # ~~~ + # + class ConcatToArray < Instruction + attr_reader :object + + def initialize(object) + @object = object + end + + def disasm(fmt) + fmt.instruction("concattoarray", [fmt.object(object)]) + end + + def to_a(_iseq) + [:concattoarray, object] + end + + def deconstruct_keys(_keys) + { object: object } + end + + def ==(other) + other.is_a?(ConcatToArray) && other.object == object + end + + def length + 2 + end + + def pops + 1 + end + + def pushes + 1 + end + + def call(vm) + array, value = vm.pop(2) + vm.push(array.concat(Array(value))) + end + end + # ### Summary # # `defineclass` defines a class. First it pops the superclass off the @@ -4472,6 +4525,52 @@ def side_effects? end end + # ### Summary + # + # `pushtoarraykwsplat` is used to append a hash literal that is being + # splatted onto an array. + # + # ### Usage + # + # ~~~ruby + # ["string", **{ foo: "bar" }] + # ~~~ + # + class PushToArrayKwSplat < Instruction + def disasm(fmt) + fmt.instruction("pushtoarraykwsplat") + end + + def to_a(_iseq) + [:pushtoarraykwsplat] + end + + def deconstruct_keys(_keys) + {} + end + + def ==(other) + other.is_a?(PushToArrayKwSplat) + end + + def length + 2 + end + + def pops + 2 + end + + def pushes + 1 + end + + def call(vm) + array, hash = vm.pop(2) + vm.push(array << hash) + end + end + # ### Summary # # `putnil` pushes a global nil object onto the stack. @@ -4759,6 +4858,54 @@ def call(vm) end end + # ### Summary + # + # `putchilledstring` pushes an unfrozen string literal onto the stack that + # acts like a frozen string. This is a migration path to frozen string + # literals as the default in the future. + # + # ### Usage + # + # ~~~ruby + # "foo" + # ~~~ + # + class PutChilledString < Instruction + attr_reader :object + + def initialize(object) + @object = object + end + + def disasm(fmt) + fmt.instruction("putchilledstring", [fmt.object(object)]) + end + + def to_a(_iseq) + [:putchilledstring, object] + end + + def deconstruct_keys(_keys) + { object: object } + end + + def ==(other) + other.is_a?(PutChilledString) && other.object == object + end + + def length + 2 + end + + def pushes + 1 + end + + def call(vm) + vm.push(object.dup) + end + end + # ### Summary # # `putstring` pushes an unfrozen string literal onto the stack. diff --git a/tasks/whitequark.rake b/tasks/whitequark.rake index 4f7ee650..6e1663aa 100644 --- a/tasks/whitequark.rake +++ b/tasks/whitequark.rake @@ -43,8 +43,13 @@ module ParseHelper # that we do not support. return if (versions & %w[3.1 3.2]).empty? - entry = caller.find { _1.include?("test_parser.rb") } - _, lineno, name = *entry.match(/(\d+):in `(.+)'/) + entry = + caller.find do |call| + call.include?("test_parser.rb") && call.match?(%r{(? "3.3" + require_relative "test_helper" module SyntaxTree diff --git a/test/fixtures/bodystmt.rb b/test/fixtures/bodystmt.rb index 4cbb8f5e..5999fdba 100644 --- a/test/fixtures/bodystmt.rb +++ b/test/fixtures/bodystmt.rb @@ -36,6 +36,7 @@ end % begin +rescue StandardError else # else end % diff --git a/test/fixtures/break.rb b/test/fixtures/break.rb index 519becda..23277f6b 100644 --- a/test/fixtures/break.rb +++ b/test/fixtures/break.rb @@ -1,37 +1,45 @@ % -break +tap { break } % -break foo +tap { break foo } % -break foo, bar +tap { break foo, bar } % -break(foo) +tap { break(foo) } % -break fooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo +tap { break fooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo } - -break( - fooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo -) +tap do + break( + fooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo + ) +end % -break(fooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo) +tap { break(fooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo) } - -break( - fooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo -) -% -break (foo), bar -% -break( - foo - bar -) -% -break foo.bar :baz do |qux| qux end +tap do + break( + fooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo + ) +end +% +tap { break (foo), bar } +% +tap do + break( + foo + bar + ) +end +% +tap { break foo.bar :baz do |qux| qux end } - -break( - foo.bar :baz do |qux| - qux - end -) -% -break :foo => "bar" +tap do + break( + foo.bar :baz do |qux| + qux + end + ) +end +% +tap { break :foo => "bar" } diff --git a/test/fixtures/ifop.rb b/test/fixtures/ifop.rb index e56eb987..f7504658 100644 --- a/test/fixtures/ifop.rb +++ b/test/fixtures/ifop.rb @@ -11,8 +11,10 @@ % foo bar ? 1 : 2 % -foooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo ? break : baz +tap { foooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo ? break : baz } - -foooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo ? - break : - baz +tap do + foooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo ? + break : + baz +end diff --git a/test/fixtures/next.rb b/test/fixtures/next.rb index 66e90028..dc159488 100644 --- a/test/fixtures/next.rb +++ b/test/fixtures/next.rb @@ -1,76 +1,82 @@ % -next +tap { next } % -next foo +tap { next foo } % -next foo, bar +tap { next foo, bar } % -next(foo) +tap { next(foo) } % -next fooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo +tap { next fooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo } - -next( - fooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo -) +tap do + next( + fooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo + ) +end % -next(fooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo) +tap { next(fooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo) } - -next( - fooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo -) -% -next (foo), bar -% -next( - foo - bar -) -% -next(1) +tap do + next( + fooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo + ) +end +% +tap { next (foo), bar } +% +tap do + next( + foo + bar + ) +end +% +tap { next(1) } - -next 1 +tap { next 1 } % -next(1.0) +tap { next(1.0) } - -next 1.0 +tap { next 1.0 } % -next($a) +tap { next($a) } - -next $a +tap { next $a } % -next(@@a) +tap { next(@@a) } - -next @@a +tap { next @@a } % -next(self) +tap { next(self) } - -next self +tap { next self } % -next(@a) +tap { next(@a) } - -next @a +tap { next @a } % -next(A) +tap { next(A) } - -next A +tap { next A } % -next([]) +tap { next([]) } - -next [] +tap { next [] } % -next([1]) +tap { next([1]) } - -next [1] +tap { next [1] } % -next([1, 2]) +tap { next([1, 2]) } - -next 1, 2 +tap { next 1, 2 } % -next fun foo do end +tap { next fun foo do end } - -next( - fun foo do - end -) -% -next :foo => "bar" +tap do + next( + fun foo do + end + ) +end diff --git a/test/fixtures/redo.rb b/test/fixtures/redo.rb index 8ab087a2..962af3d0 100644 --- a/test/fixtures/redo.rb +++ b/test/fixtures/redo.rb @@ -1,4 +1,6 @@ % -redo +tap { redo } % -redo # comment +tap do + redo # comment +end diff --git a/test/fixtures/retry.rb b/test/fixtures/retry.rb index 2b14d21a..47b6be51 100644 --- a/test/fixtures/retry.rb +++ b/test/fixtures/retry.rb @@ -1,4 +1,10 @@ % -retry +begin +rescue StandardError + retry +end % -retry # comment +begin +rescue StandardError + retry # comment +end diff --git a/test/fixtures/var_field_rassign.rb b/test/fixtures/var_field_rassign.rb index 3e019c5c..aa5ec379 100644 --- a/test/fixtures/var_field_rassign.rb +++ b/test/fixtures/var_field_rassign.rb @@ -1,6 +1,7 @@ % foo in bar % +bar = 1 foo in ^bar % foo in ^@bar diff --git a/test/fixtures/yield.rb b/test/fixtures/yield.rb index f3f023f8..3cf1e5f1 100644 --- a/test/fixtures/yield.rb +++ b/test/fixtures/yield.rb @@ -1,16 +1,30 @@ % -yield foo +def foo + yield foo +end % -yield(foo) +def foo + yield(foo) +end % -yield foo, bar +def foo + yield foo, bar +end % -yield(foo, bar) +def foo + yield(foo, bar) +end % -yield foo # comment +def foo + yield foo # comment +end % -yield(foo) # comment +def foo + yield(foo) # comment +end % -yield( # comment - foo -) +def foo + yield( # comment + foo + ) +end diff --git a/test/fixtures/yield0.rb b/test/fixtures/yield0.rb index a168c4aa..c1833bb5 100644 --- a/test/fixtures/yield0.rb +++ b/test/fixtures/yield0.rb @@ -1,4 +1,8 @@ % -yield +def foo + yield +end % -yield # comment +def foo + yield # comment +end diff --git a/test/node_test.rb b/test/node_test.rb index 19fbeed2..f2706b2c 100644 --- a/test/node_test.rb +++ b/test/node_test.rb @@ -280,7 +280,10 @@ def test_brace_block end def test_break - assert_node(Break, "break value") + at = location(chars: 6..17) + assert_node(Break, "tap { break value }", at: at) do |node| + node.block.bodystmt.body.first + end end def test_call @@ -710,7 +713,10 @@ def test_mrhs_add_star end def test_next - assert_node(Next, "next(value)") + at = location(chars: 6..17) + assert_node(Next, "tap { next(value) }", at: at) do |node| + node.block.bodystmt.body.first + end end def test_op @@ -786,7 +792,9 @@ def test_rational end def test_redo - assert_node(Redo, "redo") + assert_node(Redo, "tap { redo }", at: location(chars: 6..10)) do |node| + node.block.bodystmt.body.first + end end def test_regexp_literal @@ -833,7 +841,10 @@ def test_rest_param end def test_retry - assert_node(Retry, "retry") + at = location(chars: 15..20) + assert_node(Retry, "begin; rescue; retry; end", at: at) do |node| + node.bodystmt.rescue_clause.statements.body.first + end end def test_return @@ -949,8 +960,8 @@ def test_var_field guard_version("3.1.0") do def test_pinned_var_ref - source = "foo in ^bar" - at = location(chars: 7..11) + source = "bar = 1; foo in ^bar" + at = location(chars: 16..20) assert_node(PinnedVarRef, source, at: at, &:pattern) end @@ -1013,11 +1024,17 @@ def test_xstring_heredoc end def test_yield - assert_node(YieldNode, "yield value") + at = location(lines: 2..2, chars: 10..21) + assert_node(YieldNode, "def foo\n yield value\nend\n", at: at) do |node| + node.bodystmt.statements.body.first + end end def test_yield0 - assert_node(YieldNode, "yield") + at = location(lines: 2..2, chars: 10..15) + assert_node(YieldNode, "def foo\n yield\nend\n", at: at) do |node| + node.bodystmt.statements.body.first + end end def test_zsuper diff --git a/test/test_helper.rb b/test/test_helper.rb index 8015be14..787f819d 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -106,17 +106,15 @@ def assert_syntax_tree(node) refute_includes(json, "#<") assert_equal(type, JSON.parse(json)["type"]) - if RUBY_ENGINE != "truffleruby" - # Get a match expression from the node, then assert that it can in fact - # match the node. - # rubocop:disable all - assert(eval(<<~RUBY)) - case node - in #{node.construct_keys} - true - end - RUBY - end + # Get a match expression from the node, then assert that it can in fact + # match the node. + # rubocop:disable all + assert(eval(<<~RUBY)) + case node + in #{node.construct_keys} + true + end + RUBY end Minitest::Test.include(self) diff --git a/test/translation/parser.txt b/test/translation/parser.txt index 5e9e8d31..d732cd0d 100644 --- a/test/translation/parser.txt +++ b/test/translation/parser.txt @@ -1,1237 +1,980 @@ -!!! assert_parses_args:2249:0 -def f (foo: 1, bar: 2, **baz, &b); end -!!! assert_parses_args:2249:1 -def f (foo: 1, &b); end -!!! assert_parses_args:2249:2 -def f **baz, &b; end -!!! assert_parses_args:2249:3 -def f *, **; end -!!! assert_parses_args:2249:4 -def f a, o=1, *r, &b; end -!!! assert_parses_args:2249:5 -def f a, o=1, *r, p, &b; end -!!! assert_parses_args:2249:6 -def f a, o=1, &b; end -!!! assert_parses_args:2249:7 -def f a, o=1, p, &b; end -!!! assert_parses_args:2249:8 -def f a, *r, &b; end -!!! assert_parses_args:2249:9 -def f a, *r, p, &b; end -!!! assert_parses_args:2249:10 -def f a, &b; end -!!! assert_parses_args:2249:11 -def f o=1, *r, &b; end -!!! assert_parses_args:2249:12 -def f o=1, *r, p, &b; end -!!! assert_parses_args:2249:13 -def f o=1, &b; end -!!! assert_parses_args:2249:14 -def f o=1, p, &b; end -!!! assert_parses_args:2249:15 -def f *r, &b; end -!!! assert_parses_args:2249:16 -def f *r, p, &b; end -!!! assert_parses_args:2249:17 -def f &b; end -!!! assert_parses_args:2249:18 -def f ; end -!!! assert_parses_args:2249:19 -def f (((a))); end -!!! assert_parses_args:2249:20 -def f ((a, a1)); end -!!! assert_parses_args:2249:21 -def f ((a, *r)); end -!!! assert_parses_args:2249:22 -def f ((a, *r, p)); end -!!! assert_parses_args:2249:23 -def f ((a, *)); end -!!! assert_parses_args:2249:24 -def f ((a, *, p)); end -!!! assert_parses_args:2249:25 -def f ((*r)); end -!!! assert_parses_args:2249:26 -def f ((*r, p)); end -!!! assert_parses_args:2249:27 -def f ((*)); end -!!! assert_parses_args:2249:28 -def f ((*, p)); end -!!! assert_parses_args:2249:29 -def f foo: -; end -!!! assert_parses_args:2249:30 -def f foo: -1 -; end -!!! assert_parses_blockargs:2506:0 -f{ |a| } -!!! assert_parses_blockargs:2506:1 -f{ |a, b,| } -!!! assert_parses_blockargs:2506:2 -f{ |a| } -!!! assert_parses_blockargs:2506:3 -f{ |foo:| } -!!! assert_parses_blockargs:2506:4 -f{ } -!!! assert_parses_blockargs:2506:5 -f{ | | } -!!! assert_parses_blockargs:2506:6 -f{ |;a| } -!!! assert_parses_blockargs:2506:7 -f{ |; -a -| } -!!! assert_parses_blockargs:2506:8 -f{ || } -!!! assert_parses_blockargs:2506:9 -f{ |a| } -!!! assert_parses_blockargs:2506:10 -f{ |a, c| } -!!! assert_parses_blockargs:2506:11 -f{ |a,| } -!!! assert_parses_blockargs:2506:12 -f{ |a, &b| } -!!! assert_parses_blockargs:2506:13 -f{ |a, *s, &b| } -!!! assert_parses_blockargs:2506:14 -f{ |a, *, &b| } -!!! assert_parses_blockargs:2506:15 -f{ |a, *s| } -!!! assert_parses_blockargs:2506:16 -f{ |a, *| } -!!! assert_parses_blockargs:2506:17 -f{ |*s, &b| } -!!! assert_parses_blockargs:2506:18 -f{ |*, &b| } -!!! assert_parses_blockargs:2506:19 -f{ |*s| } -!!! assert_parses_blockargs:2506:20 -f{ |*| } -!!! assert_parses_blockargs:2506:21 -f{ |&b| } -!!! assert_parses_blockargs:2506:22 -f{ |a, o=1, o1=2, *r, &b| } -!!! assert_parses_blockargs:2506:23 -f{ |a, o=1, *r, p, &b| } -!!! assert_parses_blockargs:2506:24 -f{ |a, o=1, &b| } -!!! assert_parses_blockargs:2506:25 -f{ |a, o=1, p, &b| } -!!! assert_parses_blockargs:2506:26 -f{ |a, *r, p, &b| } -!!! assert_parses_blockargs:2506:27 -f{ |o=1, *r, &b| } -!!! assert_parses_blockargs:2506:28 -f{ |o=1, *r, p, &b| } -!!! assert_parses_blockargs:2506:29 -f{ |o=1, &b| } -!!! assert_parses_blockargs:2506:30 -f{ |o=1, p, &b| } -!!! assert_parses_blockargs:2506:31 -f{ |*r, p, &b| } -!!! assert_parses_blockargs:2506:32 -f{ |foo: 1, bar: 2, **baz, &b| } -!!! assert_parses_blockargs:2506:33 -f{ |foo: 1, &b| } -!!! assert_parses_blockargs:2506:34 -f{ |**baz, &b| } -!!! assert_parses_pattern_match:8503:0 -case foo; in self then true; end -!!! assert_parses_pattern_match:8503:1 -case foo; in 1..2 then true; end -!!! assert_parses_pattern_match:8503:2 -case foo; in 1.. then true; end -!!! assert_parses_pattern_match:8503:3 -case foo; in ..2 then true; end -!!! assert_parses_pattern_match:8503:4 -case foo; in 1...2 then true; end -!!! assert_parses_pattern_match:8503:5 -case foo; in 1... then true; end -!!! assert_parses_pattern_match:8503:6 -case foo; in ...2 then true; end -!!! assert_parses_pattern_match:8503:7 -case foo; in [*x, 1 => a, *y] then true; end -!!! assert_parses_pattern_match:8503:8 -case foo; in String(*, 1, *) then true; end -!!! assert_parses_pattern_match:8503:9 -case foo; in Array[*, 1, *] then true; end -!!! assert_parses_pattern_match:8503:10 -case foo; in *, 42, * then true; end -!!! assert_parses_pattern_match:8503:11 -case foo; in x, then nil; end -!!! assert_parses_pattern_match:8503:12 -case foo; in *x then nil; end -!!! assert_parses_pattern_match:8503:13 -case foo; in * then nil; end -!!! assert_parses_pattern_match:8503:14 -case foo; in x, y then nil; end -!!! assert_parses_pattern_match:8503:15 -case foo; in x, y, then nil; end -!!! assert_parses_pattern_match:8503:16 -case foo; in x, *y, z then nil; end -!!! assert_parses_pattern_match:8503:17 -case foo; in *x, y, z then nil; end -!!! assert_parses_pattern_match:8503:18 -case foo; in 1, "a", [], {} then nil; end -!!! assert_parses_pattern_match:8503:19 -case foo; in ->{ 42 } then true; end -!!! assert_parses_pattern_match:8503:20 -case foo; in A(1, 2) then true; end -!!! assert_parses_pattern_match:8503:21 -case foo; in A(x:) then true; end -!!! assert_parses_pattern_match:8503:22 -case foo; in A() then true; end -!!! assert_parses_pattern_match:8503:23 -case foo; in A[1, 2] then true; end -!!! assert_parses_pattern_match:8503:24 -case foo; in A[x:] then true; end -!!! assert_parses_pattern_match:8503:25 -case foo; in A[] then true; end -!!! assert_parses_pattern_match:8503:26 -case foo; in x then x; end -!!! assert_parses_pattern_match:8503:27 -case foo; in {} then true; end -!!! assert_parses_pattern_match:8503:28 -case foo; in a: 1 then true; end -!!! assert_parses_pattern_match:8503:29 -case foo; in { a: 1 } then true; end -!!! assert_parses_pattern_match:8503:30 -case foo; in { a: 1, } then true; end -!!! assert_parses_pattern_match:8503:31 -case foo; in a: then true; end -!!! assert_parses_pattern_match:8503:32 -case foo; in **a then true; end -!!! assert_parses_pattern_match:8503:33 -case foo; in ** then true; end -!!! assert_parses_pattern_match:8503:34 -case foo; in a: 1, b: 2 then true; end -!!! assert_parses_pattern_match:8503:35 -case foo; in a:, b: then true; end -!!! assert_parses_pattern_match:8503:36 -case foo; in a: 1, _a:, ** then true; end -!!! assert_parses_pattern_match:8503:37 -case foo; - in {a: 1 - } - false - ; end -!!! assert_parses_pattern_match:8503:38 -case foo; - in {a: - 2} - false - ; end -!!! assert_parses_pattern_match:8503:39 -case foo; - in {Foo: 42 - } - false - ; end -!!! assert_parses_pattern_match:8503:40 -case foo; - in a: {b:}, c: - p c - ; end -!!! assert_parses_pattern_match:8503:41 -case foo; - in {a: - } - true - ; end -!!! assert_parses_pattern_match:8503:42 -case foo; in A then true; end -!!! assert_parses_pattern_match:8503:43 -case foo; in A::B then true; end -!!! assert_parses_pattern_match:8503:44 -case foo; in ::A then true; end -!!! assert_parses_pattern_match:8503:45 -case foo; in [x] then nil; end -!!! assert_parses_pattern_match:8503:46 -case foo; in [x,] then nil; end -!!! assert_parses_pattern_match:8503:47 -case foo; in [x, y] then true; end -!!! assert_parses_pattern_match:8503:48 -case foo; in [x, y,] then true; end -!!! assert_parses_pattern_match:8503:49 -case foo; in [x, y, *] then true; end -!!! assert_parses_pattern_match:8503:50 -case foo; in [x, y, *z] then true; end -!!! assert_parses_pattern_match:8503:51 -case foo; in [x, *y, z] then true; end -!!! assert_parses_pattern_match:8503:52 -case foo; in [x, *, y] then true; end -!!! assert_parses_pattern_match:8503:53 -case foo; in [*x, y] then true; end -!!! assert_parses_pattern_match:8503:54 -case foo; in [*, x] then true; end -!!! assert_parses_pattern_match:8503:55 -case foo; in (1) then true; end -!!! assert_parses_pattern_match:8503:56 -case foo; in x if true; nil; end -!!! assert_parses_pattern_match:8503:57 -case foo; in x unless true; nil; end -!!! assert_parses_pattern_match:8503:58 -case foo; in 1; end -!!! assert_parses_pattern_match:8503:59 -case foo; in ^foo then nil; end -!!! assert_parses_pattern_match:8503:60 -case foo; in "a": then true; end -!!! assert_parses_pattern_match:8503:61 -case foo; in "#{ 'a' }": then true; end -!!! assert_parses_pattern_match:8503:62 -case foo; in "#{ %q{a} }": then true; end -!!! assert_parses_pattern_match:8503:63 -case foo; in "#{ %Q{a} }": then true; end -!!! assert_parses_pattern_match:8503:64 -case foo; in "a": 1 then true; end -!!! assert_parses_pattern_match:8503:65 -case foo; in "#{ 'a' }": 1 then true; end -!!! assert_parses_pattern_match:8503:66 -case foo; in "#{ %q{a} }": 1 then true; end -!!! assert_parses_pattern_match:8503:67 -case foo; in "#{ %Q{a} }": 1 then true; end -!!! assert_parses_pattern_match:8503:68 -case foo; in ^(42) then nil; end -!!! assert_parses_pattern_match:8503:69 -case foo; in { foo: ^(42) } then nil; end -!!! assert_parses_pattern_match:8503:70 -case foo; in ^(0+0) then nil; end -!!! assert_parses_pattern_match:8503:71 -case foo; in ^@a; end -!!! assert_parses_pattern_match:8503:72 -case foo; in ^@@TestPatternMatching; end -!!! assert_parses_pattern_match:8503:73 -case foo; in ^$TestPatternMatching; end -!!! assert_parses_pattern_match:8503:74 -case foo; in ^(1 -); end -!!! assert_parses_pattern_match:8503:75 -case foo; in 1 | 2 then true; end -!!! assert_parses_pattern_match:8503:76 -case foo; in 1 => a then true; end -!!! assert_parses_pattern_match:8503:77 -case foo; in **nil then true; end -!!! block in test_endless_comparison_method:10392:0 -def ===(other) = do_something -!!! block in test_endless_comparison_method:10392:1 -def ==(other) = do_something -!!! block in test_endless_comparison_method:10392:2 -def !=(other) = do_something -!!! block in test_endless_comparison_method:10392:3 -def <=(other) = do_something -!!! block in test_endless_comparison_method:10392:4 -def >=(other) = do_something -!!! block in test_endless_comparison_method:10392:5 -def !=(other) = do_something -!!! block in test_parser_slash_slash_n_escaping_in_literals:7327:0 -'a\ -b' -!!! block in test_parser_slash_slash_n_escaping_in_literals:7327:1 -<<-'HERE' -a\ -b -HERE -!!! block in test_parser_slash_slash_n_escaping_in_literals:7327:2 -%q{a\ -b} -!!! block in test_parser_slash_slash_n_escaping_in_literals:7327:3 -"a\ -b" -!!! block in test_parser_slash_slash_n_escaping_in_literals:7327:4 -<<-"HERE" -a\ -b -HERE -!!! block in test_parser_slash_slash_n_escaping_in_literals:7327:5 -%{a\ -b} -!!! block in test_parser_slash_slash_n_escaping_in_literals:7327:6 -%Q{a\ -b} -!!! block in test_parser_slash_slash_n_escaping_in_literals:7327:7 -%w{a\ -b} -!!! block in test_parser_slash_slash_n_escaping_in_literals:7327:8 -%W{a\ -b} -!!! block in test_parser_slash_slash_n_escaping_in_literals:7327:9 -%i{a\ -b} -!!! block in test_parser_slash_slash_n_escaping_in_literals:7327:10 -%I{a\ -b} -!!! block in test_parser_slash_slash_n_escaping_in_literals:7327:11 -:'a\ -b' -!!! block in test_parser_slash_slash_n_escaping_in_literals:7327:12 -%s{a\ -b} -!!! block in test_parser_slash_slash_n_escaping_in_literals:7327:13 -:"a\ -b" -!!! block in test_parser_slash_slash_n_escaping_in_literals:7327:14 -/a\ -b/ -!!! block in test_parser_slash_slash_n_escaping_in_literals:7327:15 -%r{a\ -b} -!!! block in test_parser_slash_slash_n_escaping_in_literals:7327:16 -%x{a\ -b} -!!! block in test_parser_slash_slash_n_escaping_in_literals:7327:17 -`a\ -b` -!!! block in test_parser_slash_slash_n_escaping_in_literals:7327:18 -<<-`HERE` -a\ -b -HERE -!!! block in test_ruby_bug_11873_a:6017:0 -a b{c d}, :e do end -!!! block in test_ruby_bug_11873_a:6017:1 -a b{c d}, 1 do end -!!! block in test_ruby_bug_11873_a:6017:2 -a b{c d}, 1.0 do end -!!! block in test_ruby_bug_11873_a:6017:3 -a b{c d}, 1.0r do end -!!! block in test_ruby_bug_11873_a:6017:4 -a b{c d}, 1.0i do end -!!! block in test_ruby_bug_11873_a:6022:0 -a b{c(d)}, :e do end -!!! block in test_ruby_bug_11873_a:6022:1 -a b{c(d)}, 1 do end -!!! block in test_ruby_bug_11873_a:6022:2 -a b{c(d)}, 1.0 do end -!!! block in test_ruby_bug_11873_a:6022:3 -a b{c(d)}, 1.0r do end -!!! block in test_ruby_bug_11873_a:6022:4 -a b{c(d)}, 1.0i do end -!!! block in test_ruby_bug_11873_a:6036:0 -a b(c d), :e do end -!!! block in test_ruby_bug_11873_a:6036:1 -a b(c d), 1 do end -!!! block in test_ruby_bug_11873_a:6036:2 -a b(c d), 1.0 do end -!!! block in test_ruby_bug_11873_a:6036:3 -a b(c d), 1.0r do end -!!! block in test_ruby_bug_11873_a:6036:4 -a b(c d), 1.0i do end -!!! block in test_ruby_bug_11873_a:6041:0 -a b(c(d)), :e do end -!!! block in test_ruby_bug_11873_a:6041:1 -a b(c(d)), 1 do end -!!! block in test_ruby_bug_11873_a:6041:2 -a b(c(d)), 1.0 do end -!!! block in test_ruby_bug_11873_a:6041:3 -a b(c(d)), 1.0r do end -!!! block in test_ruby_bug_11873_a:6041:4 -a b(c(d)), 1.0i do end -!!! test___ENCODING__:1037 +!!! test___ENCODING__:1051 __ENCODING__ -!!! test___ENCODING___legacy_:1046 +!!! test___ENCODING___legacy_:1060 __ENCODING__ -!!! test_alias:2020 +!!! test_alias:2034 alias :foo bar -!!! test_alias_gvar:2032 +!!! test_alias_gvar:2046 alias $a $b -!!! test_alias_gvar:2037 +!!! test_alias_gvar:2051 alias $a $+ -!!! test_ambiuous_quoted_label_in_ternary_operator:7204 +!!! test_ambiuous_quoted_label_in_ternary_operator:7389 a ? b & '': nil -!!! test_and:4447 +!!! test_and:4507 foo and bar -!!! test_and:4453 +!!! test_and:4513 foo && bar -!!! test_and_asgn:1748 +!!! test_and_asgn:1762 foo.a &&= 1 -!!! test_and_asgn:1758 +!!! test_and_asgn:1772 foo[0, 1] &&= 2 -!!! test_and_or_masgn:4475 +!!! test_and_or_masgn:4535 foo && (a, b = bar) -!!! test_and_or_masgn:4484 +!!! test_and_or_masgn:4544 foo || (a, b = bar) -!!! test_anonymous_blockarg:10861 +!!! test_anonymous_blockarg:11205 def foo(&); bar(&); end -!!! test_arg:2055 +!!! test_arg:2069 def f(foo); end -!!! test_arg:2066 +!!! test_arg:2080 def f(foo, bar); end -!!! test_arg_duplicate_ignored:2958 -def foo(_, _); end +!!! test_arg_combinations:2272 +def f a, o=1, *r, &b; end +!!! test_arg_combinations:2281 +def f a, o=1, *r, p, &b; end +!!! test_arg_combinations:2292 +def f a, o=1, &b; end +!!! test_arg_combinations:2300 +def f a, o=1, p, &b; end +!!! test_arg_combinations:2310 +def f a, *r, &b; end +!!! test_arg_combinations:2318 +def f a, *r, p, &b; end +!!! test_arg_combinations:2328 +def f a, &b; end +!!! test_arg_combinations:2335 +def f o=1, *r, &b; end +!!! test_arg_combinations:2343 +def f o=1, *r, p, &b; end +!!! test_arg_combinations:2353 +def f o=1, &b; end +!!! test_arg_combinations:2360 +def f o=1, p, &b; end +!!! test_arg_combinations:2369 +def f *r, &b; end +!!! test_arg_combinations:2376 +def f *r, p, &b; end +!!! test_arg_combinations:2385 +def f &b; end +!!! test_arg_combinations:2391 +def f ; end !!! test_arg_duplicate_ignored:2972 +def foo(_, _); end +!!! test_arg_duplicate_ignored:2986 def foo(_a, _a); end -!!! test_arg_label:3012 +!!! test_arg_label:3026 def foo() a:b end -!!! test_arg_label:3019 +!!! test_arg_label:3033 def foo a:b end -!!! test_arg_label:3026 +!!! test_arg_label:3040 f { || a:b } -!!! test_arg_scope:2238 +!!! test_arg_scope:2252 lambda{|;a|a} -!!! test_args_args_assocs:4077 +!!! test_args_args_assocs:4137 fun(foo, :foo => 1) -!!! test_args_args_assocs:4083 +!!! test_args_args_assocs:4143 fun(foo, :foo => 1, &baz) -!!! test_args_args_assocs_comma:4092 +!!! test_args_args_assocs_comma:4152 foo[bar, :baz => 1,] -!!! test_args_args_comma:3941 +!!! test_args_args_comma:4001 foo[bar,] -!!! test_args_args_star:3908 +!!! test_args_args_star:3968 fun(foo, *bar) -!!! test_args_args_star:3913 +!!! test_args_args_star:3973 fun(foo, *bar, &baz) -!!! test_args_assocs:4001 +!!! test_args_assocs:4061 fun(:foo => 1) -!!! test_args_assocs:4006 +!!! test_args_assocs:4066 fun(:foo => 1, &baz) -!!! test_args_assocs:4012 +!!! test_args_assocs:4072 self[:bar => 1] -!!! test_args_assocs:4021 +!!! test_args_assocs:4081 self.[]= foo, :a => 1 -!!! test_args_assocs:4031 +!!! test_args_assocs:4091 yield(:foo => 42) -!!! test_args_assocs:4039 +!!! test_args_assocs:4099 super(:foo => 42) -!!! test_args_assocs_comma:4068 +!!! test_args_assocs_comma:4128 foo[:baz => 1,] -!!! test_args_assocs_legacy:3951 +!!! test_args_assocs_legacy:4011 fun(:foo => 1) -!!! test_args_assocs_legacy:3956 +!!! test_args_assocs_legacy:4016 fun(:foo => 1, &baz) -!!! test_args_assocs_legacy:3962 +!!! test_args_assocs_legacy:4022 self[:bar => 1] -!!! test_args_assocs_legacy:3971 +!!! test_args_assocs_legacy:4031 self.[]= foo, :a => 1 -!!! test_args_assocs_legacy:3981 +!!! test_args_assocs_legacy:4041 yield(:foo => 42) -!!! test_args_assocs_legacy:3989 +!!! test_args_assocs_legacy:4049 super(:foo => 42) -!!! test_args_block_pass:3934 +!!! test_args_block_pass:3994 fun(&bar) -!!! test_args_cmd:3901 +!!! test_args_cmd:3961 fun(f bar) -!!! test_args_star:3921 +!!! test_args_star:3981 fun(*bar) -!!! test_args_star:3926 +!!! test_args_star:3986 fun(*bar, &baz) -!!! test_array_assocs:629 +!!! test_array_assocs:643 [ 1 => 2 ] -!!! test_array_assocs:637 +!!! test_array_assocs:651 [ 1, 2 => 3 ] -!!! test_array_plain:589 +!!! test_array_plain:603 [1, 2] -!!! test_array_splat:598 +!!! test_array_splat:612 [1, *foo, 2] -!!! test_array_splat:611 +!!! test_array_splat:625 [1, *foo] -!!! test_array_splat:622 +!!! test_array_splat:636 [*foo] -!!! test_array_symbols:695 +!!! test_array_symbols:709 %i[foo bar] -!!! test_array_symbols_empty:732 +!!! test_array_symbols_empty:746 %i[] -!!! test_array_symbols_empty:740 +!!! test_array_symbols_empty:754 %I() -!!! test_array_symbols_interp:706 +!!! test_array_symbols_interp:720 %I[foo #{bar}] -!!! test_array_symbols_interp:721 +!!! test_array_symbols_interp:735 %I[foo#{bar}] -!!! test_array_words:647 +!!! test_array_words:661 %w[foo bar] -!!! test_array_words_empty:682 +!!! test_array_words_empty:696 %w[] -!!! test_array_words_empty:689 +!!! test_array_words_empty:703 %W() -!!! test_array_words_interp:657 -%W[foo #{bar}] !!! test_array_words_interp:671 +%W[foo #{bar}] +!!! test_array_words_interp:685 %W[foo #{bar}foo#@baz] -!!! test_asgn_cmd:1126 +!!! test_asgn_cmd:1140 foo = m foo -!!! test_asgn_cmd:1130 +!!! test_asgn_cmd:1144 foo = bar = m foo -!!! test_asgn_mrhs:1449 +!!! test_asgn_mrhs:1463 foo = bar, 1 -!!! test_asgn_mrhs:1456 +!!! test_asgn_mrhs:1470 foo = *bar -!!! test_asgn_mrhs:1461 +!!! test_asgn_mrhs:1475 foo = baz, *bar -!!! test_back_ref:995 +!!! test_back_ref:1009 $+ -!!! test_bang:3434 +!!! test_bang:3448 !foo -!!! test_bang_cmd:3448 +!!! test_bang_cmd:3462 !m foo -!!! test_begin_cmdarg:5526 +!!! test_begin_cmdarg:5658 p begin 1.times do 1 end end -!!! test_beginless_erange_after_newline:935 +!!! test_beginless_erange_after_newline:949 foo ...100 -!!! test_beginless_irange_after_newline:923 +!!! test_beginless_irange_after_newline:937 foo ..100 -!!! test_beginless_range:903 +!!! test_beginless_range:917 ..100 -!!! test_beginless_range:912 +!!! test_beginless_range:926 ...100 -!!! test_blockarg:2187 +!!! test_block_arg_combinations:2531 +f{ } +!!! test_block_arg_combinations:2537 +f{ | | } +!!! test_block_arg_combinations:2541 +f{ |;a| } +!!! test_block_arg_combinations:2546 +f{ |; +a +| } +!!! test_block_arg_combinations:2552 +f{ || } +!!! test_block_arg_combinations:2561 +f{ |a| } +!!! test_block_arg_combinations:2571 +f{ |a, c| } +!!! test_block_arg_combinations:2580 +f{ |a,| } +!!! test_block_arg_combinations:2585 +f{ |a, &b| } +!!! test_block_arg_combinations:2599 +f{ |a, *s, &b| } +!!! test_block_arg_combinations:2610 +f{ |a, *, &b| } +!!! test_block_arg_combinations:2621 +f{ |a, *s| } +!!! test_block_arg_combinations:2631 +f{ |a, *| } +!!! test_block_arg_combinations:2640 +f{ |*s, &b| } +!!! test_block_arg_combinations:2651 +f{ |*, &b| } +!!! test_block_arg_combinations:2662 +f{ |*s| } +!!! test_block_arg_combinations:2672 +f{ |*| } +!!! test_block_arg_combinations:2678 +f{ |&b| } +!!! test_block_arg_combinations:2689 +f{ |a, o=1, o1=2, *r, &b| } +!!! test_block_arg_combinations:2700 +f{ |a, o=1, *r, p, &b| } +!!! test_block_arg_combinations:2711 +f{ |a, o=1, &b| } +!!! test_block_arg_combinations:2720 +f{ |a, o=1, p, &b| } +!!! test_block_arg_combinations:2730 +f{ |a, *r, p, &b| } +!!! test_block_arg_combinations:2740 +f{ |o=1, *r, &b| } +!!! test_block_arg_combinations:2749 +f{ |o=1, *r, p, &b| } +!!! test_block_arg_combinations:2759 +f{ |o=1, &b| } +!!! test_block_arg_combinations:2767 +f{ |o=1, p, &b| } +!!! test_block_arg_combinations:2776 +f{ |*r, p, &b| } +!!! test_block_kwarg:2867 +f{ |foo:| } +!!! test_block_kwarg_combinations:2840 +f{ |foo: 1, bar: 2, **baz, &b| } +!!! test_block_kwarg_combinations:2850 +f{ |foo: 1, &b| } +!!! test_block_kwarg_combinations:2858 +f{ |**baz, &b| } +!!! test_blockarg:2201 def f(&block); end -!!! test_break:5037 +!!! test_break:5169 break(foo) -!!! test_break:5051 +!!! test_break:5183 break foo -!!! test_break:5057 +!!! test_break:5189 break() -!!! test_break:5064 +!!! test_break:5196 break -!!! test_break_block:5072 +!!! test_break_block:5204 break fun foo do end -!!! test_bug_435:7067 +!!! test_bug_435:7252 "#{-> foo {}}" -!!! test_bug_447:7046 +!!! test_bug_447:7231 m [] do end -!!! test_bug_447:7055 +!!! test_bug_447:7240 m [], 1 do end -!!! test_bug_452:7080 +!!! test_bug_452:7265 td (1_500).toString(); td.num do; end -!!! test_bug_466:7096 +!!! test_bug_466:7281 foo "#{(1+1).to_i}" do; end -!!! test_bug_473:7113 +!!! test_bug_473:7298 m "#{[]}" -!!! test_bug_480:7124 +!!! test_bug_480:7309 m "#{}#{()}" -!!! test_bug_481:7136 +!!! test_bug_481:7321 m def x(); end; 1.tap do end -!!! test_bug_ascii_8bit_in_literal:5880 +!!! test_bug_ascii_8bit_in_literal:6031 # coding:utf-8 "\xD0\xBF\xD1\x80\xD0\xBE\xD0\xB2\xD0\xB5\xD1\x80\xD0\xBA\xD0\xB0" -!!! test_bug_cmd_string_lookahead:5752 +!!! test_bug_cmd_string_lookahead:5903 desc "foo" do end -!!! test_bug_cmdarg:5549 +!!! test_bug_cmdarg:5681 assert dogs -!!! test_bug_cmdarg:5554 +!!! test_bug_cmdarg:5686 assert do: true -!!! test_bug_cmdarg:5562 +!!! test_bug_cmdarg:5694 f x: -> do meth do end end -!!! test_bug_def_no_paren_eql_begin:5799 +!!! test_bug_def_no_paren_eql_begin:5950 def foo =begin =end end -!!! test_bug_do_block_in_call_args:5762 +!!! test_bug_do_block_in_call_args:5913 bar def foo; self.each do end end -!!! test_bug_do_block_in_cmdarg:5777 +!!! test_bug_do_block_in_cmdarg:5928 tap (proc do end) -!!! test_bug_do_block_in_hash_brace:6569 +!!! test_bug_do_block_in_hash_brace:6720 p :foo, {a: proc do end, b: proc do end} -!!! test_bug_do_block_in_hash_brace:6587 +!!! test_bug_do_block_in_hash_brace:6738 p :foo, {:a => proc do end, b: proc do end} -!!! test_bug_do_block_in_hash_brace:6605 +!!! test_bug_do_block_in_hash_brace:6756 p :foo, {"a": proc do end, b: proc do end} -!!! test_bug_do_block_in_hash_brace:6623 +!!! test_bug_do_block_in_hash_brace:6774 p :foo, {proc do end => proc do end, b: proc do end} -!!! test_bug_do_block_in_hash_brace:6643 +!!! test_bug_do_block_in_hash_brace:6794 p :foo, {** proc do end, b: proc do end} -!!! test_bug_heredoc_do:5835 +!!! test_bug_heredoc_do:5986 f <<-TABLE do TABLE end -!!! test_bug_interp_single:5789 +!!! test_bug_interp_single:5940 "#{1}" -!!! test_bug_interp_single:5793 +!!! test_bug_interp_single:5944 %W"#{1}" -!!! test_bug_lambda_leakage:6550 +!!! test_bug_lambda_leakage:6701 ->(scope) {}; scope -!!! test_bug_regex_verification:6563 +!!! test_bug_regex_verification:6714 /#)/x -!!! test_bug_rescue_empty_else:5813 +!!! test_bug_rescue_empty_else:5964 begin; rescue LoadError; else; end -!!! test_bug_while_not_parens_do:5805 +!!! test_bug_while_not_parens_do:5956 while not (true) do end -!!! test_case_cond:4844 +!!! test_case_cond:4976 case; when foo; 'foo'; end -!!! test_case_cond_else:4857 +!!! test_case_cond_else:4989 case; when foo; 'foo'; else 'bar'; end -!!! test_case_expr:4816 +!!! test_case_expr:4948 case foo; when 'bar'; bar; end -!!! test_case_expr_else:4830 +!!! test_case_expr_else:4962 case foo; when 'bar'; bar; else baz; end -!!! test_casgn_scoped:1192 +!!! test_casgn_scoped:1206 Bar::Foo = 10 -!!! test_casgn_toplevel:1181 +!!! test_casgn_toplevel:1195 ::Foo = 10 -!!! test_casgn_unscoped:1203 +!!! test_casgn_unscoped:1217 Foo = 10 -!!! test_character:248 +!!! test_character:250 ?a -!!! test_class:1827 +!!! test_class:1841 class Foo; end -!!! test_class:1837 +!!! test_class:1851 class Foo end -!!! test_class_definition_in_while_cond:6870 +!!! test_class_definition_in_while_cond:7055 while class Foo; tap do end; end; break; end -!!! test_class_definition_in_while_cond:6882 +!!! test_class_definition_in_while_cond:7067 while class Foo a = tap do end; end; break; end -!!! test_class_definition_in_while_cond:6895 +!!! test_class_definition_in_while_cond:7080 while class << self; tap do end; end; break; end -!!! test_class_definition_in_while_cond:6907 +!!! test_class_definition_in_while_cond:7092 while class << self; a = tap do end; end; break; end -!!! test_class_super:1848 +!!! test_class_super:1862 class Foo < Bar; end -!!! test_class_super_label:1860 +!!! test_class_super_label:1874 class Foo < a:b; end -!!! test_comments_before_leading_dot__27:7750 +!!! test_comments_before_leading_dot__27:7941 a # # .foo -!!! test_comments_before_leading_dot__27:7757 +!!! test_comments_before_leading_dot__27:7948 a # # .foo -!!! test_comments_before_leading_dot__27:7764 +!!! test_comments_before_leading_dot__27:7955 a # # &.foo -!!! test_comments_before_leading_dot__27:7771 +!!! test_comments_before_leading_dot__27:7962 a # # &.foo -!!! test_complex:156 +!!! test_complex:158 42i -!!! test_complex:162 +!!! test_complex:164 42ri -!!! test_complex:168 +!!! test_complex:170 42.1i -!!! test_complex:174 +!!! test_complex:176 42.1ri -!!! test_cond_begin:4686 +!!! test_cond_begin:4746 if (bar); foo; end -!!! test_cond_begin_masgn:4695 +!!! test_cond_begin_masgn:4755 if (bar; a, b = foo); end -!!! test_cond_eflipflop:4758 +!!! test_cond_eflipflop:4854 if foo...bar; end -!!! test_cond_eflipflop:4772 +!!! test_cond_eflipflop:4884 !(foo...bar) -!!! test_cond_iflipflop:4735 +!!! test_cond_eflipflop_with_beginless_range:4903 +if ...bar; end +!!! test_cond_eflipflop_with_endless_range:4893 +if foo...; end +!!! test_cond_iflipflop:4795 if foo..bar; end -!!! test_cond_iflipflop:4749 +!!! test_cond_iflipflop:4825 !(foo..bar) -!!! test_cond_match_current_line:4781 +!!! test_cond_iflipflop_with_beginless_range:4844 +if ..bar; end +!!! test_cond_iflipflop_with_endless_range:4834 +if foo..; end +!!! test_cond_match_current_line:4913 if /wat/; end -!!! test_cond_match_current_line:4801 +!!! test_cond_match_current_line:4933 !/wat/ -!!! test_const_op_asgn:1536 +!!! test_const_op_asgn:1550 A += 1 -!!! test_const_op_asgn:1542 +!!! test_const_op_asgn:1556 ::A += 1 -!!! test_const_op_asgn:1550 +!!! test_const_op_asgn:1564 B::A += 1 -!!! test_const_op_asgn:1558 +!!! test_const_op_asgn:1572 def x; self::A ||= 1; end -!!! test_const_op_asgn:1567 +!!! test_const_op_asgn:1581 def x; ::A ||= 1; end -!!! test_const_scoped:1020 +!!! test_const_scoped:1034 Bar::Foo -!!! test_const_toplevel:1011 +!!! test_const_toplevel:1025 ::Foo -!!! test_const_unscoped:1029 +!!! test_const_unscoped:1043 Foo -!!! test_control_meta_escape_chars_in_regexp__since_31:10686 +!!! test_control_meta_escape_chars_in_regexp__since_31:11030 /\c\xFF/ -!!! test_control_meta_escape_chars_in_regexp__since_31:10692 +!!! test_control_meta_escape_chars_in_regexp__since_31:11036 /\c\M-\xFF/ -!!! test_control_meta_escape_chars_in_regexp__since_31:10698 +!!! test_control_meta_escape_chars_in_regexp__since_31:11042 /\C-\xFF/ -!!! test_control_meta_escape_chars_in_regexp__since_31:10704 +!!! test_control_meta_escape_chars_in_regexp__since_31:11048 /\C-\M-\xFF/ -!!! test_control_meta_escape_chars_in_regexp__since_31:10710 +!!! test_control_meta_escape_chars_in_regexp__since_31:11054 /\M-\xFF/ -!!! test_control_meta_escape_chars_in_regexp__since_31:10716 +!!! test_control_meta_escape_chars_in_regexp__since_31:11060 /\M-\C-\xFF/ -!!! test_control_meta_escape_chars_in_regexp__since_31:10722 +!!! test_control_meta_escape_chars_in_regexp__since_31:11066 /\M-\c\xFF/ -!!! test_cpath:1807 +!!! test_cpath:1821 module ::Foo; end -!!! test_cpath:1813 +!!! test_cpath:1827 module Bar::Foo; end -!!! test_cvar:973 +!!! test_cvar:987 @@foo -!!! test_cvasgn:1106 +!!! test_cvasgn:1120 @@var = 10 -!!! test_dedenting_heredoc:297 +!!! test_dedenting_heredoc:299 p <<~E E -!!! test_dedenting_heredoc:304 +!!! test_dedenting_heredoc:306 p <<~E E -!!! test_dedenting_heredoc:311 +!!! test_dedenting_heredoc:313 p <<~E x E -!!! test_dedenting_heredoc:318 +!!! test_dedenting_heredoc:320 p <<~E ð E -!!! test_dedenting_heredoc:325 +!!! test_dedenting_heredoc:327 p <<~E x y E -!!! test_dedenting_heredoc:334 +!!! test_dedenting_heredoc:336 p <<~E x y E -!!! test_dedenting_heredoc:343 +!!! test_dedenting_heredoc:345 p <<~E x y E -!!! test_dedenting_heredoc:352 +!!! test_dedenting_heredoc:354 p <<~E x y E -!!! test_dedenting_heredoc:361 +!!! test_dedenting_heredoc:363 p <<~E x y E -!!! test_dedenting_heredoc:370 +!!! test_dedenting_heredoc:372 p <<~E x y E -!!! test_dedenting_heredoc:380 +!!! test_dedenting_heredoc:382 p <<~E x y E -!!! test_dedenting_heredoc:390 +!!! test_dedenting_heredoc:392 p <<~E x \ y E -!!! test_dedenting_heredoc:399 +!!! test_dedenting_heredoc:401 p <<~E x \ y E -!!! test_dedenting_heredoc:408 +!!! test_dedenting_heredoc:410 p <<~"E" x #{foo} E -!!! test_dedenting_heredoc:419 +!!! test_dedenting_heredoc:421 p <<~`E` x #{foo} E -!!! test_dedenting_heredoc:430 +!!! test_dedenting_heredoc:432 p <<~"E" x #{" y"} E -!!! test_dedenting_interpolating_heredoc_fake_line_continuation:459 +!!! test_dedenting_interpolating_heredoc_fake_line_continuation:461 <<~'FOO' baz\\ qux FOO -!!! test_dedenting_non_interpolating_heredoc_line_continuation:451 +!!! test_dedenting_non_interpolating_heredoc_line_continuation:453 <<~'FOO' baz\ qux FOO -!!! test_def:1899 +!!! test_def:1913 def foo; end -!!! test_def:1907 +!!! test_def:1921 def String; end -!!! test_def:1911 +!!! test_def:1925 def String=; end -!!! test_def:1915 +!!! test_def:1929 def until; end -!!! test_def:1919 +!!! test_def:1933 def BEGIN; end -!!! test_def:1923 +!!! test_def:1937 def END; end -!!! test_defined:1058 +!!! test_defined:1072 defined? foo -!!! test_defined:1064 +!!! test_defined:1078 defined?(foo) -!!! test_defined:1072 +!!! test_defined:1086 defined? @foo -!!! test_defs:1929 +!!! test_defs:1943 def self.foo; end -!!! test_defs:1937 +!!! test_defs:1951 def self::foo; end -!!! test_defs:1945 +!!! test_defs:1959 def (foo).foo; end -!!! test_defs:1949 +!!! test_defs:1963 def String.foo; end -!!! test_defs:1954 +!!! test_defs:1968 def String::foo; end -!!! test_empty_stmt:60 -!!! test_endless_method:9786 +!!! test_emit_arg_inside_procarg0_legacy:2807 +f{ |a| } +!!! test_empty_stmt:62 +!!! test_endless_comparison_method:10736:0 +def ===(other) = do_something +!!! test_endless_comparison_method:10736:1 +def ==(other) = do_something +!!! test_endless_comparison_method:10736:2 +def !=(other) = do_something +!!! test_endless_comparison_method:10736:3 +def <=(other) = do_something +!!! test_endless_comparison_method:10736:4 +def >=(other) = do_something +!!! test_endless_comparison_method:10736:5 +def !=(other) = do_something +!!! test_endless_method:10085 def foo() = 42 -!!! test_endless_method:9798 +!!! test_endless_method:10097 def inc(x) = x + 1 -!!! test_endless_method:9811 +!!! test_endless_method:10110 def obj.foo() = 42 -!!! test_endless_method:9823 +!!! test_endless_method:10122 def obj.inc(x) = x + 1 -!!! test_endless_method_command_syntax:9880 +!!! test_endless_method_command_syntax:10179 def foo = puts "Hello" -!!! test_endless_method_command_syntax:9892 +!!! test_endless_method_command_syntax:10191 def foo() = puts "Hello" -!!! test_endless_method_command_syntax:9904 +!!! test_endless_method_command_syntax:10203 def foo(x) = puts x -!!! test_endless_method_command_syntax:9917 +!!! test_endless_method_command_syntax:10216 def obj.foo = puts "Hello" -!!! test_endless_method_command_syntax:9931 +!!! test_endless_method_command_syntax:10230 def obj.foo() = puts "Hello" -!!! test_endless_method_command_syntax:9945 +!!! test_endless_method_command_syntax:10244 def rescued(x) = raise "to be caught" rescue "instance #{x}" -!!! test_endless_method_command_syntax:9964 +!!! test_endless_method_command_syntax:10263 def self.rescued(x) = raise "to be caught" rescue "class #{x}" -!!! test_endless_method_command_syntax:9985 +!!! test_endless_method_command_syntax:10284 def obj.foo(x) = puts x -!!! test_endless_method_forwarded_args_legacy:9840 +!!! test_endless_method_forwarded_args_legacy:10139 def foo(...) = bar(...) -!!! test_endless_method_with_rescue_mod:9855 +!!! test_endless_method_with_rescue_mod:10154 def m() = 1 rescue 2 -!!! test_endless_method_with_rescue_mod:9866 +!!! test_endless_method_with_rescue_mod:10165 def self.m() = 1 rescue 2 -!!! test_endless_method_without_args:10404 +!!! test_endless_method_without_args:10748 def foo = 42 -!!! test_endless_method_without_args:10412 +!!! test_endless_method_without_args:10756 def foo = 42 rescue nil -!!! test_endless_method_without_args:10423 +!!! test_endless_method_without_args:10767 def self.foo = 42 -!!! test_endless_method_without_args:10432 +!!! test_endless_method_without_args:10776 def self.foo = 42 rescue nil -!!! test_ensure:5261 +!!! test_ensure:5393 begin; meth; ensure; bar; end -!!! test_ensure_empty:5274 +!!! test_ensure_empty:5406 begin ensure end -!!! test_false:96 +!!! test_false:98 false -!!! test_float:129 +!!! test_find_pattern:10447 +case foo; in [*x, 1 => a, *y] then true; end +!!! test_find_pattern:10467 +case foo; in String(*, 1, *) then true; end +!!! test_find_pattern:10481 +case foo; in Array[*, 1, *] then true; end +!!! test_find_pattern:10495 +case foo; in *, 42, * then true; end +!!! test_float:131 1.33 -!!! test_float:134 +!!! test_float:136 -1.33 -!!! test_for:5002 +!!! test_for:5134 for a in foo do p a; end -!!! test_for:5014 +!!! test_for:5146 for a in foo; p a; end -!!! test_for_mlhs:5023 +!!! test_for_mlhs:5155 for a, b in foo; p a, b; end -!!! test_forward_arg:7899 +!!! test_forward_arg:8090 def foo(...); bar(...); end -!!! test_forward_arg_with_open_args:10745 +!!! test_forward_arg_with_open_args:11089 def foo ... end -!!! test_forward_arg_with_open_args:10752 +!!! test_forward_arg_with_open_args:11096 def foo a, b = 1, ... end -!!! test_forward_arg_with_open_args:10770 +!!! test_forward_arg_with_open_args:11114 def foo(a, ...) bar(...) end -!!! test_forward_arg_with_open_args:10781 +!!! test_forward_arg_with_open_args:11125 def foo a, ... bar(...) end -!!! test_forward_arg_with_open_args:10792 +!!! test_forward_arg_with_open_args:11136 def foo b = 1, ... bar(...) end -!!! test_forward_arg_with_open_args:10804 +!!! test_forward_arg_with_open_args:11148 def foo ...; bar(...); end -!!! test_forward_arg_with_open_args:10814 +!!! test_forward_arg_with_open_args:11158 def foo a, ...; bar(...); end -!!! test_forward_arg_with_open_args:10825 +!!! test_forward_arg_with_open_args:11169 def foo b = 1, ...; bar(...); end -!!! test_forward_arg_with_open_args:10837 +!!! test_forward_arg_with_open_args:11181 (def foo ... bar(...) end) -!!! test_forward_arg_with_open_args:10848 +!!! test_forward_arg_with_open_args:11192 (def foo ...; bar(...); end) -!!! test_forward_args_legacy:7863 +!!! test_forward_args_legacy:8054 def foo(...); bar(...); end -!!! test_forward_args_legacy:7875 +!!! test_forward_args_legacy:8066 def foo(...); super(...); end -!!! test_forward_args_legacy:7887 +!!! test_forward_args_legacy:8078 def foo(...); end -!!! test_forwarded_argument_with_kwrestarg:10962 +!!! test_forwarded_argument_with_kwrestarg:11332 def foo(argument, **); bar(argument, **); end -!!! test_forwarded_argument_with_restarg:10923 +!!! test_forwarded_argument_with_restarg:11267 def foo(argument, *); bar(argument, *); end -!!! test_forwarded_kwrestarg:10943 +!!! test_forwarded_kwrestarg:11287 def foo(**); bar(**); end -!!! test_forwarded_restarg:10905 +!!! test_forwarded_kwrestarg_with_additional_kwarg:11306 +def foo(**); bar(**, from_foo: true); end +!!! test_forwarded_restarg:11249 def foo(*); bar(*); end -!!! test_gvar:980 +!!! test_gvar:994 $foo -!!! test_gvasgn:1116 +!!! test_gvasgn:1130 $var = 10 -!!! test_hash_empty:750 +!!! test_hash_empty:764 { } -!!! test_hash_hashrocket:759 +!!! test_hash_hashrocket:773 { 1 => 2 } -!!! test_hash_hashrocket:768 +!!! test_hash_hashrocket:782 { 1 => 2, :foo => "bar" } -!!! test_hash_kwsplat:821 +!!! test_hash_kwsplat:835 { foo: 2, **bar } -!!! test_hash_label:776 +!!! test_hash_label:790 { foo: 2 } -!!! test_hash_label_end:789 +!!! test_hash_label_end:803 { 'foo': 2 } -!!! test_hash_label_end:802 +!!! test_hash_label_end:816 { 'foo': 2, 'bar': {}} -!!! test_hash_label_end:810 +!!! test_hash_label_end:824 f(a ? "a":1) -!!! test_hash_pair_value_omission:10040 +!!! test_hash_pair_value_omission:10339 {a:, b:} -!!! test_hash_pair_value_omission:10054 +!!! test_hash_pair_value_omission:10353 {puts:} -!!! test_hash_pair_value_omission:10065 +!!! test_hash_pair_value_omission:10364 +foo = 1; {foo:} +!!! test_hash_pair_value_omission:10376 +_foo = 1; {_foo:} +!!! test_hash_pair_value_omission:10388 {BAR:} -!!! test_heredoc:263 +!!! test_heredoc:265 <(**nil) {} -!!! test_kwoptarg:2124 +!!! test_kwoptarg:2138 def f(foo: 1); end -!!! test_kwrestarg_named:2135 +!!! test_kwoptarg_with_kwrestarg_and_forwarded_args:11482 +def f(a: nil, **); b(**) end +!!! test_kwrestarg_named:2149 def f(**foo); end -!!! test_kwrestarg_unnamed:2146 +!!! test_kwrestarg_unnamed:2160 def f(**); end -!!! test_lbrace_arg_after_command_args:7235 +!!! test_lbrace_arg_after_command_args:7420 let (:a) { m do; end } -!!! test_lparenarg_after_lvar__since_25:6679 +!!! test_lparenarg_after_lvar__since_25:6830 meth (-1.3).abs -!!! test_lparenarg_after_lvar__since_25:6688 +!!! test_lparenarg_after_lvar__since_25:6839 foo (-1.3).abs -!!! test_lvar:959 +!!! test_lvar:973 foo -!!! test_lvar_injecting_match:3778 +!!! test_lvar_injecting_match:3819 /(?bar)/ =~ 'bar'; match -!!! test_lvasgn:1084 +!!! test_lvasgn:1098 var = 10; var -!!! test_masgn:1247 +!!! test_marg_combinations:2454 +def f (((a))); end +!!! test_marg_combinations:2460 +def f ((a, a1)); end +!!! test_marg_combinations:2465 +def f ((a, *r)); end +!!! test_marg_combinations:2470 +def f ((a, *r, p)); end +!!! test_marg_combinations:2475 +def f ((a, *)); end +!!! test_marg_combinations:2480 +def f ((a, *, p)); end +!!! test_marg_combinations:2485 +def f ((*r)); end +!!! test_marg_combinations:2490 +def f ((*r, p)); end +!!! test_marg_combinations:2495 +def f ((*)); end +!!! test_marg_combinations:2500 +def f ((*, p)); end +!!! test_masgn:1261 foo, bar = 1, 2 -!!! test_masgn:1258 +!!! test_masgn:1272 (foo, bar) = 1, 2 -!!! test_masgn:1268 +!!! test_masgn:1282 foo, bar, baz = 1, 2 -!!! test_masgn_attr:1390 +!!! test_masgn_attr:1404 self.a, self[1, 2] = foo -!!! test_masgn_attr:1403 +!!! test_masgn_attr:1417 self::a, foo = foo -!!! test_masgn_attr:1411 +!!! test_masgn_attr:1425 self.A, foo = foo -!!! test_masgn_cmd:1439 +!!! test_masgn_cmd:1453 foo, bar = m foo -!!! test_masgn_const:1421 +!!! test_masgn_const:1435 self::A, foo = foo -!!! test_masgn_const:1429 +!!! test_masgn_const:1443 ::A, foo = foo -!!! test_masgn_nested:1365 -a, (b, c) = foo !!! test_masgn_nested:1379 +a, (b, c) = foo +!!! test_masgn_nested:1393 ((b, )) = foo -!!! test_masgn_splat:1279 +!!! test_masgn_splat:1293 @foo, @@bar = *foo -!!! test_masgn_splat:1288 +!!! test_masgn_splat:1302 a, b = *foo, bar -!!! test_masgn_splat:1296 +!!! test_masgn_splat:1310 a, *b = bar -!!! test_masgn_splat:1302 +!!! test_masgn_splat:1316 a, *b, c = bar -!!! test_masgn_splat:1313 +!!! test_masgn_splat:1327 a, * = bar -!!! test_masgn_splat:1319 +!!! test_masgn_splat:1333 a, *, c = bar -!!! test_masgn_splat:1330 +!!! test_masgn_splat:1344 *b = bar -!!! test_masgn_splat:1336 +!!! test_masgn_splat:1350 *b, c = bar -!!! test_masgn_splat:1346 +!!! test_masgn_splat:1360 * = bar -!!! test_masgn_splat:1352 +!!! test_masgn_splat:1366 *, c, d = bar -!!! test_method_definition_in_while_cond:6816 +!!! test_method_definition_in_while_cond:7001 while def foo; tap do end; end; break; end -!!! test_method_definition_in_while_cond:6828 +!!! test_method_definition_in_while_cond:7013 while def self.foo; tap do end; end; break; end -!!! test_method_definition_in_while_cond:6841 +!!! test_method_definition_in_while_cond:7026 while def foo a = tap do end; end; break; end -!!! test_method_definition_in_while_cond:6854 +!!! test_method_definition_in_while_cond:7039 while def self.foo a = tap do end; end; break; end -!!! test_module:1789 +!!! test_module:1803 module Foo; end -!!! test_multiple_pattern_matches:11086 +!!! test_multiple_args_with_trailing_comma:2786 +f{ |a, b,| } +!!! test_multiple_pattern_matches:11456 {a: 0} => a: {a: 0} => a: -!!! test_multiple_pattern_matches:11102 +!!! test_multiple_pattern_matches:11472 {a: 0} in a: {a: 0} in a: -!!! test_newline_in_hash_argument:11035 +!!! test_newline_in_hash_argument:11405 obj.set foo: 1 -!!! test_newline_in_hash_argument:11046 +!!! test_newline_in_hash_argument:11416 obj.set "foo": 1 -!!! test_newline_in_hash_argument:11057 +!!! test_newline_in_hash_argument:11427 case foo in a: 0 @@ -1240,585 +983,870 @@ in "b": 0 true end -!!! test_next:5131 +!!! test_next:5263 next(foo) -!!! test_next:5145 +!!! test_next:5277 next foo -!!! test_next:5151 +!!! test_next:5283 next() -!!! test_next:5158 +!!! test_next:5290 next -!!! test_next_block:5166 +!!! test_next_block:5298 next fun foo do end -!!! test_nil:66 +!!! test_nil:68 nil -!!! test_nil_expression:73 +!!! test_nil_expression:75 () -!!! test_nil_expression:80 +!!! test_nil_expression:82 begin end -!!! test_non_lvar_injecting_match:3793 +!!! test_non_lvar_injecting_match:3853 /#{1}(?bar)/ =~ 'bar' -!!! test_not:3462 +!!! test_not:3476 not foo -!!! test_not:3468 +!!! test_not:3482 not(foo) -!!! test_not:3474 +!!! test_not:3488 not() -!!! test_not_cmd:3488 +!!! test_not_cmd:3502 not m foo -!!! test_not_masgn__24:4672 +!!! test_not_masgn__24:4732 !(a, b = foo) -!!! test_nth_ref:1002 +!!! test_nth_ref:1016 $10 -!!! test_numbered_args_after_27:7358 +!!! test_numbered_args_after_27:7543 m { _1 + _9 } -!!! test_numbered_args_after_27:7373 +!!! test_numbered_args_after_27:7558 m do _1 + _9 end -!!! test_numbered_args_after_27:7390 +!!! test_numbered_args_after_27:7575 -> { _1 + _9} -!!! test_numbered_args_after_27:7405 +!!! test_numbered_args_after_27:7590 -> do _1 + _9 end -!!! test_numparam_outside_block:7512 +!!! test_numparam_outside_block:7697 class A; _1; end -!!! test_numparam_outside_block:7520 +!!! test_numparam_outside_block:7705 module A; _1; end -!!! test_numparam_outside_block:7528 +!!! test_numparam_outside_block:7713 class << foo; _1; end -!!! test_numparam_outside_block:7536 +!!! test_numparam_outside_block:7721 def self.m; _1; end -!!! test_numparam_outside_block:7545 +!!! test_numparam_outside_block:7730 _1 -!!! test_op_asgn:1606 +!!! test_numparam_ruby_bug_19025:10696 +p { [_1 **2] } +!!! test_op_asgn:1620 foo.a += 1 -!!! test_op_asgn:1616 +!!! test_op_asgn:1630 foo::a += 1 -!!! test_op_asgn:1622 +!!! test_op_asgn:1636 foo.A += 1 -!!! test_op_asgn_cmd:1630 +!!! test_op_asgn_cmd:1644 foo.a += m foo -!!! test_op_asgn_cmd:1636 +!!! test_op_asgn_cmd:1650 foo::a += m foo -!!! test_op_asgn_cmd:1642 +!!! test_op_asgn_cmd:1656 foo.A += m foo -!!! test_op_asgn_cmd:1654 +!!! test_op_asgn_cmd:1668 foo::A += m foo -!!! test_op_asgn_index:1664 +!!! test_op_asgn_index:1678 foo[0, 1] += 2 -!!! test_op_asgn_index_cmd:1678 +!!! test_op_asgn_index_cmd:1692 foo[0, 1] += m foo -!!! test_optarg:2074 +!!! test_optarg:2088 def f foo = 1; end -!!! test_optarg:2084 +!!! test_optarg:2098 def f(foo=1, bar=2); end -!!! test_or:4461 +!!! test_or:4521 foo or bar -!!! test_or:4467 +!!! test_or:4527 foo || bar -!!! test_or_asgn:1724 +!!! test_or_asgn:1738 foo.a ||= 1 -!!! test_or_asgn:1734 +!!! test_or_asgn:1748 foo[0, 1] ||= 2 -!!! test_parser_bug_272:6528 +!!! test_parser_bug_272:6679 a @b do |c|;end -!!! test_parser_bug_490:7151 +!!! test_parser_bug_490:7336 def m; class << self; class C; end; end; end -!!! test_parser_bug_490:7162 +!!! test_parser_bug_490:7347 def m; class << self; module M; end; end; end -!!! test_parser_bug_490:7173 +!!! test_parser_bug_490:7358 def m; class << self; A = nil; end; end -!!! test_parser_bug_507:7265 +!!! test_parser_bug_507:7450 m = -> *args do end -!!! test_parser_bug_518:7277 +!!! test_parser_bug_518:7462 class A < B end -!!! test_parser_bug_525:7287 +!!! test_parser_bug_525:7472 m1 :k => m2 do; m3() do end; end -!!! test_parser_bug_604:7737 +!!! test_parser_bug_604:7928 m a + b do end -!!! test_parser_bug_640:443 +!!! test_parser_bug_640:445 <<~FOO baz\ qux FOO -!!! test_parser_bug_645:9774 +!!! test_parser_bug_645:10073 -> (arg={}) {} -!!! test_parser_bug_830:10630 +!!! test_parser_bug_830:10974 /\(/ -!!! test_parser_drops_truncated_parts_of_squiggly_heredoc:10446 +!!! test_parser_bug_989:11684 + <<-HERE + content + HERE +!!! test_parser_drops_truncated_parts_of_squiggly_heredoc:10790 <<~HERE #{} HERE -!!! test_pattern_matching__FILE__LINE_literals:9473 +!!! test_parser_slash_slash_n_escaping_in_literals:7512:0 +'a\ +b' +!!! test_parser_slash_slash_n_escaping_in_literals:7512:1 +<<-'HERE' +a\ +b +HERE +!!! test_parser_slash_slash_n_escaping_in_literals:7512:2 +%q{a\ +b} +!!! test_parser_slash_slash_n_escaping_in_literals:7512:3 +"a\ +b" +!!! test_parser_slash_slash_n_escaping_in_literals:7512:4 +<<-"HERE" +a\ +b +HERE +!!! test_parser_slash_slash_n_escaping_in_literals:7512:5 +%{a\ +b} +!!! test_parser_slash_slash_n_escaping_in_literals:7512:6 +%Q{a\ +b} +!!! test_parser_slash_slash_n_escaping_in_literals:7512:7 +%w{a\ +b} +!!! test_parser_slash_slash_n_escaping_in_literals:7512:8 +%W{a\ +b} +!!! test_parser_slash_slash_n_escaping_in_literals:7512:9 +%i{a\ +b} +!!! test_parser_slash_slash_n_escaping_in_literals:7512:10 +%I{a\ +b} +!!! test_parser_slash_slash_n_escaping_in_literals:7512:11 +:'a\ +b' +!!! test_parser_slash_slash_n_escaping_in_literals:7512:12 +%s{a\ +b} +!!! test_parser_slash_slash_n_escaping_in_literals:7512:13 +:"a\ +b" +!!! test_parser_slash_slash_n_escaping_in_literals:7512:14 +/a\ +b/ +!!! test_parser_slash_slash_n_escaping_in_literals:7512:15 +%r{a\ +b} +!!! test_parser_slash_slash_n_escaping_in_literals:7512:16 +%x{a\ +b} +!!! test_parser_slash_slash_n_escaping_in_literals:7512:17 +`a\ +b` +!!! test_parser_slash_slash_n_escaping_in_literals:7512:18 +<<-`HERE` +a\ +b +HERE +!!! test_pattern_matching__FILE__LINE_literals:9760 case [__FILE__, __LINE__ + 1, __ENCODING__] in [__FILE__, __LINE__, __ENCODING__] end -!!! test_pattern_matching_blank_else:9390 +!!! test_pattern_matching_blank_else:9627 case 1; in 2; 3; else; end -!!! test_pattern_matching_else:9376 +!!! test_pattern_matching_const_pattern:9490 +case foo; in A(1, 2) then true; end +!!! test_pattern_matching_const_pattern:9507 +case foo; in A(x:) then true; end +!!! test_pattern_matching_const_pattern:9523 +case foo; in A() then true; end +!!! test_pattern_matching_const_pattern:9538 +case foo; in A[1, 2] then true; end +!!! test_pattern_matching_const_pattern:9555 +case foo; in A[x:] then true; end +!!! test_pattern_matching_const_pattern:9571 +case foo; in A[] then true; end +!!! test_pattern_matching_constants:9456 +case foo; in A then true; end +!!! test_pattern_matching_constants:9466 +case foo; in A::B then true; end +!!! test_pattern_matching_constants:9477 +case foo; in ::A then true; end +!!! test_pattern_matching_else:9613 case 1; in 2; 3; else; 4; end -!!! test_pattern_matching_single_line:9540 +!!! test_pattern_matching_explicit_array_match:8891 +case foo; in [x] then nil; end +!!! test_pattern_matching_explicit_array_match:8903 +case foo; in [x,] then nil; end +!!! test_pattern_matching_explicit_array_match:8915 +case foo; in [x, y] then true; end +!!! test_pattern_matching_explicit_array_match:8928 +case foo; in [x, y,] then true; end +!!! test_pattern_matching_explicit_array_match:8941 +case foo; in [x, y, *] then true; end +!!! test_pattern_matching_explicit_array_match:8955 +case foo; in [x, y, *z] then true; end +!!! test_pattern_matching_explicit_array_match:8969 +case foo; in [x, *y, z] then true; end +!!! test_pattern_matching_explicit_array_match:8983 +case foo; in [x, *, y] then true; end +!!! test_pattern_matching_explicit_array_match:8997 +case foo; in [*x, y] then true; end +!!! test_pattern_matching_explicit_array_match:9010 +case foo; in [*, x] then true; end +!!! test_pattern_matching_expr_in_paren:9443 +case foo; in (1) then true; end +!!! test_pattern_matching_hash:9025 +case foo; in {} then true; end +!!! test_pattern_matching_hash:9034 +case foo; in a: 1 then true; end +!!! test_pattern_matching_hash:9044 +case foo; in { a: 1 } then true; end +!!! test_pattern_matching_hash:9056 +case foo; in { a: 1, } then true; end +!!! test_pattern_matching_hash:9068 +case foo; in a: then true; end +!!! test_pattern_matching_hash:9080 +case foo; in **a then true; end +!!! test_pattern_matching_hash:9094 +case foo; in ** then true; end +!!! test_pattern_matching_hash:9106 +case foo; in a: 1, b: 2 then true; end +!!! test_pattern_matching_hash:9117 +case foo; in a:, b: then true; end +!!! test_pattern_matching_hash:9128 +case foo; in a: 1, _a:, ** then true; end +!!! test_pattern_matching_hash:9140 +case foo; + in {a: 1 + } + false + ; end +!!! test_pattern_matching_hash:9156 +case foo; + in {a: + 2} + false + ; end +!!! test_pattern_matching_hash:9171 +case foo; + in {Foo: 42 + } + false + ; end +!!! test_pattern_matching_hash:9186 +case foo; + in a: {b:}, c: + p c + ; end +!!! test_pattern_matching_hash:9203 +case foo; + in {a: + } + true + ; end +!!! test_pattern_matching_hash_with_string_keys:9242 +case foo; in "a": then true; end +!!! test_pattern_matching_hash_with_string_keys:9253 +case foo; in "#{ 'a' }": then true; end +!!! test_pattern_matching_hash_with_string_keys:9264 +case foo; in "#{ %q{a} }": then true; end +!!! test_pattern_matching_hash_with_string_keys:9275 +case foo; in "#{ %Q{a} }": then true; end +!!! test_pattern_matching_hash_with_string_keys:9288 +case foo; in "a": 1 then true; end +!!! test_pattern_matching_hash_with_string_keys:9297 +case foo; in "#{ 'a' }": 1 then true; end +!!! test_pattern_matching_hash_with_string_keys:9308 +case foo; in "#{ %q{a} }": 1 then true; end +!!! test_pattern_matching_hash_with_string_keys:9319 +case foo; in "#{ %Q{a} }": 1 then true; end +!!! test_pattern_matching_if_unless_modifiers:8753 +case foo; in x if true; nil; end +!!! test_pattern_matching_if_unless_modifiers:8767 +case foo; in x unless true; nil; end +!!! test_pattern_matching_implicit_array_match:8796 +case foo; in x, then nil; end +!!! test_pattern_matching_implicit_array_match:8806 +case foo; in *x then nil; end +!!! test_pattern_matching_implicit_array_match:8819 +case foo; in * then nil; end +!!! test_pattern_matching_implicit_array_match:8830 +case foo; in x, y then nil; end +!!! test_pattern_matching_implicit_array_match:8841 +case foo; in x, y, then nil; end +!!! test_pattern_matching_implicit_array_match:8852 +case foo; in x, *y, z then nil; end +!!! test_pattern_matching_implicit_array_match:8864 +case foo; in *x, y, z then nil; end +!!! test_pattern_matching_implicit_array_match:8876 +case foo; in 1, "a", [], {} then nil; end +!!! test_pattern_matching_keyword_variable:9370 +case foo; in self then true; end +!!! test_pattern_matching_lambda:9380 +case foo; in ->{ 42 } then true; end +!!! test_pattern_matching_match_alt:9587 +case foo; in 1 | 2 then true; end +!!! test_pattern_matching_match_as:9599 +case foo; in 1 => a then true; end +!!! test_pattern_matching_nil_pattern:9783 +case foo; in **nil then true; end +!!! test_pattern_matching_no_body:8745 +case foo; in 1; end +!!! test_pattern_matching_numbered_parameter:9654 +1.then { 1 in ^_1 } +!!! test_pattern_matching_pin_variable:8783 +case foo; in ^foo then nil; end +!!! test_pattern_matching_ranges:9393 +case foo; in 1..2 then true; end +!!! test_pattern_matching_ranges:9401 +case foo; in 1.. then true; end +!!! test_pattern_matching_ranges:9409 +case foo; in ..2 then true; end +!!! test_pattern_matching_ranges:9417 +case foo; in 1...2 then true; end +!!! test_pattern_matching_ranges:9425 +case foo; in 1... then true; end +!!! test_pattern_matching_ranges:9433 +case foo; in ...2 then true; end +!!! test_pattern_matching_single_line:9827 1 => [a]; a -!!! test_pattern_matching_single_line:9552 +!!! test_pattern_matching_single_line:9839 1 in [a]; a -!!! test_pattern_matching_single_line_allowed_omission_of_parentheses:9566 +!!! test_pattern_matching_single_line_allowed_omission_of_parentheses:9853 [1, 2] => a, b; a -!!! test_pattern_matching_single_line_allowed_omission_of_parentheses:9581 +!!! test_pattern_matching_single_line_allowed_omission_of_parentheses:9868 {a: 1} => a:; a -!!! test_pattern_matching_single_line_allowed_omission_of_parentheses:9596 +!!! test_pattern_matching_single_line_allowed_omission_of_parentheses:9883 [1, 2] in a, b; a -!!! test_pattern_matching_single_line_allowed_omission_of_parentheses:9611 +!!! test_pattern_matching_single_line_allowed_omission_of_parentheses:9898 {a: 1} in a:; a -!!! test_pattern_matching_single_line_allowed_omission_of_parentheses:9626 +!!! test_pattern_matching_single_line_allowed_omission_of_parentheses:9913 {key: :value} in key: value; value -!!! test_pattern_matching_single_line_allowed_omission_of_parentheses:9643 +!!! test_pattern_matching_single_line_allowed_omission_of_parentheses:9930 {key: :value} => key: value; value -!!! test_postexe:5486 +!!! test_pattern_matching_single_match:8730 +case foo; in x then x; end +!!! test_pin_expr:10800 +case foo; in ^(42) then nil; end +!!! test_pin_expr:10814 +case foo; in { foo: ^(42) } then nil; end +!!! test_pin_expr:10831 +case foo; in ^(0+0) then nil; end +!!! test_pin_expr:10847 +case foo; in ^@a; end +!!! test_pin_expr:10856 +case foo; in ^@@TestPatternMatching; end +!!! test_pin_expr:10865 +case foo; in ^$TestPatternMatching; end +!!! test_pin_expr:10874 +case foo; in ^(1 +); end +!!! test_postexe:5618 END { 1 } -!!! test_preexe:5467 +!!! test_preexe:5599 BEGIN { 1 } -!!! test_procarg0:2803 +!!! test_procarg0:2817 m { |foo| } -!!! test_procarg0:2812 +!!! test_procarg0:2826 m { |(foo, bar)| } -!!! test_range_endless:869 +!!! test_procarg0_legacy:2796 +f{ |a| } +!!! test_range_endless:883 1.. -!!! test_range_endless:877 +!!! test_range_endless:891 1... -!!! test_range_exclusive:861 +!!! test_range_exclusive:875 1...2 -!!! test_range_inclusive:853 +!!! test_range_inclusive:867 1..2 -!!! test_rational:142 +!!! test_rational:144 42r -!!! test_rational:148 +!!! test_rational:150 42.1r -!!! test_redo:5178 +!!! test_redo:5310 redo -!!! test_regex_interp:551 +!!! test_regex_interp:553 /foo#{bar}baz/ -!!! test_regex_plain:541 +!!! test_regex_plain:543 /source/im -!!! test_resbody_list:5398 +!!! test_resbody_list:5530 begin; meth; rescue Exception; bar; end -!!! test_resbody_list_mrhs:5411 +!!! test_resbody_list_mrhs:5543 begin; meth; rescue Exception, foo; bar; end -!!! test_resbody_list_var:5444 +!!! test_resbody_list_var:5576 begin; meth; rescue foo => ex; bar; end -!!! test_resbody_var:5426 +!!! test_resbody_var:5558 begin; meth; rescue => ex; bar; end -!!! test_resbody_var:5434 +!!! test_resbody_var:5566 begin; meth; rescue => @ex; bar; end -!!! test_rescue:5188 +!!! test_rescue:5320 begin; meth; rescue; foo; end -!!! test_rescue_else:5203 +!!! test_rescue_else:5335 begin; meth; rescue; foo; else; bar; end -!!! test_rescue_else_ensure:5302 +!!! test_rescue_else_ensure:5434 begin; meth; rescue; baz; else foo; ensure; bar end -!!! test_rescue_ensure:5286 +!!! test_rescue_ensure:5418 begin; meth; rescue; baz; ensure; bar; end -!!! test_rescue_in_lambda_block:6928 +!!! test_rescue_in_lambda_block:7113 -> do rescue; end -!!! test_rescue_mod:5319 +!!! test_rescue_mod:5451 meth rescue bar -!!! test_rescue_mod_asgn:5331 +!!! test_rescue_mod_asgn:5463 foo = meth rescue bar -!!! test_rescue_mod_masgn:5345 +!!! test_rescue_mod_masgn:5477 foo, bar = meth rescue [1, 2] -!!! test_rescue_mod_op_assign:5365 +!!! test_rescue_mod_op_assign:5497 foo += meth rescue bar -!!! test_rescue_without_begin_end:5381 +!!! test_rescue_without_begin_end:5513 meth do; foo; rescue; bar; end -!!! test_restarg_named:2094 +!!! test_restarg_named:2108 def f(*foo); end -!!! test_restarg_unnamed:2104 +!!! test_restarg_unnamed:2118 def f(*); end -!!! test_retry:5457 +!!! test_retry:5589 retry -!!! test_return:5084 +!!! test_return:5216 return(foo) -!!! test_return:5098 +!!! test_return:5230 return foo -!!! test_return:5104 +!!! test_return:5236 return() -!!! test_return:5111 +!!! test_return:5243 return -!!! test_return_block:5119 +!!! test_return_block:5251 return fun foo do end -!!! test_ruby_bug_10279:5905 +!!! test_ruby_bug_10279:6056 {a: if true then 42 end} -!!! test_ruby_bug_10653:5915 +!!! test_ruby_bug_10653:6066 true ? 1.tap do |n| p n end : 0 -!!! test_ruby_bug_10653:5945 +!!! test_ruby_bug_10653:6096 false ? raise {} : tap {} -!!! test_ruby_bug_10653:5958 +!!! test_ruby_bug_10653:6109 false ? raise do end : tap do end -!!! test_ruby_bug_11107:5973 +!!! test_ruby_bug_11107:6124 p ->() do a() do end end -!!! test_ruby_bug_11380:5985 +!!! test_ruby_bug_11380:6136 p -> { :hello }, a: 1 do end -!!! test_ruby_bug_11873:6353 +!!! test_ruby_bug_11873:6504 a b{c d}, "x" do end -!!! test_ruby_bug_11873:6367 +!!! test_ruby_bug_11873:6518 a b(c d), "x" do end -!!! test_ruby_bug_11873:6380 +!!! test_ruby_bug_11873:6531 a b{c(d)}, "x" do end -!!! test_ruby_bug_11873:6394 +!!! test_ruby_bug_11873:6545 a b(c(d)), "x" do end -!!! test_ruby_bug_11873:6407 +!!! test_ruby_bug_11873:6558 a b{c d}, /x/ do end -!!! test_ruby_bug_11873:6421 +!!! test_ruby_bug_11873:6572 a b(c d), /x/ do end -!!! test_ruby_bug_11873:6434 +!!! test_ruby_bug_11873:6585 a b{c(d)}, /x/ do end -!!! test_ruby_bug_11873:6448 +!!! test_ruby_bug_11873:6599 a b(c(d)), /x/ do end -!!! test_ruby_bug_11873:6461 +!!! test_ruby_bug_11873:6612 a b{c d}, /x/m do end -!!! test_ruby_bug_11873:6475 +!!! test_ruby_bug_11873:6626 a b(c d), /x/m do end -!!! test_ruby_bug_11873:6488 +!!! test_ruby_bug_11873:6639 a b{c(d)}, /x/m do end -!!! test_ruby_bug_11873:6502 +!!! test_ruby_bug_11873:6653 a b(c(d)), /x/m do end -!!! test_ruby_bug_11873_b:6050 +!!! test_ruby_bug_11873_a:6168:0 +a b{c d}, :e do end +!!! test_ruby_bug_11873_a:6168:1 +a b{c d}, 1 do end +!!! test_ruby_bug_11873_a:6168:2 +a b{c d}, 1.0 do end +!!! test_ruby_bug_11873_a:6168:3 +a b{c d}, 1.0r do end +!!! test_ruby_bug_11873_a:6168:4 +a b{c d}, 1.0i do end +!!! test_ruby_bug_11873_a:6173:0 +a b{c(d)}, :e do end +!!! test_ruby_bug_11873_a:6173:1 +a b{c(d)}, 1 do end +!!! test_ruby_bug_11873_a:6173:2 +a b{c(d)}, 1.0 do end +!!! test_ruby_bug_11873_a:6173:3 +a b{c(d)}, 1.0r do end +!!! test_ruby_bug_11873_a:6173:4 +a b{c(d)}, 1.0i do end +!!! test_ruby_bug_11873_a:6187:0 +a b(c d), :e do end +!!! test_ruby_bug_11873_a:6187:1 +a b(c d), 1 do end +!!! test_ruby_bug_11873_a:6187:2 +a b(c d), 1.0 do end +!!! test_ruby_bug_11873_a:6187:3 +a b(c d), 1.0r do end +!!! test_ruby_bug_11873_a:6187:4 +a b(c d), 1.0i do end +!!! test_ruby_bug_11873_a:6192:0 +a b(c(d)), :e do end +!!! test_ruby_bug_11873_a:6192:1 +a b(c(d)), 1 do end +!!! test_ruby_bug_11873_a:6192:2 +a b(c(d)), 1.0 do end +!!! test_ruby_bug_11873_a:6192:3 +a b(c(d)), 1.0r do end +!!! test_ruby_bug_11873_a:6192:4 +a b(c(d)), 1.0i do end +!!! test_ruby_bug_11873_b:6201 p p{p(p);p p}, tap do end -!!! test_ruby_bug_11989:6069 +!!! test_ruby_bug_11989:6220 p <<~"E" x\n y E -!!! test_ruby_bug_11990:6078 +!!! test_ruby_bug_11990:6229 p <<~E " y" x E -!!! test_ruby_bug_12073:6089 +!!! test_ruby_bug_12073:6240 a = 1; a b: 1 -!!! test_ruby_bug_12073:6102 +!!! test_ruby_bug_12073:6253 def foo raise; raise A::B, ''; end -!!! test_ruby_bug_12402:6116 +!!! test_ruby_bug_12402:6267 foo = raise(bar) rescue nil -!!! test_ruby_bug_12402:6127 +!!! test_ruby_bug_12402:6278 foo += raise(bar) rescue nil -!!! test_ruby_bug_12402:6139 +!!! test_ruby_bug_12402:6290 foo[0] += raise(bar) rescue nil -!!! test_ruby_bug_12402:6153 +!!! test_ruby_bug_12402:6304 foo.m += raise(bar) rescue nil -!!! test_ruby_bug_12402:6166 +!!! test_ruby_bug_12402:6317 foo::m += raise(bar) rescue nil -!!! test_ruby_bug_12402:6179 +!!! test_ruby_bug_12402:6330 foo.C += raise(bar) rescue nil -!!! test_ruby_bug_12402:6192 +!!! test_ruby_bug_12402:6343 foo::C ||= raise(bar) rescue nil -!!! test_ruby_bug_12402:6205 +!!! test_ruby_bug_12402:6356 foo = raise bar rescue nil -!!! test_ruby_bug_12402:6216 +!!! test_ruby_bug_12402:6367 foo += raise bar rescue nil -!!! test_ruby_bug_12402:6228 +!!! test_ruby_bug_12402:6379 foo[0] += raise bar rescue nil -!!! test_ruby_bug_12402:6242 +!!! test_ruby_bug_12402:6393 foo.m += raise bar rescue nil -!!! test_ruby_bug_12402:6255 +!!! test_ruby_bug_12402:6406 foo::m += raise bar rescue nil -!!! test_ruby_bug_12402:6268 +!!! test_ruby_bug_12402:6419 foo.C += raise bar rescue nil -!!! test_ruby_bug_12402:6281 +!!! test_ruby_bug_12402:6432 foo::C ||= raise bar rescue nil -!!! test_ruby_bug_12669:6296 +!!! test_ruby_bug_12669:6447 a = b = raise :x -!!! test_ruby_bug_12669:6305 +!!! test_ruby_bug_12669:6456 a += b = raise :x -!!! test_ruby_bug_12669:6314 +!!! test_ruby_bug_12669:6465 a = b += raise :x -!!! test_ruby_bug_12669:6323 +!!! test_ruby_bug_12669:6474 a += b += raise :x -!!! test_ruby_bug_12686:6334 +!!! test_ruby_bug_12686:6485 f (g rescue nil) -!!! test_ruby_bug_13547:7018 +!!! test_ruby_bug_13547:7203 meth[] {} -!!! test_ruby_bug_14690:7250 +!!! test_ruby_bug_14690:7435 let () { m(a) do; end } -!!! test_ruby_bug_15789:7622 +!!! test_ruby_bug_15789:7807 m ->(a = ->{_1}) {a} -!!! test_ruby_bug_15789:7636 +!!! test_ruby_bug_15789:7821 m ->(a: ->{_1}) {a} -!!! test_ruby_bug_9669:5889 +!!! test_ruby_bug_9669:6040 def a b: return end -!!! test_ruby_bug_9669:5895 +!!! test_ruby_bug_9669:6046 o = { a: 1 } -!!! test_sclass:1884 +!!! test_sclass:1898 class << foo; nil; end -!!! test_self:952 +!!! test_self:966 self -!!! test_send_attr_asgn:3528 +!!! test_send_attr_asgn:3542 foo.a = 1 -!!! test_send_attr_asgn:3536 +!!! test_send_attr_asgn:3550 foo::a = 1 -!!! test_send_attr_asgn:3544 +!!! test_send_attr_asgn:3558 foo.A = 1 -!!! test_send_attr_asgn:3552 +!!! test_send_attr_asgn:3566 foo::A = 1 -!!! test_send_attr_asgn_conditional:3751 +!!! test_send_attr_asgn_conditional:3792 a&.b = 1 -!!! test_send_binary_op:3308 +!!! test_send_binary_op:3322 foo + 1 -!!! test_send_binary_op:3314 +!!! test_send_binary_op:3328 foo - 1 -!!! test_send_binary_op:3318 +!!! test_send_binary_op:3332 foo * 1 -!!! test_send_binary_op:3322 +!!! test_send_binary_op:3336 foo / 1 -!!! test_send_binary_op:3326 +!!! test_send_binary_op:3340 foo % 1 -!!! test_send_binary_op:3330 +!!! test_send_binary_op:3344 foo ** 1 -!!! test_send_binary_op:3334 +!!! test_send_binary_op:3348 foo | 1 -!!! test_send_binary_op:3338 +!!! test_send_binary_op:3352 foo ^ 1 -!!! test_send_binary_op:3342 +!!! test_send_binary_op:3356 foo & 1 -!!! test_send_binary_op:3346 +!!! test_send_binary_op:3360 foo <=> 1 -!!! test_send_binary_op:3350 +!!! test_send_binary_op:3364 foo < 1 -!!! test_send_binary_op:3354 +!!! test_send_binary_op:3368 foo <= 1 -!!! test_send_binary_op:3358 +!!! test_send_binary_op:3372 foo > 1 -!!! test_send_binary_op:3362 +!!! test_send_binary_op:3376 foo >= 1 -!!! test_send_binary_op:3366 +!!! test_send_binary_op:3380 foo == 1 -!!! test_send_binary_op:3376 +!!! test_send_binary_op:3390 foo != 1 -!!! test_send_binary_op:3382 +!!! test_send_binary_op:3396 foo === 1 -!!! test_send_binary_op:3386 +!!! test_send_binary_op:3400 foo =~ 1 -!!! test_send_binary_op:3396 +!!! test_send_binary_op:3410 foo !~ 1 -!!! test_send_binary_op:3402 +!!! test_send_binary_op:3416 foo << 1 -!!! test_send_binary_op:3406 +!!! test_send_binary_op:3420 foo >> 1 -!!! test_send_block_chain_cmd:3201 +!!! test_send_block_chain_cmd:3215 meth 1 do end.fun bar -!!! test_send_block_chain_cmd:3212 +!!! test_send_block_chain_cmd:3226 meth 1 do end.fun(bar) -!!! test_send_block_chain_cmd:3225 +!!! test_send_block_chain_cmd:3239 meth 1 do end::fun bar -!!! test_send_block_chain_cmd:3236 +!!! test_send_block_chain_cmd:3250 meth 1 do end::fun(bar) -!!! test_send_block_chain_cmd:3249 +!!! test_send_block_chain_cmd:3263 meth 1 do end.fun bar do end -!!! test_send_block_chain_cmd:3261 +!!! test_send_block_chain_cmd:3275 meth 1 do end.fun(bar) {} -!!! test_send_block_chain_cmd:3273 +!!! test_send_block_chain_cmd:3287 meth 1 do end.fun {} -!!! test_send_block_conditional:3759 +!!! test_send_block_conditional:3800 foo&.bar {} -!!! test_send_call:3721 +!!! test_send_call:3762 foo.(1) -!!! test_send_call:3731 +!!! test_send_call:3772 foo::(1) -!!! test_send_conditional:3743 +!!! test_send_conditional:3784 a&.b -!!! test_send_index:3562 +!!! test_send_index:3576 foo[1, 2] -!!! test_send_index_asgn:3591 +!!! test_send_index_asgn:3605 foo[1, 2] = 3 -!!! test_send_index_asgn_legacy:3603 +!!! test_send_index_asgn_kwarg:3629 +foo[:kw => arg] = 3 +!!! test_send_index_asgn_kwarg_legacy:3642 +foo[:kw => arg] = 3 +!!! test_send_index_asgn_legacy:3617 foo[1, 2] = 3 -!!! test_send_index_cmd:3584 +!!! test_send_index_cmd:3598 foo[m bar] -!!! test_send_index_legacy:3573 +!!! test_send_index_legacy:3587 foo[1, 2] -!!! test_send_lambda:3615 +!!! test_send_lambda:3656 ->{ } -!!! test_send_lambda:3625 +!!! test_send_lambda:3666 -> * { } -!!! test_send_lambda:3636 +!!! test_send_lambda:3677 -> do end -!!! test_send_lambda_args:3648 +!!! test_send_lambda_args:3689 ->(a) { } -!!! test_send_lambda_args:3662 +!!! test_send_lambda_args:3703 -> (a) { } -!!! test_send_lambda_args_noparen:3686 +!!! test_send_lambda_args_noparen:3727 -> a: 1 { } -!!! test_send_lambda_args_noparen:3695 +!!! test_send_lambda_args_noparen:3736 -> a: { } -!!! test_send_lambda_args_shadow:3673 +!!! test_send_lambda_args_shadow:3714 ->(a; foo, bar) { } -!!! test_send_lambda_legacy:3707 +!!! test_send_lambda_legacy:3748 ->{ } -!!! test_send_op_asgn_conditional:3770 +!!! test_send_op_asgn_conditional:3811 a&.b &&= 1 -!!! test_send_plain:3105 +!!! test_send_plain:3119 foo.fun -!!! test_send_plain:3112 +!!! test_send_plain:3126 foo::fun -!!! test_send_plain:3119 +!!! test_send_plain:3133 foo::Fun() -!!! test_send_plain_cmd:3128 +!!! test_send_plain_cmd:3142 foo.fun bar -!!! test_send_plain_cmd:3135 +!!! test_send_plain_cmd:3149 foo::fun bar -!!! test_send_plain_cmd:3142 +!!! test_send_plain_cmd:3156 foo::Fun bar -!!! test_send_self:3044 +!!! test_send_self:3058 fun -!!! test_send_self:3050 +!!! test_send_self:3064 fun! -!!! test_send_self:3056 +!!! test_send_self:3070 fun(1) -!!! test_send_self_block:3066 +!!! test_send_self_block:3080 fun { } -!!! test_send_self_block:3070 +!!! test_send_self_block:3084 fun() { } -!!! test_send_self_block:3074 +!!! test_send_self_block:3088 fun(1) { } -!!! test_send_self_block:3078 +!!! test_send_self_block:3092 fun do end -!!! test_send_unary_op:3412 +!!! test_send_unary_op:3426 -foo -!!! test_send_unary_op:3418 +!!! test_send_unary_op:3432 +foo -!!! test_send_unary_op:3422 +!!! test_send_unary_op:3436 ~foo -!!! test_slash_newline_in_heredocs:7186 +!!! test_slash_newline_in_heredocs:7371 <<~E 1 \ 2 3 E -!!! test_slash_newline_in_heredocs:7194 +!!! test_slash_newline_in_heredocs:7379 <<-E 1 \ 2 3 E -!!! test_space_args_arg:4132 +!!! test_space_args_arg:4192 fun (1) -!!! test_space_args_arg_block:4146 +!!! test_space_args_arg_block:4206 fun (1) {} -!!! test_space_args_arg_block:4160 +!!! test_space_args_arg_block:4220 foo.fun (1) {} -!!! test_space_args_arg_block:4176 +!!! test_space_args_arg_block:4236 foo::fun (1) {} -!!! test_space_args_arg_call:4198 +!!! test_space_args_arg_call:4258 fun (1).to_i -!!! test_space_args_arg_newline:4138 +!!! test_space_args_arg_newline:4198 fun (1 ) -!!! test_space_args_block:4430 +!!! test_space_args_block:4490 fun () {} -!!! test_space_args_cmd:4125 +!!! test_space_args_cmd:4185 fun (f bar) -!!! test_string___FILE__:241 +!!! test_string___FILE__:243 __FILE__ -!!! test_string_concat:226 +!!! test_string_concat:228 "foo#@a" "bar" -!!! test_string_dvar:215 +!!! test_string_dvar:217 "#@a #@@a #$a" -!!! test_string_interp:200 +!!! test_string_interp:202 "foo#{bar}baz" -!!! test_string_plain:184 +!!! test_string_plain:186 'foobar' -!!! test_string_plain:191 +!!! test_string_plain:193 %q(foobar) -!!! test_super:3807 +!!! test_super:3867 super(foo) -!!! test_super:3815 +!!! test_super:3875 super foo -!!! test_super:3821 +!!! test_super:3881 super() -!!! test_super_block:3839 +!!! test_super_block:3899 super foo, bar do end -!!! test_super_block:3845 +!!! test_super_block:3905 super do end -!!! test_symbol_interp:484 +!!! test_symbol_interp:486 :"foo#{bar}baz" -!!! test_symbol_plain:469 +!!! test_symbol_plain:471 :foo -!!! test_symbol_plain:475 +!!! test_symbol_plain:477 :'foo' -!!! test_ternary:4605 +!!! test_ternary:4665 foo ? 1 : 2 -!!! test_ternary_ambiguous_symbol:4614 +!!! test_ternary_ambiguous_symbol:4674 t=1;(foo)?t:T -!!! test_trailing_forward_arg:8022 +!!! test_trailing_forward_arg:8237 def foo(a, b, ...); bar(a, 42, ...); end -!!! test_true:89 +!!! test_true:91 true -!!! test_unary_num_pow_precedence:3505 +!!! test_unary_num_pow_precedence:3519 +2.0 ** 10 -!!! test_unary_num_pow_precedence:3512 +!!! test_unary_num_pow_precedence:3526 -2 ** 10 -!!! test_unary_num_pow_precedence:3519 +!!! test_unary_num_pow_precedence:3533 -2.0 ** 10 -!!! test_undef:2003 +!!! test_undef:2017 undef foo, :bar, :"foo#{1}" -!!! test_unless:4529 +!!! test_unless:4589 unless foo then bar; end -!!! test_unless:4537 +!!! test_unless:4597 unless foo; bar; end -!!! test_unless_else:4573 +!!! test_unless_else:4633 unless foo then bar; else baz; end -!!! test_unless_else:4582 +!!! test_unless_else:4642 unless foo; bar; else baz; end -!!! test_unless_mod:4546 +!!! test_unless_mod:4606 bar unless foo -!!! test_until:4948 +!!! test_until:5080 until foo do meth end -!!! test_until:4955 +!!! test_until:5087 until foo; meth end -!!! test_until_mod:4963 +!!! test_until_mod:5095 meth until foo -!!! test_until_post:4978 +!!! test_until_post:5110 begin meth end until foo -!!! test_var_and_asgn:1714 +!!! test_var_and_asgn:1728 a &&= 1 -!!! test_var_op_asgn:1498 +!!! test_var_op_asgn:1512 a += 1 -!!! test_var_op_asgn:1504 +!!! test_var_op_asgn:1518 @a |= 1 -!!! test_var_op_asgn:1510 +!!! test_var_op_asgn:1524 @@var |= 10 -!!! test_var_op_asgn:1514 +!!! test_var_op_asgn:1528 def a; @@var |= 10; end -!!! test_var_op_asgn_cmd:1521 +!!! test_var_op_asgn_cmd:1535 foo += m foo -!!! test_var_or_asgn:1706 +!!! test_var_or_asgn:1720 a ||= 1 -!!! test_when_multi:4895 +!!! test_when_multi:5027 case foo; when 'bar', 'baz'; bar; end -!!! test_when_splat:4904 +!!! test_when_splat:5036 case foo; when 1, *baz; bar; when *foo; end -!!! test_when_then:4883 +!!! test_when_then:5015 case foo; when 'bar' then bar; end -!!! test_while:4924 +!!! test_while:5056 while foo do meth end -!!! test_while:4932 +!!! test_while:5064 while foo; meth end -!!! test_while_mod:4941 +!!! test_while_mod:5073 meth while foo -!!! test_while_post:4970 +!!! test_while_post:5102 begin meth end while foo -!!! test_xstring_interp:524 +!!! test_xstring_interp:526 `foo#{bar}baz` -!!! test_xstring_plain:515 +!!! test_xstring_plain:517 `foobar` -!!! test_yield:3855 +!!! test_yield:3915 yield(foo) -!!! test_yield:3863 +!!! test_yield:3923 yield foo -!!! test_yield:3869 +!!! test_yield:3929 yield() -!!! test_yield:3877 +!!! test_yield:3937 yield -!!! test_zsuper:3831 +!!! test_zsuper:3891 super diff --git a/test/translation/parser_test.rb b/test/translation/parser_test.rb index 1df98f47..dd88322e 100644 --- a/test/translation/parser_test.rb +++ b/test/translation/parser_test.rb @@ -8,109 +8,83 @@ module SyntaxTree module Translation class ParserTest < Minitest::Test - known_failures = [ - # I think this may be a bug in the parser gem's precedence calculation. - # Unary plus appears to be parsed as part of the number literal in - # CRuby, but parser is parsing it as a separate operator. - "test_unary_num_pow_precedence:3505", - - # Not much to be done about this. Basically, regular expressions with - # named capture groups that use the =~ operator inject local variables - # into the current scope. In the parser gem, it detects this and changes - # future references to that name to be a local variable instead of a - # potential method call. CRuby does not do this. - "test_lvar_injecting_match:3778", - - # This is failing because CRuby is not marking values captured in hash - # patterns as local variables, while the parser gem is. - "test_pattern_matching_hash:8971", - - # This is not actually allowed in the CRuby parser but the parser gem - # thinks it is allowed. - "test_pattern_matching_hash_with_string_keys:9016", - "test_pattern_matching_hash_with_string_keys:9027", - "test_pattern_matching_hash_with_string_keys:9038", - "test_pattern_matching_hash_with_string_keys:9060", - "test_pattern_matching_hash_with_string_keys:9071", - "test_pattern_matching_hash_with_string_keys:9082", - - # This happens with pattern matching where you're matching a literal - # value inside parentheses, which doesn't really do anything. Ripper - # doesn't capture that this value is inside a parentheses, so it's hard - # to translate properly. - "test_pattern_matching_expr_in_paren:9206", - - # These are also failing because of CRuby not marking values captured in - # hash patterns as local variables. - "test_pattern_matching_single_line_allowed_omission_of_parentheses:*", - - # I'm not even sure what this is testing, because the code is invalid in - # CRuby. - "test_control_meta_escape_chars_in_regexp__since_31:*", - ] - - todo_failures = [ - "test_dedenting_heredoc:334", - "test_dedenting_heredoc:390", - "test_dedenting_heredoc:399", - "test_slash_newline_in_heredocs:7194", - "test_parser_slash_slash_n_escaping_in_literals:*", - "test_forwarded_restarg:*", - "test_forwarded_kwrestarg:*", - "test_forwarded_argument_with_restarg:*", - "test_forwarded_argument_with_kwrestarg:*" + skips = %w[ + test_args_assocs_legacy:4041 + test_args_assocs:4091 + test_args_assocs:4091 + test_break_block:5204 + test_break:5169 + test_break:5183 + test_break:5189 + test_break:5196 + test_control_meta_escape_chars_in_regexp__since_31:* + test_dedenting_heredoc:336 + test_dedenting_heredoc:392 + test_dedenting_heredoc:401 + test_forwarded_argument_with_kwrestarg:11332 + test_forwarded_argument_with_restarg:11267 + test_forwarded_kwrestarg_with_additional_kwarg:11306 + test_forwarded_kwrestarg:11287 + test_forwarded_restarg:11249 + test_hash_pair_value_omission:10364 + test_hash_pair_value_omission:10376 + test_if_while_after_class__since_32:11374 + test_if_while_after_class__since_32:11384 + test_kwoptarg_with_kwrestarg_and_forwarded_args:11482 + test_lvar_injecting_match:3819 + test_newline_in_hash_argument:11427 + test_next_block:5298 + test_next:5263 + test_next:5277 + test_next:5283 + test_next:5290 + test_next:5290 + test_parser_slash_slash_n_escaping_in_literals:* + test_pattern_matching_explicit_array_match:8903 + test_pattern_matching_explicit_array_match:8928 + test_pattern_matching_expr_in_paren:9443 + test_pattern_matching_hash_with_string_keys:* + test_pattern_matching_hash_with_string_keys:9264 + test_pattern_matching_hash:9186 + test_pattern_matching_implicit_array_match:8796 + test_pattern_matching_implicit_array_match:8841 + test_pattern_matching_numbered_parameter:9654 + test_pattern_matching_single_line_allowed_omission_of_parentheses:9868 + test_pattern_matching_single_line_allowed_omission_of_parentheses:9898 + test_redo:5310 + test_retry:5589 + test_send_index_asgn_kwarg_legacy:3642 + test_send_index_asgn_kwarg_legacy:3642 + test_send_index_asgn_kwarg:3629 + test_send_index_asgn_kwarg:3629 + test_slash_newline_in_heredocs:7379 + test_unary_num_pow_precedence:3519 + test_yield:3915 + test_yield:3923 + test_yield:3929 + test_yield:3937 ] - current_version = RUBY_VERSION.split(".")[0..1].join(".") - - if current_version <= "2.7" - # I'm not sure why this is failing on 2.7.0, but we'll turn it off for - # now until we have more time to investigate. - todo_failures.push( + if Gem::Version.new(RUBY_VERSION) < Gem::Version.new("3.1.0") + skips.push( + "test_endless_method_forwarded_args_legacy:10139", + "test_forward_arg_with_open_args:11114", + "test_forward_arg:8090", + "test_forward_args_legacy:8054", + "test_forward_args_legacy:8066", + "test_forward_args_legacy:8078", "test_pattern_matching_hash:*", - "test_pattern_matching_single_line:9552" + "test_pattern_matching_single_line:9839", + "test_trailing_forward_arg:8237" ) end - - if current_version <= "3.0" - # In < 3.0, there are some changes to the way the parser gem handles - # forwarded args. We should eventually support this, but for now we're - # going to mark them as todo. - todo_failures.push( - "test_forward_arg:*", - "test_forward_args_legacy:*", - "test_endless_method_forwarded_args_legacy:*", - "test_trailing_forward_arg:*", - "test_forward_arg_with_open_args:10770", - ) - end - - if current_version == "3.1" - # This test actually fails on 3.1.0, even though it's marked as being - # since 3.1. So we're going to skip this test on 3.1, but leave it in - # for other versions. - known_failures.push( - "test_multiple_pattern_matches:11086", - "test_multiple_pattern_matches:11102" - ) - end - - if current_version < "3.2" || RUBY_ENGINE == "truffleruby" - known_failures.push( - "test_if_while_after_class__since_32:11004", - "test_if_while_after_class__since_32:11014", - "test_newline_in_hash_argument:11057" - ) - end - - all_failures = known_failures + todo_failures File .foreach(File.expand_path("parser.txt", __dir__), chomp: true) .slice_before { |line| line.start_with?("!!!") } .each do |(prefix, *lines)| name = prefix[4..] - next if all_failures.any? { |pattern| File.fnmatch?(pattern, name) } + next if skips.any? { |skip| File.fnmatch?(skip, name) } define_method(name) { assert_parses("#{lines.join("\n")}\n") } end From 4aad240678c4805d98b9cc33064a51ff7d3b51d9 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 10 Jul 2025 17:30:39 -0400 Subject: [PATCH 515/536] Fix up GH pages workflow --- .github/workflows/gh-pages.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 7ff5f5f1..7e4925df 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -5,8 +5,8 @@ on: branches: - main - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages permissions: @@ -39,7 +39,7 @@ jobs: rdoc --main README.md --op _site --exclude={Gemfile,Rakefile,"coverage/*","vendor/*","bin/*","test/*","tmp/*"} cp -r doc _site/doc - name: Upload artifact - uses: actions/upload-pages-artifact@v2 + uses: actions/upload-pages-artifact@v3 # Deployment job deploy: From 384408459970a56130810259ed52efe8790516a8 Mon Sep 17 00:00:00 2001 From: Nic Pillinger Date: Mon, 10 Feb 2025 15:36:30 +0000 Subject: [PATCH 516/536] pass ignore files options to language server --- lib/syntax_tree/cli.rb | 5 ++++- lib/syntax_tree/language_server.rb | 10 ++++++++-- test/language_server_test.rb | 23 +++++++++++++++++++++-- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/lib/syntax_tree/cli.rb b/lib/syntax_tree/cli.rb index e0bafce9..205dba28 100644 --- a/lib/syntax_tree/cli.rb +++ b/lib/syntax_tree/cli.rb @@ -593,7 +593,10 @@ def run(argv) when "j", "json" Json.new(options) when "lsp" - LanguageServer.new(print_width: options.print_width).run + LanguageServer.new( + print_width: options.print_width, + ignore_files: options.ignore_files + ).run return 0 when "m", "match" Match.new(options) diff --git a/lib/syntax_tree/language_server.rb b/lib/syntax_tree/language_server.rb index 6ec81030..aaa64e9a 100644 --- a/lib/syntax_tree/language_server.rb +++ b/lib/syntax_tree/language_server.rb @@ -217,11 +217,13 @@ def self.[](value) def initialize( input: $stdin, output: $stdout, - print_width: DEFAULT_PRINT_WIDTH + print_width: DEFAULT_PRINT_WIDTH, + ignore_files: [] ) @input = input.binmode @output = output.binmode @print_width = print_width + @ignore_files = ignore_files end # rubocop:disable Layout/LineLength @@ -255,8 +257,12 @@ def run store.delete(request.dig(:params, :textDocument, :uri)) when Request[method: "textDocument/formatting", id: :any, params: { textDocument: { uri: :any } }] uri = request.dig(:params, :textDocument, :uri) + filepath = uri.split("///").last + ignore = @ignore_files.any? do |glob| + File.fnmatch(glob, filepath) + end contents = store[uri] - write(id: request[:id], result: contents ? format(contents, uri.split(".").last) : nil) + write(id: request[:id], result: contents && !ignore ? format(contents, uri.split(".").last) : nil) when Request[method: "textDocument/inlayHint", id: :any, params: { textDocument: { uri: :any } }] uri = request.dig(:params, :textDocument, :uri) contents = store[uri] diff --git a/test/language_server_test.rb b/test/language_server_test.rb index f5a6ca57..54455c95 100644 --- a/test/language_server_test.rb +++ b/test/language_server_test.rb @@ -151,6 +151,24 @@ def test_formatting assert_equal("class Bar\nend\n", responses.dig(1, :result, 0, :newText)) end + def test_formatting_ignore + responses = run_server([ + Initialize.new(1), + TextDocumentDidOpen.new("file:///path/to/file.rb", "class Foo; end"), + TextDocumentFormatting.new(2, "file:///path/to/file.rb"), + Shutdown.new(3) + ], ignore_files: ["path/**/*.rb"]) + + shape = LanguageServer::Request[[ + { id: 1, result: { capabilities: Hash } }, + { id: 2, result: :any }, + { id: 3, result: {} } + ]] + + assert_operator(shape, :===, responses) + assert_nil(responses.dig(1, :result)) + end + def test_formatting_failure responses = run_server([ Initialize.new(1), @@ -322,14 +340,15 @@ def read(content) end end - def run_server(messages, print_width: DEFAULT_PRINT_WIDTH) + def run_server(messages, print_width: DEFAULT_PRINT_WIDTH, ignore_files: []) input = StringIO.new(messages.map { |message| write(message) }.join) output = StringIO.new LanguageServer.new( input: input, output: output, - print_width: print_width + print_width: print_width, + ignore_files: ignore_files ).run read(output.tap(&:rewind)) From 76dc51e036aec83d02cd1e04e279489f79ad92a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Szcz=C4=99=C5=9Bniak-Szlagowski?= Date: Fri, 19 Jul 2024 00:40:02 +0900 Subject: [PATCH 517/536] Format non-Ruby STDIN/script content too --- lib/syntax_tree/cli.rb | 31 +++++++++++++++++++++++++------ test/cli_test.rb | 26 ++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 6 deletions(-) diff --git a/lib/syntax_tree/cli.rb b/lib/syntax_tree/cli.rb index 205dba28..0baaef3d 100644 --- a/lib/syntax_tree/cli.rb +++ b/lib/syntax_tree/cli.rb @@ -63,12 +63,13 @@ def writable? class ScriptItem attr_reader :source - def initialize(source) + def initialize(source, extension) @source = source + @extension = extension end def handler - HANDLERS[".rb"] + HANDLERS[@extension] end def filepath @@ -82,8 +83,12 @@ def writable? # An item of work that correspond to the content passed in via stdin. class STDINItem + def initialize(extension) + @extension = extension + end + def handler - HANDLERS[".rb"] + HANDLERS[@extension] end def filepath @@ -457,7 +462,10 @@ def run(item) The maximum line width to use when formatting. -e SCRIPT - Parse an inline Ruby string. + Parse an inline string. + + --extension=EXTENSION + A file extension matching the content passed in via STDIN or -e. Defaults to 'rb' HELP # This represents all of the options that can be passed to the CLI. It is @@ -468,6 +476,7 @@ class Options :plugins, :print_width, :scripts, + :extension, :target_ruby_version def initialize @@ -475,6 +484,7 @@ def initialize @plugins = [] @print_width = DEFAULT_PRINT_WIDTH @scripts = [] + @extension = ".rb" @target_ruby_version = DEFAULT_RUBY_VERSION end @@ -523,6 +533,13 @@ def parser # it and add it to the list of scripts to run. opts.on("-e SCRIPT") { |script| @scripts << script } + # If there is a extension specified, then parse it and use it for + # STDIN and scripts. + opts.on("--extension=EXTENSION") do |extension| + # Both ".rb" and "rb" are going to work + @extension = ".#{extension.delete_prefix(".")}" + end + # If there is a target ruby version specified on the command line, # parse that out and use it when formatting. opts.on("--target-ruby-version=VERSION") do |version| @@ -633,9 +650,11 @@ def run(argv) end end - options.scripts.each { |script| queue << ScriptItem.new(script) } + options.scripts.each do |script| + queue << ScriptItem.new(script, options.extension) + end else - queue << STDINItem.new + queue << STDINItem.new(options.extension) end # At the end, we're going to return whether or not this worker ever diff --git a/test/cli_test.rb b/test/cli_test.rb index 20f7e0ce..200cd8d7 100644 --- a/test/cli_test.rb +++ b/test/cli_test.rb @@ -10,6 +10,10 @@ def parse(source) source * 2 end + def format(source, _print_width, **) + "Formatted #{source}" + end + def read(filepath) File.read(filepath) end @@ -202,6 +206,28 @@ def test_multiple_inline_scripts assert_equal(["1 + 1", "2 + 2"], stdio.split("\n").sort) end + def test_format_script_with_custom_handler + SyntaxTree.register_handler(".test", TestHandler.new) + stdio, = + capture_io do + SyntaxTree::CLI.run(%w[format --extension=test -e ]) + end + assert_equal("Formatted \n", stdio) + ensure + SyntaxTree::HANDLERS.delete(".test") + end + + def test_format_stdin_with_custom_handler + SyntaxTree.register_handler(".test", TestHandler.new) + stdin = $stdin + $stdin = StringIO.new("") + stdio, = capture_io { SyntaxTree::CLI.run(%w[format --extension=test]) } + assert_equal("Formatted \n", stdio) + ensure + $stdin = stdin + SyntaxTree::HANDLERS.delete(".test") + end + def test_generic_error SyntaxTree.stub(:format, ->(*) { raise }) do result = run_cli("format") From c86b6926ab7bbf47740f36abe62a9698dcc96a5e Mon Sep 17 00:00:00 2001 From: Bradley Buda Date: Thu, 4 Apr 2024 21:18:07 -0700 Subject: [PATCH 518/536] Assoc uses Identity formatter if any value is nil The existing assoc formatter has logic to identify the case where a value is nil (in a Ruby-3.1 style hash) and preserve the existing formatting. For example: `{ first:, "second" => "value" }` is correctly left as-is. However, this logic only worked if the first assoc in the container had the nil value - if a later assoc had a nil value, the Identity formatter might not be chosen which could cause the formatter to generate invalid Ruby code. As an example, this code: `{ "first" => "value", second: }` would be turned into `{ "first" => "value", :second => }`. This patch pulls the nil value check up to the top of `HashKeyFormatter.for` to ensure it takes precendence over any other formatting selections. The fixtures have been updated to cover both cases (nil value in first position, nil value in last position). Fixes #446 --- lib/syntax_tree/node.rb | 83 ++++++++++++++++++++++++----------------- test/fixtures/assoc.rb | 4 ++ 2 files changed, 53 insertions(+), 34 deletions(-) diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 5a92a5a7..4d148f35 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -1783,45 +1783,60 @@ def format_key(q, key) end end - def self.for(container) - container.assocs.each do |assoc| - if assoc.is_a?(AssocSplat) - # Splat nodes do not impact the formatting choice. - elsif assoc.value.nil? - # If the value is nil, then it has been omitted. In this case we have - # to match the existing formatting because standardizing would - # potentially break the code. For example: - # - # { first:, "second" => "value" } - # - return Identity.new - else - # Otherwise, we need to check the type of the key. If it's a label or - # dynamic symbol, we can use labels. If it's a symbol literal then it - # needs to match a certain pattern to be used as a label. If it's - # anything else, then we need to use hash rockets. - case assoc.key - when Label, DynaSymbol - # Here labels can be used. - when SymbolLiteral - # When attempting to convert a hash rocket into a hash label, - # you need to take care because only certain patterns are - # allowed. Ruby source says that they have to match keyword - # arguments to methods, but don't specify what that is. After - # some experimentation, it looks like it's: - value = assoc.key.value.value - - if !value.match?(/^[_A-Za-z]/) || value.end_with?("=") - return Rockets.new - end + class << self + def for(container) + (assocs = container.assocs).each_with_index do |assoc, index| + if assoc.is_a?(AssocSplat) + # Splat nodes do not impact the formatting choice. + elsif assoc.value.nil? + # If the value is nil, then it has been omitted. In this case we + # have to match the existing formatting because standardizing would + # potentially break the code. For example: + # + # { first:, "second" => "value" } + # + return Identity.new else - # If the value is anything else, we have to use hash rockets. - return Rockets.new + # Otherwise, we need to check the type of the key. If it's a label + # or dynamic symbol, we can use labels. If it's a symbol literal + # then it needs to match a certain pattern to be used as a label. If + # it's anything else, then we need to use hash rockets. + case assoc.key + when Label, DynaSymbol + # Here labels can be used. + when SymbolLiteral + # When attempting to convert a hash rocket into a hash label, + # you need to take care because only certain patterns are + # allowed. Ruby source says that they have to match keyword + # arguments to methods, but don't specify what that is. After + # some experimentation, it looks like it's: + value = assoc.key.value.value + + if !value.match?(/^[_A-Za-z]/) || value.end_with?("=") + if omitted_value?(assocs[(index + 1)..]) + return Identity.new + else + return Rockets.new + end + end + else + if omitted_value?(assocs[(index + 1)..]) + return Identity.new + else + return Rockets.new + end + end end end + + Labels.new end - Labels.new + private + + def omitted_value?(assocs) + assocs.any? { |assoc| !assoc.is_a?(AssocSplat) && assoc.value.nil? } + end end end diff --git a/test/fixtures/assoc.rb b/test/fixtures/assoc.rb index 0fc60e6f..83a4887a 100644 --- a/test/fixtures/assoc.rb +++ b/test/fixtures/assoc.rb @@ -48,3 +48,7 @@ { "foo #{bar}": "baz" } % { "foo=": "baz" } +% # >= 3.1.0 +{ bar => 1, baz: } +% # >= 3.1.0 +{ baz:, bar => 1 } From e1ef9ac3531312b80fb93c290753570b3066504e Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 11 Jul 2025 11:24:03 -0400 Subject: [PATCH 519/536] Track compile errors Fixes #481 --- lib/syntax_tree/parser.rb | 1 + test/parser_test.rb | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index 326d3ec7..ace077ee 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -2871,6 +2871,7 @@ def on_parse_error(error, *) alias on_assign_error on_parse_error alias on_class_name_error on_parse_error alias on_param_error on_parse_error + alias compile_error on_parse_error # :call-seq: # on_period: (String value) -> Period diff --git a/test/parser_test.rb b/test/parser_test.rb index 7ac07381..169d5b46 100644 --- a/test/parser_test.rb +++ b/test/parser_test.rb @@ -33,7 +33,7 @@ def test_parses_ripper_methods def test_errors_on_missing_token_with_location error = assert_raises(Parser::ParseError) { SyntaxTree.parse("f+\"foo") } - assert_equal(2, error.column) + assert_equal(3, error.column) end def test_errors_on_missing_end_with_location @@ -45,7 +45,7 @@ def test_errors_on_missing_regexp_ending error = assert_raises(Parser::ParseError) { SyntaxTree.parse("a =~ /foo") } - assert_equal(5, error.column) + assert_equal(6, error.column) end def test_errors_on_missing_token_without_location From 5fbd4171d73c9d258882374e97c135563f22da69 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 15 Jul 2025 10:59:58 -0400 Subject: [PATCH 520/536] Add config option to CLI Fixes #478 --- lib/syntax_tree/cli.rb | 49 +++++++++++++++++++++++++++++++++++------- test/cli_test.rb | 46 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 85 insertions(+), 10 deletions(-) diff --git a/lib/syntax_tree/cli.rb b/lib/syntax_tree/cli.rb index 0baaef3d..e3bac8f1 100644 --- a/lib/syntax_tree/cli.rb +++ b/lib/syntax_tree/cli.rb @@ -455,17 +455,26 @@ def run(item) #{Color.bold("stree write [--plugins=...] [--print-width=NUMBER] [-e SCRIPT] FILE")} Read, format, and write back the source of the given files + --ignore-files=... + A glob pattern to ignore files when processing. This can be specified + multiple times to ignore multiple patterns. + --plugins=... A comma-separated list of plugins to load. - --print-width=NUMBER + --print-width=... The maximum line width to use when formatting. - -e SCRIPT + -e ... Parse an inline string. - --extension=EXTENSION - A file extension matching the content passed in via STDIN or -e. Defaults to 'rb' + --extension=... + A file extension matching the content passed in via STDIN or -e. + Defaults to '.rb'. + + --config=... + Path to a configuration file. Defaults to .streerc in the current + working directory. HELP # This represents all of the options that can be passed to the CLI. It is @@ -563,8 +572,16 @@ class ConfigFile attr_reader :filepath - def initialize - @filepath = File.join(Dir.pwd, FILENAME) + def initialize(filepath = nil) + if filepath + if File.readable?(filepath) + @filepath = filepath + else + raise ArgumentError, "Invalid configuration file: #{filepath}" + end + else + @filepath = File.join(Dir.pwd, FILENAME) + end end def exists? @@ -582,8 +599,24 @@ class << self def run(argv) name, *arguments = argv - config_file = ConfigFile.new - arguments.unshift(*config_file.arguments) + # First, we need to check if there's a --config option specified + # so we can use the custom config file path. + config_filepath = nil + arguments.each_with_index do |arg, index| + if arg.start_with?("--config=") + config_filepath = arg.split("=", 2)[1] + arguments.delete_at(index) + break + elsif arg == "--config" && arguments[index + 1] + config_filepath = arguments[index + 1] + arguments.delete_at(index + 1) + arguments.delete_at(index) + break + end + end + + config_file = ConfigFile.new(config_filepath) + arguments = config_file.arguments.concat(arguments) options = Options.new options.parse(arguments) diff --git a/test/cli_test.rb b/test/cli_test.rb index 200cd8d7..a0d6001d 100644 --- a/test/cli_test.rb +++ b/test/cli_test.rb @@ -308,6 +308,48 @@ def test_plugin_args_with_config_file end end + def test_config_file_custom_path + with_plugin_directory do |directory| + plugin = directory.plugin("puts 'Custom config!'") + config = <<~TXT + --print-width=80 + --plugins=#{plugin} + TXT + + filepath = File.join(Dir.tmpdir, "#{SecureRandom.hex}.streerc") + with_config_file(config, filepath) do + contents = "#{"a" * 30} + #{"b" * 30}\n" + result = run_cli("format", "--config=#{filepath}", contents: contents) + + assert_equal("Custom config!\n#{contents}", result.stdio) + end + end + end + + def test_config_file_custom_path_space_separated + with_plugin_directory do |directory| + plugin = directory.plugin("puts 'Custom config space!'") + config = <<~TXT + --print-width=80 + --plugins=#{plugin} + TXT + + filepath = File.join(Dir.tmpdir, "#{SecureRandom.hex}.streerc") + with_config_file(config, filepath) do + contents = "#{"a" * 30} + #{"b" * 30}\n" + result = run_cli("format", "--config", filepath, contents: contents) + + assert_equal("Custom config space!\n#{contents}", result.stdio) + end + end + end + + def test_config_file_nonexistent_path + assert_raises(ArgumentError) do + run_cli("format", "--config=/nonexistent/path.streerc") + end + end + Result = Struct.new(:status, :stdio, :stderr, keyword_init: true) private @@ -342,8 +384,8 @@ def run_cli(command, *args, contents: :default) tempfile.unlink end - def with_config_file(contents) - filepath = File.join(Dir.pwd, SyntaxTree::CLI::ConfigFile::FILENAME) + def with_config_file(contents, filepath = nil) + filepath ||= File.join(Dir.pwd, SyntaxTree::CLI::ConfigFile::FILENAME) File.write(filepath, contents) yield From 4c9fca1d28880beee1d06ad26360fbba77e24ffe Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 16 Jul 2025 20:37:31 -0400 Subject: [PATCH 521/536] Bump to version 6.3.0 --- CHANGELOG.md | 14 ++++++++++++++ Gemfile.lock | 2 +- lib/syntax_tree/version.rb | 2 +- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1beac42f..4ad42fc8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,20 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +## [6.3.0] - 2025-07-16 + +### Added + +- The `--extension` command line option has been added to the CLI to specify what type of content is coming from stdin. +- The `--config` command line option has been added to the CLI to specify the path to the configuration file. + +### Changed + +- Fix formatting of character literals when single quotes is enabled. +- Pass ignore files option to the language server. +- Hash keys should remain unchanged when there are any omitted values in the hash. +- We now properly handle compilation errors in the parser. + ## [6.2.0] - 2023-09-20 ### Added diff --git a/Gemfile.lock b/Gemfile.lock index b855c712..9b8cbe16 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - syntax_tree (6.2.0) + syntax_tree (6.3.0) prettier_print (>= 1.2.0) GEM diff --git a/lib/syntax_tree/version.rb b/lib/syntax_tree/version.rb index 51599f77..9e80fa7b 100644 --- a/lib/syntax_tree/version.rb +++ b/lib/syntax_tree/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SyntaxTree - VERSION = "6.2.0" + VERSION = "6.3.0" end From e39427d18aac8a749fb8370598a61ecd6b337ec9 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 17 Jul 2025 09:59:26 -0400 Subject: [PATCH 522/536] Remove the translation layer The translation layer is now removed in favor of using the translation layer built into prism. --- bin/whitequark | 79 - doc/changing_structure.md | 16 - lib/syntax_tree.rb | 1 - lib/syntax_tree/translation.rb | 28 - lib/syntax_tree/translation/parser.rb | 3107 -------------------- lib/syntax_tree/translation/rubocop_ast.rb | 21 - tasks/whitequark.rake | 92 - test/translation/parser.txt | 1852 ------------ test/translation/parser_test.rb | 141 - 9 files changed, 5337 deletions(-) delete mode 100755 bin/whitequark delete mode 100644 doc/changing_structure.md delete mode 100644 lib/syntax_tree/translation.rb delete mode 100644 lib/syntax_tree/translation/parser.rb delete mode 100644 lib/syntax_tree/translation/rubocop_ast.rb delete mode 100644 tasks/whitequark.rake delete mode 100644 test/translation/parser.txt delete mode 100644 test/translation/parser_test.rb diff --git a/bin/whitequark b/bin/whitequark deleted file mode 100755 index 121bcd53..00000000 --- a/bin/whitequark +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -require "bundler/setup" -require "parser/current" - -$:.unshift(File.expand_path("../lib", __dir__)) -require "syntax_tree" - -# First, opt in to every AST feature. -Parser::Builders::Default.modernize - -# Modify the source map == check so that it doesn't check against the node -# itself so we don't get into a recursive loop. -Parser::Source::Map.prepend( - Module.new { - def ==(other) - self.class == other.class && - (instance_variables - %i[@node]).map do |ivar| - instance_variable_get(ivar) == other.instance_variable_get(ivar) - end.reduce(:&) - end - } -) - -# Next, ensure that we're comparing the nodes and also comparing the source -# ranges so that we're getting all of the necessary information. -Parser::AST::Node.prepend( - Module.new { - def ==(other) - super && (location == other.location) - end - } -) - -source = ARGF.read - -parser = Parser::CurrentRuby.new -parser.diagnostics.all_errors_are_fatal = true - -buffer = Parser::Source::Buffer.new("(string)", 1) -buffer.source = source.dup.force_encoding(parser.default_encoding) - -stree = SyntaxTree::Translation.to_parser(SyntaxTree.parse(source), buffer) -ptree = parser.parse(buffer) - -if stree == ptree - puts "Syntax trees are equivalent." -elsif stree.inspect == ptree.inspect - warn "Syntax tree locations are different." - - queue = [[stree, ptree]] - while (left, right = queue.shift) - if left.location != right.location - warn "Different node:" - pp left - - warn "Different location:" - - warn "Syntax Tree:" - pp left.location - - warn "whitequark/parser:" - pp right.location - - exit - end - - left.children.zip(right.children).each do |left_child, right_child| - queue << [left_child, right_child] if left_child.is_a?(Parser::AST::Node) - end - end -else - warn "Syntax Tree:" - pp stree - - warn "whitequark/parser:" - pp ptree -end diff --git a/doc/changing_structure.md b/doc/changing_structure.md deleted file mode 100644 index 74012f26..00000000 --- a/doc/changing_structure.md +++ /dev/null @@ -1,16 +0,0 @@ -# Changing structure - -First and foremost, changing the structure of the tree in any way is a major breaking change. It forces the consumers to update their visitors, pattern matches, and method calls. It should not be taking lightly, and can only happen on a major version change. So keep that in mind. - -That said, if you do want to change the structure of the tree, there are a few steps that you have to take. They are enumerated below. - -1. Change the structure in the required node classes. This could mean adding/removing classes or adding/removing fields. Be sure to also update the `copy` and `===` methods to be sure that they are correct. -2. Update the parser to correctly create the new structure. -3. Update any visitor methods that are affected by the change. For example, if adding a new node make sure to create the new visit method alias in the `Visitor` class. -4. Update the `FieldVisitor` class to be sure that the various serializers, pretty printers, and matchers all get updated accordingly. -5. Update the `DSL` module to be sure that folks can correctly create nodes with the new structure. -6. Ensure the formatting of the code hasn't changed. This can mostly be done by running the tests, but if there's a corner case that we don't cover that is now exposed by your change be sure to add test cases. -7. Update the translation visitors to ensure we're still translating into other ASTs correctly. -8. Update the YARV compiler visitor to ensure we're still compiling correctly. -9. Make sure we aren't referencing the previous structure in any documentation or tests. -10. Be sure to update `CHANGELOG.md` with a description of the change that you made. diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index 6c595db5..2c824f71 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -34,7 +34,6 @@ module SyntaxTree autoload :Pattern, "syntax_tree/pattern" autoload :PrettyPrintVisitor, "syntax_tree/pretty_print_visitor" autoload :Search, "syntax_tree/search" - autoload :Translation, "syntax_tree/translation" autoload :WithScope, "syntax_tree/with_scope" autoload :YARV, "syntax_tree/yarv" diff --git a/lib/syntax_tree/translation.rb b/lib/syntax_tree/translation.rb deleted file mode 100644 index 6fc96f00..00000000 --- a/lib/syntax_tree/translation.rb +++ /dev/null @@ -1,28 +0,0 @@ -# frozen_string_literal: true - -module SyntaxTree - # This module is responsible for translating the Syntax Tree syntax tree into - # other representations. - module Translation - # This method translates the given node into the representation defined by - # the whitequark/parser gem. We don't explicitly list it as a dependency - # because it's not required for the core functionality of Syntax Tree. - def self.to_parser(node, buffer) - require "parser" - require_relative "translation/parser" - - node.accept(Parser.new(buffer)) - end - - # This method translates the given node into the representation defined by - # the rubocop/rubocop-ast gem. We don't explicitly list it as a dependency - # because it's not required for the core functionality of Syntax Tree. - def self.to_rubocop_ast(node, buffer) - require "rubocop/ast" - require_relative "translation/parser" - require_relative "translation/rubocop_ast" - - node.accept(RuboCopAST.new(buffer)) - end - end -end diff --git a/lib/syntax_tree/translation/parser.rb b/lib/syntax_tree/translation/parser.rb deleted file mode 100644 index 8be4fc79..00000000 --- a/lib/syntax_tree/translation/parser.rb +++ /dev/null @@ -1,3107 +0,0 @@ -# frozen_string_literal: true - -module SyntaxTree - module Translation - # This visitor is responsible for converting the syntax tree produced by - # Syntax Tree into the syntax tree produced by the whitequark/parser gem. - class Parser < BasicVisitor - # Heredocs are represented _very_ differently in the parser gem from how - # they are represented in the Syntax Tree AST. This class is responsible - # for handling the translation. - class HeredocBuilder - Line = Struct.new(:value, :segments) - - attr_reader :node, :segments - - def initialize(node) - @node = node - @segments = [] - end - - def <<(segment) - if segment.type == :str && segments.last && - segments.last.type == :str && - !segments.last.children.first.end_with?("\n") - segments.last.children.first << segment.children.first - else - segments << segment - end - end - - def trim! - return unless node.beginning.value[2] == "~" - lines = [Line.new(+"", [])] - - segments.each do |segment| - lines.last.segments << segment - - if segment.type == :str - lines.last.value << segment.children.first - lines << Line.new(+"", []) if lines.last.value.end_with?("\n") - end - end - - lines.pop if lines.last.value.empty? - return if lines.empty? - - segments.clear - lines.each do |line| - remaining = node.dedent - - line.segments.each do |segment| - if segment.type == :str - if remaining > 0 - whitespace = segment.children.first[/^\s{0,#{remaining}}/] - segment.children.first.sub!(/^#{whitespace}/, "") - remaining -= whitespace.length - end - - if node.beginning.value[3] != "'" && segments.any? && - segments.last.type == :str && - segments.last.children.first.end_with?("\\\n") - segments.last.children.first.gsub!(/\\\n\z/, "") - segments.last.children.first.concat(segment.children.first) - elsif !segment.children.first.empty? - segments << segment - end - else - segments << segment - end - end - end - end - end - - attr_reader :buffer, :stack - - def initialize(buffer) - @buffer = buffer - @stack = [] - end - - # For each node that we visit, we keep track of it in a stack as we - # descend into its children. We do this so that child nodes can reflect on - # their parents if they need additional information about their context. - def visit(node) - stack << node - result = super - stack.pop - result - end - - visit_methods do - # Visit an AliasNode node. - def visit_alias(node) - s( - :alias, - [visit(node.left), visit(node.right)], - smap_keyword_bare( - srange_length(node.start_char, 5), - srange_node(node) - ) - ) - end - - # Visit an ARefNode. - def visit_aref(node) - if ::Parser::Builders::Default.emit_index - if node.index.nil? - s( - :index, - [visit(node.collection)], - smap_index( - srange_find(node.collection.end_char, node.end_char, "["), - srange_length(node.end_char, -1), - srange_node(node) - ) - ) - else - s( - :index, - [visit(node.collection)].concat(visit_all(node.index.parts)), - smap_index( - srange_find_between(node.collection, node.index, "["), - srange_length(node.end_char, -1), - srange_node(node) - ) - ) - end - else - if node.index.nil? - s( - :send, - [visit(node.collection), :[]], - smap_send_bare( - srange_find(node.collection.end_char, node.end_char, "[]"), - srange_node(node) - ) - ) - else - s( - :send, - [visit(node.collection), :[], *visit_all(node.index.parts)], - smap_send_bare( - srange( - srange_find_between( - node.collection, - node.index, - "[" - ).begin_pos, - node.end_char - ), - srange_node(node) - ) - ) - end - end - end - - # Visit an ARefField node. - def visit_aref_field(node) - if ::Parser::Builders::Default.emit_index - if node.index.nil? - s( - :indexasgn, - [visit(node.collection)], - smap_index( - srange_find(node.collection.end_char, node.end_char, "["), - srange_length(node.end_char, -1), - srange_node(node) - ) - ) - else - s( - :indexasgn, - [visit(node.collection)].concat(visit_all(node.index.parts)), - smap_index( - srange_find_between(node.collection, node.index, "["), - srange_length(node.end_char, -1), - srange_node(node) - ) - ) - end - else - if node.index.nil? - s( - :send, - [visit(node.collection), :[]=], - smap_send_bare( - srange_find(node.collection.end_char, node.end_char, "[]"), - srange_node(node) - ) - ) - else - s( - :send, - [visit(node.collection), :[]=].concat( - visit_all(node.index.parts) - ), - smap_send_bare( - srange( - srange_find_between( - node.collection, - node.index, - "[" - ).begin_pos, - node.end_char - ), - srange_node(node) - ) - ) - end - end - end - - # Visit an ArgBlock node. - def visit_arg_block(node) - s( - :block_pass, - [visit(node.value)], - smap_operator(srange_length(node.start_char, 1), srange_node(node)) - ) - end - - # Visit an ArgStar node. - def visit_arg_star(node) - if stack[-3].is_a?(MLHSParen) && stack[-3].contents.is_a?(MLHS) - if node.value.nil? - s(:restarg, [], smap_variable(nil, srange_node(node))) - else - s( - :restarg, - [node.value.value.to_sym], - smap_variable(srange_node(node.value), srange_node(node)) - ) - end - else - s( - :splat, - node.value.nil? ? [] : [visit(node.value)], - smap_operator( - srange_length(node.start_char, 1), - srange_node(node) - ) - ) - end - end - - # Visit an ArgsForward node. - def visit_args_forward(node) - s(:forwarded_args, [], smap(srange_node(node))) - end - - # Visit an ArrayLiteral node. - def visit_array(node) - s( - :array, - node.contents ? visit_all(node.contents.parts) : [], - if node.lbracket.nil? - smap_collection_bare(srange_node(node)) - else - smap_collection( - srange_node(node.lbracket), - srange_length(node.end_char, -1), - srange_node(node) - ) - end - ) - end - - # Visit an AryPtn node. - def visit_aryptn(node) - type = :array_pattern - children = visit_all(node.requireds) - - if node.rest.is_a?(VarField) - if !node.rest.value.nil? - children << s(:match_rest, [visit(node.rest)], nil) - elsif node.posts.empty? && - node.rest.start_char == node.rest.end_char - # Here we have an implicit rest, as in [foo,]. parser has a - # specific type for these patterns. - type = :array_pattern_with_tail - else - children << s(:match_rest, [], nil) - end - end - - if node.constant - s( - :const_pattern, - [ - visit(node.constant), - s( - type, - children + visit_all(node.posts), - smap_collection_bare( - srange(node.constant.end_char + 1, node.end_char - 1) - ) - ) - ], - smap_collection( - srange_length(node.constant.end_char, 1), - srange_length(node.end_char, -1), - srange_node(node) - ) - ) - else - s( - type, - children + visit_all(node.posts), - if buffer.source[node.start_char] == "[" - smap_collection( - srange_length(node.start_char, 1), - srange_length(node.end_char, -1), - srange_node(node) - ) - else - smap_collection_bare(srange_node(node)) - end - ) - end - end - - # Visit an Assign node. - def visit_assign(node) - target = visit(node.target) - location = - target - .location - .with_operator(srange_find_between(node.target, node.value, "=")) - .with_expression(srange_node(node)) - - s(target.type, target.children + [visit(node.value)], location) - end - - # Visit an Assoc node. - def visit_assoc(node) - if node.value.nil? - # { foo: } - expression = srange(node.start_char, node.end_char - 1) - type, location = - if node.key.value.start_with?(/[A-Z]/) - [:const, smap_constant(nil, expression, expression)] - else - [:send, smap_send_bare(expression, expression)] - end - - s( - :pair, - [ - visit(node.key), - s(type, [nil, node.key.value.chomp(":").to_sym], location) - ], - smap_operator( - srange_length(node.key.end_char, -1), - srange_node(node) - ) - ) - elsif node.key.is_a?(Label) - # { foo: 1 } - s( - :pair, - [visit(node.key), visit(node.value)], - smap_operator( - srange_length(node.key.end_char, -1), - srange_node(node) - ) - ) - elsif (operator = srange_search_between(node.key, node.value, "=>")) - # { :foo => 1 } - s( - :pair, - [visit(node.key), visit(node.value)], - smap_operator(operator, srange_node(node)) - ) - else - # { "foo": 1 } - key = visit(node.key) - key_location = - smap_collection( - key.location.begin, - srange_length(node.key.end_char - 2, 1), - srange(node.key.start_char, node.key.end_char - 1) - ) - - s( - :pair, - [s(key.type, key.children, key_location), visit(node.value)], - smap_operator( - srange_length(node.key.end_char, -1), - srange_node(node) - ) - ) - end - end - - # Visit an AssocSplat node. - def visit_assoc_splat(node) - s( - :kwsplat, - [visit(node.value)], - smap_operator(srange_length(node.start_char, 2), srange_node(node)) - ) - end - - # Visit a Backref node. - def visit_backref(node) - location = smap(srange_node(node)) - - if node.value.match?(/^\$\d+$/) - s(:nth_ref, [node.value[1..].to_i], location) - else - s(:back_ref, [node.value.to_sym], location) - end - end - - # Visit a BareAssocHash node. - def visit_bare_assoc_hash(node) - s( - if ::Parser::Builders::Default.emit_kwargs && - !stack[-2].is_a?(ArrayLiteral) - :kwargs - else - :hash - end, - visit_all(node.assocs), - smap_collection_bare(srange_node(node)) - ) - end - - # Visit a BEGINBlock node. - def visit_BEGIN(node) - s( - :preexe, - [visit(node.statements)], - smap_keyword( - srange_length(node.start_char, 5), - srange_find(node.start_char + 5, node.statements.start_char, "{"), - srange_length(node.end_char, -1), - srange_node(node) - ) - ) - end - - # Visit a Begin node. - def visit_begin(node) - location = - smap_collection( - srange_length(node.start_char, 5), - srange_length(node.end_char, -3), - srange_node(node) - ) - - if node.bodystmt.empty? - s(:kwbegin, [], location) - elsif node.bodystmt.rescue_clause.nil? && - node.bodystmt.ensure_clause.nil? && - node.bodystmt.else_clause.nil? - child = visit(node.bodystmt.statements) - - s( - :kwbegin, - child.type == :begin ? child.children : [child], - location - ) - else - s(:kwbegin, [visit(node.bodystmt)], location) - end - end - - # Visit a Binary node. - def visit_binary(node) - case node.operator - when :| - current = -2 - while stack[current].is_a?(Binary) && stack[current].operator == :| - current -= 1 - end - - if stack[current].is_a?(In) - s(:match_alt, [visit(node.left), visit(node.right)], nil) - else - visit(canonical_binary(node)) - end - when :"=>", :"&&", :and, :"||", :or - s( - { "=>": :match_as, "&&": :and, "||": :or }.fetch( - node.operator, - node.operator - ), - [visit(node.left), visit(node.right)], - smap_operator( - srange_find_between(node.left, node.right, node.operator.to_s), - srange_node(node) - ) - ) - when :=~ - # When you use a regular expression on the left hand side of a =~ - # operator and it doesn't have interpolatoin, then its named capture - # groups introduce local variables into the scope. In this case the - # parser gem has a different node (match_with_lvasgn) instead of the - # regular send. - if node.left.is_a?(RegexpLiteral) && node.left.parts.length == 1 && - node.left.parts.first.is_a?(TStringContent) - s( - :match_with_lvasgn, - [visit(node.left), visit(node.right)], - smap_operator( - srange_find_between( - node.left, - node.right, - node.operator.to_s - ), - srange_node(node) - ) - ) - else - visit(canonical_binary(node)) - end - else - visit(canonical_binary(node)) - end - end - - # Visit a BlockArg node. - def visit_blockarg(node) - if node.name.nil? - s(:blockarg, [nil], smap_variable(nil, srange_node(node))) - else - s( - :blockarg, - [node.name.value.to_sym], - smap_variable(srange_node(node.name), srange_node(node)) - ) - end - end - - # Visit a BlockVar node. - def visit_block_var(node) - shadowargs = - node.locals.map do |local| - s( - :shadowarg, - [local.value.to_sym], - smap_variable(srange_node(local), srange_node(local)) - ) - end - - params = node.params - children = - if ::Parser::Builders::Default.emit_procarg0 && node.arg0? - # There is a special node type in the parser gem for when a single - # required parameter to a block would potentially be expanded - # automatically. We handle that case here. - required = params.requireds.first - procarg0 = - if ::Parser::Builders::Default.emit_arg_inside_procarg0 && - required.is_a?(Ident) - s( - :procarg0, - [ - s( - :arg, - [required.value.to_sym], - smap_variable( - srange_node(required), - srange_node(required) - ) - ) - ], - smap_collection_bare(srange_node(required)) - ) - else - child = visit(required) - s(:procarg0, child, child.location) - end - - [procarg0] - else - visit(params).children - end - - s( - :args, - children + shadowargs, - smap_collection( - srange_length(node.start_char, 1), - srange_length(node.end_char, -1), - srange_node(node) - ) - ) - end - - # Visit a BodyStmt node. - def visit_bodystmt(node) - result = visit(node.statements) - - if node.rescue_clause - rescue_node = visit(node.rescue_clause) - - children = [result] + rescue_node.children - location = rescue_node.location - - if node.else_clause - children.pop - children << visit(node.else_clause) - - location = - smap_condition( - nil, - nil, - srange_length(node.else_clause.start_char - 3, -4), - nil, - srange( - location.expression.begin_pos, - node.else_clause.end_char - ) - ) - end - - result = s(rescue_node.type, children, location) - end - - if node.ensure_clause - ensure_node = visit(node.ensure_clause) - - expression = - ( - if result - result.location.expression.join( - ensure_node.location.expression - ) - else - ensure_node.location.expression - end - ) - location = ensure_node.location.with_expression(expression) - - result = - s(ensure_node.type, [result] + ensure_node.children, location) - end - - result - end - - # Visit a Break node. - def visit_break(node) - s( - :break, - visit_all(node.arguments.parts), - smap_keyword_bare( - srange_length(node.start_char, 5), - srange_node(node) - ) - ) - end - - # Visit a CallNode node. - def visit_call(node) - visit_command_call( - CommandCall.new( - receiver: node.receiver, - operator: node.operator, - message: node.message, - arguments: node.arguments, - block: nil, - location: node.location - ) - ) - end - - # Visit a Case node. - def visit_case(node) - clauses = [node.consequent] - while clauses.last && !clauses.last.is_a?(Else) - clauses << clauses.last.consequent - end - - else_token = - if clauses.last.is_a?(Else) - srange_length(clauses.last.start_char, 4) - end - - s( - node.consequent.is_a?(In) ? :case_match : :case, - [visit(node.value)] + clauses.map { |clause| visit(clause) }, - smap_condition( - srange_length(node.start_char, 4), - nil, - else_token, - srange_length(node.end_char, -3), - srange_node(node) - ) - ) - end - - # Visit a CHAR node. - def visit_CHAR(node) - s( - :str, - [node.value[1..]], - smap_collection( - srange_length(node.start_char, 1), - nil, - srange_node(node) - ) - ) - end - - # Visit a ClassDeclaration node. - def visit_class(node) - operator = - if node.superclass - srange_find_between(node.constant, node.superclass, "<") - end - - s( - :class, - [ - visit(node.constant), - visit(node.superclass), - visit(node.bodystmt) - ], - smap_definition( - srange_length(node.start_char, 5), - operator, - srange_node(node.constant), - srange_length(node.end_char, -3) - ).with_expression(srange_node(node)) - ) - end - - # Visit a Command node. - def visit_command(node) - visit_command_call( - CommandCall.new( - receiver: nil, - operator: nil, - message: node.message, - arguments: node.arguments, - block: node.block, - location: node.location - ) - ) - end - - # Visit a CommandCall node. - def visit_command_call(node) - children = [ - visit(node.receiver), - node.message == :call ? :call : node.message.value.to_sym - ] - - begin_token = nil - end_token = nil - - case node.arguments - when Args - children += visit_all(node.arguments.parts) - when ArgParen - case node.arguments.arguments - when nil - # skip - when ArgsForward - children << visit(node.arguments.arguments) - else - children += visit_all(node.arguments.arguments.parts) - end - - begin_token = srange_length(node.arguments.start_char, 1) - end_token = srange_length(node.arguments.end_char, -1) - end - - dot_bound = - if node.arguments - node.arguments.start_char - elsif node.block - node.block.start_char - else - node.end_char - end - - expression = - if node.arguments.is_a?(ArgParen) - srange(node.start_char, node.arguments.end_char) - elsif node.arguments.is_a?(Args) && node.arguments.parts.any? - last_part = node.arguments.parts.last - end_char = - if last_part.is_a?(Heredoc) - last_part.beginning.end_char - else - last_part.end_char - end - - srange(node.start_char, end_char) - elsif node.block - if node.receiver - srange(node.receiver.start_char, node.message.end_char) - else - srange_node(node.message) - end - else - srange_node(node) - end - - call = - s( - if node.operator.is_a?(Op) && node.operator.value == "&." - :csend - else - :send - end, - children, - smap_send( - if node.operator == :"::" - srange_find( - node.receiver.end_char, - if node.message == :call - dot_bound - else - node.message.start_char - end, - "::" - ) - elsif node.operator - srange_node(node.operator) - end, - node.message == :call ? nil : srange_node(node.message), - begin_token, - end_token, - expression - ) - ) - - if node.block - type, arguments = block_children(node.block) - - s( - type, - [call, arguments, visit(node.block.bodystmt)], - smap_collection( - srange_node(node.block.opening), - srange_length( - node.end_char, - node.block.opening.is_a?(Kw) ? -3 : -1 - ), - srange_node(node) - ) - ) - else - call - end - end - - # Visit a Const node. - def visit_const(node) - s( - :const, - [nil, node.value.to_sym], - smap_constant(nil, srange_node(node), srange_node(node)) - ) - end - - # Visit a ConstPathField node. - def visit_const_path_field(node) - if node.parent.is_a?(VarRef) && node.parent.value.is_a?(Kw) && - node.parent.value.value == "self" && node.constant.is_a?(Ident) - s(:send, [visit(node.parent), :"#{node.constant.value}="], nil) - else - s( - :casgn, - [visit(node.parent), node.constant.value.to_sym], - smap_constant( - srange_find_between(node.parent, node.constant, "::"), - srange_node(node.constant), - srange_node(node) - ) - ) - end - end - - # Visit a ConstPathRef node. - def visit_const_path_ref(node) - s( - :const, - [visit(node.parent), node.constant.value.to_sym], - smap_constant( - srange_find_between(node.parent, node.constant, "::"), - srange_node(node.constant), - srange_node(node) - ) - ) - end - - # Visit a ConstRef node. - def visit_const_ref(node) - s( - :const, - [nil, node.constant.value.to_sym], - smap_constant(nil, srange_node(node.constant), srange_node(node)) - ) - end - - # Visit a CVar node. - def visit_cvar(node) - s( - :cvar, - [node.value.to_sym], - smap_variable(srange_node(node), srange_node(node)) - ) - end - - # Visit a DefNode node. - def visit_def(node) - name = node.name.value.to_sym - args = - case node.params - when Params - child = visit(node.params) - - s( - child.type, - child.children, - smap_collection_bare(child.location&.expression) - ) - when Paren - child = visit(node.params.contents) - - s( - child.type, - child.children, - smap_collection( - srange_length(node.params.start_char, 1), - srange_length(node.params.end_char, -1), - srange_node(node.params) - ) - ) - else - s(:args, [], smap_collection_bare(nil)) - end - - location = - if node.endless? - smap_method_definition( - srange_length(node.start_char, 3), - nil, - srange_node(node.name), - nil, - srange_find_between( - (node.params || node.name), - node.bodystmt, - "=" - ), - srange_node(node) - ) - else - smap_method_definition( - srange_length(node.start_char, 3), - nil, - srange_node(node.name), - srange_length(node.end_char, -3), - nil, - srange_node(node) - ) - end - - if node.target - target = - node.target.is_a?(Paren) ? node.target.contents : node.target - - s( - :defs, - [visit(target), name, args, visit(node.bodystmt)], - smap_method_definition( - location.keyword, - srange_node(node.operator), - location.name, - location.end, - location.assignment, - location.expression - ) - ) - else - s(:def, [name, args, visit(node.bodystmt)], location) - end - end - - # Visit a Defined node. - def visit_defined(node) - paren_range = (node.start_char + 8)...node.end_char - begin_token, end_token = - if buffer.source[paren_range].include?("(") - [ - srange_find(paren_range.begin, paren_range.end, "("), - srange_length(node.end_char, -1) - ] - end - - s( - :defined?, - [visit(node.value)], - smap_keyword( - srange_length(node.start_char, 8), - begin_token, - end_token, - srange_node(node) - ) - ) - end - - # Visit a DynaSymbol node. - def visit_dyna_symbol(node) - location = - if node.quote - smap_collection( - srange_length(node.start_char, node.quote.length), - srange_length(node.end_char, -1), - srange_node(node) - ) - else - smap_collection_bare(srange_node(node)) - end - - if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) - s(:sym, ["\"#{node.parts.first.value}\"".undump.to_sym], location) - else - s(:dsym, visit_all(node.parts), location) - end - end - - # Visit an Else node. - def visit_else(node) - if node.statements.empty? && stack[-2].is_a?(Case) - s(:empty_else, [], nil) - else - visit(node.statements) - end - end - - # Visit an Elsif node. - def visit_elsif(node) - begin_start = node.predicate.end_char - begin_end = - if node.statements.empty? - node.statements.end_char - else - node.statements.body.first.start_char - end - - begin_token = - if buffer.source[begin_start...begin_end].include?("then") - srange_find(begin_start, begin_end, "then") - elsif buffer.source[begin_start...begin_end].include?(";") - srange_find(begin_start, begin_end, ";") - end - - else_token = - case node.consequent - when Elsif - srange_length(node.consequent.start_char, 5) - when Else - srange_length(node.consequent.start_char, 4) - end - - expression = srange(node.start_char, node.statements.end_char - 1) - - s( - :if, - [ - visit(node.predicate), - visit(node.statements), - visit(node.consequent) - ], - smap_condition( - srange_length(node.start_char, 5), - begin_token, - else_token, - nil, - expression - ) - ) - end - - # Visit an ENDBlock node. - def visit_END(node) - s( - :postexe, - [visit(node.statements)], - smap_keyword( - srange_length(node.start_char, 3), - srange_find(node.start_char + 3, node.statements.start_char, "{"), - srange_length(node.end_char, -1), - srange_node(node) - ) - ) - end - - # Visit an Ensure node. - def visit_ensure(node) - start_char = node.start_char - end_char = - if node.statements.empty? - start_char + 6 - else - node.statements.body.last.end_char - end - - s( - :ensure, - [visit(node.statements)], - smap_condition( - srange_length(start_char, 6), - nil, - nil, - nil, - srange(start_char, end_char) - ) - ) - end - - # Visit a Field node. - def visit_field(node) - message = - case stack[-2] - when Assign, MLHS - Ident.new( - value: "#{node.name.value}=", - location: node.name.location - ) - else - node.name - end - - visit_command_call( - CommandCall.new( - receiver: node.parent, - operator: node.operator, - message: message, - arguments: nil, - block: nil, - location: node.location - ) - ) - end - - # Visit a FloatLiteral node. - def visit_float(node) - operator = - if %w[+ -].include?(buffer.source[node.start_char]) - srange_length(node.start_char, 1) - end - - s( - :float, - [node.value.to_f], - smap_operator(operator, srange_node(node)) - ) - end - - # Visit a FndPtn node. - def visit_fndptn(node) - left, right = - [node.left, node.right].map do |child| - location = - smap_operator( - srange_length(child.start_char, 1), - srange_node(child) - ) - - if child.is_a?(VarField) && child.value.nil? - s(:match_rest, [], location) - else - s(:match_rest, [visit(child)], location) - end - end - - inner = - s( - :find_pattern, - [left, *visit_all(node.values), right], - smap_collection( - srange_length(node.start_char, 1), - srange_length(node.end_char, -1), - srange_node(node) - ) - ) - - if node.constant - s(:const_pattern, [visit(node.constant), inner], nil) - else - inner - end - end - - # Visit a For node. - def visit_for(node) - s( - :for, - [visit(node.index), visit(node.collection), visit(node.statements)], - smap_for( - srange_length(node.start_char, 3), - srange_find_between(node.index, node.collection, "in"), - srange_search_between(node.collection, node.statements, "do") || - srange_search_between(node.collection, node.statements, ";"), - srange_length(node.end_char, -3), - srange_node(node) - ) - ) - end - - # Visit a GVar node. - def visit_gvar(node) - s( - :gvar, - [node.value.to_sym], - smap_variable(srange_node(node), srange_node(node)) - ) - end - - # Visit a HashLiteral node. - def visit_hash(node) - s( - :hash, - visit_all(node.assocs), - smap_collection( - srange_length(node.start_char, 1), - srange_length(node.end_char, -1), - srange_node(node) - ) - ) - end - - # Visit a Heredoc node. - def visit_heredoc(node) - heredoc = HeredocBuilder.new(node) - - # For each part of the heredoc, if it's a string content node, split - # it into multiple string content nodes, one for each line. Otherwise, - # visit the node as normal. - node.parts.each do |part| - if part.is_a?(TStringContent) && part.value.count("\n") > 1 - index = part.start_char - lines = part.value.split("\n") - - lines.each do |line| - length = line.length + 1 - location = smap_collection_bare(srange_length(index, length)) - - heredoc << s(:str, ["#{line}\n"], location) - index += length - end - else - heredoc << visit(part) - end - end - - # Now that we have all of the pieces on the heredoc, we can trim it if - # it is a heredoc that supports trimming (i.e., it has a ~ on the - # declaration). - heredoc.trim! - - # Generate the location for the heredoc, which goes from the - # declaration to the ending delimiter. - location = - smap_heredoc( - srange_node(node.beginning), - srange( - if node.parts.empty? - node.beginning.end_char + 1 - else - node.parts.first.start_char - end, - node.ending.start_char - ), - srange(node.ending.start_char, node.ending.end_char - 1) - ) - - # Finally, decide which kind of heredoc node to generate based on its - # declaration and contents. - if node.beginning.value.match?(/`\w+`\z/) - s(:xstr, heredoc.segments, location) - elsif heredoc.segments.length == 1 - segment = heredoc.segments.first - s(segment.type, segment.children, location) - else - s(:dstr, heredoc.segments, location) - end - end - - # Visit a HshPtn node. - def visit_hshptn(node) - children = - node.keywords.map do |(keyword, value)| - next s(:pair, [visit(keyword), visit(value)], nil) if value - - case keyword - when DynaSymbol - raise if keyword.parts.length > 1 - s(:match_var, [keyword.parts.first.value.to_sym], nil) - when Label - s(:match_var, [keyword.value.chomp(":").to_sym], nil) - end - end - - if node.keyword_rest.is_a?(VarField) - children << if node.keyword_rest.value.nil? - s(:match_rest, [], nil) - elsif node.keyword_rest.value == :nil - s(:match_nil_pattern, [], nil) - else - s(:match_rest, [visit(node.keyword_rest)], nil) - end - end - - inner = s(:hash_pattern, children, nil) - if node.constant - s(:const_pattern, [visit(node.constant), inner], nil) - else - inner - end - end - - # Visit an Ident node. - def visit_ident(node) - s( - :lvar, - [node.value.to_sym], - smap_variable(srange_node(node), srange_node(node)) - ) - end - - # Visit an IfNode node. - def visit_if(node) - s( - :if, - [ - visit_predicate(node.predicate), - visit(node.statements), - visit(node.consequent) - ], - if node.modifier? - smap_keyword_bare( - srange_find_between(node.statements, node.predicate, "if"), - srange_node(node) - ) - else - begin_start = node.predicate.end_char - begin_end = - if node.statements.empty? - node.statements.end_char - else - node.statements.body.first.start_char - end - - begin_token = - if buffer.source[begin_start...begin_end].include?("then") - srange_find(begin_start, begin_end, "then") - elsif buffer.source[begin_start...begin_end].include?(";") - srange_find(begin_start, begin_end, ";") - end - - else_token = - case node.consequent - when Elsif - srange_length(node.consequent.start_char, 5) - when Else - srange_length(node.consequent.start_char, 4) - end - - smap_condition( - srange_length(node.start_char, 2), - begin_token, - else_token, - srange_length(node.end_char, -3), - srange_node(node) - ) - end - ) - end - - # Visit an IfOp node. - def visit_if_op(node) - s( - :if, - [visit(node.predicate), visit(node.truthy), visit(node.falsy)], - smap_ternary( - srange_find_between(node.predicate, node.truthy, "?"), - srange_find_between(node.truthy, node.falsy, ":"), - srange_node(node) - ) - ) - end - - # Visit an Imaginary node. - def visit_imaginary(node) - s( - :complex, - [ - # We have to do an eval here in order to get the value in case - # it's something like 42ri. to_c will not give the right value in - # that case. Maybe there's an API for this but I can't find it. - eval(node.value) - ], - smap_operator(nil, srange_node(node)) - ) - end - - # Visit an In node. - def visit_in(node) - case node.pattern - when IfNode - s( - :in_pattern, - [ - visit(node.pattern.statements), - s(:if_guard, [visit(node.pattern.predicate)], nil), - visit(node.statements) - ], - nil - ) - when UnlessNode - s( - :in_pattern, - [ - visit(node.pattern.statements), - s(:unless_guard, [visit(node.pattern.predicate)], nil), - visit(node.statements) - ], - nil - ) - else - begin_token = - srange_search_between(node.pattern, node.statements, "then") - - end_char = - if begin_token || node.statements.empty? - node.statements.end_char - 1 - else - node.statements.body.last.start_char - end - - s( - :in_pattern, - [visit(node.pattern), nil, visit(node.statements)], - smap_keyword( - srange_length(node.start_char, 2), - begin_token, - nil, - srange(node.start_char, end_char) - ) - ) - end - end - - # Visit an Int node. - def visit_int(node) - operator = - if %w[+ -].include?(buffer.source[node.start_char]) - srange_length(node.start_char, 1) - end - - s(:int, [node.value.to_i], smap_operator(operator, srange_node(node))) - end - - # Visit an IVar node. - def visit_ivar(node) - s( - :ivar, - [node.value.to_sym], - smap_variable(srange_node(node), srange_node(node)) - ) - end - - # Visit a Kw node. - def visit_kw(node) - location = smap(srange_node(node)) - - case node.value - when "__FILE__" - s(:str, [buffer.name], location) - when "__LINE__" - s( - :int, - [node.location.start_line + buffer.first_line - 1], - location - ) - when "__ENCODING__" - if ::Parser::Builders::Default.emit_encoding - s(:__ENCODING__, [], location) - else - s(:const, [s(:const, [nil, :Encoding], nil), :UTF_8], location) - end - else - s(node.value.to_sym, [], location) - end - end - - # Visit a KwRestParam node. - def visit_kwrest_param(node) - if node.name.nil? - s(:kwrestarg, [], smap_variable(nil, srange_node(node))) - else - s( - :kwrestarg, - [node.name.value.to_sym], - smap_variable(srange_node(node.name), srange_node(node)) - ) - end - end - - # Visit a Label node. - def visit_label(node) - s( - :sym, - [node.value.chomp(":").to_sym], - smap_collection_bare(srange(node.start_char, node.end_char - 1)) - ) - end - - # Visit a Lambda node. - def visit_lambda(node) - args = - node.params.is_a?(LambdaVar) ? node.params : node.params.contents - args_node = visit(args) - - type = :block - if args.empty? && (maximum = num_block_type(node.statements)) - type = :numblock - args_node = maximum - end - - begin_token, end_token = - if ( - srange = - srange_search_between(node.params, node.statements, "{") - ) - [srange, srange_length(node.end_char, -1)] - else - [ - srange_find_between(node.params, node.statements, "do"), - srange_length(node.end_char, -3) - ] - end - - selector = srange_length(node.start_char, 2) - - s( - type, - [ - if ::Parser::Builders::Default.emit_lambda - s(:lambda, [], smap(selector)) - else - s(:send, [nil, :lambda], smap_send_bare(selector, selector)) - end, - args_node, - visit(node.statements) - ], - smap_collection(begin_token, end_token, srange_node(node)) - ) - end - - # Visit a LambdaVar node. - def visit_lambda_var(node) - shadowargs = - node.locals.map do |local| - s( - :shadowarg, - [local.value.to_sym], - smap_variable(srange_node(local), srange_node(local)) - ) - end - - location = - if node.start_char == node.end_char - smap_collection_bare(nil) - elsif buffer.source[node.start_char - 1] == "(" - smap_collection( - srange_length(node.start_char, 1), - srange_length(node.end_char, -1), - srange_node(node) - ) - else - smap_collection_bare(srange_node(node)) - end - - s(:args, visit(node.params).children + shadowargs, location) - end - - # Visit an MAssign node. - def visit_massign(node) - s( - :masgn, - [visit(node.target), visit(node.value)], - smap_operator( - srange_find_between(node.target, node.value, "="), - srange_node(node) - ) - ) - end - - # Visit a MethodAddBlock node. - def visit_method_add_block(node) - case node.call - when ARef, Super, ZSuper - type, arguments = block_children(node.block) - - s( - type, - [visit(node.call), arguments, visit(node.block.bodystmt)], - smap_collection( - srange_node(node.block.opening), - srange_length( - node.block.end_char, - node.block.keywords? ? -3 : -1 - ), - srange_node(node) - ) - ) - else - visit_command_call( - CommandCall.new( - receiver: node.call.receiver, - operator: node.call.operator, - message: node.call.message, - arguments: node.call.arguments, - block: node.block, - location: node.location - ) - ) - end - end - - # Visit an MLHS node. - def visit_mlhs(node) - s( - :mlhs, - node.parts.map do |part| - if part.is_a?(Ident) - s( - :arg, - [part.value.to_sym], - smap_variable(srange_node(part), srange_node(part)) - ) - else - visit(part) - end - end, - smap_collection_bare(srange_node(node)) - ) - end - - # Visit an MLHSParen node. - def visit_mlhs_paren(node) - child = visit(node.contents) - - s( - child.type, - child.children, - smap_collection( - srange_length(node.start_char, 1), - srange_length(node.end_char, -1), - srange_node(node) - ) - ) - end - - # Visit a ModuleDeclaration node. - def visit_module(node) - s( - :module, - [visit(node.constant), visit(node.bodystmt)], - smap_definition( - srange_length(node.start_char, 6), - nil, - srange_node(node.constant), - srange_length(node.end_char, -3) - ).with_expression(srange_node(node)) - ) - end - - # Visit an MRHS node. - def visit_mrhs(node) - visit_array( - ArrayLiteral.new( - lbracket: nil, - contents: Args.new(parts: node.parts, location: node.location), - location: node.location - ) - ) - end - - # Visit a Next node. - def visit_next(node) - s( - :next, - visit_all(node.arguments.parts), - smap_keyword_bare( - srange_length(node.start_char, 4), - srange_node(node) - ) - ) - end - - # Visit a Not node. - def visit_not(node) - if node.statement.nil? - begin_token = srange_find(node.start_char, nil, "(") - end_token = srange_find(node.start_char, nil, ")") - - s( - :send, - [ - s( - :begin, - [], - smap_collection( - begin_token, - end_token, - begin_token.join(end_token) - ) - ), - :! - ], - smap_send_bare( - srange_length(node.start_char, 3), - srange_node(node) - ) - ) - else - begin_token, end_token = - if node.parentheses? - [ - srange_find( - node.start_char + 3, - node.statement.start_char, - "(" - ), - srange_length(node.end_char, -1) - ] - end - - s( - :send, - [visit(node.statement), :!], - smap_send( - nil, - srange_length(node.start_char, 3), - begin_token, - end_token, - srange_node(node) - ) - ) - end - end - - # Visit an OpAssign node. - def visit_opassign(node) - target = visit(node.target) - location = - target - .location - .with_expression(srange_node(node)) - .with_operator(srange_node(node.operator)) - - case node.operator.value - when "||=" - s(:or_asgn, [target, visit(node.value)], location) - when "&&=" - s(:and_asgn, [target, visit(node.value)], location) - else - s( - :op_asgn, - [ - target, - node.operator.value.chomp("=").to_sym, - visit(node.value) - ], - location - ) - end - end - - # Visit a Params node. - def visit_params(node) - children = [] - - children += - node.requireds.map do |required| - case required - when MLHSParen - visit(required) - else - s( - :arg, - [required.value.to_sym], - smap_variable(srange_node(required), srange_node(required)) - ) - end - end - - children += - node.optionals.map do |(name, value)| - s( - :optarg, - [name.value.to_sym, visit(value)], - smap_variable( - srange_node(name), - srange_node(name).join(srange_node(value)) - ).with_operator(srange_find_between(name, value, "=")) - ) - end - - if node.rest && !node.rest.is_a?(ExcessedComma) - children << visit(node.rest) - end - - children += - node.posts.map do |post| - s( - :arg, - [post.value.to_sym], - smap_variable(srange_node(post), srange_node(post)) - ) - end - - children += - node.keywords.map do |(name, value)| - key = name.value.chomp(":").to_sym - - if value - s( - :kwoptarg, - [key, visit(value)], - smap_variable( - srange(name.start_char, name.end_char - 1), - srange_node(name).join(srange_node(value)) - ) - ) - else - s( - :kwarg, - [key], - smap_variable( - srange(name.start_char, name.end_char - 1), - srange_node(name) - ) - ) - end - end - - case node.keyword_rest - when nil, ArgsForward - # do nothing - when :nil - children << s( - :kwnilarg, - [], - smap_variable(srange_length(node.end_char, -3), srange_node(node)) - ) - else - children << visit(node.keyword_rest) - end - - children << visit(node.block) if node.block - - if node.keyword_rest.is_a?(ArgsForward) - location = smap(srange_node(node.keyword_rest)) - - # If there are no other arguments and we have the emit_forward_arg - # option enabled, then the entire argument list is represented by a - # single forward_args node. - if children.empty? && !::Parser::Builders::Default.emit_forward_arg - return s(:forward_args, [], location) - end - - # Otherwise, we need to insert a forward_arg node into the list of - # parameters before any keyword rest or block parameters. - index = - node.requireds.length + node.optionals.length + - node.keywords.length - children.insert(index, s(:forward_arg, [], location)) - end - - location = - unless children.empty? - first = children.first.location.expression - last = children.last.location.expression - smap_collection_bare(first.join(last)) - end - - s(:args, children, location) - end - - # Visit a Paren node. - def visit_paren(node) - location = - smap_collection( - srange_length(node.start_char, 1), - srange_length(node.end_char, -1), - srange_node(node) - ) - - if node.contents.nil? || - (node.contents.is_a?(Statements) && node.contents.empty?) - s(:begin, [], location) - else - child = visit(node.contents) - child.type == :begin ? child : s(:begin, [child], location) - end - end - - # Visit a PinnedBegin node. - def visit_pinned_begin(node) - s( - :pin, - [ - s( - :begin, - [visit(node.statement)], - smap_collection( - srange_length(node.start_char + 1, 1), - srange_length(node.end_char, -1), - srange(node.start_char + 1, node.end_char) - ) - ) - ], - smap_send_bare(srange_length(node.start_char, 1), srange_node(node)) - ) - end - - # Visit a PinnedVarRef node. - def visit_pinned_var_ref(node) - s( - :pin, - [visit(node.value)], - smap_send_bare(srange_length(node.start_char, 1), srange_node(node)) - ) - end - - # Visit a Program node. - def visit_program(node) - visit(node.statements) - end - - # Visit a QSymbols node. - def visit_qsymbols(node) - parts = - node.elements.map do |element| - SymbolLiteral.new(value: element, location: element.location) - end - - visit_array( - ArrayLiteral.new( - lbracket: node.beginning, - contents: Args.new(parts: parts, location: node.location), - location: node.location - ) - ) - end - - # Visit a QWords node. - def visit_qwords(node) - visit_array( - ArrayLiteral.new( - lbracket: node.beginning, - contents: Args.new(parts: node.elements, location: node.location), - location: node.location - ) - ) - end - - # Visit a RangeNode node. - def visit_range(node) - s( - node.operator.value == ".." ? :irange : :erange, - [visit(node.left), visit(node.right)], - smap_operator(srange_node(node.operator), srange_node(node)) - ) - end - - # Visit an RAssign node. - def visit_rassign(node) - s( - node.operator.value == "=>" ? :match_pattern : :match_pattern_p, - [visit(node.value), visit(node.pattern)], - smap_operator(srange_node(node.operator), srange_node(node)) - ) - end - - # Visit a Rational node. - def visit_rational(node) - s(:rational, [node.value.to_r], smap_operator(nil, srange_node(node))) - end - - # Visit a Redo node. - def visit_redo(node) - s(:redo, [], smap_keyword_bare(srange_node(node), srange_node(node))) - end - - # Visit a RegexpLiteral node. - def visit_regexp_literal(node) - s( - :regexp, - visit_all(node.parts).push( - s( - :regopt, - node.ending.scan(/[a-z]/).sort.map(&:to_sym), - smap(srange_length(node.end_char, -(node.ending.length - 1))) - ) - ), - smap_collection( - srange_length(node.start_char, node.beginning.length), - srange_length(node.end_char - node.ending.length, 1), - srange_node(node) - ) - ) - end - - # Visit a Rescue node. - def visit_rescue(node) - # In the parser gem, there is a separation between the rescue node and - # the rescue body. They have different bounds, so we have to calculate - # those here. - start_char = node.start_char - - body_end_char = - if node.statements.empty? - start_char + 6 - else - node.statements.body.last.end_char - end - - end_char = - if node.consequent - end_node = node.consequent - end_node = end_node.consequent while end_node.consequent - - if end_node.statements.empty? - start_char + 6 - else - end_node.statements.body.last.end_char - end - else - body_end_char - end - - # These locations are reused for multiple children. - keyword = srange_length(start_char, 6) - body_expression = srange(start_char, body_end_char) - expression = srange(start_char, end_char) - - exceptions = - case node.exception&.exceptions - when nil - nil - when MRHS - visit_array( - ArrayLiteral.new( - lbracket: nil, - contents: - Args.new( - parts: node.exception.exceptions.parts, - location: node.exception.exceptions.location - ), - location: node.exception.exceptions.location - ) - ) - else - visit_array( - ArrayLiteral.new( - lbracket: nil, - contents: - Args.new( - parts: [node.exception.exceptions], - location: node.exception.exceptions.location - ), - location: node.exception.exceptions.location - ) - ) - end - - resbody = - if node.exception.nil? - s( - :resbody, - [nil, nil, visit(node.statements)], - smap_rescue_body(keyword, nil, nil, body_expression) - ) - elsif node.exception.variable.nil? - s( - :resbody, - [exceptions, nil, visit(node.statements)], - smap_rescue_body(keyword, nil, nil, body_expression) - ) - else - s( - :resbody, - [ - exceptions, - visit(node.exception.variable), - visit(node.statements) - ], - smap_rescue_body( - keyword, - srange_find( - node.start_char + 6, - node.exception.variable.start_char, - "=>" - ), - nil, - body_expression - ) - ) - end - - children = [resbody] - if node.consequent - children += visit(node.consequent).children - else - children << nil - end - - s(:rescue, children, smap_condition_bare(expression)) - end - - # Visit a RescueMod node. - def visit_rescue_mod(node) - keyword = srange_find_between(node.statement, node.value, "rescue") - - s( - :rescue, - [ - visit(node.statement), - s( - :resbody, - [nil, nil, visit(node.value)], - smap_rescue_body( - keyword, - nil, - nil, - keyword.join(srange_node(node.value)) - ) - ), - nil - ], - smap_condition_bare(srange_node(node)) - ) - end - - # Visit a RestParam node. - def visit_rest_param(node) - if node.name - s( - :restarg, - [node.name.value.to_sym], - smap_variable(srange_node(node.name), srange_node(node)) - ) - else - s(:restarg, [], smap_variable(nil, srange_node(node))) - end - end - - # Visit a Retry node. - def visit_retry(node) - s(:retry, [], smap_keyword_bare(srange_node(node), srange_node(node))) - end - - # Visit a ReturnNode node. - def visit_return(node) - s( - :return, - node.arguments ? visit_all(node.arguments.parts) : [], - smap_keyword_bare( - srange_length(node.start_char, 6), - srange_node(node) - ) - ) - end - - # Visit an SClass node. - def visit_sclass(node) - s( - :sclass, - [visit(node.target), visit(node.bodystmt)], - smap_definition( - srange_length(node.start_char, 5), - srange_find(node.start_char + 5, node.target.start_char, "<<"), - nil, - srange_length(node.end_char, -3) - ).with_expression(srange_node(node)) - ) - end - - # Visit a Statements node. - def visit_statements(node) - children = - node.body.reject do |child| - child.is_a?(Comment) || child.is_a?(EmbDoc) || - child.is_a?(EndContent) || child.is_a?(VoidStmt) - end - - case children.length - when 0 - nil - when 1 - visit(children.first) - else - s( - :begin, - visit_all(children), - smap_collection_bare( - srange(children.first.start_char, children.last.end_char) - ) - ) - end - end - - # Visit a StringConcat node. - def visit_string_concat(node) - s( - :dstr, - [visit(node.left), visit(node.right)], - smap_collection_bare(srange_node(node)) - ) - end - - # Visit a StringDVar node. - def visit_string_dvar(node) - visit(node.variable) - end - - # Visit a StringEmbExpr node. - def visit_string_embexpr(node) - s( - :begin, - visit(node.statements).then { |child| child ? [child] : [] }, - smap_collection( - srange_length(node.start_char, 2), - srange_length(node.end_char, -1), - srange_node(node) - ) - ) - end - - # Visit a StringLiteral node. - def visit_string_literal(node) - location = - if node.quote - smap_collection( - srange_length(node.start_char, node.quote.length), - srange_length(node.end_char, -1), - srange_node(node) - ) - else - smap_collection_bare(srange_node(node)) - end - - if node.parts.empty? - s(:str, [""], location) - elsif node.parts.length == 1 && node.parts.first.is_a?(TStringContent) - child = visit(node.parts.first) - s(child.type, child.children, location) - else - s(:dstr, visit_all(node.parts), location) - end - end - - # Visit a Super node. - def visit_super(node) - if node.arguments.is_a?(Args) - s( - :super, - visit_all(node.arguments.parts), - smap_keyword_bare( - srange_length(node.start_char, 5), - srange_node(node) - ) - ) - else - case node.arguments.arguments - when nil - s( - :super, - [], - smap_keyword( - srange_length(node.start_char, 5), - srange_find(node.start_char + 5, node.end_char, "("), - srange_length(node.end_char, -1), - srange_node(node) - ) - ) - when ArgsForward - s( - :super, - [visit(node.arguments.arguments)], - smap_keyword( - srange_length(node.start_char, 5), - srange_find(node.start_char + 5, node.end_char, "("), - srange_length(node.end_char, -1), - srange_node(node) - ) - ) - else - s( - :super, - visit_all(node.arguments.arguments.parts), - smap_keyword( - srange_length(node.start_char, 5), - srange_find(node.start_char + 5, node.end_char, "("), - srange_length(node.end_char, -1), - srange_node(node) - ) - ) - end - end - end - - # Visit a SymbolLiteral node. - def visit_symbol_literal(node) - begin_token = - if buffer.source[node.start_char] == ":" - srange_length(node.start_char, 1) - end - - s( - :sym, - [node.value.value.to_sym], - smap_collection(begin_token, nil, srange_node(node)) - ) - end - - # Visit a Symbols node. - def visit_symbols(node) - parts = - node.elements.map do |element| - part = element.parts.first - - if element.parts.length == 1 && part.is_a?(TStringContent) - SymbolLiteral.new(value: part, location: part.location) - else - DynaSymbol.new( - parts: element.parts, - quote: nil, - location: element.location - ) - end - end - - visit_array( - ArrayLiteral.new( - lbracket: node.beginning, - contents: Args.new(parts: parts, location: node.location), - location: node.location - ) - ) - end - - # Visit a TopConstField node. - def visit_top_const_field(node) - s( - :casgn, - [ - s(:cbase, [], smap(srange_length(node.start_char, 2))), - node.constant.value.to_sym - ], - smap_constant( - srange_length(node.start_char, 2), - srange_node(node.constant), - srange_node(node) - ) - ) - end - - # Visit a TopConstRef node. - def visit_top_const_ref(node) - s( - :const, - [ - s(:cbase, [], smap(srange_length(node.start_char, 2))), - node.constant.value.to_sym - ], - smap_constant( - srange_length(node.start_char, 2), - srange_node(node.constant), - srange_node(node) - ) - ) - end - - # Visit a TStringContent node. - def visit_tstring_content(node) - dumped = node.value.gsub(/([^[:ascii:]])/) { $1.dump[1...-1] } - - s( - :str, - ["\"#{dumped}\"".undump], - smap_collection_bare(srange_node(node)) - ) - end - - # Visit a Unary node. - def visit_unary(node) - # Special handling here for flipflops - if (paren = node.statement).is_a?(Paren) && - paren.contents.is_a?(Statements) && - paren.contents.body.length == 1 && - (range = paren.contents.body.first).is_a?(RangeNode) && - node.operator == "!" - s( - :send, - [ - s( - :begin, - [ - s( - range.operator.value == ".." ? :iflipflop : :eflipflop, - visit(range).children, - smap_operator( - srange_node(range.operator), - srange_node(range) - ) - ) - ], - smap_collection( - srange_length(paren.start_char, 1), - srange_length(paren.end_char, -1), - srange_node(paren) - ) - ), - :! - ], - smap_send_bare( - srange_length(node.start_char, 1), - srange_node(node) - ) - ) - elsif node.operator == "!" && node.statement.is_a?(RegexpLiteral) - s( - :send, - [ - s( - :match_current_line, - [visit(node.statement)], - smap(srange_node(node.statement)) - ), - :! - ], - smap_send_bare( - srange_length(node.start_char, 1), - srange_node(node) - ) - ) - else - visit(canonical_unary(node)) - end - end - - # Visit an Undef node. - def visit_undef(node) - s( - :undef, - visit_all(node.symbols), - smap_keyword_bare( - srange_length(node.start_char, 5), - srange_node(node) - ) - ) - end - - # Visit an UnlessNode node. - def visit_unless(node) - s( - :if, - [ - visit_predicate(node.predicate), - visit(node.consequent), - visit(node.statements) - ], - if node.modifier? - smap_keyword_bare( - srange_find_between(node.statements, node.predicate, "unless"), - srange_node(node) - ) - else - begin_start = node.predicate.end_char - begin_end = - if node.statements.empty? - node.statements.end_char - else - node.statements.body.first.start_char - end - - begin_token = - if buffer.source[begin_start...begin_end].include?("then") - srange_find(begin_start, begin_end, "then") - elsif buffer.source[begin_start...begin_end].include?(";") - srange_find(begin_start, begin_end, ";") - end - - else_token = - if node.consequent - srange_length(node.consequent.start_char, 4) - end - - smap_condition( - srange_length(node.start_char, 6), - begin_token, - else_token, - srange_length(node.end_char, -3), - srange_node(node) - ) - end - ) - end - - # Visit an UntilNode node. - def visit_until(node) - s( - loop_post?(node) ? :until_post : :until, - [visit(node.predicate), visit(node.statements)], - if node.modifier? - smap_keyword_bare( - srange_find_between(node.statements, node.predicate, "until"), - srange_node(node) - ) - else - smap_keyword( - srange_length(node.start_char, 5), - srange_search_between(node.predicate, node.statements, "do") || - srange_search_between(node.predicate, node.statements, ";"), - srange_length(node.end_char, -3), - srange_node(node) - ) - end - ) - end - - # Visit a VarField node. - def visit_var_field(node) - name = node.value.value.to_sym - match_var = - [stack[-3], stack[-2]].any? do |parent| - case parent - when AryPtn, FndPtn, HshPtn, In, RAssign - true - when Binary - parent.operator == :"=>" - else - false - end - end - - if match_var - s( - :match_var, - [name], - smap_variable(srange_node(node.value), srange_node(node.value)) - ) - elsif node.value.is_a?(Const) - s( - :casgn, - [nil, name], - smap_constant(nil, srange_node(node.value), srange_node(node)) - ) - else - location = smap_variable(srange_node(node), srange_node(node)) - - case node.value - when CVar - s(:cvasgn, [name], location) - when GVar - s(:gvasgn, [name], location) - when Ident - s(:lvasgn, [name], location) - when IVar - s(:ivasgn, [name], location) - when VarRef - s(:lvasgn, [name], location) - else - s(:match_rest, [], nil) - end - end - end - - # Visit a VarRef node. - def visit_var_ref(node) - visit(node.value) - end - - # Visit a VCall node. - def visit_vcall(node) - visit_command_call( - CommandCall.new( - receiver: nil, - operator: nil, - message: node.value, - arguments: nil, - block: nil, - location: node.location - ) - ) - end - - # Visit a When node. - def visit_when(node) - keyword = srange_length(node.start_char, 4) - begin_token = - if buffer.source[node.statements.start_char] == ";" - srange_length(node.statements.start_char, 1) - end - - end_char = - if node.statements.body.empty? - node.statements.end_char - else - node.statements.body.last.end_char - end - - s( - :when, - visit_all(node.arguments.parts) + [visit(node.statements)], - smap_keyword( - keyword, - begin_token, - nil, - srange(keyword.begin_pos, end_char) - ) - ) - end - - # Visit a WhileNode node. - def visit_while(node) - s( - loop_post?(node) ? :while_post : :while, - [visit(node.predicate), visit(node.statements)], - if node.modifier? - smap_keyword_bare( - srange_find_between(node.statements, node.predicate, "while"), - srange_node(node) - ) - else - smap_keyword( - srange_length(node.start_char, 5), - srange_search_between(node.predicate, node.statements, "do") || - srange_search_between(node.predicate, node.statements, ";"), - srange_length(node.end_char, -3), - srange_node(node) - ) - end - ) - end - - # Visit a Word node. - def visit_word(node) - visit_string_literal( - StringLiteral.new( - parts: node.parts, - quote: nil, - location: node.location - ) - ) - end - - # Visit a Words node. - def visit_words(node) - visit_array( - ArrayLiteral.new( - lbracket: node.beginning, - contents: Args.new(parts: node.elements, location: node.location), - location: node.location - ) - ) - end - - # Visit an XStringLiteral node. - def visit_xstring_literal(node) - s( - :xstr, - visit_all(node.parts), - smap_collection( - srange_length( - node.start_char, - buffer.source[node.start_char] == "%" ? 3 : 1 - ), - srange_length(node.end_char, -1), - srange_node(node) - ) - ) - end - - def visit_yield(node) - case node.arguments - when nil - s( - :yield, - [], - smap_keyword_bare( - srange_length(node.start_char, 5), - srange_node(node) - ) - ) - when Args - s( - :yield, - visit_all(node.arguments.parts), - smap_keyword_bare( - srange_length(node.start_char, 5), - srange_node(node) - ) - ) - else - s( - :yield, - visit_all(node.arguments.contents.parts), - smap_keyword( - srange_length(node.start_char, 5), - srange_length(node.arguments.start_char, 1), - srange_length(node.end_char, -1), - srange_node(node) - ) - ) - end - end - - # Visit a ZSuper node. - def visit_zsuper(node) - s( - :zsuper, - [], - smap_keyword_bare( - srange_length(node.start_char, 5), - srange_node(node) - ) - ) - end - end - - private - - def block_children(node) - arguments = - if node.block_var - visit(node.block_var) - else - s(:args, [], smap_collection_bare(nil)) - end - - type = :block - if !node.block_var && (maximum = num_block_type(node.bodystmt)) - type = :numblock - arguments = maximum - end - - [type, arguments] - end - - # Convert a Unary node into a canonical CommandCall node. - def canonical_unary(node) - # For integers and floats with a leading + or -, parser represents them - # as just their values with the signs attached. - if %w[+ -].include?(node.operator) && - (node.statement.is_a?(Int) || node.statement.is_a?(FloatLiteral)) - return( - node.statement.class.new( - value: "#{node.operator}#{node.statement.value}", - location: node.location - ) - ) - end - - value = { "+" => "+@", "-" => "-@" }.fetch(node.operator, node.operator) - length = node.operator.length - - CommandCall.new( - receiver: node.statement, - operator: nil, - message: - Op.new( - value: value, - location: - Location.new( - start_line: node.location.start_line, - start_char: node.start_char, - start_column: node.location.start_column, - end_line: node.location.start_line, - end_char: node.start_char + length, - end_column: node.location.start_column + length - ) - ), - arguments: nil, - block: nil, - location: node.location - ) - end - - # Convert a Binary node into a canonical CommandCall node. - def canonical_binary(node) - operator = node.operator.to_s - - start_char = node.left.end_char - end_char = node.right.start_char - - index = buffer.source[start_char...end_char].index(operator) - start_line = - node.location.start_line + - buffer.source[start_char...index].count("\n") - start_column = - index - (buffer.source[start_char...index].rindex("\n") || 0) - - op_location = - Location.new( - start_line: start_line, - start_column: start_column, - start_char: start_char + index, - end_line: start_line, - end_column: start_column + operator.length, - end_char: start_char + index + operator.length - ) - - CommandCall.new( - receiver: node.left, - operator: nil, - message: Op.new(value: operator, location: op_location), - arguments: - Args.new(parts: [node.right], location: node.right.location), - block: nil, - location: node.location - ) - end - - # When you have a begin..end while or begin..end until, it's a special - # kind of syntax that executes the block in a loop. In this case the - # parser gem has a special node type for it. - def loop_post?(node) - node.modifier? && node.statements.is_a?(Statements) && - node.statements.body.length == 1 && - node.statements.body.first.is_a?(Begin) - end - - # We need to find if we should transform this block into a numblock - # since there could be new numbered variables like _1. - def num_block_type(statements) - variables = [] - queue = [statements] - - while (child_node = queue.shift) - if child_node.is_a?(VarRef) && child_node.value.is_a?(Ident) && - child_node.value.value =~ /^_(\d+)$/ - variables << $1.to_i - end - - queue += child_node.child_nodes.compact - end - - variables.max - end - - # This method comes almost directly from the parser gem and creates a new - # parser gem node from the given s-expression. type is expected to be a - # symbol, children is expected to be an array, and location is expected to - # be a source map. - def s(type, children, location) - ::Parser::AST::Node.new(type, children, location: location) - end - - # Constructs a plain source map just for an expression. - def smap(expression) - ::Parser::Source::Map.new(expression) - end - - # Constructs a new source map for a collection. - def smap_collection(begin_token, end_token, expression) - ::Parser::Source::Map::Collection.new( - begin_token, - end_token, - expression - ) - end - - # Constructs a new source map for a collection without a begin or end. - def smap_collection_bare(expression) - smap_collection(nil, nil, expression) - end - - # Constructs a new source map for a conditional expression. - def smap_condition( - keyword, - begin_token, - else_token, - end_token, - expression - ) - ::Parser::Source::Map::Condition.new( - keyword, - begin_token, - else_token, - end_token, - expression - ) - end - - # Constructs a new source map for a conditional expression with no begin - # or end. - def smap_condition_bare(expression) - smap_condition(nil, nil, nil, nil, expression) - end - - # Constructs a new source map for a constant reference. - def smap_constant(double_colon, name, expression) - ::Parser::Source::Map::Constant.new(double_colon, name, expression) - end - - # Constructs a new source map for a class definition. - def smap_definition(keyword, operator, name, end_token) - ::Parser::Source::Map::Definition.new( - keyword, - operator, - name, - end_token - ) - end - - # Constructs a new source map for a for loop. - def smap_for(keyword, in_token, begin_token, end_token, expression) - ::Parser::Source::Map::For.new( - keyword, - in_token, - begin_token, - end_token, - expression - ) - end - - # Constructs a new source map for a heredoc. - def smap_heredoc(expression, heredoc_body, heredoc_end) - ::Parser::Source::Map::Heredoc.new( - expression, - heredoc_body, - heredoc_end - ) - end - - # Construct a source map for an index operation. - def smap_index(begin_token, end_token, expression) - ::Parser::Source::Map::Index.new(begin_token, end_token, expression) - end - - # Constructs a new source map for the use of a keyword. - def smap_keyword(keyword, begin_token, end_token, expression) - ::Parser::Source::Map::Keyword.new( - keyword, - begin_token, - end_token, - expression - ) - end - - # Constructs a new source map for the use of a keyword without a begin or - # end token. - def smap_keyword_bare(keyword, expression) - smap_keyword(keyword, nil, nil, expression) - end - - # Constructs a new source map for a method definition. - def smap_method_definition( - keyword, - operator, - name, - end_token, - assignment, - expression - ) - ::Parser::Source::Map::MethodDefinition.new( - keyword, - operator, - name, - end_token, - assignment, - expression - ) - end - - # Constructs a new source map for an operator. - def smap_operator(operator, expression) - ::Parser::Source::Map::Operator.new(operator, expression) - end - - # Constructs a source map for the body of a rescue clause. - def smap_rescue_body(keyword, assoc, begin_token, expression) - ::Parser::Source::Map::RescueBody.new( - keyword, - assoc, - begin_token, - expression - ) - end - - # Constructs a new source map for a method call. - def smap_send(dot, selector, begin_token, end_token, expression) - ::Parser::Source::Map::Send.new( - dot, - selector, - begin_token, - end_token, - expression - ) - end - - # Constructs a new source map for a method call without a begin or end. - def smap_send_bare(selector, expression) - smap_send(nil, selector, nil, nil, expression) - end - - # Constructs a new source map for a ternary expression. - def smap_ternary(question, colon, expression) - ::Parser::Source::Map::Ternary.new(question, colon, expression) - end - - # Constructs a new source map for a variable. - def smap_variable(name, expression) - ::Parser::Source::Map::Variable.new(name, expression) - end - - # Constructs a new source range from the given start and end offsets. - def srange(start_char, end_char) - ::Parser::Source::Range.new(buffer, start_char, end_char) - end - - # Constructs a new source range by finding the given needle in the given - # range of the source. If the needle is not found, returns nil. - def srange_search(start_char, end_char, needle) - index = buffer.source[start_char...end_char].index(needle) - return unless index - - offset = start_char + index - srange(offset, offset + needle.length) - end - - # Constructs a new source range by searching for the given needle between - # the end location of the start node and the start location of the end - # node. If the needle is not found, returns nil. - def srange_search_between(start_node, end_node, needle) - srange_search(start_node.end_char, end_node.start_char, needle) - end - - # Constructs a new source range by finding the given needle in the given - # range of the source. If it needle is not found, raises an error. - def srange_find(start_char, end_char, needle) - srange = srange_search(start_char, end_char, needle) - - unless srange - slice = buffer.source[start_char...end_char].inspect - raise "Could not find #{needle.inspect} in #{slice}" - end - - srange - end - - # Constructs a new source range by finding the given needle between the - # end location of the start node and the start location of the end node. - # If the needle is not found, returns raises an error. - def srange_find_between(start_node, end_node, needle) - srange_find(start_node.end_char, end_node.start_char, needle) - end - - # Constructs a new source range from the given start offset and length. - def srange_length(start_char, length) - if length > 0 - srange(start_char, start_char + length) - else - srange(start_char + length, start_char) - end - end - - # Constructs a new source range using the given node's location. - def srange_node(node) - location = node.location - srange(location.start_char, location.end_char) - end - - def visit_predicate(node) - case node - when RangeNode - s( - node.operator.value == ".." ? :iflipflop : :eflipflop, - visit(node).children, - smap_operator(srange_node(node.operator), srange_node(node)) - ) - when RegexpLiteral - s(:match_current_line, [visit(node)], smap(srange_node(node))) - when Unary - if node.operator.value == "!" && node.statement.is_a?(RegexpLiteral) - s( - :send, - [s(:match_current_line, [visit(node.statement)]), :!], - smap_send_bare(srange_node(node.operator), srange_node(node)) - ) - else - visit(node) - end - else - visit(node) - end - end - end - end -end diff --git a/lib/syntax_tree/translation/rubocop_ast.rb b/lib/syntax_tree/translation/rubocop_ast.rb deleted file mode 100644 index 53c6737b..00000000 --- a/lib/syntax_tree/translation/rubocop_ast.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -module SyntaxTree - module Translation - # This visitor is responsible for converting the syntax tree produced by - # Syntax Tree into the syntax tree produced by the rubocop/rubocop-ast gem. - class RuboCopAST < Parser - private - - # This method is effectively the same thing as the parser gem except that - # it uses the rubocop-ast specializations of the nodes. - def s(type, children, location) - ::RuboCop::AST::Builder::NODE_MAP.fetch(type, ::RuboCop::AST::Node).new( - type, - children, - location: location - ) - end - end - end -end diff --git a/tasks/whitequark.rake b/tasks/whitequark.rake deleted file mode 100644 index 6e1663aa..00000000 --- a/tasks/whitequark.rake +++ /dev/null @@ -1,92 +0,0 @@ -# frozen_string_literal: true - -# This file's purpose is to extract the examples from the whitequark/parser -# gem and generate a test file that we can use to ensure that our parser -# generates equivalent syntax trees when translating. To do this, it runs the -# parser's test suite but overrides the `assert_parses` method to collect the -# examples into a hash. Then, it writes out the hash to a file that we can use -# to generate our own tests. -# -# To run the test suite, it's important to note that we have to mirror both any -# APIs provided to the test suite (for example the ParseHelper module below). -# This is obviously relatively brittle, but it's effective for now. - -require "ast" - -module ParseHelper - # This object is going to collect all of the examples from the parser gem into - # a hash that we can use to generate our own tests. - COLLECTED = Hash.new { |hash, key| hash[key] = [] } - - include AST::Sexp - ALL_VERSIONS = %w[3.1 3.2] - - private - - def assert_context(*) - end - - def assert_diagnoses(*) - end - - def assert_diagnoses_many(*) - end - - def refute_diagnoses(*) - end - - def with_versions(*) - end - - def assert_parses(_ast, code, _source_maps = "", versions = ALL_VERSIONS) - # We're going to skip any examples that are for older Ruby versions - # that we do not support. - return if (versions & %w[3.1 3.2]).empty? - - entry = - caller.find do |call| - call.include?("test_parser.rb") && call.match?(%r{(? 1) -!!! test_args_args_assocs:4143 -fun(foo, :foo => 1, &baz) -!!! test_args_args_assocs_comma:4152 -foo[bar, :baz => 1,] -!!! test_args_args_comma:4001 -foo[bar,] -!!! test_args_args_star:3968 -fun(foo, *bar) -!!! test_args_args_star:3973 -fun(foo, *bar, &baz) -!!! test_args_assocs:4061 -fun(:foo => 1) -!!! test_args_assocs:4066 -fun(:foo => 1, &baz) -!!! test_args_assocs:4072 -self[:bar => 1] -!!! test_args_assocs:4081 -self.[]= foo, :a => 1 -!!! test_args_assocs:4091 -yield(:foo => 42) -!!! test_args_assocs:4099 -super(:foo => 42) -!!! test_args_assocs_comma:4128 -foo[:baz => 1,] -!!! test_args_assocs_legacy:4011 -fun(:foo => 1) -!!! test_args_assocs_legacy:4016 -fun(:foo => 1, &baz) -!!! test_args_assocs_legacy:4022 -self[:bar => 1] -!!! test_args_assocs_legacy:4031 -self.[]= foo, :a => 1 -!!! test_args_assocs_legacy:4041 -yield(:foo => 42) -!!! test_args_assocs_legacy:4049 -super(:foo => 42) -!!! test_args_block_pass:3994 -fun(&bar) -!!! test_args_cmd:3961 -fun(f bar) -!!! test_args_star:3981 -fun(*bar) -!!! test_args_star:3986 -fun(*bar, &baz) -!!! test_array_assocs:643 -[ 1 => 2 ] -!!! test_array_assocs:651 -[ 1, 2 => 3 ] -!!! test_array_plain:603 -[1, 2] -!!! test_array_splat:612 -[1, *foo, 2] -!!! test_array_splat:625 -[1, *foo] -!!! test_array_splat:636 -[*foo] -!!! test_array_symbols:709 -%i[foo bar] -!!! test_array_symbols_empty:746 -%i[] -!!! test_array_symbols_empty:754 -%I() -!!! test_array_symbols_interp:720 -%I[foo #{bar}] -!!! test_array_symbols_interp:735 -%I[foo#{bar}] -!!! test_array_words:661 -%w[foo bar] -!!! test_array_words_empty:696 -%w[] -!!! test_array_words_empty:703 -%W() -!!! test_array_words_interp:671 -%W[foo #{bar}] -!!! test_array_words_interp:685 -%W[foo #{bar}foo#@baz] -!!! test_asgn_cmd:1140 -foo = m foo -!!! test_asgn_cmd:1144 -foo = bar = m foo -!!! test_asgn_mrhs:1463 -foo = bar, 1 -!!! test_asgn_mrhs:1470 -foo = *bar -!!! test_asgn_mrhs:1475 -foo = baz, *bar -!!! test_back_ref:1009 -$+ -!!! test_bang:3448 -!foo -!!! test_bang_cmd:3462 -!m foo -!!! test_begin_cmdarg:5658 -p begin 1.times do 1 end end -!!! test_beginless_erange_after_newline:949 -foo -...100 -!!! test_beginless_irange_after_newline:937 -foo -..100 -!!! test_beginless_range:917 -..100 -!!! test_beginless_range:926 -...100 -!!! test_block_arg_combinations:2531 -f{ } -!!! test_block_arg_combinations:2537 -f{ | | } -!!! test_block_arg_combinations:2541 -f{ |;a| } -!!! test_block_arg_combinations:2546 -f{ |; -a -| } -!!! test_block_arg_combinations:2552 -f{ || } -!!! test_block_arg_combinations:2561 -f{ |a| } -!!! test_block_arg_combinations:2571 -f{ |a, c| } -!!! test_block_arg_combinations:2580 -f{ |a,| } -!!! test_block_arg_combinations:2585 -f{ |a, &b| } -!!! test_block_arg_combinations:2599 -f{ |a, *s, &b| } -!!! test_block_arg_combinations:2610 -f{ |a, *, &b| } -!!! test_block_arg_combinations:2621 -f{ |a, *s| } -!!! test_block_arg_combinations:2631 -f{ |a, *| } -!!! test_block_arg_combinations:2640 -f{ |*s, &b| } -!!! test_block_arg_combinations:2651 -f{ |*, &b| } -!!! test_block_arg_combinations:2662 -f{ |*s| } -!!! test_block_arg_combinations:2672 -f{ |*| } -!!! test_block_arg_combinations:2678 -f{ |&b| } -!!! test_block_arg_combinations:2689 -f{ |a, o=1, o1=2, *r, &b| } -!!! test_block_arg_combinations:2700 -f{ |a, o=1, *r, p, &b| } -!!! test_block_arg_combinations:2711 -f{ |a, o=1, &b| } -!!! test_block_arg_combinations:2720 -f{ |a, o=1, p, &b| } -!!! test_block_arg_combinations:2730 -f{ |a, *r, p, &b| } -!!! test_block_arg_combinations:2740 -f{ |o=1, *r, &b| } -!!! test_block_arg_combinations:2749 -f{ |o=1, *r, p, &b| } -!!! test_block_arg_combinations:2759 -f{ |o=1, &b| } -!!! test_block_arg_combinations:2767 -f{ |o=1, p, &b| } -!!! test_block_arg_combinations:2776 -f{ |*r, p, &b| } -!!! test_block_kwarg:2867 -f{ |foo:| } -!!! test_block_kwarg_combinations:2840 -f{ |foo: 1, bar: 2, **baz, &b| } -!!! test_block_kwarg_combinations:2850 -f{ |foo: 1, &b| } -!!! test_block_kwarg_combinations:2858 -f{ |**baz, &b| } -!!! test_blockarg:2201 -def f(&block); end -!!! test_break:5169 -break(foo) -!!! test_break:5183 -break foo -!!! test_break:5189 -break() -!!! test_break:5196 -break -!!! test_break_block:5204 -break fun foo do end -!!! test_bug_435:7252 -"#{-> foo {}}" -!!! test_bug_447:7231 -m [] do end -!!! test_bug_447:7240 -m [], 1 do end -!!! test_bug_452:7265 -td (1_500).toString(); td.num do; end -!!! test_bug_466:7281 -foo "#{(1+1).to_i}" do; end -!!! test_bug_473:7298 -m "#{[]}" -!!! test_bug_480:7309 -m "#{}#{()}" -!!! test_bug_481:7321 -m def x(); end; 1.tap do end -!!! test_bug_ascii_8bit_in_literal:6031 -# coding:utf-8 - "\xD0\xBF\xD1\x80\xD0\xBE\xD0\xB2\xD0\xB5\xD1\x80\xD0\xBA\xD0\xB0" -!!! test_bug_cmd_string_lookahead:5903 -desc "foo" do end -!!! test_bug_cmdarg:5681 -assert dogs -!!! test_bug_cmdarg:5686 -assert do: true -!!! test_bug_cmdarg:5694 -f x: -> do meth do end end -!!! test_bug_def_no_paren_eql_begin:5950 -def foo -=begin -=end -end -!!! test_bug_do_block_in_call_args:5913 -bar def foo; self.each do end end -!!! test_bug_do_block_in_cmdarg:5928 -tap (proc do end) -!!! test_bug_do_block_in_hash_brace:6720 -p :foo, {a: proc do end, b: proc do end} -!!! test_bug_do_block_in_hash_brace:6738 -p :foo, {:a => proc do end, b: proc do end} -!!! test_bug_do_block_in_hash_brace:6756 -p :foo, {"a": proc do end, b: proc do end} -!!! test_bug_do_block_in_hash_brace:6774 -p :foo, {proc do end => proc do end, b: proc do end} -!!! test_bug_do_block_in_hash_brace:6794 -p :foo, {** proc do end, b: proc do end} -!!! test_bug_heredoc_do:5986 -f <<-TABLE do -TABLE -end -!!! test_bug_interp_single:5940 -"#{1}" -!!! test_bug_interp_single:5944 -%W"#{1}" -!!! test_bug_lambda_leakage:6701 -->(scope) {}; scope -!!! test_bug_regex_verification:6714 -/#)/x -!!! test_bug_rescue_empty_else:5964 -begin; rescue LoadError; else; end -!!! test_bug_while_not_parens_do:5956 -while not (true) do end -!!! test_case_cond:4976 -case; when foo; 'foo'; end -!!! test_case_cond_else:4989 -case; when foo; 'foo'; else 'bar'; end -!!! test_case_expr:4948 -case foo; when 'bar'; bar; end -!!! test_case_expr_else:4962 -case foo; when 'bar'; bar; else baz; end -!!! test_casgn_scoped:1206 -Bar::Foo = 10 -!!! test_casgn_toplevel:1195 -::Foo = 10 -!!! test_casgn_unscoped:1217 -Foo = 10 -!!! test_character:250 -?a -!!! test_class:1841 -class Foo; end -!!! test_class:1851 -class Foo end -!!! test_class_definition_in_while_cond:7055 -while class Foo; tap do end; end; break; end -!!! test_class_definition_in_while_cond:7067 -while class Foo a = tap do end; end; break; end -!!! test_class_definition_in_while_cond:7080 -while class << self; tap do end; end; break; end -!!! test_class_definition_in_while_cond:7092 -while class << self; a = tap do end; end; break; end -!!! test_class_super:1862 -class Foo < Bar; end -!!! test_class_super_label:1874 -class Foo < a:b; end -!!! test_comments_before_leading_dot__27:7941 -a # -# -.foo -!!! test_comments_before_leading_dot__27:7948 -a # - # -.foo -!!! test_comments_before_leading_dot__27:7955 -a # -# -&.foo -!!! test_comments_before_leading_dot__27:7962 -a # - # -&.foo -!!! test_complex:158 -42i -!!! test_complex:164 -42ri -!!! test_complex:170 -42.1i -!!! test_complex:176 -42.1ri -!!! test_cond_begin:4746 -if (bar); foo; end -!!! test_cond_begin_masgn:4755 -if (bar; a, b = foo); end -!!! test_cond_eflipflop:4854 -if foo...bar; end -!!! test_cond_eflipflop:4884 -!(foo...bar) -!!! test_cond_eflipflop_with_beginless_range:4903 -if ...bar; end -!!! test_cond_eflipflop_with_endless_range:4893 -if foo...; end -!!! test_cond_iflipflop:4795 -if foo..bar; end -!!! test_cond_iflipflop:4825 -!(foo..bar) -!!! test_cond_iflipflop_with_beginless_range:4844 -if ..bar; end -!!! test_cond_iflipflop_with_endless_range:4834 -if foo..; end -!!! test_cond_match_current_line:4913 -if /wat/; end -!!! test_cond_match_current_line:4933 -!/wat/ -!!! test_const_op_asgn:1550 -A += 1 -!!! test_const_op_asgn:1556 -::A += 1 -!!! test_const_op_asgn:1564 -B::A += 1 -!!! test_const_op_asgn:1572 -def x; self::A ||= 1; end -!!! test_const_op_asgn:1581 -def x; ::A ||= 1; end -!!! test_const_scoped:1034 -Bar::Foo -!!! test_const_toplevel:1025 -::Foo -!!! test_const_unscoped:1043 -Foo -!!! test_control_meta_escape_chars_in_regexp__since_31:11030 -/\c\xFF/ -!!! test_control_meta_escape_chars_in_regexp__since_31:11036 -/\c\M-\xFF/ -!!! test_control_meta_escape_chars_in_regexp__since_31:11042 -/\C-\xFF/ -!!! test_control_meta_escape_chars_in_regexp__since_31:11048 -/\C-\M-\xFF/ -!!! test_control_meta_escape_chars_in_regexp__since_31:11054 -/\M-\xFF/ -!!! test_control_meta_escape_chars_in_regexp__since_31:11060 -/\M-\C-\xFF/ -!!! test_control_meta_escape_chars_in_regexp__since_31:11066 -/\M-\c\xFF/ -!!! test_cpath:1821 -module ::Foo; end -!!! test_cpath:1827 -module Bar::Foo; end -!!! test_cvar:987 -@@foo -!!! test_cvasgn:1120 -@@var = 10 -!!! test_dedenting_heredoc:299 -p <<~E -E -!!! test_dedenting_heredoc:306 -p <<~E - E -!!! test_dedenting_heredoc:313 -p <<~E - x -E -!!! test_dedenting_heredoc:320 -p <<~E - ð -E -!!! test_dedenting_heredoc:327 -p <<~E - x - y -E -!!! test_dedenting_heredoc:336 -p <<~E - x - y -E -!!! test_dedenting_heredoc:345 -p <<~E - x - y -E -!!! test_dedenting_heredoc:354 -p <<~E - x - y -E -!!! test_dedenting_heredoc:363 -p <<~E - x - y -E -!!! test_dedenting_heredoc:372 -p <<~E - x - -y -E -!!! test_dedenting_heredoc:382 -p <<~E - x - - y -E -!!! test_dedenting_heredoc:392 -p <<~E - x - \ y -E -!!! test_dedenting_heredoc:401 -p <<~E - x - \ y -E -!!! test_dedenting_heredoc:410 -p <<~"E" - x - #{foo} -E -!!! test_dedenting_heredoc:421 -p <<~`E` - x - #{foo} -E -!!! test_dedenting_heredoc:432 -p <<~"E" - x - #{" y"} -E -!!! test_dedenting_interpolating_heredoc_fake_line_continuation:461 -<<~'FOO' - baz\\ - qux -FOO -!!! test_dedenting_non_interpolating_heredoc_line_continuation:453 -<<~'FOO' - baz\ - qux -FOO -!!! test_def:1913 -def foo; end -!!! test_def:1921 -def String; end -!!! test_def:1925 -def String=; end -!!! test_def:1929 -def until; end -!!! test_def:1933 -def BEGIN; end -!!! test_def:1937 -def END; end -!!! test_defined:1072 -defined? foo -!!! test_defined:1078 -defined?(foo) -!!! test_defined:1086 -defined? @foo -!!! test_defs:1943 -def self.foo; end -!!! test_defs:1951 -def self::foo; end -!!! test_defs:1959 -def (foo).foo; end -!!! test_defs:1963 -def String.foo; end -!!! test_defs:1968 -def String::foo; end -!!! test_emit_arg_inside_procarg0_legacy:2807 -f{ |a| } -!!! test_empty_stmt:62 -!!! test_endless_comparison_method:10736:0 -def ===(other) = do_something -!!! test_endless_comparison_method:10736:1 -def ==(other) = do_something -!!! test_endless_comparison_method:10736:2 -def !=(other) = do_something -!!! test_endless_comparison_method:10736:3 -def <=(other) = do_something -!!! test_endless_comparison_method:10736:4 -def >=(other) = do_something -!!! test_endless_comparison_method:10736:5 -def !=(other) = do_something -!!! test_endless_method:10085 -def foo() = 42 -!!! test_endless_method:10097 -def inc(x) = x + 1 -!!! test_endless_method:10110 -def obj.foo() = 42 -!!! test_endless_method:10122 -def obj.inc(x) = x + 1 -!!! test_endless_method_command_syntax:10179 -def foo = puts "Hello" -!!! test_endless_method_command_syntax:10191 -def foo() = puts "Hello" -!!! test_endless_method_command_syntax:10203 -def foo(x) = puts x -!!! test_endless_method_command_syntax:10216 -def obj.foo = puts "Hello" -!!! test_endless_method_command_syntax:10230 -def obj.foo() = puts "Hello" -!!! test_endless_method_command_syntax:10244 -def rescued(x) = raise "to be caught" rescue "instance #{x}" -!!! test_endless_method_command_syntax:10263 -def self.rescued(x) = raise "to be caught" rescue "class #{x}" -!!! test_endless_method_command_syntax:10284 -def obj.foo(x) = puts x -!!! test_endless_method_forwarded_args_legacy:10139 -def foo(...) = bar(...) -!!! test_endless_method_with_rescue_mod:10154 -def m() = 1 rescue 2 -!!! test_endless_method_with_rescue_mod:10165 -def self.m() = 1 rescue 2 -!!! test_endless_method_without_args:10748 -def foo = 42 -!!! test_endless_method_without_args:10756 -def foo = 42 rescue nil -!!! test_endless_method_without_args:10767 -def self.foo = 42 -!!! test_endless_method_without_args:10776 -def self.foo = 42 rescue nil -!!! test_ensure:5393 -begin; meth; ensure; bar; end -!!! test_ensure_empty:5406 -begin ensure end -!!! test_false:98 -false -!!! test_find_pattern:10447 -case foo; in [*x, 1 => a, *y] then true; end -!!! test_find_pattern:10467 -case foo; in String(*, 1, *) then true; end -!!! test_find_pattern:10481 -case foo; in Array[*, 1, *] then true; end -!!! test_find_pattern:10495 -case foo; in *, 42, * then true; end -!!! test_float:131 -1.33 -!!! test_float:136 --1.33 -!!! test_for:5134 -for a in foo do p a; end -!!! test_for:5146 -for a in foo; p a; end -!!! test_for_mlhs:5155 -for a, b in foo; p a, b; end -!!! test_forward_arg:8090 -def foo(...); bar(...); end -!!! test_forward_arg_with_open_args:11089 -def foo ... -end -!!! test_forward_arg_with_open_args:11096 -def foo a, b = 1, ... -end -!!! test_forward_arg_with_open_args:11114 -def foo(a, ...) bar(...) end -!!! test_forward_arg_with_open_args:11125 -def foo a, ... - bar(...) -end -!!! test_forward_arg_with_open_args:11136 -def foo b = 1, ... - bar(...) -end -!!! test_forward_arg_with_open_args:11148 -def foo ...; bar(...); end -!!! test_forward_arg_with_open_args:11158 -def foo a, ...; bar(...); end -!!! test_forward_arg_with_open_args:11169 -def foo b = 1, ...; bar(...); end -!!! test_forward_arg_with_open_args:11181 -(def foo ... - bar(...) -end) -!!! test_forward_arg_with_open_args:11192 -(def foo ...; bar(...); end) -!!! test_forward_args_legacy:8054 -def foo(...); bar(...); end -!!! test_forward_args_legacy:8066 -def foo(...); super(...); end -!!! test_forward_args_legacy:8078 -def foo(...); end -!!! test_forwarded_argument_with_kwrestarg:11332 -def foo(argument, **); bar(argument, **); end -!!! test_forwarded_argument_with_restarg:11267 -def foo(argument, *); bar(argument, *); end -!!! test_forwarded_kwrestarg:11287 -def foo(**); bar(**); end -!!! test_forwarded_kwrestarg_with_additional_kwarg:11306 -def foo(**); bar(**, from_foo: true); end -!!! test_forwarded_restarg:11249 -def foo(*); bar(*); end -!!! test_gvar:994 -$foo -!!! test_gvasgn:1130 -$var = 10 -!!! test_hash_empty:764 -{ } -!!! test_hash_hashrocket:773 -{ 1 => 2 } -!!! test_hash_hashrocket:782 -{ 1 => 2, :foo => "bar" } -!!! test_hash_kwsplat:835 -{ foo: 2, **bar } -!!! test_hash_label:790 -{ foo: 2 } -!!! test_hash_label_end:803 -{ 'foo': 2 } -!!! test_hash_label_end:816 -{ 'foo': 2, 'bar': {}} -!!! test_hash_label_end:824 -f(a ? "a":1) -!!! test_hash_pair_value_omission:10339 -{a:, b:} -!!! test_hash_pair_value_omission:10353 -{puts:} -!!! test_hash_pair_value_omission:10364 -foo = 1; {foo:} -!!! test_hash_pair_value_omission:10376 -_foo = 1; {_foo:} -!!! test_hash_pair_value_omission:10388 -{BAR:} -!!! test_heredoc:265 -<(**nil) {} -!!! test_kwoptarg:2138 -def f(foo: 1); end -!!! test_kwoptarg_with_kwrestarg_and_forwarded_args:11482 -def f(a: nil, **); b(**) end -!!! test_kwrestarg_named:2149 -def f(**foo); end -!!! test_kwrestarg_unnamed:2160 -def f(**); end -!!! test_lbrace_arg_after_command_args:7420 -let (:a) { m do; end } -!!! test_lparenarg_after_lvar__since_25:6830 -meth (-1.3).abs -!!! test_lparenarg_after_lvar__since_25:6839 -foo (-1.3).abs -!!! test_lvar:973 -foo -!!! test_lvar_injecting_match:3819 -/(?bar)/ =~ 'bar'; match -!!! test_lvasgn:1098 -var = 10; var -!!! test_marg_combinations:2454 -def f (((a))); end -!!! test_marg_combinations:2460 -def f ((a, a1)); end -!!! test_marg_combinations:2465 -def f ((a, *r)); end -!!! test_marg_combinations:2470 -def f ((a, *r, p)); end -!!! test_marg_combinations:2475 -def f ((a, *)); end -!!! test_marg_combinations:2480 -def f ((a, *, p)); end -!!! test_marg_combinations:2485 -def f ((*r)); end -!!! test_marg_combinations:2490 -def f ((*r, p)); end -!!! test_marg_combinations:2495 -def f ((*)); end -!!! test_marg_combinations:2500 -def f ((*, p)); end -!!! test_masgn:1261 -foo, bar = 1, 2 -!!! test_masgn:1272 -(foo, bar) = 1, 2 -!!! test_masgn:1282 -foo, bar, baz = 1, 2 -!!! test_masgn_attr:1404 -self.a, self[1, 2] = foo -!!! test_masgn_attr:1417 -self::a, foo = foo -!!! test_masgn_attr:1425 -self.A, foo = foo -!!! test_masgn_cmd:1453 -foo, bar = m foo -!!! test_masgn_const:1435 -self::A, foo = foo -!!! test_masgn_const:1443 -::A, foo = foo -!!! test_masgn_nested:1379 -a, (b, c) = foo -!!! test_masgn_nested:1393 -((b, )) = foo -!!! test_masgn_splat:1293 -@foo, @@bar = *foo -!!! test_masgn_splat:1302 -a, b = *foo, bar -!!! test_masgn_splat:1310 -a, *b = bar -!!! test_masgn_splat:1316 -a, *b, c = bar -!!! test_masgn_splat:1327 -a, * = bar -!!! test_masgn_splat:1333 -a, *, c = bar -!!! test_masgn_splat:1344 -*b = bar -!!! test_masgn_splat:1350 -*b, c = bar -!!! test_masgn_splat:1360 -* = bar -!!! test_masgn_splat:1366 -*, c, d = bar -!!! test_method_definition_in_while_cond:7001 -while def foo; tap do end; end; break; end -!!! test_method_definition_in_while_cond:7013 -while def self.foo; tap do end; end; break; end -!!! test_method_definition_in_while_cond:7026 -while def foo a = tap do end; end; break; end -!!! test_method_definition_in_while_cond:7039 -while def self.foo a = tap do end; end; break; end -!!! test_module:1803 -module Foo; end -!!! test_multiple_args_with_trailing_comma:2786 -f{ |a, b,| } -!!! test_multiple_pattern_matches:11456 -{a: 0} => a: -{a: 0} => a: -!!! test_multiple_pattern_matches:11472 -{a: 0} in a: -{a: 0} in a: -!!! test_newline_in_hash_argument:11405 -obj.set foo: -1 -!!! test_newline_in_hash_argument:11416 -obj.set "foo": -1 -!!! test_newline_in_hash_argument:11427 -case foo -in a: -0 -true -in "b": -0 -true -end -!!! test_next:5263 -next(foo) -!!! test_next:5277 -next foo -!!! test_next:5283 -next() -!!! test_next:5290 -next -!!! test_next_block:5298 -next fun foo do end -!!! test_nil:68 -nil -!!! test_nil_expression:75 -() -!!! test_nil_expression:82 -begin end -!!! test_non_lvar_injecting_match:3853 -/#{1}(?bar)/ =~ 'bar' -!!! test_not:3476 -not foo -!!! test_not:3482 -not(foo) -!!! test_not:3488 -not() -!!! test_not_cmd:3502 -not m foo -!!! test_not_masgn__24:4732 -!(a, b = foo) -!!! test_nth_ref:1016 -$10 -!!! test_numbered_args_after_27:7543 -m { _1 + _9 } -!!! test_numbered_args_after_27:7558 -m do _1 + _9 end -!!! test_numbered_args_after_27:7575 --> { _1 + _9} -!!! test_numbered_args_after_27:7590 --> do _1 + _9 end -!!! test_numparam_outside_block:7697 -class A; _1; end -!!! test_numparam_outside_block:7705 -module A; _1; end -!!! test_numparam_outside_block:7713 -class << foo; _1; end -!!! test_numparam_outside_block:7721 -def self.m; _1; end -!!! test_numparam_outside_block:7730 -_1 -!!! test_numparam_ruby_bug_19025:10696 -p { [_1 **2] } -!!! test_op_asgn:1620 -foo.a += 1 -!!! test_op_asgn:1630 -foo::a += 1 -!!! test_op_asgn:1636 -foo.A += 1 -!!! test_op_asgn_cmd:1644 -foo.a += m foo -!!! test_op_asgn_cmd:1650 -foo::a += m foo -!!! test_op_asgn_cmd:1656 -foo.A += m foo -!!! test_op_asgn_cmd:1668 -foo::A += m foo -!!! test_op_asgn_index:1678 -foo[0, 1] += 2 -!!! test_op_asgn_index_cmd:1692 -foo[0, 1] += m foo -!!! test_optarg:2088 -def f foo = 1; end -!!! test_optarg:2098 -def f(foo=1, bar=2); end -!!! test_or:4521 -foo or bar -!!! test_or:4527 -foo || bar -!!! test_or_asgn:1738 -foo.a ||= 1 -!!! test_or_asgn:1748 -foo[0, 1] ||= 2 -!!! test_parser_bug_272:6679 -a @b do |c|;end -!!! test_parser_bug_490:7336 -def m; class << self; class C; end; end; end -!!! test_parser_bug_490:7347 -def m; class << self; module M; end; end; end -!!! test_parser_bug_490:7358 -def m; class << self; A = nil; end; end -!!! test_parser_bug_507:7450 -m = -> *args do end -!!! test_parser_bug_518:7462 -class A < B -end -!!! test_parser_bug_525:7472 -m1 :k => m2 do; m3() do end; end -!!! test_parser_bug_604:7928 -m a + b do end -!!! test_parser_bug_640:445 -<<~FOO - baz\ - qux -FOO -!!! test_parser_bug_645:10073 --> (arg={}) {} -!!! test_parser_bug_830:10974 -/\(/ -!!! test_parser_bug_989:11684 - <<-HERE - content - HERE -!!! test_parser_drops_truncated_parts_of_squiggly_heredoc:10790 -<<~HERE - #{} -HERE -!!! test_parser_slash_slash_n_escaping_in_literals:7512:0 -'a\ -b' -!!! test_parser_slash_slash_n_escaping_in_literals:7512:1 -<<-'HERE' -a\ -b -HERE -!!! test_parser_slash_slash_n_escaping_in_literals:7512:2 -%q{a\ -b} -!!! test_parser_slash_slash_n_escaping_in_literals:7512:3 -"a\ -b" -!!! test_parser_slash_slash_n_escaping_in_literals:7512:4 -<<-"HERE" -a\ -b -HERE -!!! test_parser_slash_slash_n_escaping_in_literals:7512:5 -%{a\ -b} -!!! test_parser_slash_slash_n_escaping_in_literals:7512:6 -%Q{a\ -b} -!!! test_parser_slash_slash_n_escaping_in_literals:7512:7 -%w{a\ -b} -!!! test_parser_slash_slash_n_escaping_in_literals:7512:8 -%W{a\ -b} -!!! test_parser_slash_slash_n_escaping_in_literals:7512:9 -%i{a\ -b} -!!! test_parser_slash_slash_n_escaping_in_literals:7512:10 -%I{a\ -b} -!!! test_parser_slash_slash_n_escaping_in_literals:7512:11 -:'a\ -b' -!!! test_parser_slash_slash_n_escaping_in_literals:7512:12 -%s{a\ -b} -!!! test_parser_slash_slash_n_escaping_in_literals:7512:13 -:"a\ -b" -!!! test_parser_slash_slash_n_escaping_in_literals:7512:14 -/a\ -b/ -!!! test_parser_slash_slash_n_escaping_in_literals:7512:15 -%r{a\ -b} -!!! test_parser_slash_slash_n_escaping_in_literals:7512:16 -%x{a\ -b} -!!! test_parser_slash_slash_n_escaping_in_literals:7512:17 -`a\ -b` -!!! test_parser_slash_slash_n_escaping_in_literals:7512:18 -<<-`HERE` -a\ -b -HERE -!!! test_pattern_matching__FILE__LINE_literals:9760 - case [__FILE__, __LINE__ + 1, __ENCODING__] - in [__FILE__, __LINE__, __ENCODING__] - end -!!! test_pattern_matching_blank_else:9627 -case 1; in 2; 3; else; end -!!! test_pattern_matching_const_pattern:9490 -case foo; in A(1, 2) then true; end -!!! test_pattern_matching_const_pattern:9507 -case foo; in A(x:) then true; end -!!! test_pattern_matching_const_pattern:9523 -case foo; in A() then true; end -!!! test_pattern_matching_const_pattern:9538 -case foo; in A[1, 2] then true; end -!!! test_pattern_matching_const_pattern:9555 -case foo; in A[x:] then true; end -!!! test_pattern_matching_const_pattern:9571 -case foo; in A[] then true; end -!!! test_pattern_matching_constants:9456 -case foo; in A then true; end -!!! test_pattern_matching_constants:9466 -case foo; in A::B then true; end -!!! test_pattern_matching_constants:9477 -case foo; in ::A then true; end -!!! test_pattern_matching_else:9613 -case 1; in 2; 3; else; 4; end -!!! test_pattern_matching_explicit_array_match:8891 -case foo; in [x] then nil; end -!!! test_pattern_matching_explicit_array_match:8903 -case foo; in [x,] then nil; end -!!! test_pattern_matching_explicit_array_match:8915 -case foo; in [x, y] then true; end -!!! test_pattern_matching_explicit_array_match:8928 -case foo; in [x, y,] then true; end -!!! test_pattern_matching_explicit_array_match:8941 -case foo; in [x, y, *] then true; end -!!! test_pattern_matching_explicit_array_match:8955 -case foo; in [x, y, *z] then true; end -!!! test_pattern_matching_explicit_array_match:8969 -case foo; in [x, *y, z] then true; end -!!! test_pattern_matching_explicit_array_match:8983 -case foo; in [x, *, y] then true; end -!!! test_pattern_matching_explicit_array_match:8997 -case foo; in [*x, y] then true; end -!!! test_pattern_matching_explicit_array_match:9010 -case foo; in [*, x] then true; end -!!! test_pattern_matching_expr_in_paren:9443 -case foo; in (1) then true; end -!!! test_pattern_matching_hash:9025 -case foo; in {} then true; end -!!! test_pattern_matching_hash:9034 -case foo; in a: 1 then true; end -!!! test_pattern_matching_hash:9044 -case foo; in { a: 1 } then true; end -!!! test_pattern_matching_hash:9056 -case foo; in { a: 1, } then true; end -!!! test_pattern_matching_hash:9068 -case foo; in a: then true; end -!!! test_pattern_matching_hash:9080 -case foo; in **a then true; end -!!! test_pattern_matching_hash:9094 -case foo; in ** then true; end -!!! test_pattern_matching_hash:9106 -case foo; in a: 1, b: 2 then true; end -!!! test_pattern_matching_hash:9117 -case foo; in a:, b: then true; end -!!! test_pattern_matching_hash:9128 -case foo; in a: 1, _a:, ** then true; end -!!! test_pattern_matching_hash:9140 -case foo; - in {a: 1 - } - false - ; end -!!! test_pattern_matching_hash:9156 -case foo; - in {a: - 2} - false - ; end -!!! test_pattern_matching_hash:9171 -case foo; - in {Foo: 42 - } - false - ; end -!!! test_pattern_matching_hash:9186 -case foo; - in a: {b:}, c: - p c - ; end -!!! test_pattern_matching_hash:9203 -case foo; - in {a: - } - true - ; end -!!! test_pattern_matching_hash_with_string_keys:9242 -case foo; in "a": then true; end -!!! test_pattern_matching_hash_with_string_keys:9253 -case foo; in "#{ 'a' }": then true; end -!!! test_pattern_matching_hash_with_string_keys:9264 -case foo; in "#{ %q{a} }": then true; end -!!! test_pattern_matching_hash_with_string_keys:9275 -case foo; in "#{ %Q{a} }": then true; end -!!! test_pattern_matching_hash_with_string_keys:9288 -case foo; in "a": 1 then true; end -!!! test_pattern_matching_hash_with_string_keys:9297 -case foo; in "#{ 'a' }": 1 then true; end -!!! test_pattern_matching_hash_with_string_keys:9308 -case foo; in "#{ %q{a} }": 1 then true; end -!!! test_pattern_matching_hash_with_string_keys:9319 -case foo; in "#{ %Q{a} }": 1 then true; end -!!! test_pattern_matching_if_unless_modifiers:8753 -case foo; in x if true; nil; end -!!! test_pattern_matching_if_unless_modifiers:8767 -case foo; in x unless true; nil; end -!!! test_pattern_matching_implicit_array_match:8796 -case foo; in x, then nil; end -!!! test_pattern_matching_implicit_array_match:8806 -case foo; in *x then nil; end -!!! test_pattern_matching_implicit_array_match:8819 -case foo; in * then nil; end -!!! test_pattern_matching_implicit_array_match:8830 -case foo; in x, y then nil; end -!!! test_pattern_matching_implicit_array_match:8841 -case foo; in x, y, then nil; end -!!! test_pattern_matching_implicit_array_match:8852 -case foo; in x, *y, z then nil; end -!!! test_pattern_matching_implicit_array_match:8864 -case foo; in *x, y, z then nil; end -!!! test_pattern_matching_implicit_array_match:8876 -case foo; in 1, "a", [], {} then nil; end -!!! test_pattern_matching_keyword_variable:9370 -case foo; in self then true; end -!!! test_pattern_matching_lambda:9380 -case foo; in ->{ 42 } then true; end -!!! test_pattern_matching_match_alt:9587 -case foo; in 1 | 2 then true; end -!!! test_pattern_matching_match_as:9599 -case foo; in 1 => a then true; end -!!! test_pattern_matching_nil_pattern:9783 -case foo; in **nil then true; end -!!! test_pattern_matching_no_body:8745 -case foo; in 1; end -!!! test_pattern_matching_numbered_parameter:9654 -1.then { 1 in ^_1 } -!!! test_pattern_matching_pin_variable:8783 -case foo; in ^foo then nil; end -!!! test_pattern_matching_ranges:9393 -case foo; in 1..2 then true; end -!!! test_pattern_matching_ranges:9401 -case foo; in 1.. then true; end -!!! test_pattern_matching_ranges:9409 -case foo; in ..2 then true; end -!!! test_pattern_matching_ranges:9417 -case foo; in 1...2 then true; end -!!! test_pattern_matching_ranges:9425 -case foo; in 1... then true; end -!!! test_pattern_matching_ranges:9433 -case foo; in ...2 then true; end -!!! test_pattern_matching_single_line:9827 -1 => [a]; a -!!! test_pattern_matching_single_line:9839 -1 in [a]; a -!!! test_pattern_matching_single_line_allowed_omission_of_parentheses:9853 -[1, 2] => a, b; a -!!! test_pattern_matching_single_line_allowed_omission_of_parentheses:9868 -{a: 1} => a:; a -!!! test_pattern_matching_single_line_allowed_omission_of_parentheses:9883 -[1, 2] in a, b; a -!!! test_pattern_matching_single_line_allowed_omission_of_parentheses:9898 -{a: 1} in a:; a -!!! test_pattern_matching_single_line_allowed_omission_of_parentheses:9913 -{key: :value} in key: value; value -!!! test_pattern_matching_single_line_allowed_omission_of_parentheses:9930 -{key: :value} => key: value; value -!!! test_pattern_matching_single_match:8730 -case foo; in x then x; end -!!! test_pin_expr:10800 -case foo; in ^(42) then nil; end -!!! test_pin_expr:10814 -case foo; in { foo: ^(42) } then nil; end -!!! test_pin_expr:10831 -case foo; in ^(0+0) then nil; end -!!! test_pin_expr:10847 -case foo; in ^@a; end -!!! test_pin_expr:10856 -case foo; in ^@@TestPatternMatching; end -!!! test_pin_expr:10865 -case foo; in ^$TestPatternMatching; end -!!! test_pin_expr:10874 -case foo; in ^(1 -); end -!!! test_postexe:5618 -END { 1 } -!!! test_preexe:5599 -BEGIN { 1 } -!!! test_procarg0:2817 -m { |foo| } -!!! test_procarg0:2826 -m { |(foo, bar)| } -!!! test_procarg0_legacy:2796 -f{ |a| } -!!! test_range_endless:883 -1.. -!!! test_range_endless:891 -1... -!!! test_range_exclusive:875 -1...2 -!!! test_range_inclusive:867 -1..2 -!!! test_rational:144 -42r -!!! test_rational:150 -42.1r -!!! test_redo:5310 -redo -!!! test_regex_interp:553 -/foo#{bar}baz/ -!!! test_regex_plain:543 -/source/im -!!! test_resbody_list:5530 -begin; meth; rescue Exception; bar; end -!!! test_resbody_list_mrhs:5543 -begin; meth; rescue Exception, foo; bar; end -!!! test_resbody_list_var:5576 -begin; meth; rescue foo => ex; bar; end -!!! test_resbody_var:5558 -begin; meth; rescue => ex; bar; end -!!! test_resbody_var:5566 -begin; meth; rescue => @ex; bar; end -!!! test_rescue:5320 -begin; meth; rescue; foo; end -!!! test_rescue_else:5335 -begin; meth; rescue; foo; else; bar; end -!!! test_rescue_else_ensure:5434 -begin; meth; rescue; baz; else foo; ensure; bar end -!!! test_rescue_ensure:5418 -begin; meth; rescue; baz; ensure; bar; end -!!! test_rescue_in_lambda_block:7113 --> do rescue; end -!!! test_rescue_mod:5451 -meth rescue bar -!!! test_rescue_mod_asgn:5463 -foo = meth rescue bar -!!! test_rescue_mod_masgn:5477 -foo, bar = meth rescue [1, 2] -!!! test_rescue_mod_op_assign:5497 -foo += meth rescue bar -!!! test_rescue_without_begin_end:5513 -meth do; foo; rescue; bar; end -!!! test_restarg_named:2108 -def f(*foo); end -!!! test_restarg_unnamed:2118 -def f(*); end -!!! test_retry:5589 -retry -!!! test_return:5216 -return(foo) -!!! test_return:5230 -return foo -!!! test_return:5236 -return() -!!! test_return:5243 -return -!!! test_return_block:5251 -return fun foo do end -!!! test_ruby_bug_10279:6056 -{a: if true then 42 end} -!!! test_ruby_bug_10653:6066 -true ? 1.tap do |n| p n end : 0 -!!! test_ruby_bug_10653:6096 -false ? raise {} : tap {} -!!! test_ruby_bug_10653:6109 -false ? raise do end : tap do end -!!! test_ruby_bug_11107:6124 -p ->() do a() do end end -!!! test_ruby_bug_11380:6136 -p -> { :hello }, a: 1 do end -!!! test_ruby_bug_11873:6504 -a b{c d}, "x" do end -!!! test_ruby_bug_11873:6518 -a b(c d), "x" do end -!!! test_ruby_bug_11873:6531 -a b{c(d)}, "x" do end -!!! test_ruby_bug_11873:6545 -a b(c(d)), "x" do end -!!! test_ruby_bug_11873:6558 -a b{c d}, /x/ do end -!!! test_ruby_bug_11873:6572 -a b(c d), /x/ do end -!!! test_ruby_bug_11873:6585 -a b{c(d)}, /x/ do end -!!! test_ruby_bug_11873:6599 -a b(c(d)), /x/ do end -!!! test_ruby_bug_11873:6612 -a b{c d}, /x/m do end -!!! test_ruby_bug_11873:6626 -a b(c d), /x/m do end -!!! test_ruby_bug_11873:6639 -a b{c(d)}, /x/m do end -!!! test_ruby_bug_11873:6653 -a b(c(d)), /x/m do end -!!! test_ruby_bug_11873_a:6168:0 -a b{c d}, :e do end -!!! test_ruby_bug_11873_a:6168:1 -a b{c d}, 1 do end -!!! test_ruby_bug_11873_a:6168:2 -a b{c d}, 1.0 do end -!!! test_ruby_bug_11873_a:6168:3 -a b{c d}, 1.0r do end -!!! test_ruby_bug_11873_a:6168:4 -a b{c d}, 1.0i do end -!!! test_ruby_bug_11873_a:6173:0 -a b{c(d)}, :e do end -!!! test_ruby_bug_11873_a:6173:1 -a b{c(d)}, 1 do end -!!! test_ruby_bug_11873_a:6173:2 -a b{c(d)}, 1.0 do end -!!! test_ruby_bug_11873_a:6173:3 -a b{c(d)}, 1.0r do end -!!! test_ruby_bug_11873_a:6173:4 -a b{c(d)}, 1.0i do end -!!! test_ruby_bug_11873_a:6187:0 -a b(c d), :e do end -!!! test_ruby_bug_11873_a:6187:1 -a b(c d), 1 do end -!!! test_ruby_bug_11873_a:6187:2 -a b(c d), 1.0 do end -!!! test_ruby_bug_11873_a:6187:3 -a b(c d), 1.0r do end -!!! test_ruby_bug_11873_a:6187:4 -a b(c d), 1.0i do end -!!! test_ruby_bug_11873_a:6192:0 -a b(c(d)), :e do end -!!! test_ruby_bug_11873_a:6192:1 -a b(c(d)), 1 do end -!!! test_ruby_bug_11873_a:6192:2 -a b(c(d)), 1.0 do end -!!! test_ruby_bug_11873_a:6192:3 -a b(c(d)), 1.0r do end -!!! test_ruby_bug_11873_a:6192:4 -a b(c(d)), 1.0i do end -!!! test_ruby_bug_11873_b:6201 -p p{p(p);p p}, tap do end -!!! test_ruby_bug_11989:6220 -p <<~"E" - x\n y -E -!!! test_ruby_bug_11990:6229 -p <<~E " y" - x -E -!!! test_ruby_bug_12073:6240 -a = 1; a b: 1 -!!! test_ruby_bug_12073:6253 -def foo raise; raise A::B, ''; end -!!! test_ruby_bug_12402:6267 -foo = raise(bar) rescue nil -!!! test_ruby_bug_12402:6278 -foo += raise(bar) rescue nil -!!! test_ruby_bug_12402:6290 -foo[0] += raise(bar) rescue nil -!!! test_ruby_bug_12402:6304 -foo.m += raise(bar) rescue nil -!!! test_ruby_bug_12402:6317 -foo::m += raise(bar) rescue nil -!!! test_ruby_bug_12402:6330 -foo.C += raise(bar) rescue nil -!!! test_ruby_bug_12402:6343 -foo::C ||= raise(bar) rescue nil -!!! test_ruby_bug_12402:6356 -foo = raise bar rescue nil -!!! test_ruby_bug_12402:6367 -foo += raise bar rescue nil -!!! test_ruby_bug_12402:6379 -foo[0] += raise bar rescue nil -!!! test_ruby_bug_12402:6393 -foo.m += raise bar rescue nil -!!! test_ruby_bug_12402:6406 -foo::m += raise bar rescue nil -!!! test_ruby_bug_12402:6419 -foo.C += raise bar rescue nil -!!! test_ruby_bug_12402:6432 -foo::C ||= raise bar rescue nil -!!! test_ruby_bug_12669:6447 -a = b = raise :x -!!! test_ruby_bug_12669:6456 -a += b = raise :x -!!! test_ruby_bug_12669:6465 -a = b += raise :x -!!! test_ruby_bug_12669:6474 -a += b += raise :x -!!! test_ruby_bug_12686:6485 -f (g rescue nil) -!!! test_ruby_bug_13547:7203 -meth[] {} -!!! test_ruby_bug_14690:7435 -let () { m(a) do; end } -!!! test_ruby_bug_15789:7807 -m ->(a = ->{_1}) {a} -!!! test_ruby_bug_15789:7821 -m ->(a: ->{_1}) {a} -!!! test_ruby_bug_9669:6040 -def a b: -return -end -!!! test_ruby_bug_9669:6046 -o = { -a: -1 -} -!!! test_sclass:1898 -class << foo; nil; end -!!! test_self:966 -self -!!! test_send_attr_asgn:3542 -foo.a = 1 -!!! test_send_attr_asgn:3550 -foo::a = 1 -!!! test_send_attr_asgn:3558 -foo.A = 1 -!!! test_send_attr_asgn:3566 -foo::A = 1 -!!! test_send_attr_asgn_conditional:3792 -a&.b = 1 -!!! test_send_binary_op:3322 -foo + 1 -!!! test_send_binary_op:3328 -foo - 1 -!!! test_send_binary_op:3332 -foo * 1 -!!! test_send_binary_op:3336 -foo / 1 -!!! test_send_binary_op:3340 -foo % 1 -!!! test_send_binary_op:3344 -foo ** 1 -!!! test_send_binary_op:3348 -foo | 1 -!!! test_send_binary_op:3352 -foo ^ 1 -!!! test_send_binary_op:3356 -foo & 1 -!!! test_send_binary_op:3360 -foo <=> 1 -!!! test_send_binary_op:3364 -foo < 1 -!!! test_send_binary_op:3368 -foo <= 1 -!!! test_send_binary_op:3372 -foo > 1 -!!! test_send_binary_op:3376 -foo >= 1 -!!! test_send_binary_op:3380 -foo == 1 -!!! test_send_binary_op:3390 -foo != 1 -!!! test_send_binary_op:3396 -foo === 1 -!!! test_send_binary_op:3400 -foo =~ 1 -!!! test_send_binary_op:3410 -foo !~ 1 -!!! test_send_binary_op:3416 -foo << 1 -!!! test_send_binary_op:3420 -foo >> 1 -!!! test_send_block_chain_cmd:3215 -meth 1 do end.fun bar -!!! test_send_block_chain_cmd:3226 -meth 1 do end.fun(bar) -!!! test_send_block_chain_cmd:3239 -meth 1 do end::fun bar -!!! test_send_block_chain_cmd:3250 -meth 1 do end::fun(bar) -!!! test_send_block_chain_cmd:3263 -meth 1 do end.fun bar do end -!!! test_send_block_chain_cmd:3275 -meth 1 do end.fun(bar) {} -!!! test_send_block_chain_cmd:3287 -meth 1 do end.fun {} -!!! test_send_block_conditional:3800 -foo&.bar {} -!!! test_send_call:3762 -foo.(1) -!!! test_send_call:3772 -foo::(1) -!!! test_send_conditional:3784 -a&.b -!!! test_send_index:3576 -foo[1, 2] -!!! test_send_index_asgn:3605 -foo[1, 2] = 3 -!!! test_send_index_asgn_kwarg:3629 -foo[:kw => arg] = 3 -!!! test_send_index_asgn_kwarg_legacy:3642 -foo[:kw => arg] = 3 -!!! test_send_index_asgn_legacy:3617 -foo[1, 2] = 3 -!!! test_send_index_cmd:3598 -foo[m bar] -!!! test_send_index_legacy:3587 -foo[1, 2] -!!! test_send_lambda:3656 -->{ } -!!! test_send_lambda:3666 --> * { } -!!! test_send_lambda:3677 --> do end -!!! test_send_lambda_args:3689 -->(a) { } -!!! test_send_lambda_args:3703 --> (a) { } -!!! test_send_lambda_args_noparen:3727 --> a: 1 { } -!!! test_send_lambda_args_noparen:3736 --> a: { } -!!! test_send_lambda_args_shadow:3714 -->(a; foo, bar) { } -!!! test_send_lambda_legacy:3748 -->{ } -!!! test_send_op_asgn_conditional:3811 -a&.b &&= 1 -!!! test_send_plain:3119 -foo.fun -!!! test_send_plain:3126 -foo::fun -!!! test_send_plain:3133 -foo::Fun() -!!! test_send_plain_cmd:3142 -foo.fun bar -!!! test_send_plain_cmd:3149 -foo::fun bar -!!! test_send_plain_cmd:3156 -foo::Fun bar -!!! test_send_self:3058 -fun -!!! test_send_self:3064 -fun! -!!! test_send_self:3070 -fun(1) -!!! test_send_self_block:3080 -fun { } -!!! test_send_self_block:3084 -fun() { } -!!! test_send_self_block:3088 -fun(1) { } -!!! test_send_self_block:3092 -fun do end -!!! test_send_unary_op:3426 --foo -!!! test_send_unary_op:3432 -+foo -!!! test_send_unary_op:3436 -~foo -!!! test_slash_newline_in_heredocs:7371 -<<~E - 1 \ - 2 - 3 -E -!!! test_slash_newline_in_heredocs:7379 -<<-E - 1 \ - 2 - 3 -E -!!! test_space_args_arg:4192 -fun (1) -!!! test_space_args_arg_block:4206 -fun (1) {} -!!! test_space_args_arg_block:4220 -foo.fun (1) {} -!!! test_space_args_arg_block:4236 -foo::fun (1) {} -!!! test_space_args_arg_call:4258 -fun (1).to_i -!!! test_space_args_arg_newline:4198 -fun (1 -) -!!! test_space_args_block:4490 -fun () {} -!!! test_space_args_cmd:4185 -fun (f bar) -!!! test_string___FILE__:243 -__FILE__ -!!! test_string_concat:228 -"foo#@a" "bar" -!!! test_string_dvar:217 -"#@a #@@a #$a" -!!! test_string_interp:202 -"foo#{bar}baz" -!!! test_string_plain:186 -'foobar' -!!! test_string_plain:193 -%q(foobar) -!!! test_super:3867 -super(foo) -!!! test_super:3875 -super foo -!!! test_super:3881 -super() -!!! test_super_block:3899 -super foo, bar do end -!!! test_super_block:3905 -super do end -!!! test_symbol_interp:486 -:"foo#{bar}baz" -!!! test_symbol_plain:471 -:foo -!!! test_symbol_plain:477 -:'foo' -!!! test_ternary:4665 -foo ? 1 : 2 -!!! test_ternary_ambiguous_symbol:4674 -t=1;(foo)?t:T -!!! test_trailing_forward_arg:8237 -def foo(a, b, ...); bar(a, 42, ...); end -!!! test_true:91 -true -!!! test_unary_num_pow_precedence:3519 -+2.0 ** 10 -!!! test_unary_num_pow_precedence:3526 --2 ** 10 -!!! test_unary_num_pow_precedence:3533 --2.0 ** 10 -!!! test_undef:2017 -undef foo, :bar, :"foo#{1}" -!!! test_unless:4589 -unless foo then bar; end -!!! test_unless:4597 -unless foo; bar; end -!!! test_unless_else:4633 -unless foo then bar; else baz; end -!!! test_unless_else:4642 -unless foo; bar; else baz; end -!!! test_unless_mod:4606 -bar unless foo -!!! test_until:5080 -until foo do meth end -!!! test_until:5087 -until foo; meth end -!!! test_until_mod:5095 -meth until foo -!!! test_until_post:5110 -begin meth end until foo -!!! test_var_and_asgn:1728 -a &&= 1 -!!! test_var_op_asgn:1512 -a += 1 -!!! test_var_op_asgn:1518 -@a |= 1 -!!! test_var_op_asgn:1524 -@@var |= 10 -!!! test_var_op_asgn:1528 -def a; @@var |= 10; end -!!! test_var_op_asgn_cmd:1535 -foo += m foo -!!! test_var_or_asgn:1720 -a ||= 1 -!!! test_when_multi:5027 -case foo; when 'bar', 'baz'; bar; end -!!! test_when_splat:5036 -case foo; when 1, *baz; bar; when *foo; end -!!! test_when_then:5015 -case foo; when 'bar' then bar; end -!!! test_while:5056 -while foo do meth end -!!! test_while:5064 -while foo; meth end -!!! test_while_mod:5073 -meth while foo -!!! test_while_post:5102 -begin meth end while foo -!!! test_xstring_interp:526 -`foo#{bar}baz` -!!! test_xstring_plain:517 -`foobar` -!!! test_yield:3915 -yield(foo) -!!! test_yield:3923 -yield foo -!!! test_yield:3929 -yield() -!!! test_yield:3937 -yield -!!! test_zsuper:3891 -super diff --git a/test/translation/parser_test.rb b/test/translation/parser_test.rb deleted file mode 100644 index dd88322e..00000000 --- a/test/translation/parser_test.rb +++ /dev/null @@ -1,141 +0,0 @@ -# frozen_string_literal: true - -require_relative "../test_helper" -require "parser/current" - -Parser::Builders::Default.modernize - -module SyntaxTree - module Translation - class ParserTest < Minitest::Test - skips = %w[ - test_args_assocs_legacy:4041 - test_args_assocs:4091 - test_args_assocs:4091 - test_break_block:5204 - test_break:5169 - test_break:5183 - test_break:5189 - test_break:5196 - test_control_meta_escape_chars_in_regexp__since_31:* - test_dedenting_heredoc:336 - test_dedenting_heredoc:392 - test_dedenting_heredoc:401 - test_forwarded_argument_with_kwrestarg:11332 - test_forwarded_argument_with_restarg:11267 - test_forwarded_kwrestarg_with_additional_kwarg:11306 - test_forwarded_kwrestarg:11287 - test_forwarded_restarg:11249 - test_hash_pair_value_omission:10364 - test_hash_pair_value_omission:10376 - test_if_while_after_class__since_32:11374 - test_if_while_after_class__since_32:11384 - test_kwoptarg_with_kwrestarg_and_forwarded_args:11482 - test_lvar_injecting_match:3819 - test_newline_in_hash_argument:11427 - test_next_block:5298 - test_next:5263 - test_next:5277 - test_next:5283 - test_next:5290 - test_next:5290 - test_parser_slash_slash_n_escaping_in_literals:* - test_pattern_matching_explicit_array_match:8903 - test_pattern_matching_explicit_array_match:8928 - test_pattern_matching_expr_in_paren:9443 - test_pattern_matching_hash_with_string_keys:* - test_pattern_matching_hash_with_string_keys:9264 - test_pattern_matching_hash:9186 - test_pattern_matching_implicit_array_match:8796 - test_pattern_matching_implicit_array_match:8841 - test_pattern_matching_numbered_parameter:9654 - test_pattern_matching_single_line_allowed_omission_of_parentheses:9868 - test_pattern_matching_single_line_allowed_omission_of_parentheses:9898 - test_redo:5310 - test_retry:5589 - test_send_index_asgn_kwarg_legacy:3642 - test_send_index_asgn_kwarg_legacy:3642 - test_send_index_asgn_kwarg:3629 - test_send_index_asgn_kwarg:3629 - test_slash_newline_in_heredocs:7379 - test_unary_num_pow_precedence:3519 - test_yield:3915 - test_yield:3923 - test_yield:3929 - test_yield:3937 - ] - - if Gem::Version.new(RUBY_VERSION) < Gem::Version.new("3.1.0") - skips.push( - "test_endless_method_forwarded_args_legacy:10139", - "test_forward_arg_with_open_args:11114", - "test_forward_arg:8090", - "test_forward_args_legacy:8054", - "test_forward_args_legacy:8066", - "test_forward_args_legacy:8078", - "test_pattern_matching_hash:*", - "test_pattern_matching_single_line:9839", - "test_trailing_forward_arg:8237" - ) - end - - File - .foreach(File.expand_path("parser.txt", __dir__), chomp: true) - .slice_before { |line| line.start_with?("!!!") } - .each do |(prefix, *lines)| - name = prefix[4..] - next if skips.any? { |skip| File.fnmatch?(skip, name) } - - define_method(name) { assert_parses("#{lines.join("\n")}\n") } - end - - private - - def assert_parses(source) - parser = ::Parser::CurrentRuby.default_parser - parser.diagnostics.consumer = ->(*) {} - - buffer = ::Parser::Source::Buffer.new("(string)", 1) - buffer.source = source - - expected = - begin - parser.parse(buffer) - rescue ::Parser::SyntaxError - # We can get a syntax error if we're parsing a fixture that was - # designed for a later Ruby version but we're running an earlier - # Ruby version. In this case we can just return early from the test. - end - - return if expected.nil? - node = SyntaxTree.parse(source) - assert_equal expected, SyntaxTree::Translation.to_parser(node, buffer) - end - end - end -end - -if ENV["PARSER_LOCATION"] - # Modify the source map == check so that it doesn't check against the node - # itself so we don't get into a recursive loop. - Parser::Source::Map.prepend( - Module.new do - def ==(other) - self.class == other.class && - (instance_variables - %i[@node]).map do |ivar| - instance_variable_get(ivar) == other.instance_variable_get(ivar) - end.reduce(:&) - end - end - ) - - # Next, ensure that we're comparing the nodes and also comparing the source - # ranges so that we're getting all of the necessary information. - Parser::AST::Node.prepend( - Module.new do - def ==(other) - super && (location == other.location) - end - end - ) -end From d1e49185d7906ead65b1aba611062d095c13415f Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 17 Jul 2025 10:23:59 -0400 Subject: [PATCH 523/536] Move YARV out of syntax tree --- exe/yarv | 63 - lib/syntax_tree.rb | 1 - lib/syntax_tree/yarv.rb | 36 - lib/syntax_tree/yarv/assembler.rb | 462 -- lib/syntax_tree/yarv/basic_block.rb | 53 - lib/syntax_tree/yarv/bf.rb | 176 - lib/syntax_tree/yarv/calldata.rb | 97 - lib/syntax_tree/yarv/compiler.rb | 2307 ------- lib/syntax_tree/yarv/control_flow_graph.rb | 257 - lib/syntax_tree/yarv/data_flow_graph.rb | 338 - lib/syntax_tree/yarv/decompiler.rb | 263 - lib/syntax_tree/yarv/disassembler.rb | 236 - lib/syntax_tree/yarv/instruction_sequence.rb | 1357 ---- lib/syntax_tree/yarv/instructions.rb | 5885 ------------------ lib/syntax_tree/yarv/legacy.rb | 340 - lib/syntax_tree/yarv/local_table.rb | 89 - lib/syntax_tree/yarv/sea_of_nodes.rb | 534 -- lib/syntax_tree/yarv/vm.rb | 628 -- test/compiler_test.rb | 533 -- test/yarv_test.rb | 517 -- 20 files changed, 14172 deletions(-) delete mode 100755 exe/yarv delete mode 100644 lib/syntax_tree/yarv.rb delete mode 100644 lib/syntax_tree/yarv/assembler.rb delete mode 100644 lib/syntax_tree/yarv/basic_block.rb delete mode 100644 lib/syntax_tree/yarv/bf.rb delete mode 100644 lib/syntax_tree/yarv/calldata.rb delete mode 100644 lib/syntax_tree/yarv/compiler.rb delete mode 100644 lib/syntax_tree/yarv/control_flow_graph.rb delete mode 100644 lib/syntax_tree/yarv/data_flow_graph.rb delete mode 100644 lib/syntax_tree/yarv/decompiler.rb delete mode 100644 lib/syntax_tree/yarv/disassembler.rb delete mode 100644 lib/syntax_tree/yarv/instruction_sequence.rb delete mode 100644 lib/syntax_tree/yarv/instructions.rb delete mode 100644 lib/syntax_tree/yarv/legacy.rb delete mode 100644 lib/syntax_tree/yarv/local_table.rb delete mode 100644 lib/syntax_tree/yarv/sea_of_nodes.rb delete mode 100644 lib/syntax_tree/yarv/vm.rb delete mode 100644 test/compiler_test.rb delete mode 100644 test/yarv_test.rb diff --git a/exe/yarv b/exe/yarv deleted file mode 100755 index 3efb23ff..00000000 --- a/exe/yarv +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -$:.unshift(File.expand_path("../lib", __dir__)) - -require "syntax_tree" - -# Require these here so that we can run binding.irb without having them require -# anything that we've already patched. -require "irb" -require "irb/completion" -require "irb/color_printer" -require "readline" - -# First, create an instance of our virtual machine. -events = - if ENV["DEBUG"] - SyntaxTree::YARV::VM::STDOUTEvents.new - else - SyntaxTree::YARV::VM::NullEvents.new - end - -vm = SyntaxTree::YARV::VM.new(events) - -# Next, set up a bunch of aliases for methods that we're going to hook into in -# order to set up our virtual machine. -class << Kernel - alias yarv_require require - alias yarv_require_relative require_relative - alias yarv_load load - alias yarv_eval eval - alias yarv_throw throw - alias yarv_catch catch -end - -# Next, patch the methods that we just aliased so that they use our virtual -# machine's versions instead. This allows us to load Ruby files and have them -# execute in our virtual machine instead of the runtime environment. -[Kernel, Kernel.singleton_class].each do |klass| - klass.define_method(:require) { |filepath| vm.require(filepath) } - - klass.define_method(:load) { |filepath| vm.load(filepath) } - - # klass.define_method(:require_relative) do |filepath| - # vm.require_relative(filepath) - # end - - # klass.define_method(:eval) do | - # source, - # binding = TOPLEVEL_BINDING, - # filename = "(eval)", - # lineno = 1 - # | - # vm.eval(source, binding, filename, lineno) - # end - - # klass.define_method(:throw) { |tag, value = nil| vm.throw(tag, value) } - - # klass.define_method(:catch) { |tag, &block| vm.catch(tag, &block) } -end - -# Finally, require the file that we want to execute. -vm.require_resolved(ARGV.shift) diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index 2c824f71..90fb7fe7 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -35,7 +35,6 @@ module SyntaxTree autoload :PrettyPrintVisitor, "syntax_tree/pretty_print_visitor" autoload :Search, "syntax_tree/search" autoload :WithScope, "syntax_tree/with_scope" - autoload :YARV, "syntax_tree/yarv" # This holds references to objects that respond to both #parse and #format # so that we can use them in the CLI. diff --git a/lib/syntax_tree/yarv.rb b/lib/syntax_tree/yarv.rb deleted file mode 100644 index bd5c54b9..00000000 --- a/lib/syntax_tree/yarv.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: true - -require "stringio" - -require_relative "yarv/basic_block" -require_relative "yarv/bf" -require_relative "yarv/calldata" -require_relative "yarv/compiler" -require_relative "yarv/control_flow_graph" -require_relative "yarv/data_flow_graph" -require_relative "yarv/decompiler" -require_relative "yarv/disassembler" -require_relative "yarv/instruction_sequence" -require_relative "yarv/instructions" -require_relative "yarv/legacy" -require_relative "yarv/local_table" -require_relative "yarv/sea_of_nodes" -require_relative "yarv/assembler" -require_relative "yarv/vm" - -module SyntaxTree - # This module provides an object representation of the YARV bytecode. - module YARV - # Compile the given source into a YARV instruction sequence. - def self.compile(source, options = Compiler::Options.new) - SyntaxTree.parse(source).accept(Compiler.new(options)) - end - - # Compile and interpret the given source. - def self.interpret(source, options = Compiler::Options.new) - iseq = RubyVM::InstructionSequence.compile(source, **options) - iseq = InstructionSequence.from(iseq.to_a) - VM.new.run_top_frame(iseq) - end - end -end diff --git a/lib/syntax_tree/yarv/assembler.rb b/lib/syntax_tree/yarv/assembler.rb deleted file mode 100644 index a48c58fd..00000000 --- a/lib/syntax_tree/yarv/assembler.rb +++ /dev/null @@ -1,462 +0,0 @@ -# frozen_string_literal: true - -module SyntaxTree - module YARV - class Assembler - class ObjectVisitor < Compiler::RubyVisitor - def visit_dyna_symbol(node) - if node.parts.empty? - :"" - else - raise CompilationError - end - end - - def visit_string_literal(node) - case node.parts.length - when 0 - "" - when 1 - raise CompilationError unless node.parts.first.is_a?(TStringContent) - node.parts.first.value - else - raise CompilationError - end - end - end - - CALLDATA_FLAGS = { - "ARGS_SPLAT" => CallData::CALL_ARGS_SPLAT, - "ARGS_BLOCKARG" => CallData::CALL_ARGS_BLOCKARG, - "FCALL" => CallData::CALL_FCALL, - "VCALL" => CallData::CALL_VCALL, - "ARGS_SIMPLE" => CallData::CALL_ARGS_SIMPLE, - "KWARG" => CallData::CALL_KWARG, - "KW_SPLAT" => CallData::CALL_KW_SPLAT, - "TAILCALL" => CallData::CALL_TAILCALL, - "SUPER" => CallData::CALL_SUPER, - "ZSUPER" => CallData::CALL_ZSUPER, - "OPT_SEND" => CallData::CALL_OPT_SEND, - "KW_SPLAT_MUT" => CallData::CALL_KW_SPLAT_MUT - }.freeze - - DEFINED_TYPES = [ - nil, - "nil", - "instance-variable", - "local-variable", - "global-variable", - "class variable", - "constant", - "method", - "yield", - "super", - "self", - "true", - "false", - "assignment", - "expression", - "ref", - "func", - "constant-from" - ].freeze - - attr_reader :lines - - def initialize(lines) - @lines = lines - end - - def assemble - iseq = InstructionSequence.new("
", "", 1, :top) - assemble_iseq(iseq, lines) - - iseq.compile! - iseq - end - - def self.assemble(source) - new(source.lines(chomp: true)).assemble - end - - def self.assemble_file(filepath) - new(File.readlines(filepath, chomp: true)).assemble - end - - private - - def assemble_iseq(iseq, lines) - labels = Hash.new { |hash, name| hash[name] = iseq.label } - line_index = 0 - - while line_index < lines.length - line = lines[line_index] - line_index += 1 - - case line.strip - when "", /^;/ - # skip over blank lines and comments - next - when /^(\w+):$/ - # create labels - iseq.push(labels[$1]) - next - when /^__END__/ - # skip over the rest of the file when we hit __END__ - return - end - - insn, operands = line.split(" ", 2) - - case insn - when "adjuststack" - iseq.adjuststack(parse_number(operands)) - when "anytostring" - iseq.anytostring - when "branchif" - iseq.branchif(labels[operands]) - when "branchnil" - iseq.branchnil(labels[operands]) - when "branchunless" - iseq.branchunless(labels[operands]) - when "checkkeyword" - kwbits_index, keyword_index = operands.split(/,\s*/) - iseq.checkkeyword( - parse_number(kwbits_index), - parse_number(keyword_index) - ) - when "checkmatch" - iseq.checkmatch(parse_number(operands)) - when "checktype" - iseq.checktype(parse_number(operands)) - when "concatarray" - iseq.concatarray - when "concatstrings" - iseq.concatstrings(parse_number(operands)) - when "defineclass" - body = parse_nested(lines[line_index..]) - line_index += body.length - - name_value, flags_value = operands.split(/,\s*/) - name = parse_symbol(name_value) - flags = parse_number(flags_value) - - class_iseq = iseq.class_child_iseq(name.to_s, 1) - assemble_iseq(class_iseq, body) - iseq.defineclass(name, class_iseq, flags) - when "defined" - type, object, message = operands.split(/,\s*/) - iseq.defined( - DEFINED_TYPES.index(type), - parse_symbol(object), - parse_string(message) - ) - when "definemethod" - body = parse_nested(lines[line_index..]) - line_index += body.length - - name = parse_symbol(operands) - method_iseq = iseq.method_child_iseq(name.to_s, 1) - assemble_iseq(method_iseq, body) - - iseq.definemethod(name, method_iseq) - when "definesmethod" - body = parse_nested(lines[line_index..]) - line_index += body.length - - name = parse_symbol(operands) - method_iseq = iseq.method_child_iseq(name.to_s, 1) - - assemble_iseq(method_iseq, body) - iseq.definesmethod(name, method_iseq) - when "dup" - iseq.dup - when "dupn" - iseq.dupn(parse_number(operands)) - when "duparray" - iseq.duparray(parse_type(operands, Array)) - when "duphash" - iseq.duphash(parse_type(operands, Hash)) - when "expandarray" - number, flags = operands.split(/,\s*/) - iseq.expandarray(parse_number(number), parse_number(flags)) - when "getblockparam" - lookup = find_local(iseq, operands) - iseq.getblockparam(lookup.index, lookup.level) - when "getblockparamproxy" - lookup = find_local(iseq, operands) - iseq.getblockparamproxy(lookup.index, lookup.level) - when "getclassvariable" - iseq.getclassvariable(parse_symbol(operands)) - when "getconstant" - iseq.getconstant(parse_symbol(operands)) - when "getglobal" - iseq.getglobal(parse_symbol(operands)) - when "getinstancevariable" - iseq.getinstancevariable(parse_symbol(operands)) - when "getlocal" - lookup = find_local(iseq, operands) - iseq.getlocal(lookup.index, lookup.level) - when "getspecial" - key, type = operands.split(/,\s*/) - iseq.getspecial(parse_number(key), parse_number(type)) - when "intern" - iseq.intern - when "invokeblock" - iseq.invokeblock( - operands ? parse_calldata(operands) : YARV.calldata(nil, 0) - ) - when "invokesuper" - calldata = - if operands - parse_calldata(operands) - else - YARV.calldata( - nil, - 0, - CallData::CALL_FCALL | CallData::CALL_ARGS_SIMPLE | - CallData::CALL_SUPER - ) - end - - block_iseq = - if lines[line_index].start_with?(" ") - body = parse_nested(lines[line_index..]) - line_index += body.length - - block_iseq = iseq.block_child_iseq(1) - assemble_iseq(block_iseq, body) - block_iseq - end - - iseq.invokesuper(calldata, block_iseq) - when "jump" - iseq.jump(labels[operands]) - when "leave" - iseq.leave - when "newarray" - iseq.newarray(parse_number(operands)) - when "newarraykwsplat" - iseq.newarraykwsplat(parse_number(operands)) - when "newhash" - iseq.newhash(parse_number(operands)) - when "newrange" - iseq.newrange(parse_options(operands, [0, 1])) - when "nop" - iseq.nop - when "objtostring" - iseq.objtostring(YARV.calldata(:to_s)) - when "once" - block_iseq = - if lines[line_index].start_with?(" ") - body = parse_nested(lines[line_index..]) - line_index += body.length - - block_iseq = iseq.block_child_iseq(1) - assemble_iseq(block_iseq, body) - block_iseq - end - - iseq.once(block_iseq, iseq.inline_storage) - when "opt_and" - iseq.send(YARV.calldata(:&, 1)) - when "opt_aref" - iseq.send(YARV.calldata(:[], 1)) - when "opt_aref_with" - iseq.opt_aref_with(parse_string(operands), YARV.calldata(:[], 1)) - when "opt_aset" - iseq.send(YARV.calldata(:[]=, 2)) - when "opt_aset_with" - iseq.opt_aset_with(parse_string(operands), YARV.calldata(:[]=, 2)) - when "opt_case_dispatch" - cdhash_value, else_label_value = operands.split(/\s*\},\s*/) - cdhash_value.sub!(/\A\{/, "") - - pairs = - cdhash_value - .split(/\s*,\s*/) - .map! { |pair| pair.split(/\s*=>\s*/) } - - cdhash = pairs.to_h { |value, nm| [parse(value), labels[nm]] } - else_label = labels[else_label_value] - - iseq.opt_case_dispatch(cdhash, else_label) - when "opt_div" - iseq.send(YARV.calldata(:/, 1)) - when "opt_empty_p" - iseq.send(YARV.calldata(:empty?)) - when "opt_eq" - iseq.send(YARV.calldata(:==, 1)) - when "opt_ge" - iseq.send(YARV.calldata(:>=, 1)) - when "opt_gt" - iseq.send(YARV.calldata(:>, 1)) - when "opt_getconstant_path" - iseq.opt_getconstant_path(parse_type(operands, Array)) - when "opt_le" - iseq.send(YARV.calldata(:<=, 1)) - when "opt_length" - iseq.send(YARV.calldata(:length)) - when "opt_lt" - iseq.send(YARV.calldata(:<, 1)) - when "opt_ltlt" - iseq.send(YARV.calldata(:<<, 1)) - when "opt_minus" - iseq.send(YARV.calldata(:-, 1)) - when "opt_mod" - iseq.send(YARV.calldata(:%, 1)) - when "opt_mult" - iseq.send(YARV.calldata(:*, 1)) - when "opt_neq" - iseq.send(YARV.calldata(:!=, 1)) - when "opt_newarray_max" - iseq.newarray(parse_number(operands)) - iseq.send(YARV.calldata(:max)) - when "opt_newarray_min" - iseq.newarray(parse_number(operands)) - iseq.send(YARV.calldata(:min)) - when "opt_nil_p" - iseq.send(YARV.calldata(:nil?)) - when "opt_not" - iseq.send(YARV.calldata(:!)) - when "opt_or" - iseq.send(YARV.calldata(:|, 1)) - when "opt_plus" - iseq.send(YARV.calldata(:+, 1)) - when "opt_regexpmatch2" - iseq.send(YARV.calldata(:=~, 1)) - when "opt_reverse" - iseq.send(YARV.calldata(:reverse)) - when "opt_send_without_block" - iseq.send(parse_calldata(operands)) - when "opt_size" - iseq.send(YARV.calldata(:size)) - when "opt_str_freeze" - iseq.putstring(parse_string(operands)) - iseq.send(YARV.calldata(:freeze)) - when "opt_str_uminus" - iseq.putstring(parse_string(operands)) - iseq.send(YARV.calldata(:-@)) - when "opt_succ" - iseq.send(YARV.calldata(:succ)) - when "pop" - iseq.pop - when "putnil" - iseq.putnil - when "putobject" - iseq.putobject(parse(operands)) - when "putself" - iseq.putself - when "putspecialobject" - iseq.putspecialobject(parse_options(operands, [1, 2, 3])) - when "putstring" - iseq.putstring(parse_string(operands)) - when "send" - block_iseq = - if lines[line_index].start_with?(" ") - body = parse_nested(lines[line_index..]) - line_index += body.length - - block_iseq = iseq.block_child_iseq(1) - assemble_iseq(block_iseq, body) - block_iseq - end - - iseq.send(parse_calldata(operands), block_iseq) - when "setblockparam" - lookup = find_local(iseq, operands) - iseq.setblockparam(lookup.index, lookup.level) - when "setconstant" - iseq.setconstant(parse_symbol(operands)) - when "setglobal" - iseq.setglobal(parse_symbol(operands)) - when "setlocal" - lookup = find_local(iseq, operands) - iseq.setlocal(lookup.index, lookup.level) - when "setn" - iseq.setn(parse_number(operands)) - when "setclassvariable" - iseq.setclassvariable(parse_symbol(operands)) - when "setinstancevariable" - iseq.setinstancevariable(parse_symbol(operands)) - when "setspecial" - iseq.setspecial(parse_number(operands)) - when "splatarray" - iseq.splatarray(parse_options(operands, [true, false])) - when "swap" - iseq.swap - when "throw" - iseq.throw(parse_number(operands)) - when "topn" - iseq.topn(parse_number(operands)) - when "toregexp" - options, length = operands.split(", ") - iseq.toregexp(parse_number(options), parse_number(length)) - when "ARG_REQ" - iseq.argument_size += 1 - iseq.local_table.plain(operands.to_sym) - when "ARG_BLOCK" - iseq.argument_options[:block_start] = iseq.argument_size - iseq.local_table.block(operands.to_sym) - iseq.argument_size += 1 - else - raise "Could not understand: #{line}" - end - end - end - - def find_local(iseq, operands) - name_string, level_string = operands.split(/,\s*/) - name = name_string.to_sym - level = level_string.to_i - - iseq.local_table.plain(name) - iseq.local_table.find(name, level) - end - - def parse(value) - program = SyntaxTree.parse(value) - raise if program.statements.body.length != 1 - - program.statements.body.first.accept(ObjectVisitor.new) - end - - def parse_options(value, options) - parse(value).tap { raise unless options.include?(_1) } - end - - def parse_type(value, type) - parse(value).tap { raise unless _1.is_a?(type) } - end - - def parse_number(value) - parse_type(value, Integer) - end - - def parse_string(value) - parse_type(value, String) - end - - def parse_symbol(value) - parse_type(value, Symbol) - end - - def parse_nested(lines) - body = lines.take_while { |line| line.match?(/^($|;| )/) } - body.map! { |line| line.delete_prefix!(" ") || +"" } - end - - def parse_calldata(value) - message, argc_value, flags_value = value.split - flags = - if flags_value - flags_value.split("|").map(&CALLDATA_FLAGS).inject(:|) - else - CallData::CALL_ARGS_SIMPLE - end - - YARV.calldata(message.to_sym, argc_value.to_i, flags) - end - end - end -end diff --git a/lib/syntax_tree/yarv/basic_block.rb b/lib/syntax_tree/yarv/basic_block.rb deleted file mode 100644 index 6798a092..00000000 --- a/lib/syntax_tree/yarv/basic_block.rb +++ /dev/null @@ -1,53 +0,0 @@ -# frozen_string_literal: true - -module SyntaxTree - module YARV - # This object represents a single basic block, wherein all contained - # instructions do not branch except for the last one. - class BasicBlock - # This is the unique identifier for this basic block. - attr_reader :id - - # This is the index into the list of instructions where this block starts. - attr_reader :block_start - - # This is the set of instructions that this block contains. - attr_reader :insns - - # This is an array of basic blocks that lead into this block. - attr_reader :incoming_blocks - - # This is an array of basic blocks that this block leads into. - attr_reader :outgoing_blocks - - def initialize(block_start, insns) - @id = "block_#{block_start}" - - @block_start = block_start - @insns = insns - - @incoming_blocks = [] - @outgoing_blocks = [] - end - - # Yield each instruction in this basic block along with its index from the - # original instruction sequence. - def each_with_length - return enum_for(:each_with_length) unless block_given? - - length = block_start - insns.each do |insn| - yield insn, length - length += insn.length - end - end - - # This method is used to verify that the basic block is well formed. It - # checks that the only instruction in this basic block that branches is - # the last instruction. - def verify - insns[0...-1].each { |insn| raise unless insn.branch_targets.empty? } - end - end - end -end diff --git a/lib/syntax_tree/yarv/bf.rb b/lib/syntax_tree/yarv/bf.rb deleted file mode 100644 index 21bc2982..00000000 --- a/lib/syntax_tree/yarv/bf.rb +++ /dev/null @@ -1,176 +0,0 @@ -# frozen_string_literal: true - -module SyntaxTree - module YARV - # Parses the given source code into a syntax tree, compiles that syntax tree - # into YARV bytecode. - class Bf - attr_reader :source - - def initialize(source) - @source = source - end - - def compile - # Set up the top-level instruction sequence that will be returned. - iseq = InstructionSequence.new("", "", 1, :top) - - # Set up the $tape global variable that will hold our state. - iseq.duphash({ 0 => 0 }) - iseq.setglobal(:$tape) - iseq.getglobal(:$tape) - iseq.putobject(0) - iseq.send(YARV.calldata(:default=, 1)) - - # Set up the $cursor global variable that will hold the current position - # in the tape. - iseq.putobject(0) - iseq.setglobal(:$cursor) - - stack = [] - source - .each_char - .chunk do |char| - # For each character, we're going to assign a type to it. This - # allows a couple of optimizations to be made by combining multiple - # instructions into single instructions, e.g., +++ becomes a single - # change_by(3) instruction. - case char - when "+", "-" - :change - when ">", "<" - :shift - when "." - :output - when "," - :input - when "[", "]" - :loop - else - :ignored - end - end - .each do |type, chunk| - # For each chunk, we're going to emit the appropriate instruction. - case type - when :change - change_by(iseq, chunk.count("+") - chunk.count("-")) - when :shift - shift_by(iseq, chunk.count(">") - chunk.count("<")) - when :output - chunk.length.times { output_char(iseq) } - when :input - chunk.length.times { input_char(iseq) } - when :loop - chunk.each do |char| - case char - when "[" - stack << loop_start(iseq) - when "]" - loop_end(iseq, *stack.pop) - end - end - end - end - - iseq.leave - iseq.compile! - iseq - end - - private - - # $tape[$cursor] += value - def change_by(iseq, value) - iseq.getglobal(:$tape) - iseq.getglobal(:$cursor) - - iseq.getglobal(:$tape) - iseq.getglobal(:$cursor) - iseq.send(YARV.calldata(:[], 1)) - - if value < 0 - iseq.putobject(-value) - iseq.send(YARV.calldata(:-, 1)) - else - iseq.putobject(value) - iseq.send(YARV.calldata(:+, 1)) - end - - iseq.send(YARV.calldata(:[]=, 2)) - iseq.pop - end - - # $cursor += value - def shift_by(iseq, value) - iseq.getglobal(:$cursor) - - if value < 0 - iseq.putobject(-value) - iseq.send(YARV.calldata(:-, 1)) - else - iseq.putobject(value) - iseq.send(YARV.calldata(:+, 1)) - end - - iseq.setglobal(:$cursor) - end - - # $stdout.putc($tape[$cursor].chr) - def output_char(iseq) - iseq.getglobal(:$stdout) - - iseq.getglobal(:$tape) - iseq.getglobal(:$cursor) - iseq.send(YARV.calldata(:[], 1)) - iseq.send(YARV.calldata(:chr)) - - iseq.send(YARV.calldata(:putc, 1)) - iseq.pop - end - - # $tape[$cursor] = $stdin.getc.ord - def input_char(iseq) - iseq.getglobal(:$tape) - iseq.getglobal(:$cursor) - - iseq.getglobal(:$stdin) - iseq.send(YARV.calldata(:getc)) - iseq.send(YARV.calldata(:ord)) - - iseq.send(YARV.calldata(:[]=, 2)) - iseq.pop - end - - # unless $tape[$cursor] == 0 - def loop_start(iseq) - start_label = iseq.label - end_label = iseq.label - - iseq.push(start_label) - iseq.getglobal(:$tape) - iseq.getglobal(:$cursor) - iseq.send(YARV.calldata(:[], 1)) - - iseq.putobject(0) - iseq.send(YARV.calldata(:==, 1)) - iseq.branchif(end_label) - - [start_label, end_label] - end - - # Jump back to the start of the loop. - def loop_end(iseq, start_label, end_label) - iseq.getglobal(:$tape) - iseq.getglobal(:$cursor) - iseq.send(YARV.calldata(:[], 1)) - - iseq.putobject(0) - iseq.send(YARV.calldata(:==, 1)) - iseq.branchunless(start_label) - - iseq.push(end_label) - end - end - end -end diff --git a/lib/syntax_tree/yarv/calldata.rb b/lib/syntax_tree/yarv/calldata.rb deleted file mode 100644 index 278a3dd9..00000000 --- a/lib/syntax_tree/yarv/calldata.rb +++ /dev/null @@ -1,97 +0,0 @@ -# frozen_string_literal: true - -module SyntaxTree - module YARV - # This is an operand to various YARV instructions that represents the - # information about a specific call site. - class CallData - flags = %i[ - CALL_ARGS_SPLAT - CALL_ARGS_BLOCKARG - CALL_FCALL - CALL_VCALL - CALL_ARGS_SIMPLE - CALL_KWARG - CALL_KW_SPLAT - CALL_TAILCALL - CALL_SUPER - CALL_ZSUPER - CALL_OPT_SEND - CALL_KW_SPLAT_MUT - ] - - # Insert the legacy CALL_BLOCKISEQ flag for Ruby 3.2 and earlier. - flags.insert(5, :CALL_BLOCKISEQ) if RUBY_VERSION < "3.3" - - # Set the flags as constants on the class. - flags.each_with_index { |name, index| const_set(name, 1 << index) } - - attr_reader :method, :argc, :flags, :kw_arg - - def initialize( - method, - argc = 0, - flags = CallData::CALL_ARGS_SIMPLE, - kw_arg = nil - ) - @method = method - @argc = argc - @flags = flags - @kw_arg = kw_arg - end - - def flag?(mask) - flags.anybits?(mask) - end - - def to_h - result = { mid: method, flag: flags, orig_argc: argc } - result[:kw_arg] = kw_arg if kw_arg - result - end - - def inspect - names = [] - names << :ARGS_SPLAT if flag?(CALL_ARGS_SPLAT) - names << :ARGS_BLOCKARG if flag?(CALL_ARGS_BLOCKARG) - names << :FCALL if flag?(CALL_FCALL) - names << :VCALL if flag?(CALL_VCALL) - names << :ARGS_SIMPLE if flag?(CALL_ARGS_SIMPLE) - names << :KWARG if flag?(CALL_KWARG) - names << :KW_SPLAT if flag?(CALL_KW_SPLAT) - names << :TAILCALL if flag?(CALL_TAILCALL) - names << :SUPER if flag?(CALL_SUPER) - names << :ZSUPER if flag?(CALL_ZSUPER) - names << :OPT_SEND if flag?(CALL_OPT_SEND) - names << :KW_SPLAT_MUT if flag?(CALL_KW_SPLAT_MUT) - - parts = [] - parts << "mid:#{method}" if method - parts << "argc:#{argc}" - parts << "kw:[#{kw_arg.join(", ")}]" if kw_arg - parts << names.join("|") if names.any? - - "" - end - - def self.from(serialized) - new( - serialized[:mid], - serialized[:orig_argc], - serialized[:flag], - serialized[:kw_arg] - ) - end - end - - # A convenience method for creating a CallData object. - def self.calldata( - method, - argc = 0, - flags = CallData::CALL_ARGS_SIMPLE, - kw_arg = nil - ) - CallData.new(method, argc, flags, kw_arg) - end - end -end diff --git a/lib/syntax_tree/yarv/compiler.rb b/lib/syntax_tree/yarv/compiler.rb deleted file mode 100644 index 0f7e7372..00000000 --- a/lib/syntax_tree/yarv/compiler.rb +++ /dev/null @@ -1,2307 +0,0 @@ -# frozen_string_literal: true - -module SyntaxTree - module YARV - # This class is an experiment in transforming Syntax Tree nodes into their - # corresponding YARV instruction sequences. It attempts to mirror the - # behavior of RubyVM::InstructionSequence.compile. - # - # You use this as with any other visitor. First you parse code into a tree, - # then you visit it with this compiler. Visiting the root node of the tree - # will return a SyntaxTree::YARV::Compiler::InstructionSequence object. - # With that object you can call #to_a on it, which will return a serialized - # form of the instruction sequence as an array. This array _should_ mirror - # the array given by RubyVM::InstructionSequence#to_a. - # - # As an example, here is how you would compile a single expression: - # - # program = SyntaxTree.parse("1 + 2") - # program.accept(SyntaxTree::YARV::Compiler.new).to_a - # - # [ - # "YARVInstructionSequence/SimpleDataFormat", - # 3, - # 1, - # 1, - # {:arg_size=>0, :local_size=>0, :stack_max=>2}, - # "", - # "", - # "", - # 1, - # :top, - # [], - # {}, - # [], - # [ - # [:putobject_INT2FIX_1_], - # [:putobject, 2], - # [:opt_plus, {:mid=>:+, :flag=>16, :orig_argc=>1}], - # [:leave] - # ] - # ] - # - # Note that this is the same output as calling: - # - # RubyVM::InstructionSequence.compile("1 + 2").to_a - # - class Compiler < BasicVisitor - # This represents a set of options that can be passed to the compiler to - # control how it compiles the code. It mirrors the options that can be - # passed to RubyVM::InstructionSequence.compile, except it only includes - # options that actually change the behavior. - class Options - def initialize( - frozen_string_literal: false, - inline_const_cache: true, - operands_unification: true, - peephole_optimization: true, - specialized_instruction: true, - tailcall_optimization: false - ) - @frozen_string_literal = frozen_string_literal - @inline_const_cache = inline_const_cache - @operands_unification = operands_unification - @peephole_optimization = peephole_optimization - @specialized_instruction = specialized_instruction - @tailcall_optimization = tailcall_optimization - end - - def to_hash - { - frozen_string_literal: @frozen_string_literal, - inline_const_cache: @inline_const_cache, - operands_unification: @operands_unification, - peephole_optimization: @peephole_optimization, - specialized_instruction: @specialized_instruction, - tailcall_optimization: @tailcall_optimization - } - end - - def frozen_string_literal! - @frozen_string_literal = true - end - - def frozen_string_literal? - @frozen_string_literal - end - - def inline_const_cache? - @inline_const_cache - end - - def operands_unification? - @operands_unification - end - - def peephole_optimization? - @peephole_optimization - end - - def specialized_instruction? - @specialized_instruction - end - - def tailcall_optimization? - @tailcall_optimization - end - end - - # This visitor is responsible for converting Syntax Tree nodes into their - # corresponding Ruby structures. This is used to convert the operands of - # some instructions like putobject that push a Ruby object directly onto - # the stack. It is only used when the entire structure can be represented - # at compile-time, as opposed to constructed at run-time. - class RubyVisitor < BasicVisitor - # This error is raised whenever a node cannot be converted into a Ruby - # object at compile-time. - class CompilationError < StandardError - end - - # This will attempt to compile the given node. If it's possible, then - # it will return the compiled object. Otherwise it will return nil. - def self.compile(node) - node.accept(new) - rescue CompilationError - end - - visit_methods do - def visit_array(node) - node.contents ? visit_all(node.contents.parts) : [] - end - - def visit_bare_assoc_hash(node) - node.assocs.to_h do |assoc| - # We can only convert regular key-value pairs. A double splat ** - # operator means it has to be converted at run-time. - raise CompilationError unless assoc.is_a?(Assoc) - [visit(assoc.key), visit(assoc.value)] - end - end - - def visit_float(node) - node.value.to_f - end - - alias visit_hash visit_bare_assoc_hash - - def visit_imaginary(node) - node.value.to_c - end - - def visit_int(node) - case (value = node.value) - when /^0b/ - value[2..].to_i(2) - when /^0o/ - value[2..].to_i(8) - when /^0d/ - value[2..].to_i - when /^0x/ - value[2..].to_i(16) - else - value.to_i - end - end - - def visit_label(node) - node.value.chomp(":").to_sym - end - - def visit_mrhs(node) - visit_all(node.parts) - end - - def visit_qsymbols(node) - node.elements.map { |element| visit(element).to_sym } - end - - def visit_qwords(node) - visit_all(node.elements) - end - - def visit_range(node) - left, right = [visit(node.left), visit(node.right)] - node.operator.value === ".." ? left..right : left...right - end - - def visit_rational(node) - node.value.to_r - end - - def visit_regexp_literal(node) - if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) - Regexp.new( - node.parts.first.value, - visit_regexp_literal_flags(node) - ) - else - # Any interpolation of expressions or variables will result in the - # regular expression being constructed at run-time. - raise CompilationError - end - end - - def visit_symbol_literal(node) - node.value.value.to_sym - end - - def visit_symbols(node) - node.elements.map { |element| visit(element).to_sym } - end - - def visit_tstring_content(node) - node.value - end - - def visit_var_ref(node) - raise CompilationError unless node.value.is_a?(Kw) - - case node.value.value - when "nil" - nil - when "true" - true - when "false" - false - else - raise CompilationError - end - end - - def visit_word(node) - if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) - node.parts.first.value - else - # Any interpolation of expressions or variables will result in the - # string being constructed at run-time. - raise CompilationError - end - end - - def visit_words(node) - visit_all(node.elements) - end - end - - # This isn't actually a visit method, though maybe it should be. It is - # responsible for converting the set of string options on a regular - # expression into its equivalent integer. - def visit_regexp_literal_flags(node) - node - .options - .chars - .inject(0) do |accum, option| - accum | - case option - when "i" - Regexp::IGNORECASE - when "x" - Regexp::EXTENDED - when "m" - Regexp::MULTILINE - else - raise "Unknown regexp option: #{option}" - end - end - end - - def visit_unsupported(_node) - raise CompilationError - end - - # Please forgive the metaprogramming here. This is used to create visit - # methods for every node that we did not explicitly handle. By default - # each of these methods will raise a CompilationError. - handled = instance_methods(false) - (Visitor.instance_methods(false) - handled).each do |method| - alias_method method, :visit_unsupported - end - end - - # These options mirror the compilation options that we currently support - # that can be also passed to RubyVM::InstructionSequence.compile. - attr_reader :options - - # The current instruction sequence that is being compiled. - attr_reader :iseq - - # A boolean to track if we're currently compiling the last statement - # within a set of statements. This information is necessary to determine - # if we need to return the value of the last statement. - attr_reader :last_statement - - def initialize(options = Options.new) - @options = options - @iseq = nil - @last_statement = false - end - - def visit_BEGIN(node) - visit(node.statements) - end - - def visit_CHAR(node) - if options.frozen_string_literal? - iseq.putobject(node.value[1..]) - else - iseq.putstring(node.value[1..]) - end - end - - def visit_END(node) - start_line = node.location.start_line - once_iseq = - with_child_iseq(iseq.block_child_iseq(start_line)) do - postexe_iseq = - with_child_iseq(iseq.block_child_iseq(start_line)) do - iseq.event(:RUBY_EVENT_B_CALL) - - *statements, last_statement = node.statements.body - visit_all(statements) - with_last_statement { visit(last_statement) } - - iseq.event(:RUBY_EVENT_B_RETURN) - iseq.leave - end - - iseq.putspecialobject(PutSpecialObject::OBJECT_VMCORE) - iseq.send( - YARV.calldata(:"core#set_postexe", 0, CallData::CALL_FCALL), - postexe_iseq - ) - iseq.leave - end - - iseq.once(once_iseq, iseq.inline_storage) - iseq.pop - end - - def visit_alias(node) - iseq.putspecialobject(PutSpecialObject::OBJECT_VMCORE) - iseq.putspecialobject(PutSpecialObject::OBJECT_CBASE) - visit(node.left) - visit(node.right) - iseq.send(YARV.calldata(:"core#set_method_alias", 3)) - end - - def visit_aref(node) - calldata = YARV.calldata(:[], 1) - visit(node.collection) - - if !options.frozen_string_literal? && - options.specialized_instruction? && (node.index.parts.length == 1) - arg = node.index.parts.first - - if arg.is_a?(StringLiteral) && (arg.parts.length == 1) - string_part = arg.parts.first - - if string_part.is_a?(TStringContent) - iseq.opt_aref_with(string_part.value, calldata) - return - end - end - end - - visit(node.index) - iseq.send(calldata) - end - - def visit_arg_block(node) - visit(node.value) - end - - def visit_arg_paren(node) - visit(node.arguments) - end - - def visit_arg_star(node) - visit(node.value) - iseq.splatarray(false) - end - - def visit_args(node) - visit_all(node.parts) - end - - def visit_array(node) - if (compiled = RubyVisitor.compile(node)) - iseq.duparray(compiled) - elsif node.contents && node.contents.parts.length == 1 && - node.contents.parts.first.is_a?(BareAssocHash) && - node.contents.parts.first.assocs.length == 1 && - node.contents.parts.first.assocs.first.is_a?(AssocSplat) - iseq.putspecialobject(PutSpecialObject::OBJECT_VMCORE) - iseq.newhash(0) - visit(node.contents.parts.first) - iseq.send(YARV.calldata(:"core#hash_merge_kwd", 2)) - iseq.newarraykwsplat(1) - else - length = 0 - - node.contents.parts.each do |part| - if part.is_a?(ArgStar) - if length > 0 - iseq.newarray(length) - length = 0 - end - - visit(part.value) - iseq.concatarray - else - visit(part) - length += 1 - end - end - - iseq.newarray(length) if length > 0 - iseq.concatarray if length > 0 && length != node.contents.parts.length - end - end - - def visit_aryptn(node) - end - - def visit_assign(node) - case node.target - when ARefField - calldata = YARV.calldata(:[]=, 2) - - if !options.frozen_string_literal? && - options.specialized_instruction? && - (node.target.index.parts.length == 1) - arg = node.target.index.parts.first - - if arg.is_a?(StringLiteral) && (arg.parts.length == 1) - string_part = arg.parts.first - - if string_part.is_a?(TStringContent) - visit(node.target.collection) - visit(node.value) - iseq.swap - iseq.topn(1) - iseq.opt_aset_with(string_part.value, calldata) - iseq.pop - return - end - end - end - - iseq.putnil - visit(node.target.collection) - visit(node.target.index) - visit(node.value) - iseq.setn(3) - iseq.send(calldata) - iseq.pop - when ConstPathField - names = constant_names(node.target) - name = names.pop - - if RUBY_VERSION >= "3.2" - iseq.opt_getconstant_path(names) - visit(node.value) - iseq.swap - iseq.topn(1) - iseq.swap - iseq.setconstant(name) - else - visit(node.value) - iseq.dup if last_statement? - iseq.opt_getconstant_path(names) - iseq.setconstant(name) - end - when Field - iseq.putnil - visit(node.target) - visit(node.value) - iseq.setn(2) - iseq.send(YARV.calldata(:"#{node.target.name.value}=", 1)) - iseq.pop - when TopConstField - name = node.target.constant.value.to_sym - - if RUBY_VERSION >= "3.2" - iseq.putobject(Object) - visit(node.value) - iseq.swap - iseq.topn(1) - iseq.swap - iseq.setconstant(name) - else - visit(node.value) - iseq.dup if last_statement? - iseq.putobject(Object) - iseq.setconstant(name) - end - when VarField - visit(node.value) - iseq.dup if last_statement? - - case node.target.value - when Const - iseq.putspecialobject(PutSpecialObject::OBJECT_CONST_BASE) - iseq.setconstant(node.target.value.value.to_sym) - when CVar - iseq.setclassvariable(node.target.value.value.to_sym) - when GVar - iseq.setglobal(node.target.value.value.to_sym) - when Ident - lookup = visit(node.target) - - if lookup.local.is_a?(LocalTable::BlockLocal) - iseq.setblockparam(lookup.index, lookup.level) - else - iseq.setlocal(lookup.index, lookup.level) - end - when IVar - iseq.setinstancevariable(node.target.value.value.to_sym) - end - end - end - - def visit_assoc(node) - visit(node.key) - visit(node.value) - end - - def visit_assoc_splat(node) - visit(node.value) - end - - def visit_backref(node) - iseq.getspecial(GetSpecial::SVAR_BACKREF, node.value[1..].to_i << 1) - end - - def visit_bare_assoc_hash(node) - if (compiled = RubyVisitor.compile(node)) - iseq.duphash(compiled) - else - visit_all(node.assocs) - end - end - - def visit_begin(node) - end - - def visit_binary(node) - case node.operator - when :"&&" - done_label = iseq.label - - visit(node.left) - iseq.dup - iseq.branchunless(done_label) - - iseq.pop - visit(node.right) - iseq.push(done_label) - when :"||" - visit(node.left) - iseq.dup - - skip_right_label = iseq.label - iseq.branchif(skip_right_label) - iseq.pop - - visit(node.right) - iseq.push(skip_right_label) - else - visit(node.left) - visit(node.right) - iseq.send(YARV.calldata(node.operator, 1)) - end - end - - def visit_block(node) - with_child_iseq(iseq.block_child_iseq(node.location.start_line)) do - iseq.event(:RUBY_EVENT_B_CALL) - visit(node.block_var) - visit(node.bodystmt) - iseq.event(:RUBY_EVENT_B_RETURN) - iseq.leave - end - end - - def visit_block_var(node) - params = node.params - - if params.requireds.length == 1 && params.optionals.empty? && - !params.rest && params.posts.empty? && params.keywords.empty? && - !params.keyword_rest && !params.block - iseq.argument_options[:ambiguous_param0] = true - end - - visit(node.params) - - node.locals.each { |local| iseq.local_table.plain(local.value.to_sym) } - end - - def visit_blockarg(node) - iseq.argument_options[:block_start] = iseq.argument_size - iseq.local_table.block(node.name.value.to_sym) - iseq.argument_size += 1 - end - - def visit_bodystmt(node) - visit(node.statements) - end - - def visit_break(node) - end - - def visit_call(node) - if node.is_a?(CallNode) - return( - visit_call( - CommandCall.new( - receiver: node.receiver, - operator: node.operator, - message: node.message, - arguments: node.arguments, - block: nil, - location: node.location - ) - ) - ) - end - - # Track whether or not this is a method call on a block proxy receiver. - # If it is, we can potentially do tailcall optimizations on it. - block_receiver = false - - if node.receiver - if node.receiver.is_a?(VarRef) - lookup = iseq.local_variable(node.receiver.value.value.to_sym) - - if lookup.local.is_a?(LocalTable::BlockLocal) - iseq.getblockparamproxy(lookup.index, lookup.level) - block_receiver = true - else - visit(node.receiver) - end - else - visit(node.receiver) - end - else - iseq.putself - end - - after_call_label = nil - if node.operator&.value == "&." - iseq.dup - after_call_label = iseq.label - iseq.branchnil(after_call_label) - end - - arg_parts = argument_parts(node.arguments) - argc = arg_parts.length - flag = 0 - - arg_parts.each do |arg_part| - case arg_part - when ArgBlock - argc -= 1 - flag |= CallData::CALL_ARGS_BLOCKARG - visit(arg_part) - when ArgStar - flag |= CallData::CALL_ARGS_SPLAT - visit(arg_part) - when ArgsForward - flag |= CallData::CALL_TAILCALL if options.tailcall_optimization? - - flag |= CallData::CALL_ARGS_SPLAT - lookup = iseq.local_table.find(:*) - iseq.getlocal(lookup.index, lookup.level) - iseq.splatarray(arg_parts.length != 1) - - flag |= CallData::CALL_ARGS_BLOCKARG - lookup = iseq.local_table.find(:&) - iseq.getblockparamproxy(lookup.index, lookup.level) - when BareAssocHash - flag |= CallData::CALL_KW_SPLAT - visit(arg_part) - else - visit(arg_part) - end - end - - block_iseq = visit(node.block) if node.block - - # If there's no block and we don't already have any special flags set, - # then we can safely call this simple arguments. Note that has to be the - # first flag we set after looking at the arguments to get the flags - # correct. - flag |= CallData::CALL_ARGS_SIMPLE if block_iseq.nil? && flag == 0 - - # If there's no receiver, then this is an "fcall". - flag |= CallData::CALL_FCALL if node.receiver.nil? - - # If we're calling a method on the passed block object and we have - # tailcall optimizations turned on, then we can set the tailcall flag. - if block_receiver && options.tailcall_optimization? - flag |= CallData::CALL_TAILCALL - end - - iseq.send( - YARV.calldata(node.message.value.to_sym, argc, flag), - block_iseq - ) - iseq.event(after_call_label) if after_call_label - end - - def visit_case(node) - visit(node.value) if node.value - - clauses = [] - else_clause = nil - current = node.consequent - - while current - clauses << current - - if (current = current.consequent).is_a?(Else) - else_clause = current - break - end - end - - branches = - clauses.map do |clause| - visit(clause.arguments) - iseq.topn(1) - iseq.send( - YARV.calldata( - :===, - 1, - CallData::CALL_FCALL | CallData::CALL_ARGS_SIMPLE - ) - ) - - label = iseq.label - iseq.branchif(label) - [clause, label] - end - - iseq.pop - else_clause ? visit(else_clause) : iseq.putnil - iseq.leave - - branches.each_with_index do |(clause, label), index| - iseq.leave if index != 0 - iseq.push(label) - iseq.pop - visit(clause) - end - end - - def visit_class(node) - name = node.constant.constant.value.to_sym - class_iseq = - with_child_iseq( - iseq.class_child_iseq(name, node.location.start_line) - ) do - iseq.event(:RUBY_EVENT_CLASS) - visit(node.bodystmt) - iseq.event(:RUBY_EVENT_END) - iseq.leave - end - - flags = DefineClass::TYPE_CLASS - - case node.constant - when ConstPathRef - flags |= DefineClass::FLAG_SCOPED - visit(node.constant.parent) - when ConstRef - iseq.putspecialobject(PutSpecialObject::OBJECT_CONST_BASE) - when TopConstRef - flags |= DefineClass::FLAG_SCOPED - iseq.putobject(Object) - end - - if node.superclass - flags |= DefineClass::FLAG_HAS_SUPERCLASS - visit(node.superclass) - else - iseq.putnil - end - - iseq.defineclass(name, class_iseq, flags) - end - - def visit_command(node) - visit_call( - CommandCall.new( - receiver: nil, - operator: nil, - message: node.message, - arguments: node.arguments, - block: node.block, - location: node.location - ) - ) - end - - def visit_command_call(node) - visit_call( - CommandCall.new( - receiver: node.receiver, - operator: node.operator, - message: node.message, - arguments: node.arguments, - block: node.block, - location: node.location - ) - ) - end - - def visit_const_path_field(node) - visit(node.parent) - end - - def visit_const_path_ref(node) - names = constant_names(node) - iseq.opt_getconstant_path(names) - end - - def visit_def(node) - name = node.name.value.to_sym - method_iseq = - iseq.method_child_iseq(name.to_s, node.location.start_line) - - with_child_iseq(method_iseq) do - visit(node.params) if node.params - iseq.event(:RUBY_EVENT_CALL) - visit(node.bodystmt) - iseq.event(:RUBY_EVENT_RETURN) - iseq.leave - end - - if node.target - visit(node.target) - iseq.definesmethod(name, method_iseq) - else - iseq.definemethod(name, method_iseq) - end - - iseq.putobject(name) - end - - def visit_defined(node) - case node.value - when Assign - # If we're assigning to a local variable, then we need to make sure - # that we put it into the local table. - if node.value.target.is_a?(VarField) && - node.value.target.value.is_a?(Ident) - iseq.local_table.plain(node.value.target.value.value.to_sym) - end - - iseq.putobject("assignment") - when VarRef - value = node.value.value - name = value.value.to_sym - - case value - when Const - iseq.putnil - iseq.defined(Defined::TYPE_CONST, name, "constant") - when CVar - iseq.putnil - iseq.defined(Defined::TYPE_CVAR, name, "class variable") - when GVar - iseq.putnil - iseq.defined(Defined::TYPE_GVAR, name, "global-variable") - when Ident - iseq.putobject("local-variable") - when IVar - iseq.definedivar(name, iseq.inline_storage, "instance-variable") - when Kw - case name - when :false - iseq.putobject("false") - when :nil - iseq.putobject("nil") - when :self - iseq.putobject("self") - when :true - iseq.putobject("true") - end - end - when VCall - iseq.putself - - name = node.value.value.value.to_sym - iseq.defined(Defined::TYPE_FUNC, name, "method") - when YieldNode - iseq.putnil - iseq.defined(Defined::TYPE_YIELD, false, "yield") - when ZSuper - iseq.putnil - iseq.defined(Defined::TYPE_ZSUPER, false, "super") - else - iseq.putobject("expression") - end - end - - def visit_dyna_symbol(node) - if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) - iseq.putobject(node.parts.first.value.to_sym) - end - end - - def visit_else(node) - visit(node.statements) - iseq.pop unless last_statement? - end - - def visit_elsif(node) - visit_if( - IfNode.new( - predicate: node.predicate, - statements: node.statements, - consequent: node.consequent, - location: node.location - ) - ) - end - - def visit_ensure(node) - end - - def visit_field(node) - visit(node.parent) - end - - def visit_float(node) - iseq.putobject(node.accept(RubyVisitor.new)) - end - - def visit_fndptn(node) - end - - def visit_for(node) - visit(node.collection) - - name = node.index.value.value.to_sym - iseq.local_table.plain(name) - - block_iseq = - with_child_iseq( - iseq.block_child_iseq(node.statements.location.start_line) - ) do - iseq.argument_options[:lead_num] ||= 0 - iseq.argument_options[:lead_num] += 1 - iseq.argument_options[:ambiguous_param0] = true - - iseq.argument_size += 1 - iseq.local_table.plain(2) - - iseq.getlocal(0, 0) - - local_variable = iseq.local_variable(name) - iseq.setlocal(local_variable.index, local_variable.level) - - iseq.event(:RUBY_EVENT_B_CALL) - iseq.nop - - visit(node.statements) - iseq.event(:RUBY_EVENT_B_RETURN) - iseq.leave - end - - iseq.send(YARV.calldata(:each, 0, 0), block_iseq) - end - - def visit_hash(node) - if (compiled = RubyVisitor.compile(node)) - iseq.duphash(compiled) - else - visit_all(node.assocs) - iseq.newhash(node.assocs.length * 2) - end - end - - def visit_hshptn(node) - end - - def visit_heredoc(node) - if node.beginning.value.end_with?("`") - visit_xstring_literal(node) - elsif node.parts.length == 1 && node.parts.first.is_a?(TStringContent) - visit(node.parts.first) - else - length = visit_string_parts(node) - iseq.concatstrings(length) - end - end - - def visit_if(node) - if node.predicate.is_a?(RangeNode) - true_label = iseq.label - false_label = iseq.label - end_label = iseq.label - - iseq.getspecial(GetSpecial::SVAR_FLIPFLOP_START, 0) - iseq.branchif(true_label) - - visit(node.predicate.left) - iseq.branchunless(end_label) - - iseq.putobject(true) - iseq.setspecial(GetSpecial::SVAR_FLIPFLOP_START) - - iseq.push(true_label) - visit(node.predicate.right) - iseq.branchunless(false_label) - - iseq.putobject(false) - iseq.setspecial(GetSpecial::SVAR_FLIPFLOP_START) - - iseq.push(false_label) - visit(node.statements) - iseq.leave - iseq.push(end_label) - iseq.putnil - else - consequent_label = iseq.label - - visit(node.predicate) - iseq.branchunless(consequent_label) - visit(node.statements) - - if last_statement? - iseq.leave - iseq.push(consequent_label) - node.consequent ? visit(node.consequent) : iseq.putnil - else - iseq.pop - - if node.consequent - done_label = iseq.label - iseq.jump(done_label) - iseq.push(consequent_label) - visit(node.consequent) - iseq.push(done_label) - else - iseq.push(consequent_label) - end - end - end - end - - def visit_if_op(node) - visit_if( - IfNode.new( - predicate: node.predicate, - statements: - Statements.new(body: [node.truthy], location: Location.default), - consequent: - Else.new( - keyword: Kw.new(value: "else", location: Location.default), - statements: - Statements.new( - body: [node.falsy], - location: Location.default - ), - location: Location.default - ), - location: Location.default - ) - ) - end - - def visit_imaginary(node) - iseq.putobject(node.accept(RubyVisitor.new)) - end - - def visit_int(node) - iseq.putobject(node.accept(RubyVisitor.new)) - end - - def visit_kwrest_param(node) - iseq.argument_options[:kwrest] = iseq.argument_size - iseq.argument_size += 1 - iseq.local_table.plain(node.name.value.to_sym) - end - - def visit_label(node) - iseq.putobject(node.accept(RubyVisitor.new)) - end - - def visit_lambda(node) - lambda_iseq = - with_child_iseq(iseq.block_child_iseq(node.location.start_line)) do - iseq.event(:RUBY_EVENT_B_CALL) - visit(node.params) - visit(node.statements) - iseq.event(:RUBY_EVENT_B_RETURN) - iseq.leave - end - - iseq.putspecialobject(PutSpecialObject::OBJECT_VMCORE) - iseq.send(YARV.calldata(:lambda, 0, CallData::CALL_FCALL), lambda_iseq) - end - - def visit_lambda_var(node) - visit_block_var(node) - end - - def visit_massign(node) - visit(node.value) - iseq.dup - visit(node.target) - end - - def visit_method_add_block(node) - visit_call( - CommandCall.new( - receiver: node.call.receiver, - operator: node.call.operator, - message: node.call.message, - arguments: node.call.arguments, - block: node.block, - location: node.location - ) - ) - end - - def visit_mlhs(node) - lookups = [] - node.parts.each do |part| - case part - when VarField - lookups << visit(part) - end - end - - iseq.expandarray(lookups.length, 0) - lookups.each { |lookup| iseq.setlocal(lookup.index, lookup.level) } - end - - def visit_module(node) - name = node.constant.constant.value.to_sym - module_iseq = - with_child_iseq( - iseq.module_child_iseq(name, node.location.start_line) - ) do - iseq.event(:RUBY_EVENT_CLASS) - visit(node.bodystmt) - iseq.event(:RUBY_EVENT_END) - iseq.leave - end - - flags = DefineClass::TYPE_MODULE - - case node.constant - when ConstPathRef - flags |= DefineClass::FLAG_SCOPED - visit(node.constant.parent) - when ConstRef - iseq.putspecialobject(PutSpecialObject::OBJECT_CONST_BASE) - when TopConstRef - flags |= DefineClass::FLAG_SCOPED - iseq.putobject(Object) - end - - iseq.putnil - iseq.defineclass(name, module_iseq, flags) - end - - def visit_mrhs(node) - if (compiled = RubyVisitor.compile(node)) - iseq.duparray(compiled) - else - visit_all(node.parts) - iseq.newarray(node.parts.length) - end - end - - def visit_next(node) - end - - def visit_not(node) - visit(node.statement) - iseq.send(YARV.calldata(:!)) - end - - def visit_opassign(node) - flag = CallData::CALL_ARGS_SIMPLE - if node.target.is_a?(ConstPathField) || node.target.is_a?(TopConstField) - flag |= CallData::CALL_FCALL - end - - case (operator = node.operator.value.chomp("=").to_sym) - when :"&&" - done_label = iseq.label - - with_opassign(node) do - iseq.dup - iseq.branchunless(done_label) - iseq.pop - visit(node.value) - end - - case node.target - when ARefField - iseq.leave - iseq.push(done_label) - iseq.setn(3) - iseq.adjuststack(3) - when ConstPathField, TopConstField - iseq.push(done_label) - iseq.swap - iseq.pop - else - iseq.push(done_label) - end - when :"||" - if node.target.is_a?(ConstPathField) || - node.target.is_a?(TopConstField) - opassign_defined(node) - iseq.swap - iseq.pop - elsif node.target.is_a?(VarField) && - [Const, CVar, GVar].include?(node.target.value.class) - opassign_defined(node) - else - skip_value_label = iseq.label - - with_opassign(node) do - iseq.dup - iseq.branchif(skip_value_label) - iseq.pop - visit(node.value) - end - - if node.target.is_a?(ARefField) - iseq.leave - iseq.push(skip_value_label) - iseq.setn(3) - iseq.adjuststack(3) - else - iseq.push(skip_value_label) - end - end - else - with_opassign(node) do - visit(node.value) - iseq.send(YARV.calldata(operator, 1, flag)) - end - end - end - - def visit_params(node) - if node.requireds.any? - iseq.argument_options[:lead_num] = 0 - - node.requireds.each do |required| - iseq.local_table.plain(required.value.to_sym) - iseq.argument_size += 1 - iseq.argument_options[:lead_num] += 1 - end - end - - node.optionals.each do |(optional, value)| - index = iseq.local_table.size - name = optional.value.to_sym - - iseq.local_table.plain(name) - iseq.argument_size += 1 - - unless iseq.argument_options.key?(:opt) - start_label = iseq.label - iseq.push(start_label) - iseq.argument_options[:opt] = [start_label] - end - - visit(value) - iseq.setlocal(index, 0) - - arg_given_label = iseq.label - iseq.push(arg_given_label) - iseq.argument_options[:opt] << arg_given_label - end - - visit(node.rest) if node.rest - - if node.posts.any? - iseq.argument_options[:post_start] = iseq.argument_size - iseq.argument_options[:post_num] = 0 - - node.posts.each do |post| - iseq.local_table.plain(post.value.to_sym) - iseq.argument_size += 1 - iseq.argument_options[:post_num] += 1 - end - end - - if node.keywords.any? - iseq.argument_options[:kwbits] = 0 - iseq.argument_options[:keyword] = [] - - keyword_bits_name = node.keyword_rest ? 3 : 2 - iseq.argument_size += 1 - keyword_bits_index = iseq.local_table.locals.size + node.keywords.size - - node.keywords.each_with_index do |(keyword, value), keyword_index| - name = keyword.value.chomp(":").to_sym - index = iseq.local_table.size - - iseq.local_table.plain(name) - iseq.argument_size += 1 - iseq.argument_options[:kwbits] += 1 - - if value.nil? - iseq.argument_options[:keyword] << name - elsif (compiled = RubyVisitor.compile(value)) - iseq.argument_options[:keyword] << [name, compiled] - else - skip_value_label = iseq.label - - iseq.argument_options[:keyword] << [name] - iseq.checkkeyword(keyword_bits_index, keyword_index) - iseq.branchif(skip_value_label) - visit(value) - iseq.setlocal(index, 0) - iseq.push(skip_value_label) - end - end - - iseq.local_table.plain(keyword_bits_name) - end - - if node.keyword_rest.is_a?(ArgsForward) - if RUBY_VERSION >= "3.2" - iseq.local_table.plain(:*) - iseq.local_table.plain(:&) - iseq.local_table.plain(:"...") - - iseq.argument_options[:rest_start] = iseq.argument_size - iseq.argument_options[:block_start] = iseq.argument_size + 1 - - iseq.argument_size += 2 - else - iseq.local_table.plain(:*) - iseq.local_table.plain(:&) - - iseq.argument_options[:rest_start] = iseq.argument_size - iseq.argument_options[:block_start] = iseq.argument_size + 1 - - iseq.argument_size += 2 - end - elsif node.keyword_rest - visit(node.keyword_rest) - end - - visit(node.block) if node.block - end - - def visit_paren(node) - visit(node.contents) - end - - def visit_pinned_begin(node) - end - - def visit_pinned_var_ref(node) - end - - def visit_program(node) - node.statements.body.each do |statement| - break unless statement.is_a?(Comment) - - if statement.value == "# frozen_string_literal: true" - options.frozen_string_literal! - end - end - - preexes = [] - statements = [] - - node.statements.body.each do |statement| - case statement - when Comment, EmbDoc, EndContent, VoidStmt - # ignore - when BEGINBlock - preexes << statement - else - statements << statement - end - end - - top_iseq = - InstructionSequence.new( - "", - "", - 1, - :top, - nil, - options - ) - - with_child_iseq(top_iseq) do - visit_all(preexes) - - if statements.empty? - iseq.putnil - else - *statements, last_statement = statements - visit_all(statements) - with_last_statement { visit(last_statement) } - end - - iseq.leave - end - - top_iseq.compile! - top_iseq - end - - def visit_qsymbols(node) - iseq.duparray(node.accept(RubyVisitor.new)) - end - - def visit_qwords(node) - if options.frozen_string_literal? - iseq.duparray(node.accept(RubyVisitor.new)) - else - visit_all(node.elements) - iseq.newarray(node.elements.length) - end - end - - def visit_range(node) - if (compiled = RubyVisitor.compile(node)) - iseq.putobject(compiled) - else - visit(node.left) - visit(node.right) - iseq.newrange(node.operator.value == ".." ? 0 : 1) - end - end - - def visit_rassign(node) - iseq.putnil - - if node.operator.is_a?(Kw) - match_label = iseq.label - - visit(node.value) - iseq.dup - - visit_pattern(node.pattern, match_label) - - iseq.pop - iseq.pop - iseq.putobject(false) - iseq.leave - - iseq.push(match_label) - iseq.adjuststack(2) - iseq.putobject(true) - else - no_key_label = iseq.label - end_leave_label = iseq.label - end_label = iseq.label - - iseq.putnil - iseq.putobject(false) - iseq.putnil - iseq.putnil - visit(node.value) - iseq.dup - - visit_pattern(node.pattern, end_label) - - # First we're going to push the core onto the stack, then we'll check - # if the value to match is truthy. If it is, we'll jump down to raise - # NoMatchingPatternKeyError. Otherwise we'll raise - # NoMatchingPatternError. - iseq.putspecialobject(PutSpecialObject::OBJECT_VMCORE) - iseq.topn(4) - iseq.branchif(no_key_label) - - # Here we're going to raise NoMatchingPatternError. - iseq.putobject(NoMatchingPatternError) - iseq.putspecialobject(PutSpecialObject::OBJECT_VMCORE) - iseq.putobject("%p: %s") - iseq.topn(4) - iseq.topn(7) - iseq.send(YARV.calldata(:"core#sprintf", 3)) - iseq.send(YARV.calldata(:"core#raise", 2)) - iseq.jump(end_leave_label) - - # Here we're going to raise NoMatchingPatternKeyError. - iseq.push(no_key_label) - iseq.putobject(NoMatchingPatternKeyError) - iseq.putspecialobject(PutSpecialObject::OBJECT_VMCORE) - iseq.putobject("%p: %s") - iseq.topn(4) - iseq.topn(7) - iseq.send(YARV.calldata(:"core#sprintf", 3)) - iseq.topn(7) - iseq.topn(9) - iseq.send( - YARV.calldata(:new, 1, CallData::CALL_KWARG, %i[matchee key]) - ) - iseq.send(YARV.calldata(:"core#raise", 1)) - - iseq.push(end_leave_label) - iseq.adjuststack(7) - iseq.putnil - iseq.leave - - iseq.push(end_label) - iseq.adjuststack(6) - iseq.putnil - end - end - - def visit_rational(node) - iseq.putobject(node.accept(RubyVisitor.new)) - end - - def visit_redo(node) - end - - def visit_regexp_literal(node) - if (compiled = RubyVisitor.compile(node)) - iseq.putobject(compiled) - else - flags = RubyVisitor.new.visit_regexp_literal_flags(node) - length = visit_string_parts(node) - iseq.toregexp(flags, length) - end - end - - def visit_rescue(node) - end - - def visit_rescue_ex(node) - end - - def visit_rescue_mod(node) - end - - def visit_rest_param(node) - iseq.local_table.plain(node.name.value.to_sym) - iseq.argument_options[:rest_start] = iseq.argument_size - iseq.argument_size += 1 - end - - def visit_retry(node) - end - - def visit_return(node) - end - - def visit_sclass(node) - visit(node.target) - iseq.putnil - - singleton_iseq = - with_child_iseq( - iseq.singleton_class_child_iseq(node.location.start_line) - ) do - iseq.event(:RUBY_EVENT_CLASS) - visit(node.bodystmt) - iseq.event(:RUBY_EVENT_END) - iseq.leave - end - - iseq.defineclass( - :singletonclass, - singleton_iseq, - DefineClass::TYPE_SINGLETON_CLASS - ) - end - - def visit_statements(node) - statements = - node.body.select do |statement| - case statement - when Comment, EmbDoc, EndContent, VoidStmt - false - else - true - end - end - - statements.empty? ? iseq.putnil : visit_all(statements) - end - - def visit_string_concat(node) - value = node.left.parts.first.value + node.right.parts.first.value - - visit_string_literal( - StringLiteral.new( - parts: [TStringContent.new(value: value, location: node.location)], - quote: node.left.quote, - location: node.location - ) - ) - end - - def visit_string_embexpr(node) - visit(node.statements) - end - - def visit_string_literal(node) - if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) - visit(node.parts.first) - else - length = visit_string_parts(node) - iseq.concatstrings(length) - end - end - - def visit_super(node) - iseq.putself - visit(node.arguments) - iseq.invokesuper( - YARV.calldata( - nil, - argument_parts(node.arguments).length, - CallData::CALL_FCALL | CallData::CALL_ARGS_SIMPLE | - CallData::CALL_SUPER - ), - nil - ) - end - - def visit_symbol_literal(node) - iseq.putobject(node.accept(RubyVisitor.new)) - end - - def visit_symbols(node) - if (compiled = RubyVisitor.compile(node)) - iseq.duparray(compiled) - else - node.elements.each do |element| - if element.parts.length == 1 && - element.parts.first.is_a?(TStringContent) - iseq.putobject(element.parts.first.value.to_sym) - else - length = visit_string_parts(element) - iseq.concatstrings(length) - iseq.intern - end - end - - iseq.newarray(node.elements.length) - end - end - - def visit_top_const_ref(node) - iseq.opt_getconstant_path(constant_names(node)) - end - - def visit_tstring_content(node) - if options.frozen_string_literal? - iseq.putobject(node.accept(RubyVisitor.new)) - else - iseq.putstring(node.accept(RubyVisitor.new)) - end - end - - def visit_unary(node) - method_id = - case node.operator - when "+", "-" - "#{node.operator}@" - else - node.operator - end - - visit_call( - CommandCall.new( - receiver: node.statement, - operator: nil, - message: Ident.new(value: method_id, location: Location.default), - arguments: nil, - block: nil, - location: Location.default - ) - ) - end - - def visit_undef(node) - node.symbols.each_with_index do |symbol, index| - iseq.pop if index != 0 - iseq.putspecialobject(PutSpecialObject::OBJECT_VMCORE) - iseq.putspecialobject(PutSpecialObject::OBJECT_CBASE) - visit(symbol) - iseq.send(YARV.calldata(:"core#undef_method", 2)) - end - end - - def visit_unless(node) - statements_label = iseq.label - - visit(node.predicate) - iseq.branchunless(statements_label) - node.consequent ? visit(node.consequent) : iseq.putnil - - if last_statement? - iseq.leave - iseq.push(statements_label) - visit(node.statements) - else - iseq.pop - - if node.consequent - done_label = iseq.label - iseq.jump(done_label) - iseq.push(statements_label) - visit(node.consequent) - iseq.push(done_label) - else - iseq.push(statements_label) - end - end - end - - def visit_until(node) - predicate_label = iseq.label - statements_label = iseq.label - - iseq.jump(predicate_label) - iseq.putnil - iseq.pop - iseq.jump(predicate_label) - - iseq.push(statements_label) - visit(node.statements) - iseq.pop - - iseq.push(predicate_label) - visit(node.predicate) - iseq.branchunless(statements_label) - iseq.putnil if last_statement? - end - - def visit_var_field(node) - case node.value - when CVar, IVar - name = node.value.value.to_sym - iseq.inline_storage_for(name) - when Ident - name = node.value.value.to_sym - - if (local_variable = iseq.local_variable(name)) - local_variable - else - iseq.local_table.plain(name) - iseq.local_variable(name) - end - end - end - - def visit_var_ref(node) - case node.value - when Const - iseq.opt_getconstant_path(constant_names(node)) - when CVar - name = node.value.value.to_sym - iseq.getclassvariable(name) - when GVar - iseq.getglobal(node.value.value.to_sym) - when Ident - lookup = iseq.local_variable(node.value.value.to_sym) - - case lookup.local - when LocalTable::BlockLocal - iseq.getblockparam(lookup.index, lookup.level) - when LocalTable::PlainLocal - iseq.getlocal(lookup.index, lookup.level) - end - when IVar - name = node.value.value.to_sym - iseq.getinstancevariable(name) - when Kw - case node.value.value - when "false" - iseq.putobject(false) - when "nil" - iseq.putnil - when "self" - iseq.putself - when "true" - iseq.putobject(true) - end - end - end - - def visit_vcall(node) - iseq.putself - iseq.send( - YARV.calldata( - node.value.value.to_sym, - 0, - CallData::CALL_FCALL | CallData::CALL_VCALL | - CallData::CALL_ARGS_SIMPLE - ) - ) - end - - def visit_when(node) - visit(node.statements) - end - - def visit_while(node) - predicate_label = iseq.label - statements_label = iseq.label - - iseq.jump(predicate_label) - iseq.putnil - iseq.pop - iseq.jump(predicate_label) - - iseq.push(statements_label) - visit(node.statements) - iseq.pop - - iseq.push(predicate_label) - visit(node.predicate) - iseq.branchif(statements_label) - iseq.putnil if last_statement? - end - - def visit_word(node) - if node.parts.length == 1 && node.parts.first.is_a?(TStringContent) - visit(node.parts.first) - else - length = visit_string_parts(node) - iseq.concatstrings(length) - end - end - - def visit_words(node) - if options.frozen_string_literal? && - (compiled = RubyVisitor.compile(node)) - iseq.duparray(compiled) - else - visit_all(node.elements) - iseq.newarray(node.elements.length) - end - end - - def visit_xstring_literal(node) - iseq.putself - length = visit_string_parts(node) - iseq.concatstrings(node.parts.length) if length > 1 - iseq.send( - YARV.calldata( - :`, - 1, - CallData::CALL_FCALL | CallData::CALL_ARGS_SIMPLE - ) - ) - end - - def visit_yield(node) - parts = argument_parts(node.arguments) - visit_all(parts) - iseq.invokeblock(YARV.calldata(nil, parts.length)) - end - - def visit_zsuper(_node) - iseq.putself - iseq.invokesuper( - YARV.calldata( - nil, - 0, - CallData::CALL_FCALL | CallData::CALL_ARGS_SIMPLE | - CallData::CALL_SUPER | CallData::CALL_ZSUPER - ), - nil - ) - end - - private - - # This is a helper that is used in places where arguments may be present - # or they may be wrapped in parentheses. It's meant to descend down the - # tree and return an array of argument nodes. - def argument_parts(node) - case node - when nil - [] - when Args - node.parts - when ArgParen - if node.arguments.is_a?(ArgsForward) - [node.arguments] - else - node.arguments.parts - end - when Paren - node.contents.parts - end - end - - # Constant names when they are being assigned or referenced come in as a - # tree, but it's more convenient to work with them as an array. This - # method converts them into that array. This is nice because it's the - # operand that goes to opt_getconstant_path in Ruby 3.2. - def constant_names(node) - current = node - names = [] - - while current.is_a?(ConstPathField) || current.is_a?(ConstPathRef) - names.unshift(current.constant.value.to_sym) - current = current.parent - end - - case current - when VarField, VarRef - names.unshift(current.value.value.to_sym) - when TopConstRef - names.unshift(current.constant.value.to_sym) - names.unshift(:"") - end - - names - end - - # For the most part when an OpAssign (operator assignment) node with a ||= - # operator is being compiled it's a matter of reading the target, checking - # if the value should be evaluated, evaluating it if so, and then writing - # the result back to the target. - # - # However, in certain kinds of assignments (X, ::X, X::Y, @@x, and $x) we - # first check if the value is defined using the defined instruction. I - # don't know why it is necessary, and suspect that it isn't. - def opassign_defined(node) - value_label = iseq.label - skip_value_label = iseq.label - - case node.target - when ConstPathField - visit(node.target.parent) - name = node.target.constant.value.to_sym - - iseq.dup - iseq.defined(Defined::TYPE_CONST_FROM, name, true) - when TopConstField - name = node.target.constant.value.to_sym - - iseq.putobject(Object) - iseq.dup - iseq.defined(Defined::TYPE_CONST_FROM, name, true) - when VarField - name = node.target.value.value.to_sym - iseq.putnil - - case node.target.value - when Const - iseq.defined(Defined::TYPE_CONST, name, true) - when CVar - iseq.defined(Defined::TYPE_CVAR, name, true) - when GVar - iseq.defined(Defined::TYPE_GVAR, name, true) - end - end - - iseq.branchunless(value_label) - - case node.target - when ConstPathField, TopConstField - iseq.dup - iseq.putobject(true) - iseq.getconstant(name) - when VarField - case node.target.value - when Const - iseq.opt_getconstant_path(constant_names(node.target)) - when CVar - iseq.getclassvariable(name) - when GVar - iseq.getglobal(name) - end - end - - iseq.dup - iseq.branchif(skip_value_label) - - iseq.pop - iseq.push(value_label) - visit(node.value) - - case node.target - when ConstPathField, TopConstField - iseq.dupn(2) - iseq.swap - iseq.setconstant(name) - when VarField - iseq.dup - - case node.target.value - when Const - iseq.putspecialobject(PutSpecialObject::OBJECT_CONST_BASE) - iseq.setconstant(name) - when CVar - iseq.setclassvariable(name) - when GVar - iseq.setglobal(name) - end - end - - iseq.push(skip_value_label) - end - - # Whenever a value is interpolated into a string-like structure, these - # three instructions are pushed. - def push_interpolate - iseq.dup - iseq.objtostring( - YARV.calldata( - :to_s, - 0, - CallData::CALL_FCALL | CallData::CALL_ARGS_SIMPLE - ) - ) - iseq.anytostring - end - - # Visit a type of pattern in a pattern match. - def visit_pattern(node, end_label) - case node - when AryPtn - length_label = iseq.label - match_failure_label = iseq.label - match_error_label = iseq.label - - # If there's a constant, then check if we match against that constant - # or not first. Branch to failure if we don't. - if node.constant - iseq.dup - visit(node.constant) - iseq.checkmatch(CheckMatch::VM_CHECKMATCH_TYPE_CASE) - iseq.branchunless(match_failure_label) - end - - # First, check if the #deconstruct cache is nil. If it is, we're going - # to call #deconstruct on the object and cache the result. - iseq.topn(2) - deconstruct_label = iseq.label - iseq.branchnil(deconstruct_label) - - # Next, ensure that the cached value was cached correctly, otherwise - # fail the match. - iseq.topn(2) - iseq.branchunless(match_failure_label) - - # Since we have a valid cached value, we can skip past the part where - # we call #deconstruct on the object. - iseq.pop - iseq.topn(1) - iseq.jump(length_label) - - # Check if the object responds to #deconstruct, fail the match - # otherwise. - iseq.event(deconstruct_label) - iseq.dup - iseq.putobject(:deconstruct) - iseq.send(YARV.calldata(:respond_to?, 1)) - iseq.setn(3) - iseq.branchunless(match_failure_label) - - # Call #deconstruct and ensure that it's an array, raise an error - # otherwise. - iseq.send(YARV.calldata(:deconstruct)) - iseq.setn(2) - iseq.dup - iseq.checktype(CheckType::TYPE_ARRAY) - iseq.branchunless(match_error_label) - - # Ensure that the deconstructed array has the correct size, fail the - # match otherwise. - iseq.push(length_label) - iseq.dup - iseq.send(YARV.calldata(:length)) - iseq.putobject(node.requireds.length) - iseq.send(YARV.calldata(:==, 1)) - iseq.branchunless(match_failure_label) - - # For each required element, check if the deconstructed array contains - # the element, otherwise jump out to the top-level match failure. - iseq.dup - node.requireds.each_with_index do |required, index| - iseq.putobject(index) - iseq.send(YARV.calldata(:[], 1)) - - case required - when VarField - lookup = visit(required) - iseq.setlocal(lookup.index, lookup.level) - else - visit(required) - iseq.checkmatch(CheckMatch::VM_CHECKMATCH_TYPE_CASE) - iseq.branchunless(match_failure_label) - end - - if index < node.requireds.length - 1 - iseq.dup - else - iseq.pop - iseq.jump(end_label) - end - end - - # Set up the routine here to raise an error to indicate that the type - # of the deconstructed array was incorrect. - iseq.push(match_error_label) - iseq.putspecialobject(PutSpecialObject::OBJECT_VMCORE) - iseq.putobject(TypeError) - iseq.putobject("deconstruct must return Array") - iseq.send(YARV.calldata(:"core#raise", 2)) - iseq.pop - - # Patch all of the match failures to jump here so that we pop a final - # value before returning to the parent node. - iseq.push(match_failure_label) - iseq.pop - when VarField - lookup = visit(node) - iseq.setlocal(lookup.index, lookup.level) - iseq.jump(end_label) - end - end - - # There are a lot of nodes in the AST that act as contains of parts of - # strings. This includes things like string literals, regular expressions, - # heredocs, etc. This method will visit all the parts of a string within - # those containers. - def visit_string_parts(node) - length = 0 - - unless node.parts.first.is_a?(TStringContent) - iseq.putobject("") - length += 1 - end - - node.parts.each do |part| - case part - when StringDVar - visit(part.variable) - push_interpolate - when StringEmbExpr - visit(part) - push_interpolate - when TStringContent - iseq.putobject(part.accept(RubyVisitor.new)) - end - - length += 1 - end - - length - end - - # The current instruction sequence that we're compiling is always stored - # on the compiler. When we descend into a node that has its own - # instruction sequence, this method can be called to temporarily set the - # new value of the instruction sequence, yield, and then set it back. - def with_child_iseq(child_iseq) - parent_iseq = iseq - - begin - @iseq = child_iseq - yield - child_iseq - ensure - @iseq = parent_iseq - end - end - - # When we're compiling the last statement of a set of statements within a - # scope, the instructions sometimes change from pops to leaves. These - # kinds of peephole optimizations can reduce the overall number of - # instructions. Therefore, we keep track of whether we're compiling the - # last statement of a scope and allow visit methods to query that - # information. - def with_last_statement - previous = @last_statement - @last_statement = true - - begin - yield - ensure - @last_statement = previous - end - end - - def last_statement? - @last_statement - end - - # OpAssign nodes can have a number of different kinds of nodes as their - # "target" (i.e., the left-hand side of the assignment). When compiling - # these nodes we typically need to first fetch the current value of the - # variable, then perform some kind of action, then store the result back - # into the variable. This method handles that by first fetching the value, - # then yielding to the block, then storing the result. - def with_opassign(node) - case node.target - when ARefField - iseq.putnil - visit(node.target.collection) - visit(node.target.index) - - iseq.dupn(2) - iseq.send(YARV.calldata(:[], 1)) - - yield - - iseq.setn(3) - iseq.send(YARV.calldata(:[]=, 2)) - iseq.pop - when ConstPathField - name = node.target.constant.value.to_sym - - visit(node.target.parent) - iseq.dup - iseq.putobject(true) - iseq.getconstant(name) - - yield - - if node.operator.value == "&&=" - iseq.dupn(2) - else - iseq.swap - iseq.topn(1) - end - - iseq.swap - iseq.setconstant(name) - when TopConstField - name = node.target.constant.value.to_sym - - iseq.putobject(Object) - iseq.dup - iseq.putobject(true) - iseq.getconstant(name) - - yield - - if node.operator.value == "&&=" - iseq.dupn(2) - else - iseq.swap - iseq.topn(1) - end - - iseq.swap - iseq.setconstant(name) - when VarField - case node.target.value - when Const - names = constant_names(node.target) - iseq.opt_getconstant_path(names) - - yield - - iseq.dup - iseq.putspecialobject(PutSpecialObject::OBJECT_CONST_BASE) - iseq.setconstant(names.last) - when CVar - name = node.target.value.value.to_sym - iseq.getclassvariable(name) - - yield - - iseq.dup - iseq.setclassvariable(name) - when GVar - name = node.target.value.value.to_sym - iseq.getglobal(name) - - yield - - iseq.dup - iseq.setglobal(name) - when Ident - local_variable = visit(node.target) - iseq.getlocal(local_variable.index, local_variable.level) - - yield - - iseq.dup - iseq.setlocal(local_variable.index, local_variable.level) - when IVar - name = node.target.value.value.to_sym - iseq.getinstancevariable(name) - - yield - - iseq.dup - iseq.setinstancevariable(name) - end - end - end - end - end -end diff --git a/lib/syntax_tree/yarv/control_flow_graph.rb b/lib/syntax_tree/yarv/control_flow_graph.rb deleted file mode 100644 index 2829bb21..00000000 --- a/lib/syntax_tree/yarv/control_flow_graph.rb +++ /dev/null @@ -1,257 +0,0 @@ -# frozen_string_literal: true - -module SyntaxTree - module YARV - # This class represents a control flow graph of a YARV instruction sequence. - # It constructs a graph of basic blocks that hold subsets of the list of - # instructions from the instruction sequence. - # - # You can use this class by calling the ::compile method and passing it a - # YARV instruction sequence. It will return a control flow graph object. - # - # iseq = RubyVM::InstructionSequence.compile("1 + 2") - # iseq = SyntaxTree::YARV::InstructionSequence.from(iseq.to_a) - # cfg = SyntaxTree::YARV::ControlFlowGraph.compile(iseq) - # - class ControlFlowGraph - # This class is responsible for creating a control flow graph from the - # given instruction sequence. - class Compiler - # This is the instruction sequence that is being compiled. - attr_reader :iseq - - # This is a hash of indices in the YARV instruction sequence that point - # to their corresponding instruction. - attr_reader :insns - - # This is a hash of labels that point to their corresponding index into - # the YARV instruction sequence. Note that this is not the same as the - # index into the list of instructions on the instruction sequence - # object. Instead, this is the index into the C array, so it includes - # operands. - attr_reader :labels - - def initialize(iseq) - @iseq = iseq - - @insns = {} - @labels = {} - - length = 0 - iseq.insns.each do |insn| - case insn - when Instruction - @insns[length] = insn - length += insn.length - when InstructionSequence::Label - @labels[insn] = length - end - end - end - - # This method is used to compile the instruction sequence into a control - # flow graph. It returns an instance of ControlFlowGraph. - def compile - blocks = build_basic_blocks - - connect_basic_blocks(blocks) - prune_basic_blocks(blocks) - - ControlFlowGraph.new(iseq, insns, blocks.values).tap(&:verify) - end - - private - - # Finds the indices of the instructions that start a basic block because - # they're either: - # - # * the start of an instruction sequence - # * the target of a branch - # * fallen through to from a branch - # - def find_basic_block_starts - block_starts = Set.new([0]) - - insns.each do |index, insn| - branch_targets = insn.branch_targets - - if branch_targets.any? - branch_targets.each do |branch_target| - block_starts.add(labels[branch_target]) - end - - block_starts.add(index + insn.length) if insn.falls_through? - end - end - - block_starts.to_a.sort - end - - # Builds up a set of basic blocks by iterating over the starts of each - # block. They are keyed by the index of their first instruction. - def build_basic_blocks - block_starts = find_basic_block_starts - - length = 0 - blocks = - iseq - .insns - .grep(Instruction) - .slice_after do |insn| - length += insn.length - block_starts.include?(length) - end - - block_starts - .zip(blocks) - .to_h do |block_start, insns| - # It's possible that we have not detected a block start but still - # have branching instructions inside of a basic block. This can - # happen if you have an unconditional jump which is followed by - # instructions that are unreachable. As of Ruby 3.2, this is - # possible with something as simple as "1 => a". In this case we - # can discard all instructions that follow branching instructions. - block_insns = - insns.slice_after { |insn| insn.branch_targets.any? }.first - - [block_start, BasicBlock.new(block_start, block_insns)] - end - end - - # Connect the blocks by letting them know which blocks are incoming and - # outgoing from each block. - def connect_basic_blocks(blocks) - blocks.each do |block_start, block| - insn = block.insns.last - - insn.branch_targets.each do |branch_target| - block.outgoing_blocks << blocks.fetch(labels[branch_target]) - end - - if (insn.branch_targets.empty? && !insn.leaves?) || - insn.falls_through? - fall_through_start = block_start + block.insns.sum(&:length) - block.outgoing_blocks << blocks.fetch(fall_through_start) - end - - block.outgoing_blocks.each do |outgoing_block| - outgoing_block.incoming_blocks << block - end - end - end - - # If there are blocks that are unreachable, we can remove them from the - # graph entirely at this point. - def prune_basic_blocks(blocks) - visited = Set.new - queue = [blocks.fetch(0)] - - until queue.empty? - current_block = queue.shift - next if visited.include?(current_block) - - visited << current_block - queue.concat(current_block.outgoing_blocks) - end - - blocks.select! { |_, block| visited.include?(block) } - end - end - - # This is the instruction sequence that this control flow graph - # corresponds to. - attr_reader :iseq - - # This is the list of instructions that this control flow graph contains. - # It is effectively the same as the list of instructions in the - # instruction sequence but with line numbers and events filtered out. - attr_reader :insns - - # This is the set of basic blocks that this control-flow graph contains. - attr_reader :blocks - - def initialize(iseq, insns, blocks) - @iseq = iseq - @insns = insns - @blocks = blocks - end - - def disasm - fmt = Disassembler.new(iseq) - fmt.puts("== cfg: #{iseq.inspect}") - - blocks.each do |block| - fmt.puts(block.id) - fmt.with_prefix(" ") do |prefix| - unless block.incoming_blocks.empty? - from = block.incoming_blocks.map(&:id) - fmt.puts("#{prefix}== from: #{from.join(", ")}") - end - - fmt.format_insns!(block.insns, block.block_start) - - to = block.outgoing_blocks.map(&:id) - to << "leaves" if block.insns.last.leaves? - fmt.puts("#{prefix}== to: #{to.join(", ")}") - end - end - - fmt.string - end - - def to_dfg - DataFlowGraph.compile(self) - end - - def to_son - to_dfg.to_son - end - - def to_mermaid - Mermaid.flowchart do |flowchart| - disasm = Disassembler::Squished.new - - blocks.each do |block| - flowchart.subgraph(block.id) do - previous = nil - - block.each_with_length do |insn, length| - node = - flowchart.node( - "node_#{length}", - "%04d %s" % [length, insn.disasm(disasm)] - ) - - flowchart.link(previous, node) if previous - previous = node - end - end - end - - blocks.each do |block| - block.outgoing_blocks.each do |outgoing| - offset = - block.block_start + block.insns.sum(&:length) - - block.insns.last.length - - from = flowchart.fetch("node_#{offset}") - to = flowchart.fetch("node_#{outgoing.block_start}") - flowchart.link(from, to) - end - end - end - end - - # This method is used to verify that the control flow graph is well - # formed. It does this by checking that each basic block is itself well - # formed. - def verify - blocks.each(&:verify) - end - - def self.compile(iseq) - Compiler.new(iseq).compile - end - end - end -end diff --git a/lib/syntax_tree/yarv/data_flow_graph.rb b/lib/syntax_tree/yarv/data_flow_graph.rb deleted file mode 100644 index aedee9ba..00000000 --- a/lib/syntax_tree/yarv/data_flow_graph.rb +++ /dev/null @@ -1,338 +0,0 @@ -# frozen_string_literal: true - -module SyntaxTree - module YARV - # Constructs a data-flow-graph of a YARV instruction sequence, via a - # control-flow-graph. Data flow is discovered locally and then globally. The - # graph only considers data flow through the stack - local variables and - # objects are considered fully escaped in this analysis. - # - # You can use this class by calling the ::compile method and passing it a - # control flow graph. It will return a data flow graph object. - # - # iseq = RubyVM::InstructionSequence.compile("1 + 2") - # iseq = SyntaxTree::YARV::InstructionSequence.from(iseq.to_a) - # cfg = SyntaxTree::YARV::ControlFlowGraph.compile(iseq) - # dfg = SyntaxTree::YARV::DataFlowGraph.compile(cfg) - # - class DataFlowGraph - # This object represents the flow of data between instructions. - class DataFlow - attr_reader :in - attr_reader :out - - def initialize - @in = [] - @out = [] - end - end - - # This represents an object that goes on the stack that is passed between - # basic blocks. - class BlockArgument - attr_reader :name - - def initialize(name) - @name = name - end - - def local? - false - end - - def to_str - name.to_s - end - end - - # This represents an object that goes on the stack that is passed between - # instructions within a basic block. - class LocalArgument - attr_reader :name, :length - - def initialize(length) - @length = length - end - - def local? - true - end - - def to_str - length.to_s - end - end - - attr_reader :cfg, :insn_flows, :block_flows - - def initialize(cfg, insn_flows, block_flows) - @cfg = cfg - @insn_flows = insn_flows - @block_flows = block_flows - end - - def blocks - cfg.blocks - end - - def disasm - fmt = Disassembler.new(cfg.iseq) - fmt.puts("== dfg: #{cfg.iseq.inspect}") - - blocks.each do |block| - fmt.puts(block.id) - fmt.with_prefix(" ") do |prefix| - unless block.incoming_blocks.empty? - from = block.incoming_blocks.map(&:id) - fmt.puts("#{prefix}== from: #{from.join(", ")}") - end - - block_flow = block_flows.fetch(block.id) - unless block_flow.in.empty? - fmt.puts("#{prefix}== in: #{block_flow.in.join(", ")}") - end - - fmt.format_insns!(block.insns, block.block_start) do |_, length| - insn_flow = insn_flows[length] - next if insn_flow.in.empty? && insn_flow.out.empty? - - fmt.print(" # ") - unless insn_flow.in.empty? - fmt.print("in: #{insn_flow.in.join(", ")}") - fmt.print("; ") unless insn_flow.out.empty? - end - - unless insn_flow.out.empty? - fmt.print("out: #{insn_flow.out.join(", ")}") - end - end - - to = block.outgoing_blocks.map(&:id) - to << "leaves" if block.insns.last.leaves? - fmt.puts("#{prefix}== to: #{to.join(", ")}") - - unless block_flow.out.empty? - fmt.puts("#{prefix}== out: #{block_flow.out.join(", ")}") - end - end - end - - fmt.string - end - - def to_son - SeaOfNodes.compile(self) - end - - def to_mermaid - Mermaid.flowchart do |flowchart| - disasm = Disassembler::Squished.new - - blocks.each do |block| - block_flow = block_flows.fetch(block.id) - graph_name = - if block_flow.in.any? - "#{block.id} #{block_flows[block.id].in.join(", ")}" - else - block.id - end - - flowchart.subgraph(graph_name) do - previous = nil - - block.each_with_length do |insn, length| - node = - flowchart.node( - "node_#{length}", - "%04d %s" % [length, insn.disasm(disasm)], - shape: :rounded - ) - - flowchart.link(previous, node, color: :red) if previous - insn_flows[length].in.each do |input| - if input.is_a?(LocalArgument) - from = flowchart.fetch("node_#{input.length}") - flowchart.link(from, node, color: :green) - end - end - - previous = node - end - end - end - - blocks.each do |block| - block.outgoing_blocks.each do |outgoing| - offset = - block.block_start + block.insns.sum(&:length) - - block.insns.last.length - - from = flowchart.fetch("node_#{offset}") - to = flowchart.fetch("node_#{outgoing.block_start}") - flowchart.link(from, to, color: :red) - end - end - end - end - - # Verify that we constructed the data flow graph correctly. - def verify - # Check that the first block has no arguments. - raise unless block_flows.fetch(blocks.first.id).in.empty? - - # Check all control flow edges between blocks pass the right number of - # arguments. - blocks.each do |block| - block_flow = block_flows.fetch(block.id) - - if block.outgoing_blocks.empty? - # With no outgoing blocks, there should be no output arguments. - raise unless block_flow.out.empty? - else - # Check with outgoing blocks... - block.outgoing_blocks.each do |outgoing_block| - outgoing_flow = block_flows.fetch(outgoing_block.id) - - # The block should have as many output arguments as the - # outgoing block has input arguments. - raise unless block_flow.out.size == outgoing_flow.in.size - end - end - end - end - - def self.compile(cfg) - Compiler.new(cfg).compile - end - - # This class is responsible for creating a data flow graph from the given - # control flow graph. - class Compiler - # This is the control flow graph that is being compiled. - attr_reader :cfg - - # This data structure will hold the data flow between instructions - # within individual basic blocks. - attr_reader :insn_flows - - # This data structure will hold the data flow between basic blocks. - attr_reader :block_flows - - def initialize(cfg) - @cfg = cfg - @insn_flows = cfg.insns.to_h { |length, _| [length, DataFlow.new] } - @block_flows = cfg.blocks.to_h { |block| [block.id, DataFlow.new] } - end - - def compile - find_internal_flow - find_external_flow - DataFlowGraph.new(cfg, insn_flows, block_flows).tap(&:verify) - end - - private - - # Find the data flow within each basic block. Using an abstract stack, - # connect from consumers of data to the producers of that data. - def find_internal_flow - cfg.blocks.each do |block| - block_flow = block_flows.fetch(block.id) - stack = [] - - # Go through each instruction in the block. - block.each_with_length do |insn, length| - insn_flow = insn_flows[length] - - # How many values will be missing from the local stack to run this - # instruction? This will be used to determine if the values that - # are being used by this instruction are coming from previous - # instructions or from previous basic blocks. - missing = insn.pops - stack.size - - # For every value the instruction pops off the stack. - insn.pops.times do - # Was the value it pops off from another basic block? - if stack.empty? - # If the stack is empty, then there aren't enough values being - # pushed from previous instructions to fulfill the needs of - # this instruction. In that case the values must be coming - # from previous basic blocks. - missing -= 1 - argument = BlockArgument.new(:"in_#{missing}") - - insn_flow.in.unshift(argument) - block_flow.in.unshift(argument) - else - # Since there are values in the stack, we can connect this - # consumer to the producer of the value. - insn_flow.in.unshift(stack.pop) - end - end - - # Record on our abstract stack that this instruction pushed - # this value onto the stack. - insn.pushes.times { stack << LocalArgument.new(length) } - end - - # Values that are left on the stack after going through all - # instructions are arguments to the basic block that we jump to. - stack.reverse_each.with_index do |producer, index| - block_flow.out << producer - - argument = BlockArgument.new(:"out_#{index}") - insn_flows[producer.length].out << argument - end - end - - # Go backwards and connect from producers to consumers. - cfg.insns.each_key do |length| - # For every instruction that produced a value used in this - # instruction... - insn_flows[length].in.each do |producer| - # If it's actually another instruction and not a basic block - # argument... - if producer.is_a?(LocalArgument) - # Record in the producing instruction that it produces a value - # used by this construction. - insn_flows[producer.length].out << LocalArgument.new(length) - end - end - end - end - - # Find the data that flows between basic blocks. - def find_external_flow - stack = [*cfg.blocks] - - until stack.empty? - block = stack.pop - block_flow = block_flows.fetch(block.id) - - block.incoming_blocks.each do |incoming_block| - incoming_flow = block_flows.fetch(incoming_block.id) - - # Does a predecessor block have fewer outputs than the successor - # has inputs? - if incoming_flow.out.size < block_flow.in.size - # If so then add arguments to pass data through from the - # incoming block's incoming blocks. - (block_flow.in.size - incoming_flow.out.size).times do |index| - name = BlockArgument.new(:"pass_#{index}") - - incoming_flow.in.unshift(name) - incoming_flow.out.unshift(name) - end - - # Since we modified the incoming block, add it back to the stack - # so it'll be considered as an outgoing block again, and - # propogate the external data flow back up the control flow - # graph. - stack << incoming_block - end - end - end - end - end - end - end -end diff --git a/lib/syntax_tree/yarv/decompiler.rb b/lib/syntax_tree/yarv/decompiler.rb deleted file mode 100644 index 6a2cddbd..00000000 --- a/lib/syntax_tree/yarv/decompiler.rb +++ /dev/null @@ -1,263 +0,0 @@ -# frozen_string_literal: true - -module SyntaxTree - module YARV - # This class is responsible for taking a compiled instruction sequence and - # walking through it to generate equivalent Ruby code. - class Decompiler - # When we're decompiling, we use a looped case statement to emulate - # jumping around in the same way the virtual machine would. This class - # provides convenience methods for generating the AST nodes that have to - # do with that label. - class BlockLabel - include DSL - attr_reader :name - - def initialize(name) - @name = name - end - - def field - VarField(Ident(name)) - end - - def ref - VarRef(Ident(name)) - end - end - - include DSL - attr_reader :iseq, :block_label - - def initialize(iseq) - @iseq = iseq - @block_label = BlockLabel.new("__block_label") - end - - def to_ruby - Program(decompile(iseq)) - end - - private - - def node_for(value) - case value - when Integer - Int(value.to_s) - when Symbol - SymbolLiteral(Ident(value.name)) - end - end - - def decompile(iseq) - label = :label_0 - clauses = {} - clause = [] - - iseq.insns.each do |insn| - case insn - when InstructionSequence::Label - unless clause.last.is_a?(Next) - clause << Assign(block_label.field, node_for(insn.name)) - end - - clauses[label] = clause - clause = [] - label = insn.name - when BranchIf - body = [ - Assign(block_label.field, node_for(insn.label.name)), - Next(Args([])) - ] - - clause << UnlessNode(clause.pop, Statements(body), nil) - when BranchUnless - body = [ - Assign(block_label.field, node_for(insn.label.name)), - Next(Args([])) - ] - - clause << IfNode(clause.pop, Statements(body), nil) - when Dup - clause << clause.last - when DupHash - assocs = - insn.object.map do |key, value| - Assoc(node_for(key), node_for(value)) - end - - clause << HashLiteral(LBrace("{"), assocs) - when GetGlobal - clause << VarRef(GVar(insn.name.name)) - when GetLocalWC0 - local = iseq.local_table.locals[insn.index] - clause << VarRef(Ident(local.name.name)) - when Jump - clause << Assign(block_label.field, node_for(insn.label.name)) - clause << Next(Args([])) - when Leave - value = Args([clause.pop]) - clause << (iseq.type != :top ? Break(value) : ReturnNode(value)) - when OptAnd, OptDiv, OptEq, OptGE, OptGT, OptLE, OptLT, OptLTLT, - OptMinus, OptMod, OptMult, OptOr, OptPlus - left, right = clause.pop(2) - clause << Binary(left, insn.calldata.method, right) - when OptAref - collection, arg = clause.pop(2) - clause << ARef(collection, Args([arg])) - when OptAset - collection, arg, value = clause.pop(3) - - clause << if value.is_a?(Binary) && value.left.is_a?(ARef) && - collection === value.left.collection && - arg === value.left.index.parts[0] - OpAssign( - ARefField(collection, Args([arg])), - Op("#{value.operator}="), - value.right - ) - else - Assign(ARefField(collection, Args([arg])), value) - end - when OptNEq - left, right = clause.pop(2) - clause << Binary(left, :"!=", right) - when OptSendWithoutBlock - method = insn.calldata.method.name - argc = insn.calldata.argc - - if insn.calldata.flag?(CallData::CALL_FCALL) - if argc == 0 - clause.pop - clause << CallNode(nil, nil, Ident(method), Args([])) - elsif argc == 1 && method.end_with?("=") - _receiver, argument = clause.pop(2) - clause << Assign( - CallNode(nil, nil, Ident(method[0..-2]), nil), - argument - ) - else - _receiver, *arguments = clause.pop(argc + 1) - clause << CallNode( - nil, - nil, - Ident(method), - ArgParen(Args(arguments)) - ) - end - else - if argc == 0 - clause << CallNode(clause.pop, Period("."), Ident(method), nil) - elsif argc == 1 && method.end_with?("=") - receiver, argument = clause.pop(2) - clause << Assign( - Field(receiver, Period("."), Ident(method[0..-2])), - argument - ) - else - receiver, *arguments = clause.pop(argc + 1) - clause << CallNode( - receiver, - Period("."), - Ident(method), - ArgParen(Args(arguments)) - ) - end - end - when Pop - # skip - when PutObject - case insn.object - when Float - clause << FloatLiteral(insn.object.inspect) - when Integer - clause << Int(insn.object.inspect) - else - raise "Unknown object type: #{insn.object.class.name}" - end - when PutObjectInt2Fix0 - clause << Int("0") - when PutObjectInt2Fix1 - clause << Int("1") - when PutSelf - clause << VarRef(Kw("self")) - when SetGlobal - target = GVar(insn.name.name) - value = clause.pop - - clause << if value.is_a?(Binary) && VarRef(target) === value.left - OpAssign(VarField(target), Op("#{value.operator}="), value.right) - else - Assign(VarField(target), value) - end - when SetLocalWC0 - target = Ident(local_name(insn.index, 0)) - value = clause.pop - - clause << if value.is_a?(Binary) && VarRef(target) === value.left - OpAssign(VarField(target), Op("#{value.operator}="), value.right) - else - Assign(VarField(target), value) - end - else - raise "Unknown instruction #{insn}" - end - end - - # If there's only one clause, then we don't need a case statement, and - # we can just disassemble the first clause. - clauses[label] = clause - return Statements(clauses.values.first) if clauses.size == 1 - - # Here we're going to build up a big case statement that will handle all - # of the different labels. - current = nil - clauses.reverse_each do |current_label, current_clause| - current = - When( - Args([node_for(current_label)]), - Statements(current_clause), - current - ) - end - switch = Case(Kw("case"), block_label.ref, current) - - # Here we're going to make sure that any locals that were established in - # the label_0 block are initialized so that scoping rules work - # correctly. - stack = [] - locals = [block_label.name] - - clauses[:label_0].each do |node| - if node.is_a?(Assign) && node.target.is_a?(VarField) && - node.target.value.is_a?(Ident) - value = node.target.value.value - next if locals.include?(value) - - stack << Assign(node.target, VarRef(Kw("nil"))) - locals << value - end - end - - # Finally, we'll set up the initial label and loop the entire case - # statement. - stack << Assign(block_label.field, node_for(:label_0)) - stack << MethodAddBlock( - CallNode(nil, nil, Ident("loop"), Args([])), - BlockNode( - Kw("do"), - nil, - BodyStmt(Statements([switch]), nil, nil, nil, nil) - ) - ) - Statements(stack) - end - - def local_name(index, level) - current = iseq - level.times { current = current.parent_iseq } - current.local_table.locals[index].name.name - end - end - end -end diff --git a/lib/syntax_tree/yarv/disassembler.rb b/lib/syntax_tree/yarv/disassembler.rb deleted file mode 100644 index dac220fd..00000000 --- a/lib/syntax_tree/yarv/disassembler.rb +++ /dev/null @@ -1,236 +0,0 @@ -# frozen_string_literal: true - -module SyntaxTree - module YARV - class Disassembler - # This class is another object that handles disassembling a YARV - # instruction sequence but it renders it without any of the extra spacing - # or alignment. - class Squished - def calldata(value) - value.inspect - end - - def enqueue(iseq) - end - - def event(name) - end - - def inline_storage(cache) - "" - end - - def instruction(name, operands = []) - operands.empty? ? name : "#{name} #{operands.join(", ")}" - end - - def label(value) - "%04d" % value.name["label_".length..] - end - - def local(index, **) - index.inspect - end - - def object(value) - value.inspect - end - end - - attr_reader :output, :queue - - attr_reader :current_prefix - attr_accessor :current_iseq - - def initialize(current_iseq = nil) - @output = StringIO.new - @queue = [] - - @current_prefix = "" - @current_iseq = current_iseq - end - - ######################################################################## - # Helpers for various instructions - ######################################################################## - - def calldata(value) - value.inspect - end - - def enqueue(iseq) - queue << iseq - end - - def event(name) - case name - when :RUBY_EVENT_B_CALL - "Bc" - when :RUBY_EVENT_B_RETURN - "Br" - when :RUBY_EVENT_CALL - "Ca" - when :RUBY_EVENT_CLASS - "Cl" - when :RUBY_EVENT_END - "En" - when :RUBY_EVENT_LINE - "Li" - when :RUBY_EVENT_RETURN - "Re" - else - raise "Unknown event: #{name}" - end - end - - def inline_storage(cache) - "" - end - - def instruction(name, operands = []) - operands.empty? ? name : "%-38s %s" % [name, operands.join(", ")] - end - - def label(value) - value.name["label_".length..] - end - - def local(index, explicit: nil, implicit: nil) - current = current_iseq - (explicit || implicit).times { current = current.parent_iseq } - - value = "#{current.local_table.name_at(index)}@#{index}" - value << ", #{explicit}" if explicit - value - end - - def object(value) - value.inspect - end - - ######################################################################## - # Entrypoints - ######################################################################## - - def format! - while (@current_iseq = queue.shift) - output << "\n" if output.pos > 0 - format_iseq(@current_iseq) - end - end - - def format_insns!(insns, length = 0) - events = [] - lines = [] - - insns.each do |insn| - case insn - when Integer - lines << insn - when Symbol - events << event(insn) - when InstructionSequence::Label - # skip - else - output << "#{current_prefix}%04d " % length - - disasm = insn.disasm(self) - output << disasm - - if lines.any? - output << " " * (65 - disasm.length) if disasm.length < 65 - elsif events.any? - output << " " * (39 - disasm.length) if disasm.length < 39 - end - - if lines.any? - output << "(%4d)" % lines.last - lines.clear - end - - if events.any? - output << "[#{events.join}]" - events.clear - end - - # A hook here to allow for custom formatting of instructions after - # the main body has been processed. - yield insn, length if block_given? - - output << "\n" - length += insn.length - end - end - end - - def print(string) - output.print(string) - end - - def puts(string) - output.puts(string) - end - - def string - output.string - end - - def with_prefix(value) - previous = @current_prefix - - begin - @current_prefix = value - yield value - ensure - @current_prefix = previous - end - end - - private - - def format_iseq(iseq) - output << "#{current_prefix}== disasm: #{iseq.inspect} " - - if iseq.catch_table.any? - output << "(catch: TRUE)\n" - output << "#{current_prefix}== catch table\n" - - with_prefix("#{current_prefix}| ") do - iseq.catch_table.each do |entry| - case entry - when InstructionSequence::CatchBreak - output << "#{current_prefix}catch type: break\n" - format_iseq(entry.iseq) - when InstructionSequence::CatchNext - output << "#{current_prefix}catch type: next\n" - when InstructionSequence::CatchRedo - output << "#{current_prefix}catch type: redo\n" - when InstructionSequence::CatchRescue - output << "#{current_prefix}catch type: rescue\n" - format_iseq(entry.iseq) - end - end - end - - output << "#{current_prefix}|#{"-" * 72}\n" - else - output << "(catch: FALSE)\n" - end - - if (local_table = iseq.local_table) && !local_table.empty? - output << "#{current_prefix}local table (size: #{local_table.size})\n" - - locals = - local_table.locals.each_with_index.map do |local, index| - "[%2d] %s@%d" % [local_table.offset(index), local.name, index] - end - - output << "#{current_prefix}#{locals.join(" ")}\n" - end - - format_insns!(iseq.insns) - end - end - end -end diff --git a/lib/syntax_tree/yarv/instruction_sequence.rb b/lib/syntax_tree/yarv/instruction_sequence.rb deleted file mode 100644 index 4f2e0d9a..00000000 --- a/lib/syntax_tree/yarv/instruction_sequence.rb +++ /dev/null @@ -1,1357 +0,0 @@ -# frozen_string_literal: true - -module SyntaxTree - # This module provides an object representation of the YARV bytecode. - module YARV - # This class is meant to mirror RubyVM::InstructionSequence. It contains a - # list of instructions along with the metadata pertaining to them. It also - # functions as a builder for the instruction sequence. - class InstructionSequence - # This provides a handle to the rb_iseq_load function, which allows you - # to pass a serialized iseq to Ruby and have it return a - # RubyVM::InstructionSequence object. - def self.iseq_load(iseq) - require "fiddle" - - @iseq_load_function ||= - Fiddle::Function.new( - Fiddle::Handle::DEFAULT["rb_iseq_load"], - [Fiddle::TYPE_VOIDP] * 3, - Fiddle::TYPE_VOIDP - ) - - Fiddle.dlunwrap(@iseq_load_function.call(Fiddle.dlwrap(iseq), 0, nil)) - rescue LoadError - raise "Could not load the Fiddle library" - rescue NameError - raise "Unable to find rb_iseq_load" - rescue Fiddle::DLError - raise "Unable to perform a dynamic load" - end - - # When the list of instructions is first being created, it's stored as a - # linked list. This is to make it easier to perform peephole optimizations - # and other transformations like instruction specialization. - class InstructionList - class Node - attr_accessor :value, :next_node - - def initialize(value, next_node = nil) - @value = value - @next_node = next_node - end - end - - include Enumerable - attr_reader :head_node, :tail_node - - def initialize - @head_node = nil - @tail_node = nil - end - - def each(&_blk) - return to_enum(__method__) unless block_given? - each_node { |node| yield node.value } - end - - def each_node - return to_enum(__method__) unless block_given? - node = head_node - - while node - yield node, node.value - node = node.next_node - end - end - - def push(instruction) - node = Node.new(instruction) - - if head_node.nil? - @head_node = node - @tail_node = node - else - @tail_node.next_node = node - @tail_node = node - end - - node - end - end - - MAGIC = "YARVInstructionSequence/SimpleDataFormat" - - # This object is used to track the size of the stack at any given time. It - # is effectively a mini symbolic interpreter. It's necessary because when - # instruction sequences get serialized they include a :stack_max field on - # them. This field is used to determine how much stack space to allocate - # for the instruction sequence. - class Stack - attr_reader :current_size, :maximum_size - - def initialize - @current_size = 0 - @maximum_size = 0 - end - - def change_by(value) - @current_size += value - @maximum_size = @current_size if @current_size > @maximum_size - end - end - - # This represents the destination of instructions that jump. Initially it - # does not track its position so that when we perform optimizations the - # indices don't get messed up. - class Label - attr_reader :name - - # When we're serializing the instruction sequence, we need to be able to - # look up the label from the branch instructions and then access the - # subsequent node. So we'll store the reference here. - attr_accessor :node - - def initialize(name = nil) - @name = name - end - - def patch!(name) - @name = name - end - - def inspect - name.inspect - end - end - - # The name of the instruction sequence. - attr_reader :name - - # The source location of the instruction sequence. - attr_reader :file, :line - - # The type of the instruction sequence. - attr_reader :type - - # The parent instruction sequence, if there is one. - attr_reader :parent_iseq - - # This is the list of information about the arguments to this - # instruction sequence. - attr_accessor :argument_size - attr_reader :argument_options - - # The catch table for this instruction sequence. - attr_reader :catch_table - - # The list of instructions for this instruction sequence. - attr_reader :insns - - # The table of local variables. - attr_reader :local_table - - # The hash of names of instance and class variables pointing to the - # index of their associated inline storage. - attr_reader :inline_storages - - # The index of the next inline storage that will be created. - attr_reader :storage_index - - # An object that will track the current size of the stack and the - # maximum size of the stack for this instruction sequence. - attr_reader :stack - - # These are various compilation options provided. - attr_reader :options - - def initialize( - name, - file, - line, - type, - parent_iseq = nil, - options = Compiler::Options.new - ) - @name = name - @file = file - @line = line - @type = type - @parent_iseq = parent_iseq - - @argument_size = 0 - @argument_options = {} - @catch_table = [] - - @local_table = LocalTable.new - @inline_storages = {} - @insns = InstructionList.new - @storage_index = 0 - @stack = Stack.new - - @options = options - end - - ########################################################################## - # Query methods - ########################################################################## - - def local_variable(name, level = 0) - if (lookup = local_table.find(name, level)) - lookup - elsif parent_iseq - parent_iseq.local_variable(name, level + 1) - end - end - - def inline_storage - storage = storage_index - @storage_index += 1 - storage - end - - def inline_storage_for(name) - inline_storages[name] = inline_storage unless inline_storages.key?(name) - - inline_storages[name] - end - - def length - insns - .each - .inject(0) do |sum, insn| - case insn - when Integer, Label, Symbol - sum - else - sum + insn.length - end - end - end - - def eval - InstructionSequence.iseq_load(to_a).eval - end - - def to_a - versions = RUBY_VERSION.split(".").map(&:to_i) - - # Dump all of the instructions into a flat list. - dumped = - insns.map do |insn| - case insn - when Integer, Symbol - insn - when Label - insn.name - else - insn.to_a(self) - end - end - - dumped_options = argument_options.dup - dumped_options[:opt].map!(&:name) if dumped_options[:opt] - - metadata = { - arg_size: argument_size, - local_size: local_table.size, - stack_max: stack.maximum_size, - node_id: -1, - node_ids: [-1] * insns.length - } - - metadata[:parser] = :prism if RUBY_VERSION >= "3.3" - - # Next, return the instruction sequence as an array. - [ - MAGIC, - versions[0], - versions[1], - 1, - metadata, - name, - file, - "", - line, - type, - local_table.names, - dumped_options, - catch_table.map(&:to_a), - dumped - ] - end - - def to_cfg - ControlFlowGraph.compile(self) - end - - def to_dfg - to_cfg.to_dfg - end - - def to_son - to_dfg.to_son - end - - def disasm - fmt = Disassembler.new - fmt.enqueue(self) - fmt.format! - fmt.string - end - - def inspect - "#:1 (#{line},0)-(#{line},0)>" - end - - # This method converts our linked list of instructions into a final array - # and performs any other compilation steps necessary. - def compile! - specialize_instructions! if options.specialized_instruction? - - catch_table.each do |catch_entry| - if !catch_entry.is_a?(CatchBreak) && catch_entry.iseq - catch_entry.iseq.compile! - end - end - - length = 0 - insns.each do |insn| - case insn - when Integer, Symbol - # skip - when Label - insn.patch!(:"label_#{length}") - when DefineClass - insn.class_iseq.compile! - length += insn.length - when DefineMethod, DefineSMethod - insn.method_iseq.compile! - length += insn.length - when InvokeSuper, Send - insn.block_iseq.compile! if insn.block_iseq - length += insn.length - when Once - insn.iseq.compile! - length += insn.length - else - length += insn.length - end - end - - @insns = insns.to_a - end - - def specialize_instructions! - insns.each_node do |node, value| - case value - when NewArray - next unless node.next_node - - next_node = node.next_node - next unless next_node.value.is_a?(Send) - next if next_node.value.block_iseq - - calldata = next_node.value.calldata - next unless calldata.flags == CallData::CALL_ARGS_SIMPLE - next unless calldata.argc == 0 - - case calldata.method - when :min - node.value = - if RUBY_VERSION < "3.3" - Legacy::OptNewArrayMin.new(value.number) - else - OptNewArraySend.new(value.number, :min) - end - - node.next_node = next_node.next_node - when :max - node.value = - if RUBY_VERSION < "3.3" - Legacy::OptNewArrayMax.new(value.number) - else - OptNewArraySend.new(value.number, :max) - end - - node.next_node = next_node.next_node - when :hash - next if RUBY_VERSION < "3.3" - node.value = OptNewArraySend.new(value.number, :hash) - node.next_node = next_node.next_node - end - when PutObject, PutString - next unless node.next_node - next if value.is_a?(PutObject) && !value.object.is_a?(String) - - next_node = node.next_node - next unless next_node.value.is_a?(Send) - next if next_node.value.block_iseq - - calldata = next_node.value.calldata - next unless calldata.flags == CallData::CALL_ARGS_SIMPLE - next unless calldata.argc == 0 - - case calldata.method - when :freeze - node.value = OptStrFreeze.new(value.object, calldata) - node.next_node = next_node.next_node - when :-@ - node.value = OptStrUMinus.new(value.object, calldata) - node.next_node = next_node.next_node - end - when Send - calldata = value.calldata - - if !value.block_iseq && - !calldata.flag?(CallData::CALL_ARGS_BLOCKARG) - # Specialize the send instruction. If it doesn't have a block - # attached, then we will replace it with an opt_send_without_block - # and do further specializations based on the called method and - # the number of arguments. - node.value = - case [calldata.method, calldata.argc] - when [:length, 0] - OptLength.new(calldata) - when [:size, 0] - OptSize.new(calldata) - when [:empty?, 0] - OptEmptyP.new(calldata) - when [:nil?, 0] - OptNilP.new(calldata) - when [:succ, 0] - OptSucc.new(calldata) - when [:!, 0] - OptNot.new(calldata) - when [:+, 1] - OptPlus.new(calldata) - when [:-, 1] - OptMinus.new(calldata) - when [:*, 1] - OptMult.new(calldata) - when [:/, 1] - OptDiv.new(calldata) - when [:%, 1] - OptMod.new(calldata) - when [:==, 1] - OptEq.new(calldata) - when [:!=, 1] - OptNEq.new(YARV.calldata(:==, 1), calldata) - when [:=~, 1] - OptRegExpMatch2.new(calldata) - when [:<, 1] - OptLT.new(calldata) - when [:<=, 1] - OptLE.new(calldata) - when [:>, 1] - OptGT.new(calldata) - when [:>=, 1] - OptGE.new(calldata) - when [:<<, 1] - OptLTLT.new(calldata) - when [:[], 1] - OptAref.new(calldata) - when [:&, 1] - OptAnd.new(calldata) - when [:|, 1] - OptOr.new(calldata) - when [:[]=, 2] - OptAset.new(calldata) - else - OptSendWithoutBlock.new(calldata) - end - end - end - end - end - - ########################################################################## - # Child instruction sequence methods - ########################################################################## - - def child_iseq(name, line, type) - InstructionSequence.new(name, file, line, type, self, options) - end - - def block_child_iseq(line) - current = self - current = current.parent_iseq while current.type == :block - child_iseq("block in #{current.name}", line, :block) - end - - def class_child_iseq(name, line) - child_iseq("", line, :class) - end - - def method_child_iseq(name, line) - child_iseq(name, line, :method) - end - - def module_child_iseq(name, line) - child_iseq("", line, :class) - end - - def singleton_class_child_iseq(line) - child_iseq("singleton class", line, :class) - end - - ########################################################################## - # Catch table methods - ########################################################################## - - class CatchEntry - attr_reader :iseq, :begin_label, :end_label, :exit_label, :restore_sp - - def initialize(iseq, begin_label, end_label, exit_label, restore_sp) - @iseq = iseq - @begin_label = begin_label - @end_label = end_label - @exit_label = exit_label - @restore_sp = restore_sp - end - end - - class CatchBreak < CatchEntry - def to_a - [ - :break, - iseq.to_a, - begin_label.name, - end_label.name, - exit_label.name, - restore_sp - ] - end - end - - class CatchEnsure < CatchEntry - def to_a - [ - :ensure, - iseq.to_a, - begin_label.name, - end_label.name, - exit_label.name - ] - end - end - - class CatchNext < CatchEntry - def to_a - [:next, nil, begin_label.name, end_label.name, exit_label.name] - end - end - - class CatchRedo < CatchEntry - def to_a - [:redo, nil, begin_label.name, end_label.name, exit_label.name] - end - end - - class CatchRescue < CatchEntry - def to_a - [ - :rescue, - iseq.to_a, - begin_label.name, - end_label.name, - exit_label.name - ] - end - end - - class CatchRetry < CatchEntry - def to_a - [:retry, nil, begin_label.name, end_label.name, exit_label.name] - end - end - - def catch_break(iseq, begin_label, end_label, exit_label, restore_sp) - catch_table << CatchBreak.new( - iseq, - begin_label, - end_label, - exit_label, - restore_sp - ) - end - - def catch_ensure(iseq, begin_label, end_label, exit_label, restore_sp) - catch_table << CatchEnsure.new( - iseq, - begin_label, - end_label, - exit_label, - restore_sp - ) - end - - def catch_next(begin_label, end_label, exit_label, restore_sp) - catch_table << CatchNext.new( - nil, - begin_label, - end_label, - exit_label, - restore_sp - ) - end - - def catch_redo(begin_label, end_label, exit_label, restore_sp) - catch_table << CatchRedo.new( - nil, - begin_label, - end_label, - exit_label, - restore_sp - ) - end - - def catch_rescue(iseq, begin_label, end_label, exit_label, restore_sp) - catch_table << CatchRescue.new( - iseq, - begin_label, - end_label, - exit_label, - restore_sp - ) - end - - def catch_retry(begin_label, end_label, exit_label, restore_sp) - catch_table << CatchRetry.new( - nil, - begin_label, - end_label, - exit_label, - restore_sp - ) - end - - ########################################################################## - # Instruction push methods - ########################################################################## - - def label - Label.new - end - - def push(value) - node = insns.push(value) - - case value - when Array, Integer, Symbol - value - when Label - value.node = node - value - else - stack.change_by(-value.pops + value.pushes) - value - end - end - - def event(name) - push(name) - end - - def adjuststack(number) - push(AdjustStack.new(number)) - end - - def anytostring - push(AnyToString.new) - end - - def branchif(label) - push(BranchIf.new(label)) - end - - def branchnil(label) - push(BranchNil.new(label)) - end - - def branchunless(label) - push(BranchUnless.new(label)) - end - - def checkkeyword(keyword_bits_index, keyword_index) - push(CheckKeyword.new(keyword_bits_index, keyword_index)) - end - - def checkmatch(type) - push(CheckMatch.new(type)) - end - - def checktype(type) - push(CheckType.new(type)) - end - - def concatarray - push(ConcatArray.new) - end - - def concatstrings(number) - push(ConcatStrings.new(number)) - end - - def concattoarray(object) - push(ConcatToArray.new(object)) - end - - def defineclass(name, class_iseq, flags) - push(DefineClass.new(name, class_iseq, flags)) - end - - def defined(type, name, message) - push(Defined.new(type, name, message)) - end - - def definedivar(name, cache, message) - if RUBY_VERSION < "3.3" - push(PutNil.new) - push(Defined.new(Defined::TYPE_IVAR, name, message)) - else - push(DefinedIVar.new(name, cache, message)) - end - end - - def definemethod(name, method_iseq) - push(DefineMethod.new(name, method_iseq)) - end - - def definesmethod(name, method_iseq) - push(DefineSMethod.new(name, method_iseq)) - end - - def dup - push(Dup.new) - end - - def duparray(object) - push(DupArray.new(object)) - end - - def duphash(object) - push(DupHash.new(object)) - end - - def dupn(number) - push(DupN.new(number)) - end - - def expandarray(length, flags) - push(ExpandArray.new(length, flags)) - end - - def getblockparam(index, level) - push(GetBlockParam.new(index, level)) - end - - def getblockparamproxy(index, level) - push(GetBlockParamProxy.new(index, level)) - end - - def getclassvariable(name) - if RUBY_VERSION < "3.0" - push(Legacy::GetClassVariable.new(name)) - else - push(GetClassVariable.new(name, inline_storage_for(name))) - end - end - - def getconstant(name) - push(GetConstant.new(name)) - end - - def getglobal(name) - push(GetGlobal.new(name)) - end - - def getinstancevariable(name) - if RUBY_VERSION < "3.2" - push(GetInstanceVariable.new(name, inline_storage_for(name))) - else - push(GetInstanceVariable.new(name, inline_storage)) - end - end - - def getlocal(index, level) - if options.operands_unification? - # Specialize the getlocal instruction based on the level of the - # local variable. If it's 0 or 1, then there's a specialized - # instruction that will look at the current scope or the parent - # scope, respectively, and requires fewer operands. - case level - when 0 - push(GetLocalWC0.new(index)) - when 1 - push(GetLocalWC1.new(index)) - else - push(GetLocal.new(index, level)) - end - else - push(GetLocal.new(index, level)) - end - end - - def getspecial(key, type) - push(GetSpecial.new(key, type)) - end - - def intern - push(Intern.new) - end - - def invokeblock(calldata) - push(InvokeBlock.new(calldata)) - end - - def invokesuper(calldata, block_iseq) - push(InvokeSuper.new(calldata, block_iseq)) - end - - def jump(label) - push(Jump.new(label)) - end - - def leave - push(Leave.new) - end - - def newarray(number) - push(NewArray.new(number)) - end - - def newarraykwsplat(number) - push(NewArrayKwSplat.new(number)) - end - - def newhash(number) - push(NewHash.new(number)) - end - - def newrange(exclude_end) - push(NewRange.new(exclude_end)) - end - - def nop - push(Nop.new) - end - - def objtostring(calldata) - push(ObjToString.new(calldata)) - end - - def once(iseq, cache) - push(Once.new(iseq, cache)) - end - - def opt_aref_with(object, calldata) - push(OptArefWith.new(object, calldata)) - end - - def opt_aset_with(object, calldata) - push(OptAsetWith.new(object, calldata)) - end - - def opt_case_dispatch(case_dispatch_hash, else_label) - push(OptCaseDispatch.new(case_dispatch_hash, else_label)) - end - - def opt_getconstant_path(names) - if RUBY_VERSION < "3.2" || !options.inline_const_cache? - cache = nil - cache_filled_label = nil - - if options.inline_const_cache? - cache = inline_storage - cache_filled_label = label - opt_getinlinecache(cache_filled_label, cache) - - if names[0] == :"" - names.shift - pop - putobject(Object) - end - elsif names[0] == :"" - names.shift - putobject(Object) - else - putnil - end - - names.each_with_index do |name, index| - putobject(index == 0) - getconstant(name) - end - - if options.inline_const_cache? - opt_setinlinecache(cache) - push(cache_filled_label) - end - else - push(OptGetConstantPath.new(names)) - end - end - - def opt_getinlinecache(label, cache) - push(Legacy::OptGetInlineCache.new(label, cache)) - end - - def opt_setinlinecache(cache) - push(Legacy::OptSetInlineCache.new(cache)) - end - - def pop - push(Pop.new) - end - - def pushtoarraykwsplat - push(PushToArrayKwSplat.new) - end - - def putchilledstring(object) - push(PutChilledString.new(object)) - end - - def putnil - push(PutNil.new) - end - - def putobject(object) - if options.operands_unification? - # Specialize the putobject instruction based on the value of the - # object. If it's 0 or 1, then there's a specialized instruction - # that will push the object onto the stack and requires fewer - # operands. - if object.eql?(0) - push(PutObjectInt2Fix0.new) - elsif object.eql?(1) - push(PutObjectInt2Fix1.new) - else - push(PutObject.new(object)) - end - else - push(PutObject.new(object)) - end - end - - def putself - push(PutSelf.new) - end - - def putspecialobject(object) - push(PutSpecialObject.new(object)) - end - - def putstring(object) - push(PutString.new(object)) - end - - def send(calldata, block_iseq = nil) - push(Send.new(calldata, block_iseq)) - end - - def setblockparam(index, level) - push(SetBlockParam.new(index, level)) - end - - def setclassvariable(name) - if RUBY_VERSION < "3.0" - push(Legacy::SetClassVariable.new(name)) - else - push(SetClassVariable.new(name, inline_storage_for(name))) - end - end - - def setconstant(name) - push(SetConstant.new(name)) - end - - def setglobal(name) - push(SetGlobal.new(name)) - end - - def setinstancevariable(name) - if RUBY_VERSION < "3.2" - push(SetInstanceVariable.new(name, inline_storage_for(name))) - else - push(SetInstanceVariable.new(name, inline_storage)) - end - end - - def setlocal(index, level) - if options.operands_unification? - # Specialize the setlocal instruction based on the level of the - # local variable. If it's 0 or 1, then there's a specialized - # instruction that will write to the current scope or the parent - # scope, respectively, and requires fewer operands. - case level - when 0 - push(SetLocalWC0.new(index)) - when 1 - push(SetLocalWC1.new(index)) - else - push(SetLocal.new(index, level)) - end - else - push(SetLocal.new(index, level)) - end - end - - def setn(number) - push(SetN.new(number)) - end - - def setspecial(key) - push(SetSpecial.new(key)) - end - - def splatarray(flag) - push(SplatArray.new(flag)) - end - - def swap - push(Swap.new) - end - - def throw(type) - push(Throw.new(type)) - end - - def topn(number) - push(TopN.new(number)) - end - - def toregexp(options, length) - push(ToRegExp.new(options, length)) - end - - # This method will create a new instruction sequence from a serialized - # RubyVM::InstructionSequence object. - def self.from(source, options = Compiler::Options.new, parent_iseq = nil) - iseq = - new(source[5], source[6], source[8], source[9], parent_iseq, options) - - # set up the labels object so that the labels are shared between the - # location in the instruction sequence and the instructions that - # reference them - labels = Hash.new { |hash, name| hash[name] = Label.new(name) } - - # set up the correct argument size - iseq.argument_size = source[4][:arg_size] - - # set up all of the locals - source[10].each { |local| iseq.local_table.plain(local) } - - # set up the argument options - iseq.argument_options.merge!(source[11]) - if iseq.argument_options[:opt] - iseq.argument_options[:opt].map! { |opt| labels[opt] } - end - - # track the child block iseqs so that our catch table can point to the - # correctly created iseqs - block_iseqs = [] - - # set up all of the instructions - source[13].each do |insn| - # add line numbers - if insn.is_a?(Integer) - iseq.push(insn) - next - end - - # add events and labels - if insn.is_a?(Symbol) - if insn.start_with?("label_") - iseq.push(labels[insn]) - else - iseq.push(insn) - end - next - end - - # add instructions, mapped to our own instruction classes - type, *opnds = insn - - case type - when :adjuststack - iseq.adjuststack(opnds[0]) - when :anytostring - iseq.anytostring - when :branchif - iseq.branchif(labels[opnds[0]]) - when :branchnil - iseq.branchnil(labels[opnds[0]]) - when :branchunless - iseq.branchunless(labels[opnds[0]]) - when :checkkeyword - iseq.checkkeyword(iseq.local_table.size - opnds[0] + 2, opnds[1]) - when :checkmatch - iseq.checkmatch(opnds[0]) - when :checktype - iseq.checktype(opnds[0]) - when :concatarray - iseq.concatarray - when :concatstrings - iseq.concatstrings(opnds[0]) - when :concattoarray - iseq.concattoarray(opnds[0]) - when :defineclass - iseq.defineclass(opnds[0], from(opnds[1], options, iseq), opnds[2]) - when :defined - iseq.defined(opnds[0], opnds[1], opnds[2]) - when :definedivar - iseq.definedivar(opnds[0], opnds[1], opnds[2]) - when :definemethod - iseq.definemethod(opnds[0], from(opnds[1], options, iseq)) - when :definesmethod - iseq.definesmethod(opnds[0], from(opnds[1], options, iseq)) - when :dup - iseq.dup - when :duparray - iseq.duparray(opnds[0]) - when :duphash - iseq.duphash(opnds[0]) - when :dupn - iseq.dupn(opnds[0]) - when :expandarray - iseq.expandarray(opnds[0], opnds[1]) - when :getblockparam, :getblockparamproxy, :getlocal, :getlocal_WC_0, - :getlocal_WC_1, :setblockparam, :setlocal, :setlocal_WC_0, - :setlocal_WC_1 - current = iseq - level = 0 - - case type - when :getlocal_WC_1, :setlocal_WC_1 - level = 1 - when :getblockparam, :getblockparamproxy, :getlocal, :setblockparam, - :setlocal - level = opnds[1] - end - - level.times { current = current.parent_iseq } - index = current.local_table.size - opnds[0] + 2 - - case type - when :getblockparam - iseq.getblockparam(index, level) - when :getblockparamproxy - iseq.getblockparamproxy(index, level) - when :getlocal, :getlocal_WC_0, :getlocal_WC_1 - iseq.getlocal(index, level) - when :setblockparam - iseq.setblockparam(index, level) - when :setlocal, :setlocal_WC_0, :setlocal_WC_1 - iseq.setlocal(index, level) - end - when :getclassvariable - iseq.push(GetClassVariable.new(opnds[0], opnds[1])) - when :getconstant - iseq.getconstant(opnds[0]) - when :getglobal - iseq.getglobal(opnds[0]) - when :getinstancevariable - iseq.push(GetInstanceVariable.new(opnds[0], opnds[1])) - when :getspecial - iseq.getspecial(opnds[0], opnds[1]) - when :intern - iseq.intern - when :invokeblock - iseq.invokeblock(CallData.from(opnds[0])) - when :invokesuper - block_iseq = opnds[1] ? from(opnds[1], options, iseq) : nil - iseq.invokesuper(CallData.from(opnds[0]), block_iseq) - when :jump - iseq.jump(labels[opnds[0]]) - when :leave - iseq.leave - when :newarray - iseq.newarray(opnds[0]) - when :newarraykwsplat - iseq.newarraykwsplat(opnds[0]) - when :newhash - iseq.newhash(opnds[0]) - when :newrange - iseq.newrange(opnds[0]) - when :nop - iseq.nop - when :objtostring - iseq.objtostring(CallData.from(opnds[0])) - when :once - iseq.once(from(opnds[0], options, iseq), opnds[1]) - when :opt_and, :opt_aref, :opt_aset, :opt_div, :opt_empty_p, :opt_eq, - :opt_ge, :opt_gt, :opt_le, :opt_length, :opt_lt, :opt_ltlt, - :opt_minus, :opt_mod, :opt_mult, :opt_nil_p, :opt_not, :opt_or, - :opt_plus, :opt_regexpmatch2, :opt_send_without_block, :opt_size, - :opt_succ - iseq.send(CallData.from(opnds[0]), nil) - when :opt_aref_with - iseq.opt_aref_with(opnds[0], CallData.from(opnds[1])) - when :opt_aset_with - iseq.opt_aset_with(opnds[0], CallData.from(opnds[1])) - when :opt_case_dispatch - hash = - opnds[0] - .each_slice(2) - .to_h - .transform_values { |value| labels[value] } - iseq.opt_case_dispatch(hash, labels[opnds[1]]) - when :opt_getconstant_path - iseq.opt_getconstant_path(opnds[0]) - when :opt_getinlinecache - iseq.opt_getinlinecache(labels[opnds[0]], opnds[1]) - when :opt_newarray_max - iseq.newarray(opnds[0]) - iseq.send(YARV.calldata(:max)) - when :opt_newarray_min - iseq.newarray(opnds[0]) - iseq.send(YARV.calldata(:min)) - when :opt_newarray_send - mid = opnds[1] - if RUBY_VERSION >= "3.4" - mid = %i[max min hash pack pack_buffer include?][mid - 1] - end - - iseq.newarray(opnds[0]) - iseq.send(CallData.new(mid)) - when :opt_neq - iseq.push( - OptNEq.new(CallData.from(opnds[0]), CallData.from(opnds[1])) - ) - when :opt_setinlinecache - iseq.opt_setinlinecache(opnds[0]) - when :opt_str_freeze - iseq.putstring(opnds[0]) - iseq.send(YARV.calldata(:freeze)) - when :opt_str_uminus - iseq.putstring(opnds[0]) - iseq.send(YARV.calldata(:-@)) - when :pop - iseq.pop - when :pushtoarraykwsplat - iseq.pushtoarraykwsplat - when :putchilledstring - iseq.putchilledstring(opnds[0]) - when :putnil - iseq.putnil - when :putobject - iseq.putobject(opnds[0]) - when :putobject_INT2FIX_0_ - iseq.putobject(0) - when :putobject_INT2FIX_1_ - iseq.putobject(1) - when :putself - iseq.putself - when :putstring - iseq.putstring(opnds[0]) - when :putspecialobject - iseq.putspecialobject(opnds[0]) - when :send - block_iseq = opnds[1] ? from(opnds[1], options, iseq) : nil - block_iseqs << block_iseq if block_iseq - iseq.send(CallData.from(opnds[0]), block_iseq) - when :setclassvariable - iseq.push(SetClassVariable.new(opnds[0], opnds[1])) - when :setconstant - iseq.setconstant(opnds[0]) - when :setglobal - iseq.setglobal(opnds[0]) - when :setinstancevariable - iseq.push(SetInstanceVariable.new(opnds[0], opnds[1])) - when :setn - iseq.setn(opnds[0]) - when :setspecial - iseq.setspecial(opnds[0]) - when :splatarray - iseq.splatarray(opnds[0]) - when :swap - iseq.swap - when :throw - iseq.throw(opnds[0]) - when :topn - iseq.topn(opnds[0]) - when :toregexp - iseq.toregexp(opnds[0], opnds[1]) - else - raise "Unknown instruction type: #{type}" - end - end - - # set up the catch table - source[12].each do |entry| - case entry[0] - when :break - if entry[1] - break_iseq = - block_iseqs.find do |block_iseq| - block_iseq.name == entry[1][5] && - block_iseq.file == entry[1][6] && - block_iseq.line == entry[1][8] - end - - iseq.catch_break( - break_iseq || from(entry[1], options, iseq), - labels[entry[2]], - labels[entry[3]], - labels[entry[4]], - entry[5] - ) - else - iseq.catch_break( - nil, - labels[entry[2]], - labels[entry[3]], - labels[entry[4]], - entry[5] - ) - end - when :ensure - iseq.catch_ensure( - from(entry[1], options, iseq), - labels[entry[2]], - labels[entry[3]], - labels[entry[4]], - entry[5] - ) - when :next - iseq.catch_next( - labels[entry[2]], - labels[entry[3]], - labels[entry[4]], - entry[5] - ) - when :rescue - iseq.catch_rescue( - from(entry[1], options, iseq), - labels[entry[2]], - labels[entry[3]], - labels[entry[4]], - entry[5] - ) - when :redo - iseq.catch_redo( - labels[entry[2]], - labels[entry[3]], - labels[entry[4]], - entry[5] - ) - when :retry - iseq.catch_retry( - labels[entry[2]], - labels[entry[3]], - labels[entry[4]], - entry[5] - ) - else - raise "unknown catch type: #{entry[0]}" - end - end - - iseq.compile! if iseq.type == :top - iseq - end - end - end -end diff --git a/lib/syntax_tree/yarv/instructions.rb b/lib/syntax_tree/yarv/instructions.rb deleted file mode 100644 index 02188dfe..00000000 --- a/lib/syntax_tree/yarv/instructions.rb +++ /dev/null @@ -1,5885 +0,0 @@ -# frozen_string_literal: true - -module SyntaxTree - module YARV - # This is a base class for all YARV instructions. It provides a few - # convenience methods for working with instructions. - class Instruction - # This method creates an instruction that represents the canonical - # (non-specialized) form of this instruction. If this instruction is not - # a specialized instruction, then this method returns `self`. - def canonical - self - end - - # This returns the size of the instruction in terms of the number of slots - # it occupies in the instruction sequence. Effectively this is 1 plus the - # number of operands. - def length - 1 - end - - # This returns the number of values that are pushed onto the stack. - def pushes - 0 - end - - # This returns the number of values that are popped off the stack. - def pops - 0 - end - - # This returns an array of labels. - def branch_targets - [] - end - - # Whether or not this instruction leaves the current frame. - def leaves? - false - end - - # Whether or not this instruction falls through to the next instruction if - # its branching fails. - def falls_through? - false - end - - # Does the instruction have side effects? Control-flow counts as a - # side-effect, as do some special-case instructions like Leave. By default - # every instruction is marked as having side effects. - def side_effects? - true - end - end - - # ### Summary - # - # `adjuststack` accepts a single integer argument and removes that many - # elements from the top of the stack. - # - # ### Usage - # - # ~~~ruby - # x = [true] - # x[0] ||= nil - # x[0] - # ~~~ - # - class AdjustStack < Instruction - attr_reader :number - - def initialize(number) - @number = number - end - - def disasm(fmt) - fmt.instruction("adjuststack", [fmt.object(number)]) - end - - def to_a(_iseq) - [:adjuststack, number] - end - - def deconstruct_keys(_keys) - { number: number } - end - - def ==(other) - other.is_a?(AdjustStack) && other.number == number - end - - def length - 2 - end - - def pops - number - end - - def call(vm) - vm.pop(number) - end - end - - # ### Summary - # - # `anytostring` ensures that the value on top of the stack is a string. - # - # It pops two values off the stack. If the first value is a string it - # pushes it back on the stack. If the first value is not a string, it uses - # Ruby's built in string coercion to coerce the second value to a string - # and then pushes that back on the stack. - # - # This is used in conjunction with `objtostring` as a fallback for when an - # object's `to_s` method does not return a string. - # - # ### Usage - # - # ~~~ruby - # "#{5}" - # ~~~ - # - class AnyToString < Instruction - def disasm(fmt) - fmt.instruction("anytostring") - end - - def to_a(_iseq) - [:anytostring] - end - - def deconstruct_keys(_keys) - {} - end - - def ==(other) - other.is_a?(AnyToString) - end - - def pops - 2 - end - - def pushes - 1 - end - - def call(vm) - original, value = vm.pop(2) - - if value.is_a?(String) - vm.push(value) - else - vm.push("#<#{original.class.name}:0000>") - end - end - end - - # ### Summary - # - # `branchif` has one argument: the jump index. It pops one value off the - # stack: the jump condition. - # - # If the value popped off the stack is true, `branchif` jumps to - # the jump index and continues executing there. - # - # ### Usage - # - # ~~~ruby - # x = true - # x ||= "foo" - # puts x - # ~~~ - # - class BranchIf < Instruction - attr_reader :label - - def initialize(label) - @label = label - end - - def disasm(fmt) - fmt.instruction("branchif", [fmt.label(label)]) - end - - def to_a(_iseq) - [:branchif, label.name] - end - - def deconstruct_keys(_keys) - { label: label } - end - - def ==(other) - other.is_a?(BranchIf) && other.label == label - end - - def length - 2 - end - - def pops - 1 - end - - def call(vm) - vm.jump(label) if vm.pop - end - - def branch_targets - [label] - end - - def falls_through? - true - end - end - - # ### Summary - # - # `branchnil` has one argument: the jump index. It pops one value off the - # stack: the jump condition. - # - # If the value popped off the stack is nil, `branchnil` jumps to - # the jump index and continues executing there. - # - # ### Usage - # - # ~~~ruby - # x = nil - # if x&.to_s - # puts "hi" - # end - # ~~~ - # - class BranchNil < Instruction - attr_reader :label - - def initialize(label) - @label = label - end - - def disasm(fmt) - fmt.instruction("branchnil", [fmt.label(label)]) - end - - def to_a(_iseq) - [:branchnil, label.name] - end - - def deconstruct_keys(_keys) - { label: label } - end - - def ==(other) - other.is_a?(BranchNil) && other.label == label - end - - def length - 2 - end - - def pops - 1 - end - - def call(vm) - vm.jump(label) if vm.pop.nil? - end - - def branch_targets - [label] - end - - def falls_through? - true - end - end - - # ### Summary - # - # `branchunless` has one argument: the jump index. It pops one value off - # the stack: the jump condition. - # - # If the value popped off the stack is false or nil, `branchunless` jumps - # to the jump index and continues executing there. - # - # ### Usage - # - # ~~~ruby - # if 2 + 3 - # puts "foo" - # end - # ~~~ - # - class BranchUnless < Instruction - attr_reader :label - - def initialize(label) - @label = label - end - - def disasm(fmt) - fmt.instruction("branchunless", [fmt.label(label)]) - end - - def to_a(_iseq) - [:branchunless, label.name] - end - - def deconstruct_keys(_keys) - { label: label } - end - - def ==(other) - other.is_a?(BranchUnless) && other.label == label - end - - def length - 2 - end - - def pops - 1 - end - - def call(vm) - vm.jump(label) unless vm.pop - end - - def branch_targets - [label] - end - - def falls_through? - true - end - end - - # ### Summary - # - # `checkkeyword` checks if a keyword was passed at the callsite that - # called into the method represented by the instruction sequence. It has - # two arguments: the index of the local variable that stores the keywords - # metadata and the index of the keyword within that metadata. It pushes - # a boolean onto the stack indicating whether or not the keyword was - # given. - # - # ### Usage - # - # ~~~ruby - # def evaluate(value: rand) - # value - # end - # - # evaluate(value: 3) - # ~~~ - # - class CheckKeyword < Instruction - attr_reader :keyword_bits_index, :keyword_index - - def initialize(keyword_bits_index, keyword_index) - @keyword_bits_index = keyword_bits_index - @keyword_index = keyword_index - end - - def disasm(fmt) - fmt.instruction( - "checkkeyword", - [fmt.object(keyword_bits_index), fmt.object(keyword_index)] - ) - end - - def to_a(iseq) - [ - :checkkeyword, - iseq.local_table.offset(keyword_bits_index), - keyword_index - ] - end - - def deconstruct_keys(_keys) - { keyword_bits_index: keyword_bits_index, keyword_index: keyword_index } - end - - def ==(other) - other.is_a?(CheckKeyword) && - other.keyword_bits_index == keyword_bits_index && - other.keyword_index == keyword_index - end - - def length - 3 - end - - def pushes - 1 - end - - def call(vm) - vm.push(vm.local_get(keyword_bits_index, 0)[keyword_index]) - end - end - - # ### Summary - # - # `checkmatch` checks if the current pattern matches the current value. It - # pops the target and the pattern off the stack and pushes a boolean onto - # the stack if it matches or not. - # - # ### Usage - # - # ~~~ruby - # foo in Foo - # ~~~ - # - class CheckMatch < Instruction - VM_CHECKMATCH_TYPE_WHEN = 1 - VM_CHECKMATCH_TYPE_CASE = 2 - VM_CHECKMATCH_TYPE_RESCUE = 3 - VM_CHECKMATCH_TYPE_MASK = 0x03 - VM_CHECKMATCH_ARRAY = 0x04 - - attr_reader :type - - def initialize(type) - @type = type - end - - def disasm(fmt) - fmt.instruction("checkmatch", [fmt.object(type)]) - end - - def to_a(_iseq) - [:checkmatch, type] - end - - def deconstruct_keys(_keys) - { type: type } - end - - def ==(other) - other.is_a?(CheckMatch) && other.type == type - end - - def length - 2 - end - - def pops - 2 - end - - def pushes - 1 - end - - def call(vm) - target, pattern = vm.pop(2) - - vm.push( - if type & VM_CHECKMATCH_ARRAY > 0 - pattern.any? { |item| check?(item, target) } - else - check?(pattern, target) - end - ) - end - - private - - def check?(pattern, target) - case type & VM_CHECKMATCH_TYPE_MASK - when VM_CHECKMATCH_TYPE_WHEN - pattern - when VM_CHECKMATCH_TYPE_CASE - pattern === target - when VM_CHECKMATCH_TYPE_RESCUE - unless pattern.is_a?(Module) - raise TypeError, "class or module required for rescue clause" - end - - pattern === target - end - end - end - - # ### Summary - # - # `checktype` checks if the value on top of the stack is of a certain type. - # The type is the only argument. It pops the value off the stack and pushes - # a boolean onto the stack indicating whether or not the value is of the - # given type. - # - # ### Usage - # - # ~~~ruby - # foo in [bar] - # ~~~ - # - class CheckType < Instruction - TYPE_OBJECT = 0x01 - TYPE_CLASS = 0x02 - TYPE_MODULE = 0x03 - TYPE_FLOAT = 0x04 - TYPE_STRING = 0x05 - TYPE_REGEXP = 0x06 - TYPE_ARRAY = 0x07 - TYPE_HASH = 0x08 - TYPE_STRUCT = 0x09 - TYPE_BIGNUM = 0x0a - TYPE_FILE = 0x0b - TYPE_DATA = 0x0c - TYPE_MATCH = 0x0d - TYPE_COMPLEX = 0x0e - TYPE_RATIONAL = 0x0f - TYPE_NIL = 0x11 - TYPE_TRUE = 0x12 - TYPE_FALSE = 0x13 - TYPE_SYMBOL = 0x14 - TYPE_FIXNUM = 0x15 - TYPE_UNDEF = 0x16 - - attr_reader :type - - def initialize(type) - @type = type - end - - def disasm(fmt) - name = - case type - when TYPE_OBJECT - "T_OBJECT" - when TYPE_CLASS - "T_CLASS" - when TYPE_MODULE - "T_MODULE" - when TYPE_FLOAT - "T_FLOAT" - when TYPE_STRING - "T_STRING" - when TYPE_REGEXP - "T_REGEXP" - when TYPE_ARRAY - "T_ARRAY" - when TYPE_HASH - "T_HASH" - when TYPE_STRUCT - "T_STRUCT" - when TYPE_BIGNUM - "T_BIGNUM" - when TYPE_FILE - "T_FILE" - when TYPE_DATA - "T_DATA" - when TYPE_MATCH - "T_MATCH" - when TYPE_COMPLEX - "T_COMPLEX" - when TYPE_RATIONAL - "T_RATIONAL" - when TYPE_NIL - "T_NIL" - when TYPE_TRUE - "T_TRUE" - when TYPE_FALSE - "T_FALSE" - when TYPE_SYMBOL - "T_SYMBOL" - when TYPE_FIXNUM - "T_FIXNUM" - when TYPE_UNDEF - "T_UNDEF" - end - - fmt.instruction("checktype", [name]) - end - - def to_a(_iseq) - [:checktype, type] - end - - def deconstruct_keys(_keys) - { type: type } - end - - def ==(other) - other.is_a?(CheckType) && other.type == type - end - - def length - 2 - end - - def pops - 1 - end - - def pushes - # TODO: This is incorrect. The instruction only pushes a single value - # onto the stack. However, if this is set to 1, we no longer match the - # output of RubyVM::InstructionSequence. So leaving this here until we - # can investigate further. - 2 - end - - def call(vm) - object = vm.pop - result = - case type - when TYPE_OBJECT - raise NotImplementedError, "checktype TYPE_OBJECT" - when TYPE_CLASS - object.is_a?(Class) - when TYPE_MODULE - object.is_a?(Module) - when TYPE_FLOAT - object.is_a?(Float) - when TYPE_STRING - object.is_a?(String) - when TYPE_REGEXP - object.is_a?(Regexp) - when TYPE_ARRAY - object.is_a?(Array) - when TYPE_HASH - object.is_a?(Hash) - when TYPE_STRUCT - object.is_a?(Struct) - when TYPE_BIGNUM - raise NotImplementedError, "checktype TYPE_BIGNUM" - when TYPE_FILE - object.is_a?(File) - when TYPE_DATA - raise NotImplementedError, "checktype TYPE_DATA" - when TYPE_MATCH - raise NotImplementedError, "checktype TYPE_MATCH" - when TYPE_COMPLEX - object.is_a?(Complex) - when TYPE_RATIONAL - object.is_a?(Rational) - when TYPE_NIL - object.nil? - when TYPE_TRUE - object == true - when TYPE_FALSE - object == false - when TYPE_SYMBOL - object.is_a?(Symbol) - when TYPE_FIXNUM - object.is_a?(Integer) - when TYPE_UNDEF - raise NotImplementedError, "checktype TYPE_UNDEF" - end - - vm.push(result) - end - end - - # ### Summary - # - # `concatarray` concatenates the two Arrays on top of the stack. - # - # It coerces the two objects at the top of the stack into Arrays by - # calling `to_a` if necessary, and makes sure to `dup` the first Array if - # it was already an Array, to avoid mutating it when concatenating. - # - # ### Usage - # - # ~~~ruby - # [1, *2] - # ~~~ - # - class ConcatArray < Instruction - def disasm(fmt) - fmt.instruction("concatarray") - end - - def to_a(_iseq) - [:concatarray] - end - - def deconstruct_keys(_keys) - {} - end - - def ==(other) - other.is_a?(ConcatArray) - end - - def pops - 2 - end - - def pushes - 1 - end - - def call(vm) - left, right = vm.pop(2) - vm.push([*left, *right]) - end - end - - # ### Summary - # - # `concatstrings` pops a number of strings from the stack joins them - # together into a single string and pushes that string back on the stack. - # - # This does no coercion and so is always used in conjunction with - # `objtostring` and `anytostring` to ensure the stack contents are always - # strings. - # - # ### Usage - # - # ~~~ruby - # "#{5}" - # ~~~ - # - class ConcatStrings < Instruction - attr_reader :number - - def initialize(number) - @number = number - end - - def disasm(fmt) - fmt.instruction("concatstrings", [fmt.object(number)]) - end - - def to_a(_iseq) - [:concatstrings, number] - end - - def deconstruct_keys(_keys) - { number: number } - end - - def ==(other) - other.is_a?(ConcatStrings) && other.number == number - end - - def length - 2 - end - - def pops - number - end - - def pushes - 1 - end - - def call(vm) - vm.push(vm.pop(number).join) - end - end - - # ### Summary - # - # `concattoarray` pops a single value off the stack and attempts to concat - # it to the Array on top of the stack. If the value is not an Array, it - # will be coerced into one. - # - # ### Usage - # - # ~~~ruby - # [1, *2] - # ~~~ - # - class ConcatToArray < Instruction - attr_reader :object - - def initialize(object) - @object = object - end - - def disasm(fmt) - fmt.instruction("concattoarray", [fmt.object(object)]) - end - - def to_a(_iseq) - [:concattoarray, object] - end - - def deconstruct_keys(_keys) - { object: object } - end - - def ==(other) - other.is_a?(ConcatToArray) && other.object == object - end - - def length - 2 - end - - def pops - 1 - end - - def pushes - 1 - end - - def call(vm) - array, value = vm.pop(2) - vm.push(array.concat(Array(value))) - end - end - - # ### Summary - # - # `defineclass` defines a class. First it pops the superclass off the - # stack, then it pops the object off the stack that the class should be - # defined under. It has three arguments: the name of the constant, the - # instruction sequence associated with the class, and various flags that - # indicate if it is a singleton class, a module, or a regular class. - # - # ### Usage - # - # ~~~ruby - # class Foo - # end - # ~~~ - # - class DefineClass < Instruction - TYPE_CLASS = 0 - TYPE_SINGLETON_CLASS = 1 - TYPE_MODULE = 2 - FLAG_SCOPED = 8 - FLAG_HAS_SUPERCLASS = 16 - - attr_reader :name, :class_iseq, :flags - - def initialize(name, class_iseq, flags) - @name = name - @class_iseq = class_iseq - @flags = flags - end - - def disasm(fmt) - fmt.enqueue(class_iseq) - fmt.instruction( - "defineclass", - [fmt.object(name), class_iseq.name, fmt.object(flags)] - ) - end - - def to_a(_iseq) - [:defineclass, name, class_iseq.to_a, flags] - end - - def deconstruct_keys(_keys) - { name: name, class_iseq: class_iseq, flags: flags } - end - - def ==(other) - other.is_a?(DefineClass) && other.name == name && - other.class_iseq == class_iseq && other.flags == flags - end - - def length - 4 - end - - def pops - 2 - end - - def pushes - 1 - end - - def call(vm) - object, superclass = vm.pop(2) - - if name == :singletonclass - vm.push(vm.run_class_frame(class_iseq, object.singleton_class)) - elsif object.const_defined?(name) - vm.push(vm.run_class_frame(class_iseq, object.const_get(name))) - elsif flags & TYPE_MODULE > 0 - clazz = Module.new - object.const_set(name, clazz) - vm.push(vm.run_class_frame(class_iseq, clazz)) - else - clazz = - if flags & FLAG_HAS_SUPERCLASS > 0 - Class.new(superclass) - else - Class.new - end - - object.const_set(name, clazz) - vm.push(vm.run_class_frame(class_iseq, clazz)) - end - end - end - - # ### Summary - # - # `defined` checks if the top value of the stack is defined. If it is, it - # pushes its value onto the stack. Otherwise it pushes `nil`. - # - # ### Usage - # - # ~~~ruby - # defined?(x) - # ~~~ - # - class Defined < Instruction - TYPE_NIL = 1 - TYPE_IVAR = 2 - TYPE_LVAR = 3 - TYPE_GVAR = 4 - TYPE_CVAR = 5 - TYPE_CONST = 6 - TYPE_METHOD = 7 - TYPE_YIELD = 8 - TYPE_ZSUPER = 9 - TYPE_SELF = 10 - TYPE_TRUE = 11 - TYPE_FALSE = 12 - TYPE_ASGN = 13 - TYPE_EXPR = 14 - TYPE_REF = 15 - TYPE_FUNC = 16 - TYPE_CONST_FROM = 17 - - attr_reader :type, :name, :message - - def initialize(type, name, message) - @type = type - @name = name - @message = message - end - - def disasm(fmt) - type_name = - case type - when TYPE_NIL - "nil" - when TYPE_IVAR - "ivar" - when TYPE_LVAR - "lvar" - when TYPE_GVAR - "gvar" - when TYPE_CVAR - "cvar" - when TYPE_CONST - "const" - when TYPE_METHOD - "method" - when TYPE_YIELD - "yield" - when TYPE_ZSUPER - "zsuper" - when TYPE_SELF - "self" - when TYPE_TRUE - "true" - when TYPE_FALSE - "false" - when TYPE_ASGN - "asgn" - when TYPE_EXPR - "expr" - when TYPE_REF - "ref" - when TYPE_FUNC - "func" - when TYPE_CONST_FROM - "constant-from" - end - - fmt.instruction( - "defined", - [type_name, fmt.object(name), fmt.object(message)] - ) - end - - def to_a(_iseq) - [:defined, type, name, message] - end - - def deconstruct_keys(_keys) - { type: type, name: name, message: message } - end - - def ==(other) - other.is_a?(Defined) && other.type == type && other.name == name && - other.message == message - end - - def length - 4 - end - - def pops - 1 - end - - def pushes - 1 - end - - def call(vm) - object = vm.pop - - result = - case type - when TYPE_NIL, TYPE_SELF, TYPE_TRUE, TYPE_FALSE, TYPE_ASGN, TYPE_EXPR - message - when TYPE_IVAR - message if vm.frame._self.instance_variable_defined?(name) - when TYPE_LVAR - raise NotImplementedError, "defined TYPE_LVAR" - when TYPE_GVAR - message if global_variables.include?(name) - when TYPE_CVAR - clazz = vm.frame._self - clazz = clazz.singleton_class unless clazz.is_a?(Module) - message if clazz.class_variable_defined?(name) - when TYPE_CONST - clazz = vm.frame._self - clazz = clazz.singleton_class unless clazz.is_a?(Module) - message if clazz.const_defined?(name) - when TYPE_METHOD - raise NotImplementedError, "defined TYPE_METHOD" - when TYPE_YIELD - raise NotImplementedError, "defined TYPE_YIELD" - when TYPE_ZSUPER - raise NotImplementedError, "defined TYPE_ZSUPER" - when TYPE_REF - raise NotImplementedError, "defined TYPE_REF" - when TYPE_FUNC - message if object.respond_to?(name, true) - when TYPE_CONST_FROM - defined = - vm.frame.nesting.any? { |scope| scope.const_defined?(name, true) } - message if defined - end - - vm.push(result) - end - end - - # ### Summary - # - # `definedivar` checks if an instance variable is defined. It is a - # specialization of the `defined` instruction. It accepts three arguments: - # the name of the instance variable, an inline cache, and the string that - # should be pushed onto the stack in the event that the instance variable - # is defined. - # - # ### Usage - # - # ~~~ruby - # defined?(@value) - # ~~~ - # - class DefinedIVar < Instruction - attr_reader :name, :cache, :message - - def initialize(name, cache, message) - @name = name - @cache = cache - @message = message - end - - def disasm(fmt) - fmt.instruction( - "definedivar", - [fmt.object(name), fmt.inline_storage(cache), fmt.object(message)] - ) - end - - def to_a(_iseq) - [:definedivar, name, cache, message] - end - - def deconstruct_keys(_keys) - { name: name, cache: cache, message: message } - end - - def ==(other) - other.is_a?(DefinedIVar) && other.name == name && - other.cache == cache && other.message == message - end - - def length - 4 - end - - def pushes - 1 - end - - def call(vm) - result = (message if vm.frame._self.instance_variable_defined?(name)) - - vm.push(result) - end - end - - # ### Summary - # - # `definemethod` defines a method on the class of the current value of - # `self`. It accepts two arguments. The first is the name of the method - # being defined. The second is the instruction sequence representing the - # body of the method. - # - # ### Usage - # - # ~~~ruby - # def value = "value" - # ~~~ - # - class DefineMethod < Instruction - attr_reader :method_name, :method_iseq - - def initialize(method_name, method_iseq) - @method_name = method_name - @method_iseq = method_iseq - end - - def disasm(fmt) - fmt.enqueue(method_iseq) - fmt.instruction( - "definemethod", - [fmt.object(method_name), method_iseq.name] - ) - end - - def to_a(_iseq) - [:definemethod, method_name, method_iseq.to_a] - end - - def deconstruct_keys(_keys) - { method_name: method_name, method_iseq: method_iseq } - end - - def ==(other) - other.is_a?(DefineMethod) && other.method_name == method_name && - other.method_iseq == method_iseq - end - - def length - 3 - end - - def call(vm) - name = method_name - nesting = vm.frame.nesting - iseq = method_iseq - - vm - .frame - ._self - .__send__(:define_method, name) do |*args, **kwargs, &block| - vm.run_method_frame( - name, - nesting, - iseq, - self, - *args, - **kwargs, - &block - ) - end - end - end - - # ### Summary - # - # `definesmethod` defines a method on the singleton class of the current - # value of `self`. It accepts two arguments. The first is the name of the - # method being defined. The second is the instruction sequence representing - # the body of the method. It pops the object off the stack that the method - # should be defined on. - # - # ### Usage - # - # ~~~ruby - # def self.value = "value" - # ~~~ - # - class DefineSMethod < Instruction - attr_reader :method_name, :method_iseq - - def initialize(method_name, method_iseq) - @method_name = method_name - @method_iseq = method_iseq - end - - def disasm(fmt) - fmt.enqueue(method_iseq) - fmt.instruction( - "definesmethod", - [fmt.object(method_name), method_iseq.name] - ) - end - - def to_a(_iseq) - [:definesmethod, method_name, method_iseq.to_a] - end - - def deconstruct_keys(_keys) - { method_name: method_name, method_iseq: method_iseq } - end - - def ==(other) - other.is_a?(DefineSMethod) && other.method_name == method_name && - other.method_iseq == method_iseq - end - - def length - 3 - end - - def pops - 1 - end - - def call(vm) - name = method_name - nesting = vm.frame.nesting - iseq = method_iseq - - vm - .frame - ._self - .__send__(:define_singleton_method, name) do |*args, **kwargs, &block| - vm.run_method_frame( - name, - nesting, - iseq, - self, - *args, - **kwargs, - &block - ) - end - end - end - - # ### Summary - # - # `dup` copies the top value of the stack and pushes it onto the stack. - # - # ### Usage - # - # ~~~ruby - # $global = 5 - # ~~~ - # - class Dup < Instruction - def disasm(fmt) - fmt.instruction("dup") - end - - def to_a(_iseq) - [:dup] - end - - def deconstruct_keys(_keys) - {} - end - - def ==(other) - other.is_a?(Dup) - end - - def pops - 1 - end - - def pushes - 2 - end - - def call(vm) - vm.push(vm.stack.last.dup) - end - - def side_effects? - false - end - end - - # ### Summary - # - # `duparray` dups an Array literal and pushes it onto the stack. - # - # ### Usage - # - # ~~~ruby - # [true] - # ~~~ - # - class DupArray < Instruction - attr_reader :object - - def initialize(object) - @object = object - end - - def disasm(fmt) - fmt.instruction("duparray", [fmt.object(object)]) - end - - def to_a(_iseq) - [:duparray, object] - end - - def deconstruct_keys(_keys) - { object: object } - end - - def ==(other) - other.is_a?(DupArray) && other.object == object - end - - def length - 2 - end - - def pushes - 1 - end - - def call(vm) - vm.push(object.dup) - end - end - - # ### Summary - # - # `duphash` dups a Hash literal and pushes it onto the stack. - # - # ### Usage - # - # ~~~ruby - # { a: 1 } - # ~~~ - # - class DupHash < Instruction - attr_reader :object - - def initialize(object) - @object = object - end - - def disasm(fmt) - fmt.instruction("duphash", [fmt.object(object)]) - end - - def to_a(_iseq) - [:duphash, object] - end - - def deconstruct_keys(_keys) - { object: object } - end - - def ==(other) - other.is_a?(DupHash) && other.object == object - end - - def length - 2 - end - - def pushes - 1 - end - - def call(vm) - vm.push(object.dup) - end - end - - # ### Summary - # - # `dupn` duplicates the top `n` stack elements. - # - # ### Usage - # - # ~~~ruby - # Object::X ||= true - # ~~~ - # - class DupN < Instruction - attr_reader :number - - def initialize(number) - @number = number - end - - def disasm(fmt) - fmt.instruction("dupn", [fmt.object(number)]) - end - - def to_a(_iseq) - [:dupn, number] - end - - def deconstruct_keys(_keys) - { number: number } - end - - def ==(other) - other.is_a?(DupN) && other.number == number - end - - def length - 2 - end - - def pushes - number - end - - def call(vm) - values = vm.pop(number) - vm.push(*values) - vm.push(*values) - end - end - - # ### Summary - # - # `expandarray` looks at the top of the stack, and if the value is an array - # it replaces it on the stack with `number` elements of the array, or `nil` - # if the elements are missing. - # - # ### Usage - # - # ~~~ruby - # x, = [true, false, nil] - # ~~~ - # - class ExpandArray < Instruction - attr_reader :number, :flags - - def initialize(number, flags) - @number = number - @flags = flags - end - - def disasm(fmt) - fmt.instruction("expandarray", [fmt.object(number), fmt.object(flags)]) - end - - def to_a(_iseq) - [:expandarray, number, flags] - end - - def deconstruct_keys(_keys) - { number: number, flags: flags } - end - - def ==(other) - other.is_a?(ExpandArray) && other.number == number && - other.flags == flags - end - - def length - 3 - end - - def pops - 1 - end - - def pushes - number - end - - def call(vm) - object = vm.pop - object = - if Array === object - object.dup - elsif object.respond_to?(:to_ary, true) - object.to_ary - else - [object] - end - - splat_flag = flags & 0x01 > 0 - postarg_flag = flags & 0x02 > 0 - - if number == 0 && splat_flag == 0 - # no space left on stack - elsif postarg_flag - values = [] - - if number > object.size - (number - object.size).times { values.push(nil) } - end - [number, object.size].min.times { values.push(object.pop) } - values.push(object.to_a) if splat_flag - - values.each { |item| vm.push(item) } - else - values = [] - - [number, object.size].min.times { values.push(object.shift) } - if number > values.size - (number - values.size).times { values.push(nil) } - end - values.push(object.to_a) if splat_flag - - values.reverse_each { |item| vm.push(item) } - end - end - end - - # ### Summary - # - # `getblockparam` is a similar instruction to `getlocal` in that it looks - # for a local variable in the current instruction sequence's local table and - # walks recursively up the parent instruction sequences until it finds it. - # The local it retrieves, however, is a special block local that was passed - # to the current method. It pushes the value of the block local onto the - # stack. - # - # ### Usage - # - # ~~~ruby - # def foo(&block) - # block - # end - # ~~~ - # - class GetBlockParam < Instruction - attr_reader :index, :level - - def initialize(index, level) - @index = index - @level = level - end - - def disasm(fmt) - fmt.instruction("getblockparam", [fmt.local(index, explicit: level)]) - end - - def to_a(iseq) - current = iseq - level.times { current = iseq.parent_iseq } - [:getblockparam, current.local_table.offset(index), level] - end - - def deconstruct_keys(_keys) - { index: index, level: level } - end - - def ==(other) - other.is_a?(GetBlockParam) && other.index == index && - other.level == level - end - - def length - 3 - end - - def pushes - 1 - end - - def call(vm) - vm.push(vm.local_get(index, level)) - end - end - - # ### Summary - # - # `getblockparamproxy` is almost the same as `getblockparam` except that it - # pushes a proxy object onto the stack instead of the actual value of the - # block local. This is used when a method is being called on the block - # local. - # - # ### Usage - # - # ~~~ruby - # def foo(&block) - # block.call - # end - # ~~~ - # - class GetBlockParamProxy < Instruction - attr_reader :index, :level - - def initialize(index, level) - @index = index - @level = level - end - - def disasm(fmt) - fmt.instruction( - "getblockparamproxy", - [fmt.local(index, explicit: level)] - ) - end - - def to_a(iseq) - current = iseq - level.times { current = iseq.parent_iseq } - [:getblockparamproxy, current.local_table.offset(index), level] - end - - def deconstruct_keys(_keys) - { index: index, level: level } - end - - def ==(other) - other.is_a?(GetBlockParamProxy) && other.index == index && - other.level == level - end - - def length - 3 - end - - def pushes - 1 - end - - def call(vm) - vm.push(vm.local_get(index, level)) - end - end - - # ### Summary - # - # `getclassvariable` looks for a class variable in the current class and - # pushes its value onto the stack. It uses an inline cache to reduce the - # need to lookup the class variable in the class hierarchy every time. - # - # ### Usage - # - # ~~~ruby - # @@class_variable - # ~~~ - # - class GetClassVariable < Instruction - attr_reader :name, :cache - - def initialize(name, cache) - @name = name - @cache = cache - end - - def disasm(fmt) - fmt.instruction( - "getclassvariable", - [fmt.object(name), fmt.inline_storage(cache)] - ) - end - - def to_a(_iseq) - [:getclassvariable, name, cache] - end - - def deconstruct_keys(_keys) - { name: name, cache: cache } - end - - def ==(other) - other.is_a?(GetClassVariable) && other.name == name && - other.cache == cache - end - - def length - 3 - end - - def pushes - 1 - end - - def call(vm) - clazz = vm.frame._self - clazz = clazz.class unless clazz.is_a?(Class) - vm.push(clazz.class_variable_get(name)) - end - end - - # ### Summary - # - # `getconstant` performs a constant lookup and pushes the value of the - # constant onto the stack. It pops both the class it should look in and - # whether or not it should look globally as well. - # - # ### Usage - # - # ~~~ruby - # Constant - # ~~~ - # - class GetConstant < Instruction - attr_reader :name - - def initialize(name) - @name = name - end - - def disasm(fmt) - fmt.instruction("getconstant", [fmt.object(name)]) - end - - def to_a(_iseq) - [:getconstant, name] - end - - def deconstruct_keys(_keys) - { name: name } - end - - def ==(other) - other.is_a?(GetConstant) && other.name == name - end - - def length - 2 - end - - def pops - 2 - end - - def pushes - 1 - end - - def call(vm) - const_base, allow_nil = vm.pop(2) - - if const_base - if const_base.const_defined?(name) - vm.push(const_base.const_get(name)) - return - end - elsif const_base.nil? && allow_nil - vm.frame.nesting.reverse_each do |clazz| - if clazz.const_defined?(name) - vm.push(clazz.const_get(name)) - return - end - end - end - - raise NameError, "uninitialized constant #{name}" - end - end - - # ### Summary - # - # `getglobal` pushes the value of a global variables onto the stack. - # - # ### Usage - # - # ~~~ruby - # $$ - # ~~~ - # - class GetGlobal < Instruction - attr_reader :name - - def initialize(name) - @name = name - end - - def disasm(fmt) - fmt.instruction("getglobal", [fmt.object(name)]) - end - - def to_a(_iseq) - [:getglobal, name] - end - - def deconstruct_keys(_keys) - { name: name } - end - - def ==(other) - other.is_a?(GetGlobal) && other.name == name - end - - def length - 2 - end - - def pushes - 1 - end - - def call(vm) - # Evaluating the name of the global variable because there isn't a - # reflection API for global variables. - vm.push(eval(name.to_s, binding, __FILE__, __LINE__)) - end - end - - # ### Summary - # - # `getinstancevariable` pushes the value of an instance variable onto the - # stack. It uses an inline cache to avoid having to look up the instance - # variable in the class hierarchy every time. - # - # This instruction has two forms, but both have the same structure. Before - # Ruby 3.2, the inline cache corresponded to both the get and set - # instructions and could be shared. Since Ruby 3.2, it uses object shapes - # instead so the caches are unique per instruction. - # - # ### Usage - # - # ~~~ruby - # @instance_variable - # ~~~ - # - class GetInstanceVariable < Instruction - attr_reader :name, :cache - - def initialize(name, cache) - @name = name - @cache = cache - end - - def disasm(fmt) - fmt.instruction( - "getinstancevariable", - [fmt.object(name), fmt.inline_storage(cache)] - ) - end - - def to_a(_iseq) - [:getinstancevariable, name, cache] - end - - def deconstruct_keys(_keys) - { name: name, cache: cache } - end - - def ==(other) - other.is_a?(GetInstanceVariable) && other.name == name && - other.cache == cache - end - - def length - 3 - end - - def pushes - 1 - end - - def call(vm) - method = Object.instance_method(:instance_variable_get) - vm.push(method.bind(vm.frame._self).call(name)) - end - end - - # ### Summary - # - # `getlocal` fetches the value of a local variable from a frame determined - # by the level and index arguments. The level is the number of frames back - # to look and the index is the index in the local table. It pushes the value - # it finds onto the stack. - # - # ### Usage - # - # ~~~ruby - # value = 5 - # tap { tap { value } } - # ~~~ - # - class GetLocal < Instruction - attr_reader :index, :level - - def initialize(index, level) - @index = index - @level = level - end - - def disasm(fmt) - fmt.instruction("getlocal", [fmt.local(index, explicit: level)]) - end - - def to_a(iseq) - current = iseq - level.times { current = current.parent_iseq } - [:getlocal, current.local_table.offset(index), level] - end - - def deconstruct_keys(_keys) - { index: index, level: level } - end - - def ==(other) - other.is_a?(GetLocal) && other.index == index && other.level == level - end - - def length - 3 - end - - def pushes - 1 - end - - def call(vm) - vm.push(vm.local_get(index, level)) - end - end - - # ### Summary - # - # `getlocal_WC_0` is a specialized version of the `getlocal` instruction. It - # fetches the value of a local variable from the current frame determined by - # the index given as its only argument. - # - # ### Usage - # - # ~~~ruby - # value = 5 - # value - # ~~~ - # - class GetLocalWC0 < Instruction - attr_reader :index - - def initialize(index) - @index = index - end - - def disasm(fmt) - fmt.instruction("getlocal_WC_0", [fmt.local(index, implicit: 0)]) - end - - def to_a(iseq) - [:getlocal_WC_0, iseq.local_table.offset(index)] - end - - def deconstruct_keys(_keys) - { index: index } - end - - def ==(other) - other.is_a?(GetLocalWC0) && other.index == index - end - - def length - 2 - end - - def pushes - 1 - end - - def canonical - GetLocal.new(index, 0) - end - - def call(vm) - canonical.call(vm) - end - end - - # ### Summary - # - # `getlocal_WC_1` is a specialized version of the `getlocal` instruction. It - # fetches the value of a local variable from the parent frame determined by - # the index given as its only argument. - # - # ### Usage - # - # ~~~ruby - # value = 5 - # self.then { value } - # ~~~ - # - class GetLocalWC1 < Instruction - attr_reader :index - - def initialize(index) - @index = index - end - - def disasm(fmt) - fmt.instruction("getlocal_WC_1", [fmt.local(index, implicit: 1)]) - end - - def to_a(iseq) - [:getlocal_WC_1, iseq.parent_iseq.local_table.offset(index)] - end - - def deconstruct_keys(_keys) - { index: index } - end - - def ==(other) - other.is_a?(GetLocalWC1) && other.index == index - end - - def length - 2 - end - - def pushes - 1 - end - - def canonical - GetLocal.new(index, 1) - end - - def call(vm) - canonical.call(vm) - end - end - - # ### Summary - # - # `getspecial` pushes the value of a special local variable onto the stack. - # - # ### Usage - # - # ~~~ruby - # 1 if (a == 1) .. (b == 2) - # ~~~ - # - class GetSpecial < Instruction - SVAR_LASTLINE = 0 # $_ - SVAR_BACKREF = 1 # $~ - SVAR_FLIPFLOP_START = 2 # flipflop - - attr_reader :key, :type - - def initialize(key, type) - @key = key - @type = type - end - - def disasm(fmt) - fmt.instruction("getspecial", [fmt.object(key), fmt.object(type)]) - end - - def to_a(_iseq) - [:getspecial, key, type] - end - - def deconstruct_keys(_keys) - { key: key, type: type } - end - - def ==(other) - other.is_a?(GetSpecial) && other.key == key && other.type == type - end - - def length - 3 - end - - def pushes - 1 - end - - def call(vm) - case key - when SVAR_LASTLINE - raise NotImplementedError, "getspecial SVAR_LASTLINE" - when SVAR_BACKREF - raise NotImplementedError, "getspecial SVAR_BACKREF" - when SVAR_FLIPFLOP_START - vm.frame_svar.svars[SVAR_FLIPFLOP_START] - end - end - end - - # ### Summary - # - # `intern` converts the top element of the stack to a symbol and pushes the - # symbol onto the stack. - # - # ### Usage - # - # ~~~ruby - # :"#{"foo"}" - # ~~~ - # - class Intern < Instruction - def disasm(fmt) - fmt.instruction("intern") - end - - def to_a(_iseq) - [:intern] - end - - def deconstruct_keys(_keys) - {} - end - - def ==(other) - other.is_a?(Intern) - end - - def pops - 1 - end - - def pushes - 1 - end - - def call(vm) - vm.push(vm.pop.to_sym) - end - end - - # ### Summary - # - # `invokeblock` invokes the block given to the current method. It pops the - # arguments for the block off the stack and pushes the result of running the - # block onto the stack. - # - # ### Usage - # - # ~~~ruby - # def foo - # yield - # end - # ~~~ - # - class InvokeBlock < Instruction - attr_reader :calldata - - def initialize(calldata) - @calldata = calldata - end - - def disasm(fmt) - fmt.instruction("invokeblock", [fmt.calldata(calldata)]) - end - - def to_a(_iseq) - [:invokeblock, calldata.to_h] - end - - def deconstruct_keys(_keys) - { calldata: calldata } - end - - def ==(other) - other.is_a?(InvokeBlock) && other.calldata == calldata - end - - def length - 2 - end - - def pops - calldata.argc - end - - def pushes - 1 - end - - def call(vm) - vm.push(vm.frame_yield.block.call(*vm.pop(calldata.argc))) - end - end - - # ### Summary - # - # `invokesuper` is similar to the `send` instruction, except that it calls - # the super method. It pops the receiver and arguments off the stack and - # pushes the return value onto the stack. - # - # ### Usage - # - # ~~~ruby - # def foo - # super - # end - # ~~~ - # - class InvokeSuper < Instruction - attr_reader :calldata, :block_iseq - - def initialize(calldata, block_iseq) - @calldata = calldata - @block_iseq = block_iseq - end - - def disasm(fmt) - fmt.enqueue(block_iseq) if block_iseq - fmt.instruction( - "invokesuper", - [fmt.calldata(calldata), block_iseq&.name || "nil"] - ) - end - - def to_a(_iseq) - [:invokesuper, calldata.to_h, block_iseq&.to_a] - end - - def deconstruct_keys(_keys) - { calldata: calldata, block_iseq: block_iseq } - end - - def ==(other) - other.is_a?(InvokeSuper) && other.calldata == calldata && - other.block_iseq == block_iseq - end - - def pops - argb = (calldata.flag?(CallData::CALL_ARGS_BLOCKARG) ? 1 : 0) - argb + calldata.argc + 1 - end - - def pushes - 1 - end - - def call(vm) - block = - if (iseq = block_iseq) - frame = vm.frame - ->(*args, **kwargs, &blk) do - vm.run_block_frame(iseq, frame, *args, **kwargs, &blk) - end - end - - keywords = - if calldata.kw_arg - calldata.kw_arg.zip(vm.pop(calldata.kw_arg.length)).to_h - else - {} - end - - arguments = vm.pop(calldata.argc) - receiver = vm.pop - - method = receiver.method(vm.frame.name).super_method - vm.push(method.call(*arguments, **keywords, &block)) - end - end - - # ### Summary - # - # `jump` unconditionally jumps to the label given as its only argument. - # - # ### Usage - # - # ~~~ruby - # x = 0 - # if x == 0 - # puts "0" - # else - # puts "2" - # end - # ~~~ - # - class Jump < Instruction - attr_reader :label - - def initialize(label) - @label = label - end - - def disasm(fmt) - fmt.instruction("jump", [fmt.label(label)]) - end - - def to_a(_iseq) - [:jump, label.name] - end - - def deconstruct_keys(_keys) - { label: label } - end - - def ==(other) - other.is_a?(Jump) && other.label == label - end - - def length - 2 - end - - def call(vm) - vm.jump(label) - end - - def branch_targets - [label] - end - end - - # ### Summary - # - # `leave` exits the current frame. - # - # ### Usage - # - # ~~~ruby - # ;; - # ~~~ - # - class Leave < Instruction - def disasm(fmt) - fmt.instruction("leave") - end - - def to_a(_iseq) - [:leave] - end - - def deconstruct_keys(_keys) - {} - end - - def ==(other) - other.is_a?(Leave) - end - - def pops - 1 - end - - def pushes - # TODO: This is wrong. It should be 1. But it's 0 for now because - # otherwise the stack size is incorrectly calculated. - 0 - end - - def call(vm) - vm.leave - end - - def leaves? - true - end - end - - # ### Summary - # - # `newarray` puts a new array initialized with `number` values from the - # stack. It pops `number` values off the stack and pushes the array onto the - # stack. - # - # ### Usage - # - # ~~~ruby - # ["string"] - # ~~~ - # - class NewArray < Instruction - attr_reader :number - - def initialize(number) - @number = number - end - - def disasm(fmt) - fmt.instruction("newarray", [fmt.object(number)]) - end - - def to_a(_iseq) - [:newarray, number] - end - - def deconstruct_keys(_keys) - { number: number } - end - - def ==(other) - other.is_a?(NewArray) && other.number == number - end - - def length - 2 - end - - def pops - number - end - - def pushes - 1 - end - - def call(vm) - vm.push(vm.pop(number)) - end - end - - # ### Summary - # - # `newarraykwsplat` is a specialized version of `newarray` that takes a ** - # splat argument. It pops `number` values off the stack and pushes the array - # onto the stack. - # - # ### Usage - # - # ~~~ruby - # ["string", **{ foo: "bar" }] - # ~~~ - # - class NewArrayKwSplat < Instruction - attr_reader :number - - def initialize(number) - @number = number - end - - def disasm(fmt) - fmt.instruction("newarraykwsplat", [fmt.object(number)]) - end - - def to_a(_iseq) - [:newarraykwsplat, number] - end - - def deconstruct_keys(_keys) - { number: number } - end - - def ==(other) - other.is_a?(NewArrayKwSplat) && other.number == number - end - - def length - 2 - end - - def pops - number - end - - def pushes - 1 - end - - def call(vm) - vm.push(vm.pop(number)) - end - end - - # ### Summary - # - # `newhash` puts a new hash onto the stack, using `number` elements from the - # stack. `number` needs to be even. It pops `number` elements off the stack - # and pushes a hash onto the stack. - # - # ### Usage - # - # ~~~ruby - # def foo(key, value) - # { key => value } - # end - # ~~~ - # - class NewHash < Instruction - attr_reader :number - - def initialize(number) - @number = number - end - - def disasm(fmt) - fmt.instruction("newhash", [fmt.object(number)]) - end - - def to_a(_iseq) - [:newhash, number] - end - - def deconstruct_keys(_keys) - { number: number } - end - - def ==(other) - other.is_a?(NewHash) && other.number == number - end - - def length - 2 - end - - def pops - number - end - - def pushes - 1 - end - - def call(vm) - vm.push(vm.pop(number).each_slice(2).to_h) - end - end - - # ### Summary - # - # `newrange` creates a new range object from the top two values on the - # stack. It pops both of them off, and then pushes on the new range. It - # takes one argument which is 0 if the end is included or 1 if the end value - # is excluded. - # - # ### Usage - # - # ~~~ruby - # x = 0 - # y = 1 - # p (x..y), (x...y) - # ~~~ - # - class NewRange < Instruction - attr_reader :exclude_end - - def initialize(exclude_end) - @exclude_end = exclude_end - end - - def disasm(fmt) - fmt.instruction("newrange", [fmt.object(exclude_end)]) - end - - def to_a(_iseq) - [:newrange, exclude_end] - end - - def deconstruct_keys(_keys) - { exclude_end: exclude_end } - end - - def ==(other) - other.is_a?(NewRange) && other.exclude_end == exclude_end - end - - def length - 2 - end - - def pops - 2 - end - - def pushes - 1 - end - - def call(vm) - vm.push(Range.new(*vm.pop(2), exclude_end == 1)) - end - end - - # ### Summary - # - # `nop` is a no-operation instruction. It is used to pad the instruction - # sequence so there is a place for other instructions to jump to. - # - # ### Usage - # - # ~~~ruby - # raise rescue true - # ~~~ - # - class Nop < Instruction - def disasm(fmt) - fmt.instruction("nop") - end - - def to_a(_iseq) - [:nop] - end - - def deconstruct_keys(_keys) - {} - end - - def ==(other) - other.is_a?(Nop) - end - - def call(vm) - end - - def side_effects? - false - end - end - - # ### Summary - # - # `objtostring` pops a value from the stack, calls `to_s` on that value and - # then pushes the result back to the stack. - # - # It has various fast paths for classes like String, Symbol, Module, Class, - # etc. For everything else it calls `to_s`. - # - # ### Usage - # - # ~~~ruby - # "#{5}" - # ~~~ - # - class ObjToString < Instruction - attr_reader :calldata - - def initialize(calldata) - @calldata = calldata - end - - def disasm(fmt) - fmt.instruction("objtostring", [fmt.calldata(calldata)]) - end - - def to_a(_iseq) - [:objtostring, calldata.to_h] - end - - def deconstruct_keys(_keys) - { calldata: calldata } - end - - def ==(other) - other.is_a?(ObjToString) && other.calldata == calldata - end - - def length - 2 - end - - def pops - 1 - end - - def pushes - 1 - end - - def call(vm) - vm.push(vm.pop.to_s) - end - end - - # ### Summary - # - # `once` is an instruction that wraps an instruction sequence and ensures - # that is it only ever executed once for the lifetime of the program. It - # uses a cache to ensure that it is only executed once. It pushes the result - # of running the instruction sequence onto the stack. - # - # ### Usage - # - # ~~~ruby - # END { puts "END" } - # ~~~ - # - class Once < Instruction - attr_reader :iseq, :cache - - def initialize(iseq, cache) - @iseq = iseq - @cache = cache - end - - def disasm(fmt) - fmt.enqueue(iseq) - fmt.instruction("once", [iseq.name, fmt.inline_storage(cache)]) - end - - def to_a(_iseq) - [:once, iseq.to_a, cache] - end - - def deconstruct_keys(_keys) - { iseq: iseq, cache: cache } - end - - def ==(other) - other.is_a?(Once) && other.iseq == iseq && other.cache == cache - end - - def length - 3 - end - - def pushes - 1 - end - - def call(vm) - return if @executed - vm.push(vm.run_block_frame(iseq, vm.frame)) - @executed = true - end - end - - # ### Summary - # - # `opt_and` is a specialization of the `opt_send_without_block` instruction - # that occurs when the `&` operator is used. There is a fast path for if - # both operands are integers. It pops both the receiver and the argument off - # the stack and pushes on the result. - # - # ### Usage - # - # ~~~ruby - # 2 & 3 - # ~~~ - # - class OptAnd < Instruction - attr_reader :calldata - - def initialize(calldata) - @calldata = calldata - end - - def disasm(fmt) - fmt.instruction("opt_and", [fmt.calldata(calldata)]) - end - - def to_a(_iseq) - [:opt_and, calldata.to_h] - end - - def deconstruct_keys(_keys) - { calldata: calldata } - end - - def ==(other) - other.is_a?(OptAnd) && other.calldata == calldata - end - - def length - 2 - end - - def pops - 2 - end - - def pushes - 1 - end - - def canonical - Send.new(calldata, nil) - end - - def call(vm) - canonical.call(vm) - end - end - - # ### Summary - # - # `opt_aref` is a specialization of the `opt_send_without_block` instruction - # that occurs when the `[]` operator is used. There are fast paths if the - # receiver is an integer, array, or hash. - # - # ### Usage - # - # ~~~ruby - # 7[2] - # ~~~ - # - class OptAref < Instruction - attr_reader :calldata - - def initialize(calldata) - @calldata = calldata - end - - def disasm(fmt) - fmt.instruction("opt_aref", [fmt.calldata(calldata)]) - end - - def to_a(_iseq) - [:opt_aref, calldata.to_h] - end - - def deconstruct_keys(_keys) - { calldata: calldata } - end - - def ==(other) - other.is_a?(OptAref) && other.calldata == calldata - end - - def length - 2 - end - - def pops - 2 - end - - def pushes - 1 - end - - def canonical - Send.new(calldata, nil) - end - - def call(vm) - canonical.call(vm) - end - end - - # ### Summary - # - # `opt_aref_with` is a specialization of the `opt_aref` instruction that - # occurs when the `[]` operator is used with a string argument known at - # compile time. There are fast paths if the receiver is a hash. It pops the - # receiver off the stack and pushes on the result. - # - # ### Usage - # - # ~~~ruby - # { 'test' => true }['test'] - # ~~~ - # - class OptArefWith < Instruction - attr_reader :object, :calldata - - def initialize(object, calldata) - @object = object - @calldata = calldata - end - - def disasm(fmt) - fmt.instruction( - "opt_aref_with", - [fmt.object(object), fmt.calldata(calldata)] - ) - end - - def to_a(_iseq) - [:opt_aref_with, object, calldata.to_h] - end - - def deconstruct_keys(_keys) - { object: object, calldata: calldata } - end - - def ==(other) - other.is_a?(OptArefWith) && other.object == object && - other.calldata == calldata - end - - def length - 3 - end - - def pops - 1 - end - - def pushes - 1 - end - - def call(vm) - vm.push(vm.pop[object]) - end - end - - # ### Summary - # - # `opt_aset` is an instruction for setting the hash value by the key in - # the `recv[obj] = set` format. It is a specialization of the - # `opt_send_without_block` instruction. It pops the receiver, the key, and - # the value off the stack and pushes on the result. - # - # ### Usage - # - # ~~~ruby - # {}[:key] = value - # ~~~ - # - class OptAset < Instruction - attr_reader :calldata - - def initialize(calldata) - @calldata = calldata - end - - def disasm(fmt) - fmt.instruction("opt_aset", [fmt.calldata(calldata)]) - end - - def to_a(_iseq) - [:opt_aset, calldata.to_h] - end - - def deconstruct_keys(_keys) - { calldata: calldata } - end - - def ==(other) - other.is_a?(OptAset) && other.calldata == calldata - end - - def length - 2 - end - - def pops - 3 - end - - def pushes - 1 - end - - def canonical - Send.new(calldata, nil) - end - - def call(vm) - canonical.call(vm) - end - end - - # ### Summary - # - # `opt_aset_with` is an instruction for setting the hash value by the known - # string key in the `recv[obj] = set` format. It pops the receiver and the - # value off the stack and pushes on the result. - # - # ### Usage - # - # ~~~ruby - # {}["key"] = value - # ~~~ - # - class OptAsetWith < Instruction - attr_reader :object, :calldata - - def initialize(object, calldata) - @object = object - @calldata = calldata - end - - def disasm(fmt) - fmt.instruction( - "opt_aset_with", - [fmt.object(object), fmt.calldata(calldata)] - ) - end - - def to_a(_iseq) - [:opt_aset_with, object, calldata.to_h] - end - - def deconstruct_keys(_keys) - { object: object, calldata: calldata } - end - - def ==(other) - other.is_a?(OptAsetWith) && other.object == object && - other.calldata == calldata - end - - def length - 3 - end - - def pops - 2 - end - - def pushes - 1 - end - - def call(vm) - hash, value = vm.pop(2) - vm.push(hash[object] = value) - end - end - - # ### Summary - # - # `opt_case_dispatch` is a branch instruction that moves the control flow - # for case statements that have clauses where they can all be used as hash - # keys for an internal hash. - # - # It has two arguments: the `case_dispatch_hash` and an `else_label`. It - # pops one value off the stack: a hash key. `opt_case_dispatch` looks up the - # key in the `case_dispatch_hash` and jumps to the corresponding label if - # there is one. If there is no value in the `case_dispatch_hash`, - # `opt_case_dispatch` jumps to the `else_label` index. - # - # ### Usage - # - # ~~~ruby - # case 1 - # when 1 - # puts "foo" - # else - # puts "bar" - # end - # ~~~ - # - class OptCaseDispatch < Instruction - attr_reader :case_dispatch_hash, :else_label - - def initialize(case_dispatch_hash, else_label) - @case_dispatch_hash = case_dispatch_hash - @else_label = else_label - end - - def disasm(fmt) - fmt.instruction( - "opt_case_dispatch", - ["", fmt.label(else_label)] - ) - end - - def to_a(_iseq) - [ - :opt_case_dispatch, - case_dispatch_hash.flat_map { |key, value| [key, value.name] }, - else_label.name - ] - end - - def deconstruct_keys(_keys) - { case_dispatch_hash: case_dispatch_hash, else_label: else_label } - end - - def ==(other) - other.is_a?(OptCaseDispatch) && - other.case_dispatch_hash == case_dispatch_hash && - other.else_label == else_label - end - - def length - 3 - end - - def pops - 1 - end - - def call(vm) - vm.jump(case_dispatch_hash.fetch(vm.pop, else_label)) - end - - def branch_targets - case_dispatch_hash.values.push(else_label) - end - - def falls_through? - true - end - end - - # ### Summary - # - # `opt_div` is a specialization of the `opt_send_without_block` instruction - # that occurs when the `/` operator is used. There are fast paths for if - # both operands are integers, or if both operands are floats. It pops both - # the receiver and the argument off the stack and pushes on the result. - # - # ### Usage - # - # ~~~ruby - # 2 / 3 - # ~~~ - # - class OptDiv < Instruction - attr_reader :calldata - - def initialize(calldata) - @calldata = calldata - end - - def disasm(fmt) - fmt.instruction("opt_div", [fmt.calldata(calldata)]) - end - - def to_a(_iseq) - [:opt_div, calldata.to_h] - end - - def deconstruct_keys(_keys) - { calldata: calldata } - end - - def ==(other) - other.is_a?(OptDiv) && other.calldata == calldata - end - - def length - 2 - end - - def pops - 2 - end - - def pushes - 1 - end - - def canonical - Send.new(calldata, nil) - end - - def call(vm) - canonical.call(vm) - end - end - - # ### Summary - # - # `opt_empty_p` is an optimization applied when the method `empty?` is - # called. It pops the receiver off the stack and pushes on the result of the - # method call. - # - # ### Usage - # - # ~~~ruby - # "".empty? - # ~~~ - # - class OptEmptyP < Instruction - attr_reader :calldata - - def initialize(calldata) - @calldata = calldata - end - - def disasm(fmt) - fmt.instruction("opt_empty_p", [fmt.calldata(calldata)]) - end - - def to_a(_iseq) - [:opt_empty_p, calldata.to_h] - end - - def deconstruct_keys(_keys) - { calldata: calldata } - end - - def ==(other) - other.is_a?(OptEmptyP) && other.calldata == calldata - end - - def length - 2 - end - - def pops - 1 - end - - def pushes - 1 - end - - def canonical - Send.new(calldata, nil) - end - - def call(vm) - canonical.call(vm) - end - end - - # ### Summary - # - # `opt_eq` is a specialization of the `opt_send_without_block` instruction - # that occurs when the == operator is used. Fast paths exist when both - # operands are integers, floats, symbols or strings. It pops both the - # receiver and the argument off the stack and pushes on the result. - # - # ### Usage - # - # ~~~ruby - # 2 == 2 - # ~~~ - # - class OptEq < Instruction - attr_reader :calldata - - def initialize(calldata) - @calldata = calldata - end - - def disasm(fmt) - fmt.instruction("opt_eq", [fmt.calldata(calldata)]) - end - - def to_a(_iseq) - [:opt_eq, calldata.to_h] - end - - def deconstruct_keys(_keys) - { calldata: calldata } - end - - def ==(other) - other.is_a?(OptEq) && other.calldata == calldata - end - - def length - 2 - end - - def pops - 2 - end - - def pushes - 1 - end - - def canonical - Send.new(calldata, nil) - end - - def call(vm) - canonical.call(vm) - end - end - - # ### Summary - # - # `opt_ge` is a specialization of the `opt_send_without_block` instruction - # that occurs when the >= operator is used. Fast paths exist when both - # operands are integers or floats. It pops both the receiver and the - # argument off the stack and pushes on the result. - # - # ### Usage - # - # ~~~ruby - # 4 >= 3 - # ~~~ - # - class OptGE < Instruction - attr_reader :calldata - - def initialize(calldata) - @calldata = calldata - end - - def disasm(fmt) - fmt.instruction("opt_ge", [fmt.calldata(calldata)]) - end - - def to_a(_iseq) - [:opt_ge, calldata.to_h] - end - - def deconstruct_keys(_keys) - { calldata: calldata } - end - - def ==(other) - other.is_a?(OptGE) && other.calldata == calldata - end - - def length - 2 - end - - def pops - 2 - end - - def pushes - 1 - end - - def canonical - Send.new(calldata, nil) - end - - def call(vm) - canonical.call(vm) - end - end - - # ### Summary - # - # `opt_getconstant_path` performs a constant lookup on a chain of constant - # names. It accepts as its argument an array of constant names, and pushes - # the value of the constant onto the stack. - # - # ### Usage - # - # ~~~ruby - # ::Object - # ~~~ - # - class OptGetConstantPath < Instruction - attr_reader :names - - def initialize(names) - @names = names - end - - def disasm(fmt) - cache = "" - fmt.instruction("opt_getconstant_path", [cache]) - end - - def to_a(_iseq) - [:opt_getconstant_path, names] - end - - def deconstruct_keys(_keys) - { names: names } - end - - def ==(other) - other.is_a?(OptGetConstantPath) && other.names == names - end - - def length - 2 - end - - def pushes - 1 - end - - def call(vm) - current = vm.frame._self - current = current.class unless current.is_a?(Class) - - names.each do |name| - current = name == :"" ? Object : current.const_get(name) - end - - vm.push(current) - end - end - - # ### Summary - # - # `opt_gt` is a specialization of the `opt_send_without_block` instruction - # that occurs when the > operator is used. Fast paths exist when both - # operands are integers or floats. It pops both the receiver and the - # argument off the stack and pushes on the result. - # - # ### Usage - # - # ~~~ruby - # 4 > 3 - # ~~~ - # - class OptGT < Instruction - attr_reader :calldata - - def initialize(calldata) - @calldata = calldata - end - - def disasm(fmt) - fmt.instruction("opt_gt", [fmt.calldata(calldata)]) - end - - def to_a(_iseq) - [:opt_gt, calldata.to_h] - end - - def deconstruct_keys(_keys) - { calldata: calldata } - end - - def ==(other) - other.is_a?(OptGT) && other.calldata == calldata - end - - def length - 2 - end - - def pops - 2 - end - - def pushes - 1 - end - - def canonical - Send.new(calldata, nil) - end - - def call(vm) - canonical.call(vm) - end - end - - # ### Summary - # - # `opt_le` is a specialization of the `opt_send_without_block` instruction - # that occurs when the <= operator is used. Fast paths exist when both - # operands are integers or floats. It pops both the receiver and the - # argument off the stack and pushes on the result. - # - # ### Usage - # - # ~~~ruby - # 3 <= 4 - # ~~~ - # - class OptLE < Instruction - attr_reader :calldata - - def initialize(calldata) - @calldata = calldata - end - - def disasm(fmt) - fmt.instruction("opt_le", [fmt.calldata(calldata)]) - end - - def to_a(_iseq) - [:opt_le, calldata.to_h] - end - - def deconstruct_keys(_keys) - { calldata: calldata } - end - - def ==(other) - other.is_a?(OptLE) && other.calldata == calldata - end - - def length - 2 - end - - def pops - 2 - end - - def pushes - 1 - end - - def canonical - Send.new(calldata, nil) - end - - def call(vm) - canonical.call(vm) - end - end - - # ### Summary - # - # `opt_length` is a specialization of `opt_send_without_block`, when the - # `length` method is called. There are fast paths when the receiver is - # either a string, hash, or array. It pops the receiver off the stack and - # pushes on the result of the method call. - # - # ### Usage - # - # ~~~ruby - # "".length - # ~~~ - # - class OptLength < Instruction - attr_reader :calldata - - def initialize(calldata) - @calldata = calldata - end - - def disasm(fmt) - fmt.instruction("opt_length", [fmt.calldata(calldata)]) - end - - def to_a(_iseq) - [:opt_length, calldata.to_h] - end - - def deconstruct_keys(_keys) - { calldata: calldata } - end - - def ==(other) - other.is_a?(OptLength) && other.calldata == calldata - end - - def length - 2 - end - - def pops - 1 - end - - def pushes - 1 - end - - def canonical - Send.new(calldata, nil) - end - - def call(vm) - canonical.call(vm) - end - end - - # ### Summary - # - # `opt_lt` is a specialization of the `opt_send_without_block` instruction - # that occurs when the < operator is used. Fast paths exist when both - # operands are integers or floats. It pops both the receiver and the - # argument off the stack and pushes on the result. - # - # ### Usage - # - # ~~~ruby - # 3 < 4 - # ~~~ - # - class OptLT < Instruction - attr_reader :calldata - - def initialize(calldata) - @calldata = calldata - end - - def disasm(fmt) - fmt.instruction("opt_lt", [fmt.calldata(calldata)]) - end - - def to_a(_iseq) - [:opt_lt, calldata.to_h] - end - - def deconstruct_keys(_keys) - { calldata: calldata } - end - - def ==(other) - other.is_a?(OptLT) && other.calldata == calldata - end - - def length - 2 - end - - def pops - 2 - end - - def pushes - 1 - end - - def canonical - Send.new(calldata, nil) - end - - def call(vm) - canonical.call(vm) - end - end - - # ### Summary - # - # `opt_ltlt` is a specialization of the `opt_send_without_block` instruction - # that occurs when the `<<` operator is used. Fast paths exists when the - # receiver is either a String or an Array. It pops both the receiver and the - # argument off the stack and pushes on the result. - # - # ### Usage - # - # ~~~ruby - # "" << 2 - # ~~~ - # - class OptLTLT < Instruction - attr_reader :calldata - - def initialize(calldata) - @calldata = calldata - end - - def disasm(fmt) - fmt.instruction("opt_ltlt", [fmt.calldata(calldata)]) - end - - def to_a(_iseq) - [:opt_ltlt, calldata.to_h] - end - - def deconstruct_keys(_keys) - { calldata: calldata } - end - - def ==(other) - other.is_a?(OptLTLT) && other.calldata == calldata - end - - def length - 2 - end - - def pops - 2 - end - - def pushes - 1 - end - - def canonical - Send.new(calldata, nil) - end - - def call(vm) - canonical.call(vm) - end - end - - # ### Summary - # - # `opt_minus` is a specialization of the `opt_send_without_block` - # instruction that occurs when the `-` operator is used. There are fast - # paths for if both operands are integers or if both operands are floats. It - # pops both the receiver and the argument off the stack and pushes on the - # result. - # - # ### Usage - # - # ~~~ruby - # 3 - 2 - # ~~~ - # - class OptMinus < Instruction - attr_reader :calldata - - def initialize(calldata) - @calldata = calldata - end - - def disasm(fmt) - fmt.instruction("opt_minus", [fmt.calldata(calldata)]) - end - - def to_a(_iseq) - [:opt_minus, calldata.to_h] - end - - def deconstruct_keys(_keys) - { calldata: calldata } - end - - def ==(other) - other.is_a?(OptMinus) && other.calldata == calldata - end - - def length - 2 - end - - def pops - 2 - end - - def pushes - 1 - end - - def canonical - Send.new(calldata, nil) - end - - def call(vm) - canonical.call(vm) - end - end - - # ### Summary - # - # `opt_mod` is a specialization of the `opt_send_without_block` instruction - # that occurs when the `%` operator is used. There are fast paths for if - # both operands are integers or if both operands are floats. It pops both - # the receiver and the argument off the stack and pushes on the result. - # - # ### Usage - # - # ~~~ruby - # 4 % 2 - # ~~~ - # - class OptMod < Instruction - attr_reader :calldata - - def initialize(calldata) - @calldata = calldata - end - - def disasm(fmt) - fmt.instruction("opt_mod", [fmt.calldata(calldata)]) - end - - def to_a(_iseq) - [:opt_mod, calldata.to_h] - end - - def deconstruct_keys(_keys) - { calldata: calldata } - end - - def ==(other) - other.is_a?(OptMod) && other.calldata == calldata - end - - def length - 2 - end - - def pops - 2 - end - - def pushes - 1 - end - - def canonical - Send.new(calldata, nil) - end - - def call(vm) - canonical.call(vm) - end - end - - # ### Summary - # - # `opt_mult` is a specialization of the `opt_send_without_block` instruction - # that occurs when the `*` operator is used. There are fast paths for if - # both operands are integers or floats. It pops both the receiver and the - # argument off the stack and pushes on the result. - # - # ### Usage - # - # ~~~ruby - # 3 * 2 - # ~~~ - # - class OptMult < Instruction - attr_reader :calldata - - def initialize(calldata) - @calldata = calldata - end - - def disasm(fmt) - fmt.instruction("opt_mult", [fmt.calldata(calldata)]) - end - - def to_a(_iseq) - [:opt_mult, calldata.to_h] - end - - def deconstruct_keys(_keys) - { calldata: calldata } - end - - def ==(other) - other.is_a?(OptMult) && other.calldata == calldata - end - - def length - 2 - end - - def pops - 2 - end - - def pushes - 1 - end - - def canonical - Send.new(calldata, nil) - end - - def call(vm) - canonical.call(vm) - end - end - - # ### Summary - # - # `opt_neq` is an optimization that tests whether two values at the top of - # the stack are not equal by testing their equality and calling the `!` on - # the result. This allows `opt_neq` to use the fast paths optimized in - # `opt_eq` when both operands are Integers, Floats, Symbols, or Strings. It - # pops both the receiver and the argument off the stack and pushes on the - # result. - # - # ### Usage - # - # ~~~ruby - # 2 != 2 - # ~~~ - # - class OptNEq < Instruction - attr_reader :eq_calldata, :neq_calldata - - def initialize(eq_calldata, neq_calldata) - @eq_calldata = eq_calldata - @neq_calldata = neq_calldata - end - - def disasm(fmt) - fmt.instruction( - "opt_neq", - [fmt.calldata(eq_calldata), fmt.calldata(neq_calldata)] - ) - end - - def to_a(_iseq) - [:opt_neq, eq_calldata.to_h, neq_calldata.to_h] - end - - def deconstruct_keys(_keys) - { eq_calldata: eq_calldata, neq_calldata: neq_calldata } - end - - def ==(other) - other.is_a?(OptNEq) && other.eq_calldata == eq_calldata && - other.neq_calldata == neq_calldata - end - - def length - 3 - end - - def pops - 2 - end - - def pushes - 1 - end - - def call(vm) - receiver, argument = vm.pop(2) - vm.push(receiver != argument) - end - end - - # ### Summary - # - # `opt_newarray_send` is a specialization that occurs when a dynamic array - # literal is created and immediately sent the `min`, `max`, or `hash` - # methods. It pops the values of the array off the stack and pushes on the - # result of the method call. - # - # ### Usage - # - # ~~~ruby - # [a, b, c].max - # ~~~ - # - class OptNewArraySend < Instruction - attr_reader :number, :method - - def initialize(number, method) - @number = number - @method = method - end - - def disasm(fmt) - fmt.instruction( - "opt_newarray_send", - [fmt.object(number), fmt.object(method)] - ) - end - - def to_a(_iseq) - [:opt_newarray_send, number, method] - end - - def deconstruct_keys(_keys) - { number: number, method: method } - end - - def ==(other) - other.is_a?(OptNewArraySend) && other.number == number && - other.method == method - end - - def length - 3 - end - - def pops - number - end - - def pushes - 1 - end - - def call(vm) - vm.push(vm.pop(number).__send__(method)) - end - end - - # ### Summary - # - # `opt_nil_p` is an optimization applied when the method `nil?` is called. - # It returns true immediately when the receiver is `nil` and defers to the - # `nil?` method in other cases. It pops the receiver off the stack and - # pushes on the result. - # - # ### Usage - # - # ~~~ruby - # "".nil? - # ~~~ - # - class OptNilP < Instruction - attr_reader :calldata - - def initialize(calldata) - @calldata = calldata - end - - def disasm(fmt) - fmt.instruction("opt_nil_p", [fmt.calldata(calldata)]) - end - - def to_a(_iseq) - [:opt_nil_p, calldata.to_h] - end - - def deconstruct_keys(_keys) - { calldata: calldata } - end - - def ==(other) - other.is_a?(OptNilP) && other.calldata == calldata - end - - def length - 2 - end - - def pops - 1 - end - - def pushes - 1 - end - - def canonical - Send.new(calldata, nil) - end - - def call(vm) - canonical.call(vm) - end - end - - # ### Summary - # - # `opt_not` negates the value on top of the stack by calling the `!` method - # on it. It pops the receiver off the stack and pushes on the result. - # - # ### Usage - # - # ~~~ruby - # !true - # ~~~ - # - class OptNot < Instruction - attr_reader :calldata - - def initialize(calldata) - @calldata = calldata - end - - def disasm(fmt) - fmt.instruction("opt_not", [fmt.calldata(calldata)]) - end - - def to_a(_iseq) - [:opt_not, calldata.to_h] - end - - def deconstruct_keys(_keys) - { calldata: calldata } - end - - def ==(other) - other.is_a?(OptNot) && other.calldata == calldata - end - - def length - 2 - end - - def pops - 1 - end - - def pushes - 1 - end - - def canonical - Send.new(calldata, nil) - end - - def call(vm) - canonical.call(vm) - end - end - - # ### Summary - # - # `opt_or` is a specialization of the `opt_send_without_block` instruction - # that occurs when the `|` operator is used. There is a fast path for if - # both operands are integers. It pops both the receiver and the argument off - # the stack and pushes on the result. - # - # ### Usage - # - # ~~~ruby - # 2 | 3 - # ~~~ - # - class OptOr < Instruction - attr_reader :calldata - - def initialize(calldata) - @calldata = calldata - end - - def disasm(fmt) - fmt.instruction("opt_or", [fmt.calldata(calldata)]) - end - - def to_a(_iseq) - [:opt_or, calldata.to_h] - end - - def deconstruct_keys(_keys) - { calldata: calldata } - end - - def ==(other) - other.is_a?(OptOr) && other.calldata == calldata - end - - def length - 2 - end - - def pops - 2 - end - - def pushes - 1 - end - - def canonical - Send.new(calldata, nil) - end - - def call(vm) - canonical.call(vm) - end - end - - # ### Summary - # - # `opt_plus` is a specialization of the `opt_send_without_block` instruction - # that occurs when the `+` operator is used. There are fast paths for if - # both operands are integers, floats, strings, or arrays. It pops both the - # receiver and the argument off the stack and pushes on the result. - # - # ### Usage - # - # ~~~ruby - # 2 + 3 - # ~~~ - # - class OptPlus < Instruction - attr_reader :calldata - - def initialize(calldata) - @calldata = calldata - end - - def disasm(fmt) - fmt.instruction("opt_plus", [fmt.calldata(calldata)]) - end - - def to_a(_iseq) - [:opt_plus, calldata.to_h] - end - - def deconstruct_keys(_keys) - { calldata: calldata } - end - - def ==(other) - other.is_a?(OptPlus) && other.calldata == calldata - end - - def length - 2 - end - - def pops - 2 - end - - def pushes - 1 - end - - def canonical - Send.new(calldata, nil) - end - - def call(vm) - canonical.call(vm) - end - end - - # ### Summary - # - # `opt_regexpmatch2` is a specialization of the `opt_send_without_block` - # instruction that occurs when the `=~` operator is used. It pops both the - # receiver and the argument off the stack and pushes on the result. - # - # ### Usage - # - # ~~~ruby - # /a/ =~ "a" - # ~~~ - # - class OptRegExpMatch2 < Instruction - attr_reader :calldata - - def initialize(calldata) - @calldata = calldata - end - - def disasm(fmt) - fmt.instruction("opt_regexpmatch2", [fmt.calldata(calldata)]) - end - - def to_a(_iseq) - [:opt_regexpmatch2, calldata.to_h] - end - - def deconstruct_keys(_keys) - { calldata: calldata } - end - - def ==(other) - other.is_a?(OptRegExpMatch2) && other.calldata == calldata - end - - def length - 2 - end - - def pops - 2 - end - - def pushes - 1 - end - - def canonical - Send.new(calldata, nil) - end - - def call(vm) - canonical.call(vm) - end - end - - # ### Summary - # - # `opt_send_without_block` is a specialization of the send instruction that - # occurs when a method is being called without a block. It pops the receiver - # and the arguments off the stack and pushes on the result. - # - # ### Usage - # - # ~~~ruby - # puts "Hello, world!" - # ~~~ - # - class OptSendWithoutBlock < Instruction - attr_reader :calldata - - def initialize(calldata) - @calldata = calldata - end - - def disasm(fmt) - fmt.instruction("opt_send_without_block", [fmt.calldata(calldata)]) - end - - def to_a(_iseq) - [:opt_send_without_block, calldata.to_h] - end - - def deconstruct_keys(_keys) - { calldata: calldata } - end - - def ==(other) - other.is_a?(OptSendWithoutBlock) && other.calldata == calldata - end - - def length - 2 - end - - def pops - 1 + calldata.argc - end - - def pushes - 1 - end - - def canonical - Send.new(calldata, nil) - end - - def call(vm) - canonical.call(vm) - end - end - - # ### Summary - # - # `opt_size` is a specialization of `opt_send_without_block`, when the - # `size` method is called. There are fast paths when the receiver is either - # a string, hash, or array. It pops the receiver off the stack and pushes on - # the result. - # - # ### Usage - # - # ~~~ruby - # "".size - # ~~~ - # - class OptSize < Instruction - attr_reader :calldata - - def initialize(calldata) - @calldata = calldata - end - - def disasm(fmt) - fmt.instruction("opt_size", [fmt.calldata(calldata)]) - end - - def to_a(_iseq) - [:opt_size, calldata.to_h] - end - - def deconstruct_keys(_keys) - { calldata: calldata } - end - - def ==(other) - other.is_a?(OptSize) && other.calldata == calldata - end - - def length - 2 - end - - def pops - 1 - end - - def pushes - 1 - end - - def canonical - Send.new(calldata, nil) - end - - def call(vm) - canonical.call(vm) - end - end - - # ### Summary - # - # `opt_str_freeze` pushes a frozen known string value with no interpolation - # onto the stack using the #freeze method. If the method gets overridden, - # this will fall back to a send. - # - # ### Usage - # - # ~~~ruby - # "hello".freeze - # ~~~ - # - class OptStrFreeze < Instruction - attr_reader :object, :calldata - - def initialize(object, calldata) - @object = object - @calldata = calldata - end - - def disasm(fmt) - fmt.instruction( - "opt_str_freeze", - [fmt.object(object), fmt.calldata(calldata)] - ) - end - - def to_a(_iseq) - [:opt_str_freeze, object, calldata.to_h] - end - - def deconstruct_keys(_keys) - { object: object, calldata: calldata } - end - - def ==(other) - other.is_a?(OptStrFreeze) && other.object == object && - other.calldata == calldata - end - - def length - 3 - end - - def pushes - 1 - end - - def call(vm) - vm.push(object.freeze) - end - end - - # ### Summary - # - # `opt_str_uminus` pushes a frozen known string value with no interpolation - # onto the stack. If the method gets overridden, this will fall back to a - # send. - # - # ### Usage - # - # ~~~ruby - # -"string" - # ~~~ - # - class OptStrUMinus < Instruction - attr_reader :object, :calldata - - def initialize(object, calldata) - @object = object - @calldata = calldata - end - - def disasm(fmt) - fmt.instruction( - "opt_str_uminus", - [fmt.object(object), fmt.calldata(calldata)] - ) - end - - def to_a(_iseq) - [:opt_str_uminus, object, calldata.to_h] - end - - def deconstruct_keys(_keys) - { object: object, calldata: calldata } - end - - def ==(other) - other.is_a?(OptStrUMinus) && other.object == object && - other.calldata == calldata - end - - def length - 3 - end - - def pushes - 1 - end - - def call(vm) - vm.push(-object) - end - end - - # ### Summary - # - # `opt_succ` is a specialization of the `opt_send_without_block` instruction - # when the method being called is `succ`. Fast paths exist when the receiver - # is either a String or a Fixnum. It pops the receiver off the stack and - # pushes on the result. - # - # ### Usage - # - # ~~~ruby - # "".succ - # ~~~ - # - class OptSucc < Instruction - attr_reader :calldata - - def initialize(calldata) - @calldata = calldata - end - - def disasm(fmt) - fmt.instruction("opt_succ", [fmt.calldata(calldata)]) - end - - def to_a(_iseq) - [:opt_succ, calldata.to_h] - end - - def deconstruct_keys(_keys) - { calldata: calldata } - end - - def ==(other) - other.is_a?(OptSucc) && other.calldata == calldata - end - - def length - 2 - end - - def pops - 1 - end - - def pushes - 1 - end - - def canonical - Send.new(calldata, nil) - end - - def call(vm) - canonical.call(vm) - end - end - - # ### Summary - # - # `pop` pops the top value off the stack. - # - # ### Usage - # - # ~~~ruby - # a ||= 2 - # ~~~ - # - class Pop < Instruction - def disasm(fmt) - fmt.instruction("pop") - end - - def to_a(_iseq) - [:pop] - end - - def deconstruct_keys(_keys) - {} - end - - def ==(other) - other.is_a?(Pop) - end - - def pops - 1 - end - - def call(vm) - vm.pop - end - - def side_effects? - false - end - end - - # ### Summary - # - # `pushtoarraykwsplat` is used to append a hash literal that is being - # splatted onto an array. - # - # ### Usage - # - # ~~~ruby - # ["string", **{ foo: "bar" }] - # ~~~ - # - class PushToArrayKwSplat < Instruction - def disasm(fmt) - fmt.instruction("pushtoarraykwsplat") - end - - def to_a(_iseq) - [:pushtoarraykwsplat] - end - - def deconstruct_keys(_keys) - {} - end - - def ==(other) - other.is_a?(PushToArrayKwSplat) - end - - def length - 2 - end - - def pops - 2 - end - - def pushes - 1 - end - - def call(vm) - array, hash = vm.pop(2) - vm.push(array << hash) - end - end - - # ### Summary - # - # `putnil` pushes a global nil object onto the stack. - # - # ### Usage - # - # ~~~ruby - # nil - # ~~~ - # - class PutNil < Instruction - def disasm(fmt) - fmt.instruction("putnil") - end - - def to_a(_iseq) - [:putnil] - end - - def deconstruct_keys(_keys) - {} - end - - def ==(other) - other.is_a?(PutNil) - end - - def pushes - 1 - end - - def canonical - PutObject.new(nil) - end - - def call(vm) - canonical.call(vm) - end - - def side_effects? - false - end - end - - # ### Summary - # - # `putobject` pushes a known value onto the stack. - # - # ### Usage - # - # ~~~ruby - # 5 - # ~~~ - # - class PutObject < Instruction - attr_reader :object - - def initialize(object) - @object = object - end - - def disasm(fmt) - fmt.instruction("putobject", [fmt.object(object)]) - end - - def to_a(_iseq) - [:putobject, object] - end - - def deconstruct_keys(_keys) - { object: object } - end - - def ==(other) - other.is_a?(PutObject) && other.object == object - end - - def length - 2 - end - - def pushes - 1 - end - - def call(vm) - vm.push(object) - end - - def side_effects? - false - end - end - - # ### Summary - # - # `putobject_INT2FIX_0_` pushes 0 on the stack. It is a specialized - # instruction resulting from the operand unification optimization. It is - # equivalent to `putobject 0`. - # - # ### Usage - # - # ~~~ruby - # 0 - # ~~~ - # - class PutObjectInt2Fix0 < Instruction - def disasm(fmt) - fmt.instruction("putobject_INT2FIX_0_") - end - - def to_a(_iseq) - [:putobject_INT2FIX_0_] - end - - def deconstruct_keys(_keys) - {} - end - - def ==(other) - other.is_a?(PutObjectInt2Fix0) - end - - def pushes - 1 - end - - def canonical - PutObject.new(0) - end - - def call(vm) - canonical.call(vm) - end - - def side_effects? - false - end - end - - # ### Summary - # - # `putobject_INT2FIX_1_` pushes 1 on the stack. It is a specialized - # instruction resulting from the operand unification optimization. It is - # equivalent to `putobject 1`. - # - # ### Usage - # - # ~~~ruby - # 1 - # ~~~ - # - class PutObjectInt2Fix1 < Instruction - def disasm(fmt) - fmt.instruction("putobject_INT2FIX_1_") - end - - def to_a(_iseq) - [:putobject_INT2FIX_1_] - end - - def deconstruct_keys(_keys) - {} - end - - def ==(other) - other.is_a?(PutObjectInt2Fix1) - end - - def pushes - 1 - end - - def canonical - PutObject.new(1) - end - - def call(vm) - canonical.call(vm) - end - - def side_effects? - false - end - end - - # ### Summary - # - # `putself` pushes the current value of self onto the stack. - # - # ### Usage - # - # ~~~ruby - # puts "Hello, world!" - # ~~~ - # - class PutSelf < Instruction - def disasm(fmt) - fmt.instruction("putself") - end - - def to_a(_iseq) - [:putself] - end - - def deconstruct_keys(_keys) - {} - end - - def ==(other) - other.is_a?(PutSelf) - end - - def pushes - 1 - end - - def call(vm) - vm.push(vm.frame._self) - end - - def side_effects? - false - end - end - - # ### Summary - # - # `putspecialobject` pushes one of three special objects onto the stack. - # These are either the VM core special object, the class base special - # object, or the constant base special object. - # - # ### Usage - # - # ~~~ruby - # alias foo bar - # ~~~ - # - class PutSpecialObject < Instruction - OBJECT_VMCORE = 1 - OBJECT_CBASE = 2 - OBJECT_CONST_BASE = 3 - - attr_reader :object - - def initialize(object) - @object = object - end - - def disasm(fmt) - fmt.instruction("putspecialobject", [fmt.object(object)]) - end - - def to_a(_iseq) - [:putspecialobject, object] - end - - def deconstruct_keys(_keys) - { object: object } - end - - def ==(other) - other.is_a?(PutSpecialObject) && other.object == object - end - - def length - 2 - end - - def pushes - 1 - end - - def call(vm) - case object - when OBJECT_VMCORE - vm.push(vm.frozen_core) - when OBJECT_CBASE - value = vm.frame._self - value = value.singleton_class unless value.is_a?(Class) - vm.push(value) - when OBJECT_CONST_BASE - vm.push(vm.const_base) - end - end - end - - # ### Summary - # - # `putchilledstring` pushes an unfrozen string literal onto the stack that - # acts like a frozen string. This is a migration path to frozen string - # literals as the default in the future. - # - # ### Usage - # - # ~~~ruby - # "foo" - # ~~~ - # - class PutChilledString < Instruction - attr_reader :object - - def initialize(object) - @object = object - end - - def disasm(fmt) - fmt.instruction("putchilledstring", [fmt.object(object)]) - end - - def to_a(_iseq) - [:putchilledstring, object] - end - - def deconstruct_keys(_keys) - { object: object } - end - - def ==(other) - other.is_a?(PutChilledString) && other.object == object - end - - def length - 2 - end - - def pushes - 1 - end - - def call(vm) - vm.push(object.dup) - end - end - - # ### Summary - # - # `putstring` pushes an unfrozen string literal onto the stack. - # - # ### Usage - # - # ~~~ruby - # "foo" - # ~~~ - # - class PutString < Instruction - attr_reader :object - - def initialize(object) - @object = object - end - - def disasm(fmt) - fmt.instruction("putstring", [fmt.object(object)]) - end - - def to_a(_iseq) - [:putstring, object] - end - - def deconstruct_keys(_keys) - { object: object } - end - - def ==(other) - other.is_a?(PutString) && other.object == object - end - - def length - 2 - end - - def pushes - 1 - end - - def call(vm) - vm.push(object.dup) - end - end - - # ### Summary - # - # `send` invokes a method with an optional block. It pops its receiver and - # the arguments for the method off the stack and pushes the return value - # onto the stack. It has two arguments: the calldata for the call site and - # the optional block instruction sequence. - # - # ### Usage - # - # ~~~ruby - # "hello".tap { |i| p i } - # ~~~ - # - class Send < Instruction - attr_reader :calldata, :block_iseq - - def initialize(calldata, block_iseq) - @calldata = calldata - @block_iseq = block_iseq - end - - def disasm(fmt) - fmt.enqueue(block_iseq) if block_iseq - fmt.instruction( - "send", - [fmt.calldata(calldata), block_iseq&.name || "nil"] - ) - end - - def to_a(_iseq) - [:send, calldata.to_h, block_iseq&.to_a] - end - - def deconstruct_keys(_keys) - { calldata: calldata, block_iseq: block_iseq } - end - - def ==(other) - other.is_a?(Send) && other.calldata == calldata && - other.block_iseq == block_iseq - end - - def length - 3 - end - - def pops - argb = (calldata.flag?(CallData::CALL_ARGS_BLOCKARG) ? 1 : 0) - argb + calldata.argc + 1 - end - - def pushes - 1 - end - - def call(vm) - block = - if (iseq = block_iseq) - frame = vm.frame - ->(*args, **kwargs, &blk) do - vm.run_block_frame(iseq, frame, *args, **kwargs, &blk) - end - elsif calldata.flag?(CallData::CALL_ARGS_BLOCKARG) - vm.pop - end - - keywords = - if calldata.kw_arg - calldata.kw_arg.zip(vm.pop(calldata.kw_arg.length)).to_h - else - {} - end - - arguments = vm.pop(calldata.argc) - receiver = vm.pop - - vm.push( - receiver.__send__(calldata.method, *arguments, **keywords, &block) - ) - end - end - - # ### Summary - # - # `setblockparam` sets the value of a block local variable on a frame - # determined by the level and index arguments. The level is the number of - # frames back to look and the index is the index in the local table. It pops - # the value it is setting off the stack. - # - # ### Usage - # - # ~~~ruby - # def foo(&bar) - # bar = baz - # end - # ~~~ - # - class SetBlockParam < Instruction - attr_reader :index, :level - - def initialize(index, level) - @index = index - @level = level - end - - def disasm(fmt) - fmt.instruction("setblockparam", [fmt.local(index, explicit: level)]) - end - - def to_a(iseq) - current = iseq - level.times { current = current.parent_iseq } - [:setblockparam, current.local_table.offset(index), level] - end - - def deconstruct_keys(_keys) - { index: index, level: level } - end - - def ==(other) - other.is_a?(SetBlockParam) && other.index == index && - other.level == level - end - - def length - 3 - end - - def pops - 1 - end - - def call(vm) - vm.local_set(index, level, vm.pop) - end - end - - # ### Summary - # - # `setclassvariable` looks for a class variable in the current class and - # sets its value to the value it pops off the top of the stack. It uses an - # inline cache to reduce the need to lookup the class variable in the class - # hierarchy every time. - # - # ### Usage - # - # ~~~ruby - # @@class_variable = 1 - # ~~~ - # - class SetClassVariable < Instruction - attr_reader :name, :cache - - def initialize(name, cache) - @name = name - @cache = cache - end - - def disasm(fmt) - fmt.instruction( - "setclassvariable", - [fmt.object(name), fmt.inline_storage(cache)] - ) - end - - def to_a(_iseq) - [:setclassvariable, name, cache] - end - - def deconstruct_keys(_keys) - { name: name, cache: cache } - end - - def ==(other) - other.is_a?(SetClassVariable) && other.name == name && - other.cache == cache - end - - def length - 3 - end - - def pops - 1 - end - - def call(vm) - clazz = vm.frame._self - clazz = clazz.class unless clazz.is_a?(Class) - clazz.class_variable_set(name, vm.pop) - end - end - - # ### Summary - # - # `setconstant` pops two values off the stack: the value to set the - # constant to and the constant base to set it in. - # - # ### Usage - # - # ~~~ruby - # Constant = 1 - # ~~~ - # - class SetConstant < Instruction - attr_reader :name - - def initialize(name) - @name = name - end - - def disasm(fmt) - fmt.instruction("setconstant", [fmt.object(name)]) - end - - def to_a(_iseq) - [:setconstant, name] - end - - def deconstruct_keys(_keys) - { name: name } - end - - def ==(other) - other.is_a?(SetConstant) && other.name == name - end - - def length - 2 - end - - def pops - 2 - end - - def call(vm) - value, parent = vm.pop(2) - parent.const_set(name, value) - end - end - - # ### Summary - # - # `setglobal` sets the value of a global variable to a value popped off the - # top of the stack. - # - # ### Usage - # - # ~~~ruby - # $global = 5 - # ~~~ - # - class SetGlobal < Instruction - attr_reader :name - - def initialize(name) - @name = name - end - - def disasm(fmt) - fmt.instruction("setglobal", [fmt.object(name)]) - end - - def to_a(_iseq) - [:setglobal, name] - end - - def deconstruct_keys(_keys) - { name: name } - end - - def ==(other) - other.is_a?(SetGlobal) && other.name == name - end - - def length - 2 - end - - def pops - 1 - end - - def call(vm) - # Evaluating the name of the global variable because there isn't a - # reflection API for global variables. - eval("#{name} = vm.pop", binding, __FILE__, __LINE__) - end - end - - # ### Summary - # - # `setinstancevariable` pops a value off the top of the stack and then sets - # the instance variable associated with the instruction to that value. - # - # This instruction has two forms, but both have the same structure. Before - # Ruby 3.2, the inline cache corresponded to both the get and set - # instructions and could be shared. Since Ruby 3.2, it uses object shapes - # instead so the caches are unique per instruction. - # - # ### Usage - # - # ~~~ruby - # @instance_variable = 1 - # ~~~ - # - class SetInstanceVariable < Instruction - attr_reader :name, :cache - - def initialize(name, cache) - @name = name - @cache = cache - end - - def disasm(fmt) - fmt.instruction( - "setinstancevariable", - [fmt.object(name), fmt.inline_storage(cache)] - ) - end - - def to_a(_iseq) - [:setinstancevariable, name, cache] - end - - def deconstruct_keys(_keys) - { name: name, cache: cache } - end - - def ==(other) - other.is_a?(SetInstanceVariable) && other.name == name && - other.cache == cache - end - - def length - 3 - end - - def pops - 1 - end - - def call(vm) - method = Object.instance_method(:instance_variable_set) - method.bind(vm.frame._self).call(name, vm.pop) - end - end - - # ### Summary - # - # `setlocal` sets the value of a local variable on a frame determined by the - # level and index arguments. The level is the number of frames back to - # look and the index is the index in the local table. It pops the value it - # is setting off the stack. - # - # ### Usage - # - # ~~~ruby - # value = 5 - # tap { tap { value = 10 } } - # ~~~ - # - class SetLocal < Instruction - attr_reader :index, :level - - def initialize(index, level) - @index = index - @level = level - end - - def disasm(fmt) - fmt.instruction("setlocal", [fmt.local(index, explicit: level)]) - end - - def to_a(iseq) - current = iseq - level.times { current = current.parent_iseq } - [:setlocal, current.local_table.offset(index), level] - end - - def deconstruct_keys(_keys) - { index: index, level: level } - end - - def ==(other) - other.is_a?(SetLocal) && other.index == index && other.level == level - end - - def length - 3 - end - - def pops - 1 - end - - def call(vm) - vm.local_set(index, level, vm.pop) - end - end - - # ### Summary - # - # `setlocal_WC_0` is a specialized version of the `setlocal` instruction. It - # sets the value of a local variable on the current frame to the value at - # the top of the stack as determined by the index given as its only - # argument. - # - # ### Usage - # - # ~~~ruby - # value = 5 - # ~~~ - # - class SetLocalWC0 < Instruction - attr_reader :index - - def initialize(index) - @index = index - end - - def disasm(fmt) - fmt.instruction("setlocal_WC_0", [fmt.local(index, implicit: 0)]) - end - - def to_a(iseq) - [:setlocal_WC_0, iseq.local_table.offset(index)] - end - - def deconstruct_keys(_keys) - { index: index } - end - - def ==(other) - other.is_a?(SetLocalWC0) && other.index == index - end - - def length - 2 - end - - def pops - 1 - end - - def canonical - SetLocal.new(index, 0) - end - - def call(vm) - canonical.call(vm) - end - end - - # ### Summary - # - # `setlocal_WC_1` is a specialized version of the `setlocal` instruction. It - # sets the value of a local variable on the parent frame to the value at the - # top of the stack as determined by the index given as its only argument. - # - # ### Usage - # - # ~~~ruby - # value = 5 - # self.then { value = 10 } - # ~~~ - # - class SetLocalWC1 < Instruction - attr_reader :index - - def initialize(index) - @index = index - end - - def disasm(fmt) - fmt.instruction("setlocal_WC_1", [fmt.local(index, implicit: 1)]) - end - - def to_a(iseq) - [:setlocal_WC_1, iseq.parent_iseq.local_table.offset(index)] - end - - def deconstruct_keys(_keys) - { index: index } - end - - def ==(other) - other.is_a?(SetLocalWC1) && other.index == index - end - - def length - 2 - end - - def pops - 1 - end - - def canonical - SetLocal.new(index, 1) - end - - def call(vm) - canonical.call(vm) - end - end - - # ### Summary - # - # `setn` sets a value in the stack to a value popped off the top of the - # stack. It then pushes that value onto the top of the stack as well. - # - # ### Usage - # - # ~~~ruby - # {}[:key] = 'val' - # ~~~ - # - class SetN < Instruction - attr_reader :number - - def initialize(number) - @number = number - end - - def disasm(fmt) - fmt.instruction("setn", [fmt.object(number)]) - end - - def to_a(_iseq) - [:setn, number] - end - - def deconstruct_keys(_keys) - { number: number } - end - - def ==(other) - other.is_a?(SetN) && other.number == number - end - - def length - 2 - end - - def pops - 1 - end - - def pushes - 1 - end - - def call(vm) - vm.stack[-number - 1] = vm.stack.last - end - end - - # ### Summary - # - # `setspecial` pops a value off the top of the stack and sets a special - # local variable to that value. The special local variable is determined by - # the key given as its only argument. - # - # ### Usage - # - # ~~~ruby - # baz if (foo == 1) .. (bar == 1) - # ~~~ - # - class SetSpecial < Instruction - attr_reader :key - - def initialize(key) - @key = key - end - - def disasm(fmt) - fmt.instruction("setspecial", [fmt.object(key)]) - end - - def to_a(_iseq) - [:setspecial, key] - end - - def deconstruct_keys(_keys) - { key: key } - end - - def ==(other) - other.is_a?(SetSpecial) && other.key == key - end - - def length - 2 - end - - def pops - 1 - end - - def call(vm) - case key - when GetSpecial::SVAR_LASTLINE - raise NotImplementedError, "setspecial SVAR_LASTLINE" - when GetSpecial::SVAR_BACKREF - raise NotImplementedError, "setspecial SVAR_BACKREF" - when GetSpecial::SVAR_FLIPFLOP_START - vm.frame_svar.svars[GetSpecial::SVAR_FLIPFLOP_START] - end - end - end - - # ### Summary - # - # `splatarray` coerces the array object at the top of the stack into Array - # by calling `to_a`. It pushes a duplicate of the array if there is a flag, - # and the original array if there isn't one. - # - # ### Usage - # - # ~~~ruby - # x = *(5) - # ~~~ - # - class SplatArray < Instruction - attr_reader :flag - - def initialize(flag) - @flag = flag - end - - def disasm(fmt) - fmt.instruction("splatarray", [fmt.object(flag)]) - end - - def to_a(_iseq) - [:splatarray, flag] - end - - def deconstruct_keys(_keys) - { flag: flag } - end - - def ==(other) - other.is_a?(SplatArray) && other.flag == flag - end - - def length - 2 - end - - def pops - 1 - end - - def pushes - 1 - end - - def call(vm) - value = vm.pop - - vm.push( - if Array === value - value.instance_of?(Array) ? value.dup : Array[*value] - elsif value.nil? - value.to_a - else - if value.respond_to?(:to_a, true) - result = value.to_a - - if result.nil? - [value] - elsif !result.is_a?(Array) - raise TypeError, "expected to_a to return an Array" - end - else - [value] - end - end - ) - end - end - - # ### Summary - # - # `swap` swaps the top two elements in the stack. - # - # ### TracePoint - # - # `swap` does not dispatch any events. - # - # ### Usage - # - # ~~~ruby - # !!defined?([[]]) - # ~~~ - # - class Swap < Instruction - def disasm(fmt) - fmt.instruction("swap") - end - - def to_a(_iseq) - [:swap] - end - - def deconstruct_keys(_keys) - {} - end - - def ==(other) - other.is_a?(Swap) - end - - def pops - 2 - end - - def pushes - 2 - end - - def call(vm) - left, right = vm.pop(2) - vm.push(right, left) - end - end - - # ### Summary - # - # `throw` pops a value off the top of the stack and throws it. It is caught - # using the instruction sequence's (or an ancestor's) catch table. It pushes - # on the result of throwing the value. - # - # ### Usage - # - # ~~~ruby - # [1, 2, 3].map { break 2 } - # ~~~ - # - class Throw < Instruction - RUBY_TAG_NONE = 0x0 - RUBY_TAG_RETURN = 0x1 - RUBY_TAG_BREAK = 0x2 - RUBY_TAG_NEXT = 0x3 - RUBY_TAG_RETRY = 0x4 - RUBY_TAG_REDO = 0x5 - RUBY_TAG_RAISE = 0x6 - RUBY_TAG_THROW = 0x7 - RUBY_TAG_FATAL = 0x8 - - VM_THROW_NO_ESCAPE_FLAG = 0x8000 - VM_THROW_STATE_MASK = 0xff - - attr_reader :type - - def initialize(type) - @type = type - end - - def disasm(fmt) - fmt.instruction("throw", [fmt.object(type)]) - end - - def to_a(_iseq) - [:throw, type] - end - - def deconstruct_keys(_keys) - { type: type } - end - - def ==(other) - other.is_a?(Throw) && other.type == type - end - - def length - 2 - end - - def pops - 1 - end - - def pushes - 1 - end - - def call(vm) - state = type & VM_THROW_STATE_MASK - value = vm.pop - - case state - when RUBY_TAG_NONE - case value - when nil - # do nothing - when Exception - raise value - else - raise NotImplementedError - end - when RUBY_TAG_RETURN - raise VM::ReturnError.new(value, error_backtrace(vm)) - when RUBY_TAG_BREAK - raise VM::BreakError.new(value, error_backtrace(vm)) - when RUBY_TAG_NEXT - raise VM::NextError.new(value, error_backtrace(vm)) - else - raise NotImplementedError, "Unknown throw kind #{state}" - end - end - - private - - def error_backtrace(vm) - backtrace = [] - current = vm.frame - - while current - backtrace << "#{current.iseq.file}:#{current.line}:in" \ - "`#{current.iseq.name}'" - current = current.parent - end - - [*backtrace, *caller] - end - end - - # ### Summary - # - # `topn` pushes a single value onto the stack that is a copy of the value - # within the stack that is `number` of slots down from the top. - # - # ### Usage - # - # ~~~ruby - # case 3 - # when 1..5 - # puts "foo" - # end - # ~~~ - # - class TopN < Instruction - attr_reader :number - - def initialize(number) - @number = number - end - - def disasm(fmt) - fmt.instruction("topn", [fmt.object(number)]) - end - - def to_a(_iseq) - [:topn, number] - end - - def deconstruct_keys(_keys) - { number: number } - end - - def ==(other) - other.is_a?(TopN) && other.number == number - end - - def length - 2 - end - - def pushes - 1 - end - - def call(vm) - vm.push(vm.stack[-number - 1]) - end - end - - # ### Summary - # - # `toregexp` pops a number of values off the stack, combines them into a new - # regular expression, and pushes the new regular expression onto the stack. - # - # ### Usage - # - # ~~~ruby - # /foo #{bar}/ - # ~~~ - # - class ToRegExp < Instruction - attr_reader :options, :length - - def initialize(options, length) - @options = options - @length = length - end - - def disasm(fmt) - fmt.instruction("toregexp", [fmt.object(options), fmt.object(length)]) - end - - def to_a(_iseq) - [:toregexp, options, length] - end - - def deconstruct_keys(_keys) - { options: options, length: length } - end - - def ==(other) - other.is_a?(ToRegExp) && other.options == options && - other.length == length - end - - def pops - length - end - - def pushes - 1 - end - - def call(vm) - vm.push(Regexp.new(vm.pop(length).join, options)) - end - end - end -end diff --git a/lib/syntax_tree/yarv/legacy.rb b/lib/syntax_tree/yarv/legacy.rb deleted file mode 100644 index 8715993a..00000000 --- a/lib/syntax_tree/yarv/legacy.rb +++ /dev/null @@ -1,340 +0,0 @@ -# frozen_string_literal: true - -module SyntaxTree - module YARV - # This module contains the instructions that used to be a part of YARV but - # have been replaced or removed in more recent versions. - module Legacy - # ### Summary - # - # `getclassvariable` looks for a class variable in the current class and - # pushes its value onto the stack. - # - # This version of the `getclassvariable` instruction is no longer used - # since in Ruby 3.0 it gained an inline cache.` - # - # ### Usage - # - # ~~~ruby - # @@class_variable - # ~~~ - # - class GetClassVariable < Instruction - attr_reader :name - - def initialize(name) - @name = name - end - - def disasm(fmt) - fmt.instruction("getclassvariable", [fmt.object(name)]) - end - - def to_a(_iseq) - [:getclassvariable, name] - end - - def deconstruct_keys(_keys) - { name: name } - end - - def ==(other) - other.is_a?(GetClassVariable) && other.name == name - end - - def length - 2 - end - - def pushes - 1 - end - - def canonical - YARV::GetClassVariable.new(name, nil) - end - - def call(vm) - canonical.call(vm) - end - end - - # ### Summary - # - # `opt_getinlinecache` is a wrapper around a series of `putobject` and - # `getconstant` instructions that allows skipping past them if the inline - # cache is currently set. It pushes the value of the cache onto the stack - # if it is set, otherwise it pushes `nil`. - # - # This instruction is no longer used since in Ruby 3.2 it was replaced by - # the consolidated `opt_getconstant_path` instruction. - # - # ### Usage - # - # ~~~ruby - # Constant - # ~~~ - # - class OptGetInlineCache < Instruction - attr_reader :label, :cache - - def initialize(label, cache) - @label = label - @cache = cache - end - - def disasm(fmt) - fmt.instruction( - "opt_getinlinecache", - [fmt.label(label), fmt.inline_storage(cache)] - ) - end - - def to_a(_iseq) - [:opt_getinlinecache, label.name, cache] - end - - def deconstruct_keys(_keys) - { label: label, cache: cache } - end - - def ==(other) - other.is_a?(OptGetInlineCache) && other.label == label && - other.cache == cache - end - - def length - 3 - end - - def pushes - 1 - end - - def call(vm) - vm.push(nil) - end - - def branch_targets - [label] - end - - def falls_through? - true - end - end - - # ### Summary - # - # `opt_newarray_max` is a specialization that occurs when the `max` method - # is called on an array literal. It pops the values of the array off the - # stack and pushes on the result. - # - # ### Usage - # - # ~~~ruby - # [a, b, c].max - # ~~~ - # - class OptNewArrayMax < Instruction - attr_reader :number - - def initialize(number) - @number = number - end - - def disasm(fmt) - fmt.instruction("opt_newarray_max", [fmt.object(number)]) - end - - def to_a(_iseq) - [:opt_newarray_max, number] - end - - def deconstruct_keys(_keys) - { number: number } - end - - def ==(other) - other.is_a?(OptNewArrayMax) && other.number == number - end - - def length - 2 - end - - def pops - number - end - - def pushes - 1 - end - - def call(vm) - vm.push(vm.pop(number).max) - end - end - - # ### Summary - # - # `opt_newarray_min` is a specialization that occurs when the `min` method - # is called on an array literal. It pops the values of the array off the - # stack and pushes on the result. - # - # ### Usage - # - # ~~~ruby - # [a, b, c].min - # ~~~ - # - class OptNewArrayMin < Instruction - attr_reader :number - - def initialize(number) - @number = number - end - - def disasm(fmt) - fmt.instruction("opt_newarray_min", [fmt.object(number)]) - end - - def to_a(_iseq) - [:opt_newarray_min, number] - end - - def deconstruct_keys(_keys) - { number: number } - end - - def ==(other) - other.is_a?(OptNewArrayMin) && other.number == number - end - - def length - 2 - end - - def pops - number - end - - def pushes - 1 - end - - def call(vm) - vm.push(vm.pop(number).min) - end - end - - # ### Summary - # - # `opt_setinlinecache` sets an inline cache for a constant lookup. It pops - # the value it should set off the top of the stack. It uses this value to - # set the cache. It then pushes that value back onto the top of the stack. - # - # This instruction is no longer used since in Ruby 3.2 it was replaced by - # the consolidated `opt_getconstant_path` instruction. - # - # ### Usage - # - # ~~~ruby - # Constant - # ~~~ - # - class OptSetInlineCache < Instruction - attr_reader :cache - - def initialize(cache) - @cache = cache - end - - def disasm(fmt) - fmt.instruction("opt_setinlinecache", [fmt.inline_storage(cache)]) - end - - def to_a(_iseq) - [:opt_setinlinecache, cache] - end - - def deconstruct_keys(_keys) - { cache: cache } - end - - def ==(other) - other.is_a?(OptSetInlineCache) && other.cache == cache - end - - def length - 2 - end - - def pops - 1 - end - - def pushes - 1 - end - - def call(vm) - end - end - - # ### Summary - # - # `setclassvariable` looks for a class variable in the current class and - # sets its value to the value it pops off the top of the stack. - # - # This version of the `setclassvariable` instruction is no longer used - # since in Ruby 3.0 it gained an inline cache. - # - # ### Usage - # - # ~~~ruby - # @@class_variable = 1 - # ~~~ - # - class SetClassVariable < Instruction - attr_reader :name - - def initialize(name) - @name = name - end - - def disasm(fmt) - fmt.instruction("setclassvariable", [fmt.object(name)]) - end - - def to_a(_iseq) - [:setclassvariable, name] - end - - def deconstruct_keys(_keys) - { name: name } - end - - def ==(other) - other.is_a?(SetClassVariable) && other.name == name - end - - def length - 2 - end - - def pops - 1 - end - - def canonical - YARV::SetClassVariable.new(name, nil) - end - - def call(vm) - canonical.call(vm) - end - end - end - end -end diff --git a/lib/syntax_tree/yarv/local_table.rb b/lib/syntax_tree/yarv/local_table.rb deleted file mode 100644 index 54cc55ad..00000000 --- a/lib/syntax_tree/yarv/local_table.rb +++ /dev/null @@ -1,89 +0,0 @@ -# frozen_string_literal: true - -module SyntaxTree - module YARV - # This represents every local variable associated with an instruction - # sequence. There are two kinds of locals: plain locals that are what you - # expect, and block proxy locals, which represent local variables - # associated with blocks that were passed into the current instruction - # sequence. - class LocalTable - # A local representing a block passed into the current instruction - # sequence. - class BlockLocal - attr_reader :name - - def initialize(name) - @name = name - end - end - - # A regular local variable. - class PlainLocal - attr_reader :name - - def initialize(name) - @name = name - end - end - - # The result of looking up a local variable in the current local table. - class Lookup - attr_reader :local, :index, :level - - def initialize(local, index, level) - @local = local - @index = index - @level = level - end - end - - attr_reader :locals - - def initialize - @locals = [] - end - - def empty? - locals.empty? - end - - def find(name, level = 0) - index = locals.index { |local| local.name == name } - Lookup.new(locals[index], index, level) if index - end - - def has?(name) - locals.any? { |local| local.name == name } - end - - def names - locals.map(&:name) - end - - def name_at(index) - locals[index].name - end - - def size - locals.length - end - - # Add a BlockLocal to the local table. - def block(name) - locals << BlockLocal.new(name) unless has?(name) - end - - # Add a PlainLocal to the local table. - def plain(name) - locals << PlainLocal.new(name) unless has?(name) - end - - # This is the offset from the top of the stack where this local variable - # lives. - def offset(index) - size - (index - 3) - 1 - end - end - end -end diff --git a/lib/syntax_tree/yarv/sea_of_nodes.rb b/lib/syntax_tree/yarv/sea_of_nodes.rb deleted file mode 100644 index 33ef14f7..00000000 --- a/lib/syntax_tree/yarv/sea_of_nodes.rb +++ /dev/null @@ -1,534 +0,0 @@ -# frozen_string_literal: true - -module SyntaxTree - module YARV - # A sea of nodes is an intermediate representation used by a compiler to - # represent both control and data flow in the same graph. The way we use it - # allows us to have the vertices of the graph represent either an - # instruction in the instruction sequence or a synthesized node that we add - # to the graph. The edges of the graph represent either control flow or data - # flow. - class SeaOfNodes - # This object represents a node in the graph that holds a YARV - # instruction. - class InsnNode - attr_reader :inputs, :outputs, :insn, :offset - - def initialize(insn, offset) - @inputs = [] - @outputs = [] - - @insn = insn - @offset = offset - end - - def id - offset - end - - def label - "%04d %s" % [offset, insn.disasm(Disassembler::Squished.new)] - end - end - - # Phi nodes are used to represent the merging of data flow from multiple - # incoming blocks. - class PhiNode - attr_reader :inputs, :outputs, :id - - def initialize(id) - @inputs = [] - @outputs = [] - @id = id - end - - def label - "#{id} φ" - end - end - - # Merge nodes are present in any block that has multiple incoming blocks. - # It provides a place for Phi nodes to attach their results. - class MergeNode - attr_reader :inputs, :outputs, :id - - def initialize(id) - @inputs = [] - @outputs = [] - @id = id - end - - def label - "#{id} ψ" - end - end - - # The edge of a graph represents either control flow or data flow. - class Edge - TYPES = %i[data control info].freeze - - attr_reader :from - attr_reader :to - attr_reader :type - attr_reader :label - - def initialize(from, to, type, label) - raise unless TYPES.include?(type) - - @from = from - @to = to - @type = type - @label = label - end - end - - # A subgraph represents the local data and control flow of a single basic - # block. - class SubGraph - attr_reader :first_fixed, :last_fixed, :inputs, :outputs - - def initialize(first_fixed, last_fixed, inputs, outputs) - @first_fixed = first_fixed - @last_fixed = last_fixed - @inputs = inputs - @outputs = outputs - end - end - - # The compiler is responsible for taking a data flow graph and turning it - # into a sea of nodes. - class Compiler - attr_reader :dfg, :nodes - - def initialize(dfg) - @dfg = dfg - @nodes = [] - - # We need to put a unique ID on the synthetic nodes in the graph, so - # we keep a counter that we increment any time we create a new - # synthetic node. - @id_counter = 999 - end - - def compile - local_graphs = {} - dfg.blocks.each do |block| - local_graphs[block.id] = create_local_graph(block) - end - - connect_local_graphs_control(local_graphs) - connect_local_graphs_data(local_graphs) - cleanup_phi_nodes - cleanup_insn_nodes - - SeaOfNodes.new(dfg, nodes, local_graphs).tap(&:verify) - end - - private - - # Counter for synthetic nodes. - def id_counter - @id_counter += 1 - end - - # Create a sub-graph for a single basic block - block block argument - # inputs and outputs will be left dangling, to be connected later. - def create_local_graph(block) - block_flow = dfg.block_flows.fetch(block.id) - - # A map of instructions to nodes. - insn_nodes = {} - - # Create a node for each instruction in the block. - block.each_with_length do |insn, offset| - node = InsnNode.new(insn, offset) - insn_nodes[offset] = node - nodes << node - end - - # The first and last node in the sub-graph, and the last fixed node. - previous_fixed = nil - first_fixed = nil - last_fixed = nil - - # The merge node for the phi nodes to attach to. - merge_node = nil - - # If there is more than one predecessor and we have basic block - # arguments coming in, then we need a merge node for the phi nodes to - # attach to. - if block.incoming_blocks.size > 1 && !block_flow.in.empty? - merge_node = MergeNode.new(id_counter) - nodes << merge_node - - previous_fixed = merge_node - first_fixed = merge_node - last_fixed = merge_node - end - - # Connect local control flow (only nodes with side effects.) - block.each_with_length do |insn, length| - if insn.side_effects? - insn_node = insn_nodes[length] - connect previous_fixed, insn_node, :control if previous_fixed - previous_fixed = insn_node - first_fixed ||= insn_node - last_fixed = insn_node - end - end - - # Connect basic block arguments. - inputs = {} - outputs = {} - block_flow.in.each do |arg| - # Each basic block argument gets a phi node. Even if there's only - # one predecessor! We'll tidy this up later. - phi = PhiNode.new(id_counter) - connect(phi, merge_node, :info) if merge_node - nodes << phi - inputs[arg] = phi - - block.each_with_length do |_, consumer_offset| - consumer_flow = dfg.insn_flows[consumer_offset] - consumer_flow.in.each_with_index do |producer, input_index| - if producer == arg - connect(phi, insn_nodes[consumer_offset], :data, input_index) - end - end - end - - block_flow.out.each { |out| outputs[out] = phi if out == arg } - end - - # Connect local dataflow from consumers back to producers. - block.each_with_length do |_, consumer_offset| - consumer_flow = dfg.insn_flows.fetch(consumer_offset) - consumer_flow.in.each_with_index do |producer, input_index| - if producer.local? - connect( - insn_nodes[producer.length], - insn_nodes[consumer_offset], - :data, - input_index - ) - end - end - end - - # Connect dataflow from producers that leaves the block. - block.each_with_length do |_, producer_pc| - dfg - .insn_flows - .fetch(producer_pc) - .out - .each do |consumer| - unless consumer.local? - # This is an argument to the successor block - not to an - # instruction here. - outputs[consumer.name] = insn_nodes[producer_pc] - end - end - end - - # A graph with only side-effect free instructions will currently have - # no fixed nodes! In that case just use the first instruction's node - # for both first and last. But it's a bug that it'll appear in the - # control flow path! - SubGraph.new( - first_fixed || insn_nodes[block.block_start], - last_fixed || insn_nodes[block.block_start], - inputs, - outputs - ) - end - - # Connect control flow that flows between basic blocks. - def connect_local_graphs_control(local_graphs) - dfg.blocks.each do |predecessor| - predecessor_last = local_graphs[predecessor.id].last_fixed - predecessor.outgoing_blocks.each_with_index do |successor, index| - label = - if index > 0 && - index == (predecessor.outgoing_blocks.length - 1) - # If there are multiple outgoing blocks from this block, then - # the last one is a fallthrough. Otherwise it's a branch. - :fallthrough - else - :"branch#{index}" - end - - connect( - predecessor_last, - local_graphs[successor.id].first_fixed, - :control, - label - ) - end - end - end - - # Connect data flow that flows between basic blocks. - def connect_local_graphs_data(local_graphs) - dfg.blocks.each do |predecessor| - arg_outs = local_graphs[predecessor.id].outputs.values - arg_outs.each_with_index do |arg_out, arg_n| - predecessor.outgoing_blocks.each do |successor| - successor_graph = local_graphs[successor.id] - arg_in = successor_graph.inputs.values[arg_n] - - # We're connecting to a phi node, so we may need a special - # label. - raise unless arg_in.is_a?(PhiNode) - - label = - case arg_out - when InsnNode - # Instructions that go into a phi node are labelled by the - # offset of last instruction in the block that executed - # them. This way you know which value to use for the phi, - # based on the last instruction you executed. - dfg.blocks.find do |block| - block_start = block.block_start - block_end = - block_start + block.insns.sum(&:length) - - block.insns.last.length - - if (block_start..block_end).cover?(arg_out.offset) - break block_end - end - end - when PhiNode - # Phi nodes to phi nodes are not labelled. - else - raise - end - - connect(arg_out, arg_in, :data, label) - end - end - end - end - - # We don't always build things in an optimal way. Go back and fix up - # some mess we left. Ideally we wouldn't create these problems in the - # first place. - def cleanup_phi_nodes - nodes.dup.each do |node| # dup because we're mutating - next unless node.is_a?(PhiNode) - - if node.inputs.size == 1 - # Remove phi nodes with a single input. - connect_over(node) - remove(node) - elsif node.inputs.map(&:from).uniq.size == 1 - # Remove phi nodes where all inputs are the same. - producer_edge = node.inputs.first - consumer_edge = node.outputs.find { |e| !e.to.is_a?(MergeNode) } - connect( - producer_edge.from, - consumer_edge.to, - :data, - consumer_edge.label - ) - remove(node) - end - end - end - - # Eliminate as many unnecessary nodes as we can. - def cleanup_insn_nodes - nodes.dup.each do |node| - next unless node.is_a?(InsnNode) - - case node.insn - when AdjustStack - # If there are any inputs to the adjust stack that are immediately - # discarded, we can remove them from the input list. - number = node.insn.number - - node.inputs.dup.each do |input_edge| - next if input_edge.type != :data - - from = input_edge.from - next unless from.is_a?(InsnNode) - - if from.inputs.empty? && from.outputs.size == 1 - number -= 1 - remove(input_edge.from) - elsif from.insn.is_a?(Dup) - number -= 1 - connect_over(from) - remove(from) - - new_edge = node.inputs.last - new_edge.from.outputs.delete(new_edge) - node.inputs.delete(new_edge) - end - end - - if number == 0 - connect_over(node) - remove(node) - else - next_node = - if number == 1 - InsnNode.new(Pop.new, node.offset) - else - InsnNode.new(AdjustStack.new(number), node.offset) - end - - next_node.inputs.concat(node.inputs) - next_node.outputs.concat(node.outputs) - - # Dynamically finding the index of the node in the nodes array - # because we're mutating the array as we go. - nodes[nodes.index(node)] = next_node - end - when Jump - # When you have a jump instruction that only has one input and one - # output, you can just connect over top of it and remove it. - if node.inputs.size == 1 && node.outputs.size == 1 - connect_over(node) - remove(node) - end - when Pop - from = node.inputs.find { |edge| edge.type == :data }.from - next unless from.is_a?(InsnNode) - - removed = - if from.inputs.empty? && from.outputs.size == 1 - remove(from) - true - elsif from.insn.is_a?(Dup) - connect_over(from) - remove(from) - - new_edge = node.inputs.last - new_edge.from.outputs.delete(new_edge) - node.inputs.delete(new_edge) - true - else - false - end - - if removed - connect_over(node) - remove(node) - end - end - end - end - - # Connect one node to another. - def connect(from, to, type, label = nil) - raise if from == to - raise if !to.is_a?(PhiNode) && type == :data && label.nil? - - edge = Edge.new(from, to, type, label) - from.outputs << edge - to.inputs << edge - end - - # Connect all of the inputs to all of the outputs of a node. - def connect_over(node) - node.inputs.each do |producer_edge| - node.outputs.each do |consumer_edge| - connect( - producer_edge.from, - consumer_edge.to, - producer_edge.type, - producer_edge.label - ) - end - end - end - - # Remove a node from the graph. - def remove(node) - node.inputs.each do |producer_edge| - producer_edge.from.outputs.reject! { |edge| edge.to == node } - end - - node.outputs.each do |consumer_edge| - consumer_edge.to.inputs.reject! { |edge| edge.from == node } - end - - nodes.delete(node) - end - end - - attr_reader :dfg, :nodes, :local_graphs - - def initialize(dfg, nodes, local_graphs) - @dfg = dfg - @nodes = nodes - @local_graphs = local_graphs - end - - def to_mermaid - Mermaid.flowchart do |flowchart| - nodes.each do |node| - flowchart.node("node_#{node.id}", node.label, shape: :rounded) - end - - nodes.each do |producer| - producer.outputs.each do |consumer_edge| - label = - if !consumer_edge.label - # No label. - elsif consumer_edge.to.is_a?(PhiNode) - # Edges into phi nodes are labelled by the offset of the - # instruction going into the merge. - "%04d" % consumer_edge.label - else - consumer_edge.label.to_s - end - - flowchart.link( - flowchart.fetch("node_#{producer.id}"), - flowchart.fetch("node_#{consumer_edge.to.id}"), - label, - type: consumer_edge.type == :info ? :dotted : :directed, - color: { data: :green, control: :red }[consumer_edge.type] - ) - end - end - end - end - - def verify - # Verify edge labels. - nodes.each do |node| - # Not talking about phi nodes right now. - next if node.is_a?(PhiNode) - - if node.is_a?(InsnNode) && node.insn.branch_targets.any? && - !node.insn.is_a?(Leave) - # A branching node must have at least one branch edge and - # potentially a fallthrough edge coming out. - - labels = node.outputs.map(&:label).sort - raise if labels[0] != :branch0 - raise if labels[1] != :fallthrough && labels.size > 2 - else - labels = node.inputs.filter { |e| e.type == :data }.map(&:label) - next if labels.empty? - - # No nil labels - raise if labels.any?(&:nil?) - - # Labels should start at zero. - raise unless labels.min.zero? - - # Labels should be contiguous. - raise unless labels.sort == (labels.min..labels.max).to_a - end - end - end - - def self.compile(dfg) - Compiler.new(dfg).compile - end - end - end -end diff --git a/lib/syntax_tree/yarv/vm.rb b/lib/syntax_tree/yarv/vm.rb deleted file mode 100644 index b303944d..00000000 --- a/lib/syntax_tree/yarv/vm.rb +++ /dev/null @@ -1,628 +0,0 @@ -# frozen_string_literal: true - -require "forwardable" - -module SyntaxTree - # This module provides an object representation of the YARV bytecode. - module YARV - class VM - class Jump - attr_reader :label - - def initialize(label) - @label = label - end - end - - class Leave - attr_reader :value - - def initialize(value) - @value = value - end - end - - class Frame - attr_reader :iseq, :parent, :stack_index, :_self, :nesting, :svars - attr_accessor :line, :pc - - def initialize(iseq, parent, stack_index, _self, nesting) - @iseq = iseq - @parent = parent - @stack_index = stack_index - @_self = _self - @nesting = nesting - - @svars = {} - @line = iseq.line - @pc = 0 - end - end - - class TopFrame < Frame - def initialize(iseq) - super(iseq, nil, 0, TOPLEVEL_BINDING.eval("self"), [Object]) - end - end - - class BlockFrame < Frame - def initialize(iseq, parent, stack_index) - super(iseq, parent, stack_index, parent._self, parent.nesting) - end - end - - class MethodFrame < Frame - attr_reader :name, :block - - def initialize(iseq, nesting, parent, stack_index, _self, name, block) - super(iseq, parent, stack_index, _self, nesting) - @name = name - @block = block - end - end - - class ClassFrame < Frame - def initialize(iseq, parent, stack_index, _self) - super(iseq, parent, stack_index, _self, parent.nesting + [_self]) - end - end - - class RescueFrame < Frame - def initialize(iseq, parent, stack_index) - super(iseq, parent, stack_index, parent._self, parent.nesting) - end - end - - class ThrownError < StandardError - attr_reader :value - - def initialize(value, backtrace) - super("This error was thrown by the Ruby VM.") - @value = value - set_backtrace(backtrace) - end - end - - class ReturnError < ThrownError - end - - class BreakError < ThrownError - end - - class NextError < ThrownError - end - - class FrozenCore - define_method("core#hash_merge_kwd") { |left, right| left.merge(right) } - - define_method("core#hash_merge_ptr") do |hash, *values| - hash.merge(values.each_slice(2).to_h) - end - - define_method("core#set_method_alias") do |clazz, new_name, old_name| - clazz.alias_method(new_name, old_name) - end - - define_method("core#set_variable_alias") do |new_name, old_name| - # Using eval here since there isn't a reflection API to be able to - # alias global variables. - eval("alias #{new_name} #{old_name}", binding, __FILE__, __LINE__) - end - - define_method("core#set_postexe") { |&block| END { block.call } } - - define_method("core#undef_method") do |clazz, name| - clazz.undef_method(name) - nil - end - end - - # This is the main entrypoint for events firing in the VM, which allows - # us to implement tracing. - class NullEvents - def publish_frame_change(frame) - end - - def publish_instruction(iseq, insn) - end - - def publish_stack_change(stack) - end - - def publish_tracepoint(event) - end - end - - # This is a simple implementation of tracing that prints to STDOUT. - class STDOUTEvents - attr_reader :disassembler - - def initialize - @disassembler = Disassembler.new - end - - def publish_frame_change(frame) - puts "%-16s %s" % ["frame-change", "#{frame.iseq.file}@#{frame.line}"] - end - - def publish_instruction(iseq, insn) - disassembler.current_iseq = iseq - puts "%-16s %s" % ["instruction", insn.disasm(disassembler)] - end - - def publish_stack_change(stack) - puts "%-16s %s" % ["stack-change", stack.values.inspect] - end - - def publish_tracepoint(event) - puts "%-16s %s" % ["tracepoint", event.inspect] - end - end - - # This represents the global VM stack. It effectively is an array, but - # wraps mutating functions with instrumentation. - class Stack - attr_reader :events, :values - - def initialize(events) - @events = events - @values = [] - end - - def concat(...) - values.concat(...).tap { events.publish_stack_change(self) } - end - - def last - values.last - end - - def length - values.length - end - - def push(...) - values.push(...).tap { events.publish_stack_change(self) } - end - - def pop(...) - values.pop(...).tap { events.publish_stack_change(self) } - end - - def slice!(...) - values.slice!(...).tap { events.publish_stack_change(self) } - end - - def [](...) - values.[](...) - end - - def []=(...) - values.[]=(...).tap { events.publish_stack_change(self) } - end - end - - FROZEN_CORE = FrozenCore.new.freeze - - extend Forwardable - - attr_reader :events - - attr_reader :stack - def_delegators :stack, :push, :pop - - attr_reader :frame - - def initialize(events = NullEvents.new) - @events = events - @stack = Stack.new(events) - @frame = nil - end - - def self.run(iseq) - new.run_top_frame(iseq) - end - - ########################################################################## - # Helper methods for frames - ########################################################################## - - def run_frame(frame) - # First, set the current frame to the given value. - previous = @frame - @frame = frame - events.publish_frame_change(@frame) - - # Next, set up the local table for the frame. This is actually incorrect - # as it could use the values already on the stack, but for now we're - # just doing this for simplicity. - stack.concat(Array.new(frame.iseq.local_table.size)) - - # Yield so that some frame-specific setup can be done. - start_label = yield if block_given? - frame.pc = frame.iseq.insns.index(start_label) if start_label - - # Finally we can execute the instructions one at a time. If they return - # jumps or leaves we will handle those appropriately. - loop do - case (insn = frame.iseq.insns[frame.pc]) - when Integer - frame.line = insn - frame.pc += 1 - when Symbol - events.publish_tracepoint(insn) - frame.pc += 1 - when InstructionSequence::Label - # skip labels - frame.pc += 1 - else - begin - events.publish_instruction(frame.iseq, insn) - result = insn.call(self) - rescue ReturnError => error - raise if frame.iseq.type != :method - - stack.slice!(frame.stack_index..) - @frame = frame.parent - events.publish_frame_change(@frame) - - return error.value - rescue BreakError => error - raise if frame.iseq.type != :block - - catch_entry = - find_catch_entry(frame, InstructionSequence::CatchBreak) - raise unless catch_entry - - stack.slice!( - ( - frame.stack_index + frame.iseq.local_table.size + - catch_entry.restore_sp - ).. - ) - @frame = frame - events.publish_frame_change(@frame) - - frame.pc = frame.iseq.insns.index(catch_entry.exit_label) - push(result = error.value) - rescue NextError => error - raise if frame.iseq.type != :block - - catch_entry = - find_catch_entry(frame, InstructionSequence::CatchNext) - raise unless catch_entry - - stack.slice!( - ( - frame.stack_index + frame.iseq.local_table.size + - catch_entry.restore_sp - ).. - ) - @frame = frame - events.publish_frame_change(@frame) - - frame.pc = frame.iseq.insns.index(catch_entry.exit_label) - push(result = error.value) - rescue Exception => error - catch_entry = - find_catch_entry(frame, InstructionSequence::CatchRescue) - raise unless catch_entry - - stack.slice!( - ( - frame.stack_index + frame.iseq.local_table.size + - catch_entry.restore_sp - ).. - ) - @frame = frame - events.publish_frame_change(@frame) - - frame.pc = frame.iseq.insns.index(catch_entry.exit_label) - push(result = run_rescue_frame(catch_entry.iseq, frame, error)) - end - - case result - when Jump - frame.pc = frame.iseq.insns.index(result.label) + 1 - when Leave - # this shouldn't be necessary, but is because we're not handling - # the stack correctly at the moment - stack.slice!(frame.stack_index..) - - # restore the previous frame - @frame = previous || frame.parent - events.publish_frame_change(@frame) if @frame - - return result.value - else - frame.pc += 1 - end - end - end - end - - def find_catch_entry(frame, type) - iseq = frame.iseq - iseq.catch_table.find do |catch_entry| - next unless catch_entry.is_a?(type) - - begin_pc = iseq.insns.index(catch_entry.begin_label) - end_pc = iseq.insns.index(catch_entry.end_label) - - (begin_pc...end_pc).cover?(frame.pc) - end - end - - def run_top_frame(iseq) - run_frame(TopFrame.new(iseq)) - end - - def run_block_frame(iseq, frame, *args, **kwargs, &block) - run_frame(BlockFrame.new(iseq, frame, stack.length)) do - setup_arguments(iseq, args, kwargs, block) - end - end - - def run_class_frame(iseq, clazz) - run_frame(ClassFrame.new(iseq, frame, stack.length, clazz)) - end - - def run_method_frame(name, nesting, iseq, _self, *args, **kwargs, &block) - run_frame( - MethodFrame.new( - iseq, - nesting, - frame, - stack.length, - _self, - name, - block - ) - ) { setup_arguments(iseq, args, kwargs, block) } - end - - def run_rescue_frame(iseq, frame, error) - run_frame(RescueFrame.new(iseq, frame, stack.length)) do - local_set(0, 0, error) - nil - end - end - - def setup_arguments(iseq, args, kwargs, block) - locals = [*args] - local_index = 0 - start_label = nil - - # First, set up all of the leading arguments. These are positional and - # required arguments at the start of the argument list. - if (lead_num = iseq.argument_options[:lead_num]) - lead_num.times do - local_set(local_index, 0, locals.shift) - local_index += 1 - end - end - - # Next, set up all of the optional arguments. The opt array contains - # the labels that the frame should start at if the optional is - # present. The last element of the array is the label that the frame - # should start at if all of the optional arguments are present. - if (opt = iseq.argument_options[:opt]) - opt[0...-1].each do |label| - if locals.empty? - start_label = label - break - else - local_set(local_index, 0, locals.shift) - local_index += 1 - end - - start_label = opt.last if start_label.nil? - end - end - - # If there is a splat argument, then we'll set that up here. It will - # grab up all of the remaining positional arguments. - if (rest_start = iseq.argument_options[:rest_start]) - if (post_start = iseq.argument_options[:post_start]) - length = post_start - rest_start - local_set(local_index, 0, locals[0...length]) - locals = locals[length..] - else - local_set(local_index, 0, locals.dup) - locals.clear - end - local_index += 1 - end - - # Next, set up any post arguments. These are positional arguments that - # come after the splat argument. - if (post_num = iseq.argument_options[:post_num]) - post_num.times do - local_set(local_index, 0, locals.shift) - local_index += 1 - end - end - - if (keyword_option = iseq.argument_options[:keyword]) - # First, set up the keyword bits array. - keyword_bits = - keyword_option.map do |config| - kwargs.key?(config.is_a?(Array) ? config[0] : config) - end - - iseq.local_table.locals.each_with_index do |local, index| - # If this is the keyword bits local, then set it appropriately. - if local.name.is_a?(Integer) - local_set(index, 0, keyword_bits) - next - end - - # First, find the configuration for this local in the keywords - # list if it exists. - name = local.name - config = - keyword_option.find do |keyword| - keyword.is_a?(Array) ? keyword[0] == name : keyword == name - end - - # If the configuration doesn't exist, then the local is not a - # keyword local. - next unless config - - if !config.is_a?(Array) - # required keyword - local_set(index, 0, kwargs.fetch(name)) - elsif !config[1].nil? - # optional keyword with embedded default value - local_set(index, 0, kwargs.fetch(name, config[1])) - else - # optional keyword with expression default value - local_set(index, 0, kwargs[name]) - end - end - end - - local_set(local_index, 0, block) if iseq.argument_options[:block_start] - - start_label - end - - ########################################################################## - # Helper methods for instructions - ########################################################################## - - def const_base - frame.nesting.last - end - - def frame_at(level) - current = frame - level.times { current = current.parent } - current - end - - def frame_svar - current = frame - current = current.parent while current.is_a?(BlockFrame) - current - end - - def frame_yield - current = frame - current = current.parent until current.is_a?(MethodFrame) - current - end - - def frozen_core - FROZEN_CORE - end - - def jump(label) - Jump.new(label) - end - - def leave - Leave.new(pop) - end - - def local_get(index, level) - stack[frame_at(level).stack_index + index] - end - - def local_set(index, level, value) - stack[frame_at(level).stack_index + index] = value - end - - ########################################################################## - # Methods for overriding runtime behavior - ########################################################################## - - DLEXT = ".#{RbConfig::CONFIG["DLEXT"]}" - SOEXT = ".#{RbConfig::CONFIG["SOEXT"]}" - - def require_resolved(filepath) - $LOADED_FEATURES << filepath - iseq = RubyVM::InstructionSequence.compile_file(filepath) - run_top_frame(InstructionSequence.from(iseq.to_a)) - end - - def require_internal(filepath, loading: false) - case (extname = File.extname(filepath)) - when "" - # search for all the extensions - searching = filepath - extensions = ["", ".rb", DLEXT, SOEXT] - when ".rb", DLEXT, SOEXT - # search only for the given extension name - searching = File.basename(filepath, extname) - extensions = [extname] - else - # we don't handle these extensions, raise a load error - raise LoadError, "cannot load such file -- #{filepath}" - end - - if filepath.start_with?("/") - # absolute path, search only in the given directory - directories = [File.dirname(searching)] - searching = File.basename(searching) - else - # relative path, search in the load path - directories = $LOAD_PATH - end - - directories.each do |directory| - extensions.each do |extension| - absolute_path = File.join(directory, "#{searching}#{extension}") - next unless File.exist?(absolute_path) - - if !loading && $LOADED_FEATURES.include?(absolute_path) - return false - elsif extension == ".rb" - require_resolved(absolute_path) - return true - elsif loading - return Kernel.send(:yarv_load, filepath) - else - return Kernel.send(:yarv_require, filepath) - end - end - end - - if loading - Kernel.send(:yarv_load, filepath) - else - Kernel.send(:yarv_require, filepath) - end - end - - def require(filepath) - require_internal(filepath, loading: false) - end - - def require_relative(filepath) - Kernel.yarv_require_relative(filepath) - end - - def load(filepath) - require_internal(filepath, loading: true) - end - - def eval( - source, - binding = TOPLEVEL_BINDING, - filename = "(eval)", - lineno = 1 - ) - Kernel.yarv_eval(source, binding, filename, lineno) - end - - def throw(tag, value = nil) - Kernel.throw(tag, value) - end - - def catch(tag, &block) - Kernel.catch(tag, &block) - end - end - end -end diff --git a/test/compiler_test.rb b/test/compiler_test.rb deleted file mode 100644 index 6cf8999e..00000000 --- a/test/compiler_test.rb +++ /dev/null @@ -1,533 +0,0 @@ -# frozen_string_literal: true - -return unless defined?(RubyVM::InstructionSequence) -return if RUBY_VERSION < "3.1" || RUBY_VERSION > "3.3" - -require_relative "test_helper" - -module SyntaxTree - class CompilerTest < Minitest::Test - CASES = [ - # Hooks - "BEGIN { a = 1 }", - "a = 1; END { a = 1 }; a", - # Various literals placed on the stack - "true", - "false", - "nil", - "self", - "0", - "1", - "2", - "1.0", - "1i", - "1r", - "1..2", - "1...2", - "(1)", - "%w[foo bar baz]", - "%W[foo bar baz]", - "%i[foo bar baz]", - "%I[foo bar baz]", - "{ foo: 1, bar: 1.0, baz: 1i }", - "'foo'", - "\"foo\"", - "\"foo\#{bar}\"", - "\"foo\#@bar\"", - "%q[foo]", - "%Q[foo]", - <<~RUBY, - "foo" \\ - "bar" - RUBY - <<~RUBY, - < 2", - "1 >= 2", - "1 == 2", - "1 != 2", - "1 & 2", - "1 | 2", - "1 << 2", - "1 ^ 2", - "foo.empty?", - "foo.length", - "foo.nil?", - "foo.size", - "foo.succ", - "/foo/ =~ \"foo\" && $1", - "\"foo\".freeze", - "\"foo\".freeze(1)", - "-\"foo\"", - "\"foo\".-@", - "\"foo\".-@(1)", - # Various method calls - "foo?", - "foo.bar", - "foo.bar(baz)", - "foo bar", - "foo.bar baz", - "foo(*bar)", - "foo(**bar)", - "foo(&bar)", - "foo.bar = baz", - "not foo", - "!foo", - "~foo", - "+foo", - "-foo", - "`foo`", - "`foo \#{bar} baz`", - # Local variables - "foo", - "foo = 1", - "foo = 1; bar = 2; baz = 3", - "foo = 1; foo", - "foo += 1", - "foo -= 1", - "foo *= 1", - "foo /= 1", - "foo %= 1", - "foo &= 1", - "foo |= 1", - "foo &&= 1", - "foo ||= 1", - "foo <<= 1", - "foo ^= 1", - "foo, bar = 1, 2", - "foo, bar, = 1, 2", - "foo, bar, baz = 1, 2", - "foo, bar = 1, 2, 3", - "foo = 1, 2, 3", - "foo, * = 1, 2, 3", - # Instance variables - "@foo", - "@foo = 1", - "@foo = 1; @bar = 2; @baz = 3", - "@foo = 1; @foo", - "@foo += 1", - "@foo -= 1", - "@foo *= 1", - "@foo /= 1", - "@foo %= 1", - "@foo &= 1", - "@foo |= 1", - "@foo &&= 1", - "@foo ||= 1", - "@foo <<= 1", - "@foo ^= 1", - # Class variables - "@@foo", - "@@foo = 1", - "@@foo = 1; @@bar = 2; @@baz = 3", - "@@foo = 1; @@foo", - "@@foo += 1", - "@@foo -= 1", - "@@foo *= 1", - "@@foo /= 1", - "@@foo %= 1", - "@@foo &= 1", - "@@foo |= 1", - "@@foo &&= 1", - "@@foo ||= 1", - "@@foo <<= 1", - "@@foo ^= 1", - # Global variables - "$foo", - "$foo = 1", - "$foo = 1; $bar = 2; $baz = 3", - "$foo = 1; $foo", - "$foo += 1", - "$foo -= 1", - "$foo *= 1", - "$foo /= 1", - "$foo %= 1", - "$foo &= 1", - "$foo |= 1", - "$foo &&= 1", - "$foo ||= 1", - "$foo <<= 1", - "$foo ^= 1", - # Index access - "foo[bar]", - "foo[bar] = 1", - "foo[bar] += 1", - "foo[bar] -= 1", - "foo[bar] *= 1", - "foo[bar] /= 1", - "foo[bar] %= 1", - "foo[bar] &= 1", - "foo[bar] |= 1", - "foo[bar] &&= 1", - "foo[bar] ||= 1", - "foo[bar] <<= 1", - "foo[bar] ^= 1", - "foo['true']", - "foo['true'] = 1", - # Constants (single) - "Foo", - "Foo = 1", - "Foo += 1", - "Foo -= 1", - "Foo *= 1", - "Foo /= 1", - "Foo %= 1", - "Foo &= 1", - "Foo |= 1", - "Foo &&= 1", - "Foo ||= 1", - "Foo <<= 1", - "Foo ^= 1", - # Constants (top) - "::Foo", - "::Foo = 1", - "::Foo += 1", - "::Foo -= 1", - "::Foo *= 1", - "::Foo /= 1", - "::Foo %= 1", - "::Foo &= 1", - "::Foo |= 1", - "::Foo &&= 1", - "::Foo ||= 1", - "::Foo <<= 1", - "::Foo ^= 1", - # Constants (nested) - "Foo::Bar::Baz", - "Foo::Bar::Baz += 1", - "Foo::Bar::Baz -= 1", - "Foo::Bar::Baz *= 1", - "Foo::Bar::Baz /= 1", - "Foo::Bar::Baz %= 1", - "Foo::Bar::Baz &= 1", - "Foo::Bar::Baz |= 1", - "Foo::Bar::Baz &&= 1", - "Foo::Bar::Baz ||= 1", - "Foo::Bar::Baz <<= 1", - "Foo::Bar::Baz ^= 1", - # Constants (top nested) - "::Foo::Bar::Baz", - "::Foo::Bar::Baz = 1", - "::Foo::Bar::Baz += 1", - "::Foo::Bar::Baz -= 1", - "::Foo::Bar::Baz *= 1", - "::Foo::Bar::Baz /= 1", - "::Foo::Bar::Baz %= 1", - "::Foo::Bar::Baz &= 1", - "::Foo::Bar::Baz |= 1", - "::Foo::Bar::Baz &&= 1", - "::Foo::Bar::Baz ||= 1", - "::Foo::Bar::Baz <<= 1", - "::Foo::Bar::Baz ^= 1", - # Constants (calls) - "Foo::Bar.baz", - "::Foo::Bar.baz", - "Foo::Bar.baz = 1", - "::Foo::Bar.baz = 1", - # Control flow - "foo&.bar", - "foo&.bar(1)", - "foo&.bar 1, 2, 3", - "foo&.bar {}", - "foo && bar", - "foo || bar", - "if foo then bar end", - "if foo then bar else baz end", - "if foo then bar elsif baz then qux end", - "foo if bar", - "unless foo then bar end", - "unless foo then bar else baz end", - "foo unless bar", - "foo while bar", - "while foo do bar end", - "foo until bar", - "until foo do bar end", - "for i in [1, 2, 3] do i end", - "foo ? bar : baz", - "case foo when bar then 1 end", - "case foo when bar then 1 else 2 end", - "baz if (foo == 1) .. (bar == 1)", - # Constructed values - "foo..bar", - "foo...bar", - "[1, 1.0, 1i, 1r]", - "[foo, bar, baz]", - "[@foo, @bar, @baz]", - "[@@foo, @@bar, @@baz]", - "[$foo, $bar, $baz]", - "%W[foo \#{bar} baz]", - "%I[foo \#{bar} baz]", - "[foo, bar] + [baz, qux]", - "[foo, bar, *baz, qux]", - "{ foo: bar, baz: qux }", - "{ :foo => bar, :baz => qux }", - "{ foo => bar, baz => qux }", - "%s[foo]", - "[$1, $2, $3, $4, $5, $6, $7, $8, $9]", - "/foo \#{bar} baz/", - "%r{foo \#{bar} baz}", - "[1, 2, 3].max", - "[foo, bar, baz].max", - "[foo, bar, baz].max(1)", - "[1, 2, 3].min", - "[foo, bar, baz].min", - "[foo, bar, baz].min(1)", - "[1, 2, 3].hash", - "[foo, bar, baz].hash", - "[foo, bar, baz].hash(1)", - "[1, 2, 3].foo", - "[foo, bar, baz].foo", - "[foo, bar, baz].foo(1)", - "[**{ x: true }][0][:x]", - # Core method calls - "alias foo bar", - "alias :foo :bar", - "super", - "super(1)", - "super(1, 2, 3)", - "undef foo", - "undef :foo", - "undef foo, bar, baz", - "undef :foo, :bar, :baz", - "def foo; yield; end", - "def foo; yield(1); end", - "def foo; yield(1, 2, 3); end", - # defined? usage - "defined?(foo)", - "defined?(\"foo\")", - "defined?(:foo)", - "defined?(@foo)", - "defined?(@@foo)", - "defined?($foo)", - "defined?(Foo)", - "defined?(yield)", - "defined?(super)", - "foo = 1; defined?(foo)", - "defined?(self)", - "defined?(true)", - "defined?(false)", - "defined?(nil)", - "defined?(foo = 1)", - # Ignored content - ";;;", - "# comment", - "=begin\nfoo\n=end", - <<~RUBY, - __END__ - RUBY - # Method definitions - "def foo; end", - "def foo(bar); end", - "def foo(bar, baz); end", - "def foo(bar = 1); end", - "def foo(bar = 1, baz = 2); end", - "def foo(*bar); end", - "def foo(bar, *baz); end", - "def foo(*bar, baz, qux); end", - "def foo(bar, *baz, qux); end", - "def foo(bar, baz, *qux, quaz); end", - "def foo(bar, baz, &qux); end", - "def foo(bar, *baz, &qux); end", - "def foo(&qux); qux; end", - "def foo(&qux); qux.call; end", - "def foo(&qux); qux = bar; end", - "def foo(bar:); end", - "def foo(bar:, baz:); end", - "def foo(bar: 1); end", - "def foo(bar: 1, baz: 2); end", - "def foo(bar: baz); end", - "def foo(bar: 1, baz: qux); end", - "def foo(bar: qux, baz: 1); end", - "def foo(bar: baz, qux: qaz); end", - "def foo(**rest); end", - "def foo(bar:, **rest); end", - "def foo(bar:, baz:, **rest); end", - "def foo(bar: 1, **rest); end", - "def foo(bar: 1, baz: 2, **rest); end", - "def foo(bar: baz, **rest); end", - "def foo(bar: 1, baz: qux, **rest); end", - "def foo(bar: qux, baz: 1, **rest); end", - "def foo(bar: baz, qux: qaz, **rest); end", - "def foo(...); end", - "def foo(bar, ...); end", - "def foo(...); bar(...); end", - "def foo(bar, ...); baz(1, 2, 3, ...); end", - "def self.foo; end", - "def foo.bar(baz); end", - # Class/module definitions - "module Foo; end", - "module ::Foo; end", - "module Foo::Bar; end", - "module ::Foo::Bar; end", - "module Foo; module Bar; end; end", - "class Foo; end", - "class ::Foo; end", - "class Foo::Bar; end", - "class ::Foo::Bar; end", - "class Foo; class Bar; end; end", - "class Foo < Baz; end", - "class ::Foo < Baz; end", - "class Foo::Bar < Baz; end", - "class ::Foo::Bar < Baz; end", - "class Foo; class Bar < Baz; end; end", - "class Foo < baz; end", - "class << Object; end", - "class << ::String; end", - # Block - "foo do end", - "foo {}", - "foo do |bar| end", - "foo { |bar| }", - "foo { |bar; baz| }", - "-> do end", - "-> {}", - "-> (bar) do end", - "-> (bar) {}", - "-> (bar; baz) { }", - # Pattern matching - "foo in bar", - "foo in [bar]", - "foo in [bar, baz]", - "foo in [1, 2, 3, bar, 4, 5, 6, baz]", - "foo in Foo[1, 2, 3, bar, 4, 5, 6, baz]", - "foo => bar" - ] - - # These are the combinations of instructions that we're going to test. - OPTIONS = [ - YARV::Compiler::Options.new, - YARV::Compiler::Options.new(frozen_string_literal: true), - YARV::Compiler::Options.new(operands_unification: false), - # TODO: have this work when peephole optimizations are turned off. - # YARV::Compiler::Options.new(peephole_optimization: false), - YARV::Compiler::Options.new(specialized_instruction: false), - YARV::Compiler::Options.new(inline_const_cache: false), - YARV::Compiler::Options.new(tailcall_optimization: true) - ] - - OPTIONS.each do |options| - suffix = options.to_hash.map { |key, value| "#{key}=#{value}" }.join("&") - - CASES.each do |source| - define_method(:"test_compiles_#{source}_(#{suffix})") do - assert_compiles(source, options) - end - - define_method(:"test_loads_#{source}_(#{suffix})") do - assert_loads(source, options) - end - - define_method(:"test_disasms_#{source}_(#{suffix})") do - assert_disasms(source, options) - end - end - end - - def test_evaluation - assert_evaluates 5, "2 + 3" - assert_evaluates 5, "a = 2; b = 3; a + b" - end - - private - - def serialize_iseq(iseq) - serialized = iseq.to_a - - serialized[4].delete(:node_id) - serialized[4].delete(:code_location) - serialized[4].delete(:node_ids) - - serialized[13] = serialized[13].filter_map do |insn| - case insn - when Array - insn.map do |operand| - if operand.is_a?(Array) && - operand[0] == YARV::InstructionSequence::MAGIC - serialize_iseq(operand) - else - operand - end - end - when Integer, :RUBY_EVENT_LINE - # ignore these for now - else - insn - end - end - - serialized - end - - # Check that the compiled instruction sequence matches the expected - # instruction sequence. - def assert_compiles(source, options) - assert_equal( - serialize_iseq(RubyVM::InstructionSequence.compile(source, **options)), - serialize_iseq(YARV.compile(source, options)) - ) - end - - # Check that the compiled instruction sequence matches the instruction - # sequence created directly from the compiled instruction sequence. - def assert_loads(source, options) - compiled = RubyVM::InstructionSequence.compile(source, **options) - - assert_equal( - serialize_iseq(compiled), - serialize_iseq(YARV::InstructionSequence.from(compiled.to_a, options)) - ) - end - - # Check that we can successfully disasm the compiled instruction sequence. - def assert_disasms(source, options) - compiled = RubyVM::InstructionSequence.compile(source, **options) - yarv = YARV::InstructionSequence.from(compiled.to_a, options) - assert_kind_of String, yarv.disasm - end - - def assert_evaluates(expected, source) - assert_equal expected, YARV.compile(source).eval - end - end -end diff --git a/test/yarv_test.rb b/test/yarv_test.rb deleted file mode 100644 index 78622434..00000000 --- a/test/yarv_test.rb +++ /dev/null @@ -1,517 +0,0 @@ -# frozen_string_literal: true - -return if !defined?(RubyVM::InstructionSequence) || RUBY_VERSION < "3.1" -require_relative "test_helper" - -module SyntaxTree - class YARVTest < Minitest::Test - CASES = { - "0" => "return 0\n", - "1" => "return 1\n", - "2" => "return 2\n", - "1.0" => "return 1.0\n", - "1 + 2" => "return 1 + 2\n", - "1 - 2" => "return 1 - 2\n", - "1 * 2" => "return 1 * 2\n", - "1 / 2" => "return 1 / 2\n", - "1 % 2" => "return 1 % 2\n", - "1 < 2" => "return 1 < 2\n", - "1 <= 2" => "return 1 <= 2\n", - "1 > 2" => "return 1 > 2\n", - "1 >= 2" => "return 1 >= 2\n", - "1 == 2" => "return 1 == 2\n", - "1 != 2" => "return 1 != 2\n", - "1 & 2" => "return 1 & 2\n", - "1 | 2" => "return 1 | 2\n", - "1 << 2" => "return 1 << 2\n", - "1 >> 2" => "return 1.>>(2)\n", - "1 ** 2" => "return 1.**(2)\n", - "a = 1; a" => "a = 1\nreturn a\n" - }.freeze - - CASES.each do |source, expected| - define_method("test_disassemble_#{source}") do - assert_decompiles(expected, source) - end - end - - def test_bf - hello_world = - "++++++++[>++++[>++>+++>+++>+<<<<-]>+>+>->>+[<]<-]" \ - ">>.>---.+++++++..+++.>>.<-.<.+++.------.--------.>>+.>++." - - iseq = YARV::Bf.new(hello_world).compile - stdout, = capture_io { iseq.eval } - assert_equal "Hello World!\n", stdout - - Formatter.format(hello_world, YARV::Decompiler.new(iseq).to_ruby) - end - - # rubocop:disable Layout/LineLength - EMULATION_CASES = { - # adjuststack - "x = [true]; x[0] ||= nil; x[0]" => true, - # anytostring - "\"\#{5}\"" => "5", - "class A2Str; def to_s; 1; end; end; \"\#{A2Str.new}\"" => - "#", - # branchif - "x = true; x ||= \"foo\"; x" => true, - # branchnil - "x = nil; if x&.to_s; 'hi'; else; 'bye'; end" => "bye", - # branchunless - "if 2 + 3; 'hi'; else; 'bye'; end" => "hi", - # checkkeyword - # "def evaluate(value: rand); value.floor; end; evaluate" => 0, - # checkmatch - "'foo' in String" => true, - "case 1; when *[1, 2, 3]; true; end" => true, - # checktype - "['foo'] in [String]" => true, - # concatarray - "[1, *2]" => [1, 2], - # concatstrings - "\"\#{7}\"" => "7", - # defineclass - "class DefineClass; def bar; end; end" => :bar, - "module DefineModule; def bar; end; end" => :bar, - "class << self; self; end" => - TOPLEVEL_BINDING.eval("self").singleton_class, - # defined - "defined?(1)" => "expression", - "defined?(foo = 1)" => "assignment", - "defined?(Object)" => "constant", - # definemethod - "def definemethod = 5; definemethod" => 5, - # definesmethod - "def self.definesmethod = 5; self.definesmethod" => 5, - # dup - "$global = 5" => 5, - # duparray - "[true]" => [true], - # duphash - "{ a: 1 }" => { - a: 1 - }, - # dupn - "Object::X ||= true" => true, - # expandarray - "x, = [true, false, nil]" => [true, false, nil], - "*, x = [true, false, nil]" => [true, false, nil], - # getblockparam - "def getblockparam(&block); block; end; getblockparam { 1 }.call" => 1, - # getblockparamproxy - "def getblockparamproxy(&block); block.call; end; getblockparamproxy { 1 }" => - 1, - # getclassvariable - "class CVar; @@foo = 5; end; class << CVar; @@foo; end" => 5, - # getconstant - "Object" => Object, - # getglobal - "$$" => $$, - # getinstancevariable - "@foo = 5; @foo" => 5, - # getlocal - "value = 5; self.then { self.then { self.then { value } } }" => 5, - # getlocalwc0 - "value = 5; value" => 5, - # getlocalwc1 - "value = 5; self.then { value }" => 5, - # getspecial - "1 if (2 == 2) .. (3 == 3)" => 1, - # intern - ":\"foo\#{1}\"" => :foo1, - # invokeblock - "def invokeblock = yield; invokeblock { 1 }" => 1, - # invokesuper - <<~RUBY => 2, - class Parent - def value - 1 - end - end - - class Child < Parent - def value - super + 1 - end - end - - Child.new.value - RUBY - # jump - "x = 0; if x == 0 then 1 else 2 end" => 1, - # newarray - "[\"value\"]" => ["value"], - # newarraykwsplat - "[\"string\", **{ foo: \"bar\" }]" => ["string", { foo: "bar" }], - # newhash - "def newhash(key, value) = { key => value }; newhash(1, 2)" => { - 1 => 2 - }, - # newrange - "x = 0; y = 1; (x..y).to_a" => [0, 1], - # nop - # objtostring - "\"\#{6}\"" => "6", - # once - "/\#{1}/o" => /1/o, - # opt_and - "0b0110 & 0b1011" => 0b0010, - # opt_aref - "x = [1, 2, 3]; x[1]" => 2, - # opt_aref_with - "x = { \"a\" => 1 }; x[\"a\"]" => 1, - # opt_aset - "x = [1, 2, 3]; x[1] = 4; x" => [1, 4, 3], - # opt_aset_with - "x = { \"a\" => 1 }; x[\"a\"] = 2; x" => { - "a" => 2 - }, - # opt_case_dispatch - <<~RUBY => "foo", - case 1 - when 1 - "foo" - else - "bar" - end - RUBY - # opt_div - "5 / 2" => 2, - # opt_empty_p - "[].empty?" => true, - # opt_eq - "1 == 1" => true, - # opt_ge - "1 >= 1" => true, - # opt_getconstant_path - "::Object" => Object, - # opt_gt - "1 > 1" => false, - # opt_le - "1 <= 1" => true, - # opt_length - "[1, 2, 3].length" => 3, - # opt_lt - "1 < 1" => false, - # opt_ltlt - "\"\" << 2" => "\u0002", - # opt_minus - "1 - 1" => 0, - # opt_mod - "5 % 2" => 1, - # opt_mult - "5 * 2" => 10, - # opt_neq - "1 != 1" => false, - # opt_newarray_max - "def opt_newarray_max(a, b, c) = [a, b, c].max; opt_newarray_max(1, 2, 3)" => - 3, - # opt_newarray_min - "def opt_newarray_min(a, b, c) = [a, b, c].min; opt_newarray_min(1, 2, 3)" => - 1, - # opt_nil_p - "nil.nil?" => true, - # opt_not - "!true" => false, - # opt_or - "0b0110 | 0b1011" => 0b1111, - # opt_plus - "1 + 1" => 2, - # opt_regexpmatch2 - "/foo/ =~ \"~~~foo\"" => 3, - # opt_send_without_block - "5.to_s" => "5", - # opt_size - "[1, 2, 3].size" => 3, - # opt_str_freeze - "\"foo\".freeze" => "foo", - # opt_str_uminus - "-\"foo\"" => -"foo", - # opt_succ - "1.succ" => 2, - # pop - "a ||= 2; a" => 2, - # putnil - "[nil]" => [nil], - # putobject - "2" => 2, - # putobject_INT2FIX_0_ - "0" => 0, - # putobject_INT2FIX_1_ - "1" => 1, - # putself - "self" => TOPLEVEL_BINDING.eval("self"), - # putspecialobject - "[class Undef; def foo = 1; undef foo; end]" => [nil], - # putstring - "\"foo\"" => "foo", - # send - "\"hello\".then { |value| value }" => "hello", - # setblockparam - "def setblockparam(&bar); bar = -> { 1 }; bar.call; end; setblockparam" => - 1, - # setclassvariable - "class CVarSet; @@foo = 1; end; class << CVarSet; @@foo = 10; end" => 10, - # setconstant - "SetConstant = 1" => 1, - # setglobal - "$global = 10" => 10, - # setinstancevariable - "@ivar = 5" => 5, - # setlocal - "x = 5; tap { tap { tap { x = 10 } } }; x" => 10, - # setlocal_WC_0 - "x = 5; x" => 5, - # setlocal_WC_1 - "x = 5; tap { x = 10 }; x" => 10, - # setn - "{}[:key] = 'value'" => "value", - # setspecial - "1 if (1 == 1) .. (2 == 2)" => 1, - # splatarray - "x = *(5)" => [5], - # swap - "!!defined?([[]])" => true, - # throw - # topn - "case 3; when 1..5; 'foo'; end" => "foo", - # toregexp - "/abc \#{1 + 2} def/" => /abc 3 def/ - }.freeze - # rubocop:enable Layout/LineLength - - EMULATION_CASES.each do |source, expected| - define_method("test_emulate_#{source}") do - assert_emulates(expected, source) - end - end - - ObjectSpace.each_object(YARV::Instruction.singleton_class) do |instruction| - next if instruction == YARV::Instruction - - define_method("test_instruction_interface_#{instruction.name}") do - methods = instruction.instance_methods(false) - assert_empty(%i[disasm to_a deconstruct_keys call ==] - methods) - end - end - - def test_cfg - iseq = RubyVM::InstructionSequence.compile("100 + (14 < 0 ? -1 : +1)") - iseq = SyntaxTree::YARV::InstructionSequence.from(iseq.to_a) - cfg = SyntaxTree::YARV::ControlFlowGraph.compile(iseq) - - assert_equal(<<~DISASM, cfg.disasm) - == cfg: #@:1 (1,0)-(1,0)> - block_0 - 0000 putobject 100 - 0002 putobject 14 - 0004 putobject_INT2FIX_0_ - 0005 opt_lt - 0007 branchunless 13 - == to: block_13, block_9 - block_9 - == from: block_0 - 0009 putobject -1 - 0011 jump 14 - == to: block_14 - block_13 - == from: block_0 - 0013 putobject_INT2FIX_1_ - == to: block_14 - block_14 - == from: block_9, block_13 - 0014 opt_plus - 0016 leave - == to: leaves - DISASM - end - - def test_dfg - iseq = RubyVM::InstructionSequence.compile("100 + (14 < 0 ? -1 : +1)") - iseq = SyntaxTree::YARV::InstructionSequence.from(iseq.to_a) - cfg = SyntaxTree::YARV::ControlFlowGraph.compile(iseq) - dfg = SyntaxTree::YARV::DataFlowGraph.compile(cfg) - - assert_equal(<<~DISASM, dfg.disasm) - == dfg: #@:1 (1,0)-(1,0)> - block_0 - 0000 putobject 100 # out: out_0 - 0002 putobject 14 # out: 5 - 0004 putobject_INT2FIX_0_ # out: 5 - 0005 opt_lt # in: 2, 4; out: 7 - 0007 branchunless 13 # in: 5 - == to: block_13, block_9 - == out: 0 - block_9 - == from: block_0 - == in: pass_0 - 0009 putobject -1 # out: out_0 - 0011 jump 14 - == to: block_14 - == out: pass_0, 9 - block_13 - == from: block_0 - == in: pass_0 - 0013 putobject_INT2FIX_1_ # out: out_0 - == to: block_14 - == out: pass_0, 13 - block_14 - == from: block_9, block_13 - == in: in_0, in_1 - 0014 opt_plus # in: in_0, in_1; out: 16 - 0016 leave # in: 14 - == to: leaves - DISASM - end - - def test_son - iseq = RubyVM::InstructionSequence.compile("(14 < 0 ? -1 : +1) + 100") - iseq = SyntaxTree::YARV::InstructionSequence.from(iseq.to_a) - cfg = SyntaxTree::YARV::ControlFlowGraph.compile(iseq) - dfg = SyntaxTree::YARV::DataFlowGraph.compile(cfg) - son = SyntaxTree::YARV::SeaOfNodes.compile(dfg) - - assert_equal(<<~MERMAID, son.to_mermaid) - flowchart TD - node_0("0000 putobject 14") - node_2("0002 putobject_INT2FIX_0_") - node_3("0003 opt_lt <calldata!mid:<, argc:1, ARGS_SIMPLE>") - node_5("0005 branchunless 0011") - node_7("0007 putobject -1") - node_11("0011 putobject_INT2FIX_1_") - node_12("0012 putobject 100") - node_14("0014 opt_plus <calldata!mid:+, argc:1, ARGS_SIMPLE>") - node_16("0016 leave") - node_1000("1000 ψ") - node_1001("1001 φ") - node_0 -- "0" --> node_3 - node_2 -- "1" --> node_3 - node_3 --> node_5 - node_3 -- "0" --> node_5 - node_5 -- "branch0" --> node_11 - node_5 -- "fallthrough" --> node_1000 - node_7 -- "0009" --> node_1001 - node_11 -- "branch0" --> node_1000 - node_11 -- "0011" --> node_1001 - node_12 -- "1" --> node_14 - node_14 --> node_16 - node_14 -- "0" --> node_16 - node_1000 --> node_14 - node_1001 -.-> node_1000 - node_1001 -- "0" --> node_14 - linkStyle 0 stroke:green - linkStyle 1 stroke:green - linkStyle 2 stroke:red - linkStyle 3 stroke:green - linkStyle 4 stroke:red - linkStyle 5 stroke:red - linkStyle 6 stroke:green - linkStyle 7 stroke:red - linkStyle 8 stroke:green - linkStyle 9 stroke:green - linkStyle 10 stroke:red - linkStyle 11 stroke:green - linkStyle 12 stroke:red - linkStyle 14 stroke:green - MERMAID - end - - def test_son_indirect_basic_block_argument - iseq = RubyVM::InstructionSequence.compile("100 + (14 < 0 ? -1 : +1)") - iseq = SyntaxTree::YARV::InstructionSequence.from(iseq.to_a) - cfg = SyntaxTree::YARV::ControlFlowGraph.compile(iseq) - dfg = SyntaxTree::YARV::DataFlowGraph.compile(cfg) - son = SyntaxTree::YARV::SeaOfNodes.compile(dfg) - - assert_equal(<<~MERMAID, son.to_mermaid) - flowchart TD - node_0("0000 putobject 100") - node_2("0002 putobject 14") - node_4("0004 putobject_INT2FIX_0_") - node_5("0005 opt_lt <calldata!mid:<, argc:1, ARGS_SIMPLE>") - node_7("0007 branchunless 0013") - node_9("0009 putobject -1") - node_13("0013 putobject_INT2FIX_1_") - node_14("0014 opt_plus <calldata!mid:+, argc:1, ARGS_SIMPLE>") - node_16("0016 leave") - node_1002("1002 ψ") - node_1004("1004 φ") - node_0 -- "0" --> node_14 - node_2 -- "0" --> node_5 - node_4 -- "1" --> node_5 - node_5 --> node_7 - node_5 -- "0" --> node_7 - node_7 -- "branch0" --> node_13 - node_7 -- "fallthrough" --> node_1002 - node_9 -- "0011" --> node_1004 - node_13 -- "branch0" --> node_1002 - node_13 -- "0013" --> node_1004 - node_14 --> node_16 - node_14 -- "0" --> node_16 - node_1002 --> node_14 - node_1004 -.-> node_1002 - node_1004 -- "1" --> node_14 - linkStyle 0 stroke:green - linkStyle 1 stroke:green - linkStyle 2 stroke:green - linkStyle 3 stroke:red - linkStyle 4 stroke:green - linkStyle 5 stroke:red - linkStyle 6 stroke:red - linkStyle 7 stroke:green - linkStyle 8 stroke:red - linkStyle 9 stroke:green - linkStyle 10 stroke:red - linkStyle 11 stroke:green - linkStyle 12 stroke:red - linkStyle 14 stroke:green - MERMAID - end - - private - - def assert_decompiles(expected, source) - ruby = YARV::Decompiler.new(YARV.compile(source)).to_ruby - actual = Formatter.format(source, ruby) - assert_equal expected, actual - end - - def assert_emulates(expected, source) - ruby_iseq = RubyVM::InstructionSequence.compile(source) - yarv_iseq = YARV::InstructionSequence.from(ruby_iseq.to_a) - - exercise_iseq(yarv_iseq) - result = SyntaxTree::YARV::VM.new.run_top_frame(yarv_iseq) - assert_equal(expected, result) - end - - def exercise_iseq(iseq) - iseq.disasm - iseq.to_a - - iseq.insns.each do |insn| - case insn - when YARV::InstructionSequence::Label, Integer, Symbol - next - end - - insn.pushes - insn.pops - insn.canonical - - case insn - when YARV::DefineClass - exercise_iseq(insn.class_iseq) - when YARV::DefineMethod, YARV::DefineSMethod - exercise_iseq(insn.method_iseq) - when YARV::InvokeSuper, YARV::Send - exercise_iseq(insn.block_iseq) if insn.block_iseq - when YARV::Once - exercise_iseq(insn.iseq) - end - end - end - end -end From d37f62ee7967af64ad1c7cc09182093f7485fa12 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Aug 2025 19:01:01 +0000 Subject: [PATCH 524/536] Bump rubocop from 1.78.0 to 1.79.2 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.78.0 to 1.79.2. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.78.0...v1.79.2) --- updated-dependencies: - dependency-name: rubocop dependency-version: 1.79.2 dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 9b8cbe16..c8f4f1be 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -10,12 +10,12 @@ GEM ast (2.4.3) docile (1.4.1) fiddle (1.1.8) - json (2.12.2) + json (2.13.2) language_server-protocol (3.17.0.5) lint_roller (1.1.0) minitest (5.25.5) parallel (1.27.0) - parser (3.3.8.0) + parser (3.3.9.0) ast (~> 2.4.1) racc prettier_print (1.2.1) @@ -23,8 +23,8 @@ GEM racc (1.8.1) rainbow (3.1.1) rake (13.3.0) - regexp_parser (2.10.0) - rubocop (1.78.0) + regexp_parser (2.11.0) + rubocop (1.79.2) json (~> 2.3) language_server-protocol (~> 3.17.0.2) lint_roller (~> 1.1.0) @@ -32,10 +32,10 @@ GEM parser (>= 3.3.0.2) rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 2.9.3, < 3.0) - rubocop-ast (>= 1.45.1, < 2.0) + rubocop-ast (>= 1.46.0, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 4.0) - rubocop-ast (1.45.1) + rubocop-ast (1.46.0) parser (>= 3.3.7.2) prism (~> 1.4) ruby-progressbar (1.13.0) From c152a45825594f1e492be54bd50b22bac99772a8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 21:35:04 +0000 Subject: [PATCH 525/536] Bump actions/checkout from 4 to 5 Bumps [actions/checkout](https://github.com/actions/checkout) from 4 to 5. - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/gh-pages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 7e4925df..b20082bd 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Setup Pages uses: actions/configure-pages@v5 - name: Set up Ruby From d991a96df36415555cde6ad30910870b5895ae8a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 22 Aug 2025 17:38:52 +0000 Subject: [PATCH 526/536] Bump rubocop from 1.79.2 to 1.80.0 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.79.2 to 1.80.0. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.79.2...v1.80.0) --- updated-dependencies: - dependency-name: rubocop dependency-version: 1.80.0 dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index c8f4f1be..80d7ecdf 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -23,8 +23,8 @@ GEM racc (1.8.1) rainbow (3.1.1) rake (13.3.0) - regexp_parser (2.11.0) - rubocop (1.79.2) + regexp_parser (2.11.2) + rubocop (1.80.0) json (~> 2.3) language_server-protocol (~> 3.17.0.2) lint_roller (~> 1.1.0) @@ -45,7 +45,7 @@ GEM simplecov_json_formatter (~> 0.1) simplecov-html (0.13.1) simplecov_json_formatter (0.1.4) - unicode-display_width (3.1.4) + unicode-display_width (3.1.5) unicode-emoji (~> 4.0, >= 4.0.4) unicode-emoji (4.0.4) From dcb32f9788d2ed5d66731557b7f9b26a01a508e6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 22 Aug 2025 17:56:40 +0000 Subject: [PATCH 527/536] Bump actions/upload-pages-artifact from 3 to 4 Bumps [actions/upload-pages-artifact](https://github.com/actions/upload-pages-artifact) from 3 to 4. - [Release notes](https://github.com/actions/upload-pages-artifact/releases) - [Commits](https://github.com/actions/upload-pages-artifact/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/upload-pages-artifact dependency-version: '4' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/gh-pages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index b20082bd..02f75a3e 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -39,7 +39,7 @@ jobs: rdoc --main README.md --op _site --exclude={Gemfile,Rakefile,"coverage/*","vendor/*","bin/*","test/*","tmp/*"} cp -r doc _site/doc - name: Upload artifact - uses: actions/upload-pages-artifact@v3 + uses: actions/upload-pages-artifact@v4 # Deployment job deploy: From abdf7cabbd22b4397a0a40f70c01912e87d167bb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 Aug 2025 23:24:14 +0000 Subject: [PATCH 528/536] Bump rubocop from 1.80.0 to 1.80.1 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.80.0 to 1.80.1. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.80.0...v1.80.1) --- updated-dependencies: - dependency-name: rubocop dependency-version: 1.80.1 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 80d7ecdf..b65a0992 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -24,7 +24,7 @@ GEM rainbow (3.1.1) rake (13.3.0) regexp_parser (2.11.2) - rubocop (1.80.0) + rubocop (1.80.1) json (~> 2.3) language_server-protocol (~> 3.17.0.2) lint_roller (~> 1.1.0) From e981ee86c37704cfd9bac0c5f995127b3b63b448 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 4 Sep 2025 04:03:30 +0000 Subject: [PATCH 529/536] Bump rubocop from 1.80.1 to 1.80.2 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.80.1 to 1.80.2. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.80.1...v1.80.2) --- updated-dependencies: - dependency-name: rubocop dependency-version: 1.80.2 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index b65a0992..ac9d30be 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -24,7 +24,7 @@ GEM rainbow (3.1.1) rake (13.3.0) regexp_parser (2.11.2) - rubocop (1.80.1) + rubocop (1.80.2) json (~> 2.3) language_server-protocol (~> 3.17.0.2) lint_roller (~> 1.1.0) From daa50f95347f2a21a4935df0739ab25b455a288f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 26 Sep 2025 17:03:04 +0000 Subject: [PATCH 530/536] Bump rubocop from 1.80.2 to 1.81.1 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.80.2 to 1.81.1. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.80.2...v1.81.1) --- updated-dependencies: - dependency-name: rubocop dependency-version: 1.81.1 dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index ac9d30be..bfc2ed60 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -10,7 +10,7 @@ GEM ast (2.4.3) docile (1.4.1) fiddle (1.1.8) - json (2.13.2) + json (2.15.0) language_server-protocol (3.17.0.5) lint_roller (1.1.0) minitest (5.25.5) @@ -19,12 +19,12 @@ GEM ast (~> 2.4.1) racc prettier_print (1.2.1) - prism (1.4.0) + prism (1.5.1) racc (1.8.1) rainbow (3.1.1) rake (13.3.0) - regexp_parser (2.11.2) - rubocop (1.80.2) + regexp_parser (2.11.3) + rubocop (1.81.1) json (~> 2.3) language_server-protocol (~> 3.17.0.2) lint_roller (~> 1.1.0) @@ -32,10 +32,10 @@ GEM parser (>= 3.3.0.2) rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 2.9.3, < 3.0) - rubocop-ast (>= 1.46.0, < 2.0) + rubocop-ast (>= 1.47.1, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 4.0) - rubocop-ast (1.46.0) + rubocop-ast (1.47.1) parser (>= 3.3.7.2) prism (~> 1.4) ruby-progressbar (1.13.0) @@ -45,9 +45,9 @@ GEM simplecov_json_formatter (~> 0.1) simplecov-html (0.13.1) simplecov_json_formatter (0.1.4) - unicode-display_width (3.1.5) - unicode-emoji (~> 4.0, >= 4.0.4) - unicode-emoji (4.0.4) + unicode-display_width (3.2.0) + unicode-emoji (~> 4.1) + unicode-emoji (4.1.0) PLATFORMS arm64-darwin-21 From 044a2379bd09efb80b5c25fd0116e48eae4d2606 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 Oct 2025 17:03:02 +0000 Subject: [PATCH 531/536] Bump minitest from 5.25.5 to 5.26.0 Bumps [minitest](https://github.com/minitest/minitest) from 5.25.5 to 5.26.0. - [Changelog](https://github.com/minitest/minitest/blob/master/History.rdoc) - [Commits](https://github.com/minitest/minitest/compare/v5.25.5...v5.26.0) --- updated-dependencies: - dependency-name: minitest dependency-version: 5.26.0 dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index bfc2ed60..9ff472f3 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -13,7 +13,7 @@ GEM json (2.15.0) language_server-protocol (3.17.0.5) lint_roller (1.1.0) - minitest (5.25.5) + minitest (5.26.0) parallel (1.27.0) parser (3.3.9.0) ast (~> 2.4.1) From 606a7c8a4425f18a061bf854a86960f43827338d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Oct 2025 17:03:16 +0000 Subject: [PATCH 532/536] Bump rubocop from 1.81.1 to 1.81.6 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.81.1 to 1.81.6. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.81.1...v1.81.6) --- updated-dependencies: - dependency-name: rubocop dependency-version: 1.81.6 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 9ff472f3..6d6096ee 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -10,7 +10,7 @@ GEM ast (2.4.3) docile (1.4.1) fiddle (1.1.8) - json (2.15.0) + json (2.15.1) language_server-protocol (3.17.0.5) lint_roller (1.1.0) minitest (5.26.0) @@ -19,12 +19,12 @@ GEM ast (~> 2.4.1) racc prettier_print (1.2.1) - prism (1.5.1) + prism (1.6.0) racc (1.8.1) rainbow (3.1.1) rake (13.3.0) regexp_parser (2.11.3) - rubocop (1.81.1) + rubocop (1.81.6) json (~> 2.3) language_server-protocol (~> 3.17.0.2) lint_roller (~> 1.1.0) From acaf2788e75cf088ba54b33a99c3f9bff7efc51f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 29 Oct 2025 17:03:19 +0000 Subject: [PATCH 533/536] Bump rake from 13.3.0 to 13.3.1 Bumps [rake](https://github.com/ruby/rake) from 13.3.0 to 13.3.1. - [Release notes](https://github.com/ruby/rake/releases) - [Changelog](https://github.com/ruby/rake/blob/master/History.rdoc) - [Commits](https://github.com/ruby/rake/compare/v13.3.0...v13.3.1) --- updated-dependencies: - dependency-name: rake dependency-version: 13.3.1 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 6d6096ee..7be345b8 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -22,7 +22,7 @@ GEM prism (1.6.0) racc (1.8.1) rainbow (3.1.1) - rake (13.3.0) + rake (13.3.1) regexp_parser (2.11.3) rubocop (1.81.6) json (~> 2.3) From a3f184b56e56d95ff1bf4f8f733443498dda6600 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 31 Oct 2025 17:03:04 +0000 Subject: [PATCH 534/536] Bump rubocop from 1.81.6 to 1.81.7 Bumps [rubocop](https://github.com/rubocop/rubocop) from 1.81.6 to 1.81.7. - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.81.6...v1.81.7) --- updated-dependencies: - dependency-name: rubocop dependency-version: 1.81.7 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 7be345b8..e130b65c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -10,12 +10,12 @@ GEM ast (2.4.3) docile (1.4.1) fiddle (1.1.8) - json (2.15.1) + json (2.15.2) language_server-protocol (3.17.0.5) lint_roller (1.1.0) minitest (5.26.0) parallel (1.27.0) - parser (3.3.9.0) + parser (3.3.10.0) ast (~> 2.4.1) racc prettier_print (1.2.1) @@ -24,7 +24,7 @@ GEM rainbow (3.1.1) rake (13.3.1) regexp_parser (2.11.3) - rubocop (1.81.6) + rubocop (1.81.7) json (~> 2.3) language_server-protocol (~> 3.17.0.2) lint_roller (~> 1.1.0) From 6d9e196c56ecf55c8501f012b3ddfcaef5e46505 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Nov 2025 17:39:59 +0000 Subject: [PATCH 535/536] Bump minitest from 5.26.0 to 5.26.1 Bumps [minitest](https://github.com/minitest/minitest) from 5.26.0 to 5.26.1. - [Changelog](https://github.com/minitest/minitest/blob/master/History.rdoc) - [Commits](https://github.com/minitest/minitest/compare/v5.26.0...v5.26.1) --- updated-dependencies: - dependency-name: minitest dependency-version: 5.26.1 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 7be345b8..58f4df81 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -13,7 +13,7 @@ GEM json (2.15.1) language_server-protocol (3.17.0.5) lint_roller (1.1.0) - minitest (5.26.0) + minitest (5.26.1) parallel (1.27.0) parser (3.3.9.0) ast (~> 2.4.1) From 1aab3bf8aad7070b214db08f33730103e16522e5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 20 Nov 2025 17:05:07 +0000 Subject: [PATCH 536/536] Bump actions/checkout from 5 to 6 Bumps [actions/checkout](https://github.com/actions/checkout) from 5 to 6. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/gh-pages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 02f75a3e..a9fce541 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Setup Pages uses: actions/configure-pages@v5 - name: Set up Ruby