diff --git a/CHANGELOG.md b/CHANGELOG.md index 7baae519d..02aba9f08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,16 @@ Changelog ========= -Not released (2021-11-25) +Not released (2021-11-29) ------------------------- +Features implemented: + * ruby31.y: parse anonymous block argument. (#833) (Ilya Bylich) + * ruby31.y: parse forward argument without parentheses (#832) (Ilya Bylich) + +v3.0.3.0 (2021-11-25) +--------------------- + API modifications: * Bump maintenance branches to 3.0.3, 2.7.5, and 2.6.9 (#829) (Koichi ITO) diff --git a/doc/AST_FORMAT.md b/doc/AST_FORMAT.md index 4c72f9157..b4bcc972e 100644 --- a/doc/AST_FORMAT.md +++ b/doc/AST_FORMAT.md @@ -1014,6 +1014,17 @@ Format: Begin of the `expression` points to `&`. + +### Anonymous block argument + +Format: + +~~~ +(blockarg nil) +"&" + ~ expression +~~~ + ### Auto-expanding proc argument (1.9) In Ruby 1.9 and later, when a proc-like closure (i.e. a closure @@ -1391,6 +1402,15 @@ Used when passing expression as block `foo(&bar)` ~~~~ expression ~~~ +### Passing expression as anonymous block `foo(&)` + +~~~ +(send nil :foo (int 1) (block-pass nil)) +"foo(1, &)" + ^ operator + ~ expression +~~~ + ### "Stabby lambda" ~~~ diff --git a/lib/parser/builders/default.rb b/lib/parser/builders/default.rb index b36308e4e..fe563e9eb 100644 --- a/lib/parser/builders/default.rb +++ b/lib/parser/builders/default.rb @@ -879,8 +879,14 @@ def alias(alias_t, to, from) def args(begin_t, args, end_t, check_args=true) args = check_duplicate_args(args) if check_args - n(:args, args, - collection_map(begin_t, args, end_t)) + validate_no_forward_arg_after_restarg(args) + + map = collection_map(begin_t, args, end_t) + if !self.class.emit_forward_arg && args.length == 1 && args[0].type == :forward_arg + n(:forward_args, [], map) + else + n(:args, args, map) + end end def numargs(max_numparam) @@ -967,9 +973,12 @@ def shadowarg(name_t) end def blockarg(amper_t, name_t) - check_reserved_for_numparam(value(name_t), loc(name_t)) + if !name_t.nil? + check_reserved_for_numparam(value(name_t), loc(name_t)) + end - n(:blockarg, [ value(name_t).to_sym ], + arg_name = name_t ? value(name_t).to_sym : nil + n(:blockarg, [ arg_name ], arg_prefix_map(amper_t, name_t)) end @@ -1744,6 +1753,21 @@ def check_duplicate_arg(this_arg, map={}) end end + def validate_no_forward_arg_after_restarg(args) + restarg = nil + forward_arg = nil + args.each do |arg| + case arg.type + when :restarg then restarg = arg + when :forward_arg then forward_arg = arg + end + end + + if !forward_arg.nil? && !restarg.nil? + diagnostic :error, :forward_arg_after_restarg, nil, forward_arg.loc.expression, [restarg.loc.expression] + end + end + def check_assignment_to_numparam(name, loc) # MRI < 2.7 treats numbered parameters as regular variables # and so it's allowed to perform assignments like `_1 = 42`. diff --git a/lib/parser/context.rb b/lib/parser/context.rb index 432c83178..ef741135f 100644 --- a/lib/parser/context.rb +++ b/lib/parser/context.rb @@ -9,6 +9,9 @@ module Parser # + :sclass - in the singleton class body (class << obj; end) # + :def - in the method body (def m; end) # + :defs - in the singleton method body (def self.m; end) + # + :def_open_args - in the arglist of the method definition + # keep in mind that it's set **only** after reducing the first argument, + # if you need to handle the first argument check `lex_state == expr_fname` # + :block - in the block body (tap {}) # + :lambda - in the lambda body (-> {}) # @@ -64,5 +67,9 @@ def in_lambda? def in_dynamic_block? in_block? || in_lambda? end + + def in_def_open_args? + @stack.last == :def_open_args + end end end diff --git a/lib/parser/lexer.rl b/lib/parser/lexer.rl index 5ef1222bb..69c2403c7 100644 --- a/lib/parser/lexer.rl +++ b/lib/parser/lexer.rl @@ -937,7 +937,11 @@ class Parser::Lexer # b" # must be parsed as "ab" current_literal.extend_string(tok.gsub("\\\n".freeze, ''.freeze), @ts, @te) - elsif current_literal.regexp? && @version < 31 + elsif current_literal.regexp? && @version >= 31 && %w[c C m M].include?(escaped_char) + # Ruby >= 3.1 escapes \c- and \m chars, that's the only escape sequence + # supported by regexes so far, so it needs a separate branch. + current_literal.extend_string(@escape, @ts, @te) + elsif current_literal.regexp? # Regular expressions should include escape sequences in their # escaped form. On the other hand, escaped newlines are removed (in cases like "\\C-\\\n\\M-x") current_literal.extend_string(tok.gsub("\\\n".freeze, ''.freeze), @ts, @te) @@ -1438,6 +1442,18 @@ class Parser::Lexer => { emit(:tLABEL, tok(@ts, @te - 2), @ts, @te - 1) fhold; fnext expr_labelarg; fbreak; }; + '...' c_nl + => { + if @version >= 31 + emit(:tBDOT3, '...'.freeze, @ts, @te - 1) + emit(:tNL, "\n".freeze, @te - 1, @te) + fnext expr_end; fbreak; + else + p -= 4; + fhold; fgoto expr_end; + end + }; + w_space_comment; c_any @@ -2042,19 +2058,38 @@ class Parser::Lexer fnext expr_beg; fbreak; }; - '...' + '...' c_nl? => { + # Here we scan and conditionally emit "\n": + # + if it's there + # + and emitted we do nothing + # + and not emitted we return `p` to "\n" to process it on the next scan + # + if it's not there we do nothing + followed_by_nl = @te - 1 == @newline_s + nl_emitted = false + dots_te = followed_by_nl ? @te - 1 : @te + if @version >= 30 if @lambda_stack.any? && @lambda_stack.last + 1 == @paren_nest # To reject `->(...)` like `->...` - emit(:tDOT3) + emit(:tDOT3, '...'.freeze, @ts, dots_te) else - emit(:tBDOT3) + emit(:tBDOT3, '...'.freeze, @ts, dots_te) + + if @version >= 31 && followed_by_nl && @context.in_def_open_args? + emit(:tNL, @te - 1, @te) + nl_emitted = true + end end elsif @version >= 27 - emit(:tBDOT3) + emit(:tBDOT3, '...'.freeze, @ts, dots_te) else - emit(:tDOT3) + emit(:tDOT3, '...'.freeze, @ts, dots_te) + end + + if followed_by_nl && !nl_emitted + # return "\n" to process it on the next scan + fhold; end fnext expr_beg; fbreak; diff --git a/lib/parser/messages.rb b/lib/parser/messages.rb index 1853b1542..7d59d8a3b 100644 --- a/lib/parser/messages.rb +++ b/lib/parser/messages.rb @@ -77,6 +77,8 @@ module Parser :duplicate_pattern_key => 'duplicate hash pattern key %{name}', :endless_setter => 'setter method cannot be defined in an endless method definition', :invalid_id_to_get => 'identifier %{identifier} is not valid to get', + :forward_arg_after_restarg => '... after rest argument', + :no_anonymous_blockarg => 'no anonymous block parameter', # Parser warnings :useless_else => 'else without rescue is useless', diff --git a/lib/parser/ruby31.y b/lib/parser/ruby31.y index dda280be9..8daa9d48f 100644 --- a/lib/parser/ruby31.y +++ b/lib/parser/ruby31.y @@ -1159,6 +1159,14 @@ rule { result = @builder.block_pass(val[0], val[1]) } + | tAMPER + { + if !@static_env.declared_anonymous_blockarg? + diagnostic :error, :no_anonymous_blockarg, nil, val[0] + end + + result = @builder.block_pass(val[0], nil) + } opt_block_arg: tCOMMA block_arg { @@ -2698,20 +2706,6 @@ f_opt_paren_args: f_paren_args { result = @builder.args(val[0], val[1], val[2]) - @lexer.state = :expr_value - } - | tLPAREN2 f_arg tCOMMA args_forward rparen - { - args = [ *val[1], @builder.forward_arg(val[3]) ] - result = @builder.args(val[0], args, val[4]) - - @static_env.declare_forward_args - } - | tLPAREN2 args_forward rparen - { - result = @builder.forward_only_args(val[0], val[1], val[2]) - @static_env.declare_forward_args - @lexer.state = :expr_value } @@ -2719,9 +2713,11 @@ f_opt_paren_args: f_paren_args | { result = @lexer.in_kwarg @lexer.in_kwarg = true + @context.push(:def_open_args) } f_args term { + @context.pop @lexer.in_kwarg = val[0] result = @builder.args(nil, val[1], nil) } @@ -2742,6 +2738,11 @@ f_opt_paren_args: f_paren_args { result = [ val[0] ] } + | args_forward + { + @static_env.declare_forward_args + result = [ @builder.forward_arg(val[0]) ] + } opt_args_tail: tCOMMA args_tail { @@ -3019,6 +3020,12 @@ f_opt_paren_args: f_paren_args result = @builder.blockarg(val[0], val[1]) } + | blkarg_mark + { + @static_env.declare_anonymous_blockarg + + result = @builder.blockarg(val[0], nil) + } opt_f_block_arg: tCOMMA f_block_arg { diff --git a/lib/parser/static_environment.rb b/lib/parser/static_environment.rb index 5c5516e2c..f6b5406be 100644 --- a/lib/parser/static_environment.rb +++ b/lib/parser/static_environment.rb @@ -4,6 +4,7 @@ module Parser class StaticEnvironment FORWARD_ARGS = :FORWARD_ARGS + ANONYMOUS_BLOCKARG = :ANONYMOUS_BLOCKARG def initialize reset @@ -52,6 +53,14 @@ def declared_forward_args? declared?(FORWARD_ARGS) end + def declare_anonymous_blockarg + declare(ANONYMOUS_BLOCKARG) + end + + def declared_anonymous_blockarg? + declared?(ANONYMOUS_BLOCKARG) + end + def empty? @stack.empty? end diff --git a/lib/parser/version.rb b/lib/parser/version.rb index 8d7516f2a..07c4d160e 100644 --- a/lib/parser/version.rb +++ b/lib/parser/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module Parser - VERSION = '3.0.3.0' + VERSION = '3.0.3.1' end diff --git a/test/test_parser.rb b/test/test_parser.rb index 63b7008f8..57b67a7f1 100644 --- a/test/test_parser.rb +++ b/test/test_parser.rb @@ -6742,6 +6742,18 @@ def test_context_defs end end + def test_context_def_open_args + assert_context( + [:def, :def_open_args], + %q{def foo a = get_context; end}, + SINCE_3_1) + + assert_context( + [:defs, :def_open_args], + %q{def self.foo a = get_context; end}, + SINCE_3_1) + end + def test_context_cmd_brace_block [ 'tap foo { get_context }', @@ -10673,7 +10685,61 @@ def test_warn_on_duplicate_hash_key SINCE_3_1) end - def test_control_meta_escape_chars_in_regexp + def test_parser_bug_830 + assert_parses( + s(:regexp, + s(:str, "\\("), + s(:regopt)), + %q{/\(/}, + %q{}, + ALL_VERSIONS) + end + + def test_control_meta_escape_chars_in_regexp__before_31 + assert_parses( + s(:regexp, s(:str, "\\c\\xFF"), s(:regopt)), + %q{/\c\xFF/}.dup.force_encoding('ascii-8bit'), + %q{}, + ALL_VERSIONS - SINCE_3_1) + + assert_parses( + s(:regexp, s(:str, "\\c\\M-\\xFF"), s(:regopt)), + %q{/\c\M-\xFF/}.dup.force_encoding('ascii-8bit'), + %q{}, + ALL_VERSIONS - SINCE_3_1) + + assert_parses( + s(:regexp, s(:str, "\\C-\\xFF"), s(:regopt)), + %q{/\C-\xFF/}.dup.force_encoding('ascii-8bit'), + %q{}, + ALL_VERSIONS - SINCE_3_1) + + assert_parses( + s(:regexp, s(:str, "\\C-\\M-\\xFF"), s(:regopt)), + %q{/\C-\M-\xFF/}.dup.force_encoding('ascii-8bit'), + %q{}, + ALL_VERSIONS - SINCE_3_1) + + assert_parses( + s(:regexp, s(:str, "\\M-\\xFF"), s(:regopt)), + %q{/\M-\xFF/}.dup.force_encoding('ascii-8bit'), + %q{}, + ALL_VERSIONS - SINCE_3_1) + + assert_parses( + s(:regexp, s(:str, "\\M-\\C-\\xFF"), s(:regopt)), + %q{/\M-\C-\xFF/}.dup.force_encoding('ascii-8bit'), + %q{}, + ALL_VERSIONS - SINCE_3_1) + + assert_parses( + s(:regexp, s(:str, "\\M-\\c\\xFF"), s(:regopt)), + %q{/\M-\c\xFF/}.dup.force_encoding('ascii-8bit'), + %q{}, + ALL_VERSIONS - SINCE_3_1) + end + + def test_control_meta_escape_chars_in_regexp__since_31 x9f = "\x9F".dup.force_encoding('ascii-8bit') assert_parses( @@ -10718,4 +10784,72 @@ def test_control_meta_escape_chars_in_regexp %q{}, SINCE_3_1) end + + def test_forward_arg_with_open_args + assert_diagnoses_many( + [ + [:warning, :triple_dot_at_eol], + [:error, :unexpected_token, { :token => 'tDOT3' }], + ], + %Q{def foo ...\nend}, + SINCE_2_7 - SINCE_3_1) + + assert_diagnoses_many( + [ + [:error, :unexpected_token, { :token => 'tBDOT3' }], + ], + %Q{def foo a, b = 1, ...\nend}, + SINCE_2_7 - SINCE_3_1) + + assert_parses( + s(:def, :foo, + s(:args, s(:forward_arg)), nil), + %Q{def foo ...\nend}, + %q{ ~~~ expression (args.forward_arg)}, + SINCE_3_1) + + assert_parses( + s(:def, :foo, + s(:args, + s(:arg, :a), + s(:optarg, :b, + s(:int, 1)), + s(:forward_arg)), nil), + %Q{def foo a, b = 1, ...\nend}, + %q{ ~~~ expression (args.forward_arg)}, + SINCE_3_1) + + assert_diagnoses( + [:error, :forward_arg_after_restarg], + %Q{def foo *rest, ...\nend}, + %q{ ~~~ location + | ~~~~~ highlights (0)}, + SINCE_3_1) + end + + def test_anonymous_blockarg + assert_parses( + s(:def, :foo, + s(:args, + s(:blockarg, nil)), + s(:send, nil, :bar, + s(:block_pass, nil))), + %q{def foo(&); bar(&); end}, + %q{ ~ expression (args.blockarg) + | ~ operator (send.block_pass) + | ~ expression (send.block_pass)}, + SINCE_3_1) + + assert_diagnoses( + [:error, :no_anonymous_blockarg], + %q{def foo(); bar(&); end}, + %q{ ^ location}, + SINCE_3_1) + + assert_diagnoses( + [:error, :unexpected_token, { :token => 'tINTEGER' }], + %q{def foo(&0); end}, + %q{ ^ location}, + SINCE_3_1) + end end