Jsx ast (#7286)

* Intial exploration of JSX ast

Map child expressions

Initial mapping of Pexp_jsx_fragment to 0

Correct location in mapping

Update analysis for jsx_fragment

Remove unused code

Print something for ml print

Commit invalid test results for reference

Try improve printing

Correct fragment range, try and print comments

Indent jsx

Process comments from children inside fragment

Attach comments to fragment tags

Fix comment

Improve comment formatting

Print single element on same line

Update comment

WIP: Debug

More debugging

Works

Fix some jsx printing

Fix the test

Clean up

Update tests with location changes

* Initial mapping from0

* Format code

* Fix jsx fragment mapping

* Remove fragment

* Update test output

* Introducing Pexp_jsx_unary_element & Pexp_jsx_container_element

* Refactor fragment transformation.

* Initial transform of Pexp_jsx_unary_element in automatic mode

* Make props for unary element in automatic mode.

* Initial custom component unary tag

* lowercase container element with children.

* Uppercase container elements

* Streamline automatic element calls

* lowercase container element in classic mode

* Deal with uppercase tags in classic mode.

* Remove old code

* Improve recovery of incomplete jsx tags.
Port frontend completion.

* Correct range of incomplete jsx elements

* Update semantic tokens

* Make the closing tag optional for jsx_container_element.

* Add tighter pattern match for edge case in jsx props completion.

* Update analysis tests for jsx elements ast.

* print_jsx_unary_tag

* Rough print_jsx_container_tag

* Add ml printing

* Wing the sexp thing

* First step towards ast mapping

* prop punning conversion

* Map prop value

* Map prop spreading

* Initial container element mapping.

* Try support children spreading

* Only print space when there are props

* Add space after children.

* Restore braces in props and children

* Inline is_jsx_expression and remove old code

* Better indentation of children

* Handle unary tag

* Refactor

* WIP: Fix unary tag comments handling

* Fix comments inside prop expression

* Formats

* Fix closing tag indentation

* Fix closing tag indentation for cases with break vs inline

* Handle some edge cases

* Refactor

* WIP: Handle punning

* Fix optional printing with braces

* WIP: Handle comments attachment in a different way

* Handle empty props

* Handle props spread

* Fix optional with punning

* Indent props if they don't fit on one line.

* Fix indentation of children when props are multiline.

* Refactor to Pexp_jsx_element

* Don't always append make when uppercase component

* Use Doc.line for child spreading

* Exotic prop name

* Don't create record if only spreading prop, use that expression instead.

* Key prop can be optional

* Keep comment after opening greater than

* Print prop value with comments

* Update generic jsx completion tests

* Revert isJsxComponent

* Assign comment to the opening element tag name.

* Attach comments to closing > and closing tag

* Extract helpers to parsetree_viewer

* setup to walk jsx props

* Correct range for prop spreading

* Complete walk_jsx_prop

* Finish comment assignment for container elements

* Revisit comment attachment for unary tags

* Improve behaviour for comments in unary tag

* Keep location as is for prop spreading

* print leading comments of closing unary token

* Attach comments between empty container tag

* comments between opening and closing tag

* Correct opening_greater_than_doc

* Deal with container element comment edge cases.

* Ensure comments inside braced property values are correct.

* More accurate locations

* Update syntax roundtrip test files

* Update mapping test results

* comment after prop name

* For them older camls

* More older caml stuff

* Remove unreached code in typecore

* Remove leftover comments

* Preserve lines between jsx elements

* Preserve newline between jsx children, streamline jsx fragment children printing

* Update analysis expected

* Another location change in expected

* One more update?

* Remove leftover comments

* Remove obsolete comment

* Ensure fragment starts after arrow

* Add changelog entry

* Add missing Pexp_await

---------

Co-authored-by: Shulhi Sapli <shulhi@gmail.com>

Author: nojaf

CHANGELOG.md

@@ -12,6 +12,9 @@
 
 # 12.0.0-alpha.12 (Unreleased)
 
+#### :house: Internal
+- Better representation of JSX in AST . https://github.com/rescript-lang/rescript/pull/7286
+
 # 12.0.0-alpha.11
 
 #### :bug: Bug fix

analysis/src/CompletionFrontEnd.ml

@@ -1233,8 +1233,6 @@ let completionWithParser1 ~currentFile ~debug ~offset ~path ~posCursor
                          then ValueOrField
                          else Value);
                     }))
-        | Pexp_construct ({txt = Lident ("::" | "()")}, _) ->
-          (* Ignore list expressions, used in JSX, unit, and more *) ()
         | Pexp_construct (lid, eOpt) -> (
           let lidPath = flattenLidCheckDot lid in
           if debug then
@@ -1325,10 +1323,29 @@ let completionWithParser1 ~currentFile ~debug ~offset ~path ~posCursor
                         inJsx = !inJsxContext;
                       }))
             | None -> ())
-        | Pexp_apply {funct = {pexp_desc = Pexp_ident compName}; args}
-          when Res_parsetree_viewer.is_jsx_expression expr ->
+        | Pexp_jsx_element
+            ( Jsx_unary_element
+                {
+                  jsx_unary_element_tag_name = compName;
+                  jsx_unary_element_props = props;
+                }
+            | Jsx_container_element
+                {
+                  jsx_container_element_tag_name_start = compName;
+                  jsx_container_element_props = props;
+                } ) ->
           inJsxContext := true;
-          let jsxProps = CompletionJsx.extractJsxProps ~compName ~args in
+          let children =
+            match expr.pexp_desc with
+            | Pexp_jsx_element
+                (Jsx_container_element
+                   {jsx_container_element_children = children}) ->
+              children
+            | _ -> JSXChildrenItems []
+          in
+          let jsxProps =
+            CompletionJsx.extractJsxProps ~compName ~props ~children
+          in
           let compNamePath = flattenLidCheckDot ~jsx:true compName in
           if debug then
             Printf.printf "JSX <%s:%s %s> _children:%s\n"
@@ -1345,10 +1362,21 @@ let completionWithParser1 ~currentFile ~debug ~offset ~path ~posCursor
               | None -> "None"
               | Some childrenPosStart -> Pos.toString childrenPosStart);
           let jsxCompletable =
-            CompletionJsx.findJsxPropsCompletable ~jsxProps
-              ~endPos:(Loc.end_ expr.pexp_loc) ~posBeforeCursor
-              ~posAfterCompName:(Loc.end_ compName.loc)
-              ~firstCharBeforeCursorNoWhite ~charAtCursor
+            match expr.pexp_desc with
+            | Pexp_jsx_element
+                (Jsx_container_element
+                   {
+                     jsx_container_element_closing_tag = None;
+                     jsx_container_element_children =
+                       JSXChildrenSpreading _ | JSXChildrenItems (_ :: _);
+                   }) ->
+              (* This is a weird edge case where there is no closing tag but there are children *)
+              None
+            | _ ->
+              CompletionJsx.findJsxPropsCompletable ~jsxProps
+                ~endPos:(Loc.end_ expr.pexp_loc) ~posBeforeCursor
+                ~posAfterCompName:(Loc.end_ compName.loc)
+                ~firstCharBeforeCursorNoWhite ~charAtCursor
           in
           if jsxCompletable <> None then setResultOpt jsxCompletable
           else if compName.loc |> Loc.hasPos ~pos:posBeforeCursor then

analysis/src/CompletionJsx.ml

@@ -455,40 +455,39 @@ let findJsxPropsCompletable ~jsxProps ~endPos ~posBeforeCursor
   in
   loop jsxProps.props
 
-let extractJsxProps ~(compName : Longident.t Location.loc) ~args =
-  let thisCaseShouldNotHappen =
-    {
-      compName = Location.mknoloc (Longident.Lident "");
-      props = [];
-      childrenStart = None;
-    }
+let extractJsxProps ~(compName : Longident.t Location.loc) ~props ~children =
+  let open Parsetree in
+  let childrenStart =
+    match children with
+    | JSXChildrenItems [] -> None
+    | JSXChildrenSpreading child | JSXChildrenItems (child :: _) ->
+      if child.pexp_loc.loc_ghost then None else Some (Loc.start child.pexp_loc)
   in
-  let rec processProps ~acc args =
-    match args with
-    | (Asttypes.Labelled {txt = "children"}, {Parsetree.pexp_loc}) :: _ ->
-      {
-        compName;
-        props = List.rev acc;
-        childrenStart =
-          (if pexp_loc.loc_ghost then None else Some (Loc.start pexp_loc));
-      }
-    | ( (Labelled {txt = s; loc} | Optional {txt = s; loc}),
-        (eProp : Parsetree.expression) )
-      :: rest -> (
-      let namedArgLoc = if loc = Location.none then None else Some loc in
-      match namedArgLoc with
-      | Some loc ->
-        processProps
-          ~acc:
-            ({
-               name = s;
-               posStart = Loc.start loc;
-               posEnd = Loc.end_ loc;
-               exp = eProp;
-             }
-            :: acc)
-          rest
-      | None -> processProps ~acc rest)
-    | _ -> thisCaseShouldNotHappen
+  let props =
+    props
+    |> List.map (function
+         | JSXPropPunning (_, name) ->
+           {
+             name = name.txt;
+             posStart = Loc.start name.loc;
+             posEnd = Loc.end_ name.loc;
+             exp =
+               Ast_helper.Exp.ident ~loc:name.loc
+                 {txt = Longident.Lident name.txt; loc = name.loc};
+           }
+         | JSXPropValue (name, _, value) ->
+           {
+             name = name.txt;
+             posStart = Loc.start name.loc;
+             posEnd = Loc.end_ name.loc;
+             exp = value;
+           }
+         | JSXPropSpreading (loc, expr) ->
+           {
+             name = "_spreadProps";
+             posStart = Loc.start loc;
+             posEnd = Loc.end_ loc;
+             exp = expr;
+           })
   in
-  args |> processProps ~acc:[]
+  {compName; props; childrenStart}

analysis/src/SemanticTokens.ml

@@ -120,9 +120,7 @@ let emitLongident ?(backwards = false) ?(jsx = false)
   let rec flatten acc lid =
     match lid with
     | Longident.Lident txt -> txt :: acc
-    | Ldot (lid, txt) ->
-      let acc = if jsx && txt = "createElement" then acc else txt :: acc in
-      flatten acc lid
+    | Ldot (lid, txt) -> flatten (txt :: acc) lid
     | _ -> acc
   in
   let rec loop pos segments =
@@ -247,8 +245,8 @@ let command ~debug ~emitter ~path =
                ~posEnd:(Some (Loc.end_ loc))
                ~lid ~debug;
       Ast_iterator.default_iterator.expr iterator e
-    | Pexp_apply {funct = {pexp_desc = Pexp_ident lident; pexp_loc}; args}
-      when Res_parsetree_viewer.is_jsx_expression e ->
+    | Pexp_jsx_element (Jsx_unary_element {jsx_unary_element_tag_name = lident})
+      ->
       (*
          Angled brackets:
           - These are handled in the grammar:  <>  </>  </  />
@@ -258,46 +256,56 @@ let command ~debug ~emitter ~path =
           - handled like other Longitent.t, except lowercase id is marked Token.JsxLowercase
       *)
       emitter (* --> <div... *)
-      |> emitJsxTag ~debug ~name:"<"
-           ~pos:
-             (let pos = Loc.start e.pexp_loc in
-              (fst pos, snd pos - 1 (* the AST skips the loc of < somehow *)));
-      emitter |> emitJsxOpen ~lid:lident.txt ~debug ~loc:pexp_loc;
-
-      let posOfGreatherthanAfterProps =
-        let rec loop = function
-          | (Asttypes.Labelled {txt = "children"}, {Parsetree.pexp_loc}) :: _ ->
-            Loc.start pexp_loc
-          | _ :: args -> loop args
-          | [] -> (* should not happen *) (-1, -1)
-        in
-
-        loop args
-      in
-      let posOfFinalGreatherthan =
-        let pos = Loc.end_ e.pexp_loc in
-        (fst pos, snd pos - 1)
-      in
-      let selfClosing =
-        fst posOfGreatherthanAfterProps == fst posOfFinalGreatherthan
-        && snd posOfGreatherthanAfterProps + 1 == snd posOfFinalGreatherthan
-        (* there's an off-by one somehow in the AST *)
-      in
-      (if not selfClosing then
-         let lineStart, colStart = Loc.start pexp_loc in
-         let lineEnd, colEnd = Loc.end_ pexp_loc in
-         let length = if lineStart = lineEnd then colEnd - colStart else 0 in
-         let lineEndWhole, colEndWhole = Loc.end_ e.pexp_loc in
-         if length > 0 && colEndWhole > length then (
-           emitter
-           |> emitJsxClose ~debug ~lid:lident.txt
-                ~pos:(lineEndWhole, colEndWhole - 1);
-           emitter (* <foo ...props > <-- *)
-           |> emitJsxTag ~debug ~name:">" ~pos:posOfGreatherthanAfterProps;
-           emitter (* <foo> ... </foo> <-- *)
-           |> emitJsxTag ~debug ~name:">" ~pos:posOfFinalGreatherthan));
-
-      args |> List.iter (fun (_lbl, arg) -> iterator.expr iterator arg)
+      |> emitJsxTag ~debug ~name:"<" ~pos:(Loc.start e.pexp_loc);
+      emitter |> emitJsxOpen ~lid:lident.txt ~debug ~loc:lident.loc;
+      let closing_line, closing_column = Loc.end_ e.pexp_loc in
+      emitter (* <foo ...props /> <-- *)
+      |> emitJsxTag ~debug ~name:"/>" ~pos:(closing_line, closing_column - 2)
+      (* minus two for /> *)
+    | Pexp_jsx_element
+        (Jsx_container_element
+           {
+             jsx_container_element_tag_name_start = lident;
+             jsx_container_element_opening_tag_end = posOfGreatherthanAfterProps;
+             jsx_container_element_children = children;
+             jsx_container_element_closing_tag = closing_tag_opt;
+           }) ->
+      (* opening tag *)
+      emitter (* --> <div... *)
+      |> emitJsxTag ~debug ~name:"<" ~pos:(Loc.start e.pexp_loc);
+      emitter |> emitJsxOpen ~lid:lident.txt ~debug ~loc:lident.loc;
+      emitter (* <foo ...props > <-- *)
+      |> emitJsxTag ~debug ~name:">"
+           ~pos:(Pos.ofLexing posOfGreatherthanAfterProps);
+
+      (* children *)
+      (match children with
+      | Parsetree.JSXChildrenSpreading child -> iterator.expr iterator child
+      | Parsetree.JSXChildrenItems children ->
+        List.iter (iterator.expr iterator) children);
+
+      (* closing tag *)
+      closing_tag_opt
+      |> Option.iter
+           (fun
+             {
+               (* </ *)
+               Parsetree.jsx_closing_container_tag_start = closing_less_than;
+               (* name *)
+               jsx_closing_container_tag_name = tag_name_end;
+               (* > *)
+               jsx_closing_container_tag_end = final_greather_than;
+             }
+           ->
+             emitter
+             |> emitJsxTag ~debug ~name:"</"
+                  ~pos:(Pos.ofLexing closing_less_than);
+             emitter
+             |> emitJsxClose ~debug ~lid:lident.txt
+                  ~pos:(Loc.start tag_name_end.loc);
+             emitter (* <foo> ... </foo> <-- *)
+             |> emitJsxTag ~debug ~name:">"
+                  ~pos:(Pos.ofLexing final_greather_than))
     | Pexp_apply
         {
           funct =

analysis/src/Utils.ml

@@ -112,6 +112,7 @@ let identifyPexp pexp =
   | Pexp_extension _ -> "Pexp_extension"
   | Pexp_open _ -> "Pexp_open"
   | Pexp_await _ -> "Pexp_await"
+  | Pexp_jsx_element _ -> "Pexp_jsx_element"
 
 let identifyPpat pat =
   match pat with

compiler/frontend/bs_ast_mapper.ml

@@ -292,6 +292,20 @@ module M = struct
 end
 
 module E = struct
+  let map_jsx_children sub = function
+    | JSXChildrenSpreading e -> JSXChildrenSpreading (sub.expr sub e)
+    | JSXChildrenItems xs -> JSXChildrenItems (List.map (sub.expr sub) xs)
+
+  let map_jsx_prop sub = function
+    | JSXPropPunning (optional, name) ->
+      JSXPropPunning (optional, map_loc sub name)
+    | JSXPropValue (name, optional, value) ->
+      JSXPropValue (map_loc sub name, optional, sub.expr sub value)
+    | JSXPropSpreading (loc, e) ->
+      JSXPropSpreading (sub.location sub loc, sub.expr sub e)
+
+  let map_jsx_props sub = List.map (map_jsx_prop sub)
+
   (* Value expressions for the core language *)
 
   let map sub {pexp_loc = loc; pexp_desc = desc; pexp_attributes = attrs} =
@@ -367,6 +381,31 @@ module E = struct
       open_ ~loc ~attrs ovf (map_loc sub lid) (sub.expr sub e)
     | Pexp_extension x -> extension ~loc ~attrs (sub.extension sub x)
     | Pexp_await e -> await ~loc ~attrs (sub.expr sub e)
+    | Pexp_jsx_element
+        (Jsx_fragment
+           {
+             jsx_fragment_opening = o;
+             jsx_fragment_children = children;
+             jsx_fragment_closing = c;
+           }) ->
+      jsx_fragment o (map_jsx_children sub children) c
+    | Pexp_jsx_element
+        (Jsx_unary_element
+           {jsx_unary_element_tag_name = name; jsx_unary_element_props = props})
+      ->
+      jsx_unary_element ~loc ~attrs name (map_jsx_props sub props)
+    | Pexp_jsx_element
+        (Jsx_container_element
+           {
+             jsx_container_element_tag_name_start = name;
+             jsx_container_element_opening_tag_end = ote;
+             jsx_container_element_props = props;
+             jsx_container_element_children = children;
+             jsx_container_element_closing_tag = closing_tag;
+           }) ->
+      jsx_container_element ~loc ~attrs name (map_jsx_props sub props) ote
+        (map_jsx_children sub children)
+        closing_tag
 end
 
 module P = struct

compiler/ml/ast_helper.ml

@@ -181,7 +181,59 @@ module Exp = struct
   let open_ ?loc ?attrs a b c = mk ?loc ?attrs (Pexp_open (a, b, c))
   let extension ?loc ?attrs a = mk ?loc ?attrs (Pexp_extension a)
   let await ?loc ?attrs a = mk ?loc ?attrs (Pexp_await a)
+  let jsx_fragment ?loc ?attrs a b c =
+    mk ?loc ?attrs
+      (Pexp_jsx_element
+         (Jsx_fragment
+            {
+              jsx_fragment_opening = a;
+              jsx_fragment_children = b;
+              jsx_fragment_closing = c;
+            }))
+  let jsx_unary_element ?loc ?attrs a b =
+    mk ?loc ?attrs
+      (Pexp_jsx_element
+         (Jsx_unary_element
+            {jsx_unary_element_tag_name = a; jsx_unary_element_props = b}))
+
+  let jsx_container_element ?loc ?attrs a b c d e =
+    mk ?loc ?attrs
+      (Pexp_jsx_element
+         (Jsx_container_element
+            {
+              jsx_container_element_tag_name_start = a;
+              jsx_container_element_props = b;
+              jsx_container_element_opening_tag_end = c;
+              jsx_container_element_children = d;
+              jsx_container_element_closing_tag = e;
+            }))
+
   let case lhs ?guard rhs = {pc_lhs = lhs; pc_guard = guard; pc_rhs = rhs}
+
+  let make_list_expression loc seq ext_opt =
+    let rec handle_seq = function
+      | [] -> (
+        match ext_opt with
+        | Some ext -> ext
+        | None ->
+          let loc = {loc with Location.loc_ghost = true} in
+          let nil = Location.mkloc (Longident.Lident "[]") loc in
+          construct ~loc nil None)
+      | e1 :: el ->
+        let exp_el = handle_seq el in
+        let loc =
+          Location.
+            {
+              loc_start = e1.Parsetree.pexp_loc.Location.loc_start;
+              loc_end = exp_el.pexp_loc.loc_end;
+              loc_ghost = false;
+            }
+        in
+        let arg = tuple ~loc [e1; exp_el] in
+        construct ~loc (Location.mkloc (Longident.Lident "::") loc) (Some arg)
+    in
+    let expr = handle_seq seq in
+    {expr with pexp_loc = loc}
 end
 
 module Mty = struct