Move rescript-editor-analysis and rescript-tools into compiler repo (#7000)

* Dune: explicitly set package to "rescript" before adding more packages

* Add package "analysis" from rescript-vscode repo

* Analysis: remove vendored libs that are already in the compiler repo

* Add Tast_iterator module from rescript-vscode repo

* Remove class stuff from Tast_iterator

* Format Tast_iterator

* Fix DeadValue.ml

* No more Config.uncurried

* Analysis: use default print width instead of taking it from syntax cli

* Make pexp_attributes mutable

* Add package "tools" from rescript-vscode repo

* Format OCaml code

* Mark Res_multi_printer.default_print_width as live

* Format

* Do not check in tools/analysis test artifacts

* Add rescript-editor-analysis and rescript-tools to rescript npm package

* Fix path for analysis/tools binaries in test scripts

* Run analysis tests in CI

* analysis: get rid of PervasivesU

* Nullable is now @unboxed

* More pervasives

* Analysis tests: use rescript from this repo and adapt

* Output changes that are ok

* Tools tests: use rescript from this repo

* commit OK changes to analysis output

* commit more OK changes to test output

* get CodeLens to a good enough state

* fix jsx prop completion broken by first class regexp literal support

* use predef paths so weird type lookups do not happen

* commit OK change

* commit OK output

* Fix gitignore

* commit OK output (JSON type changes)

* Do not run analysis tests on Windows

* Add opam exec

* print module name instead of full file name in debug statement

* dont use deprecated functions

* update generic jsx transform test

* update incremental typechecking test

* Fix uppercase exotic idents

* Don't run analysis tests on Linux ARM

* Move tools and analysis tests to tests folder

* CHANGELOG

* Add rescript-tools cli wrapper script

* Remove obsolete Makefile

* Ship RescriptTools module with the rescript npm package

---------

Co-authored-by: Gabriel Nordeborn <gabbe.nord@gmail.com>

Author: cknitt

.github/workflows/ci.yml

@@ -320,6 +320,10 @@ jobs:
       - name: Check for diffs in tests folder
         run: git diff --ignore-cr-at-eol --exit-code tests
 
+      - name: Run analysis / tools tests
+        if: runner.os != 'Windows' && matrix.os != 'buildjet-2vcpu-ubuntu-2204-arm'
+        run: opam exec -- make -C tests/analysis_tests test && make -C tests/tools_tests test
+
       - name: Run gentype tests
         if: runner.os != 'Windows'
         run: make -C tests/gentype_tests/typescript-react-example clean test

.gitignore

@@ -76,3 +76,8 @@ playground/compiler.js
 
 rewatch/target/
 rewatch/rewatch
+
+tests/tools_tests/**/*.res.js
+tests/tools_tests/lib
+tests/analysis_tests*/lib
+tests/analysis_tests/**/*.bs.js

CHANGELOG.md

@@ -17,11 +17,12 @@
 - Introduce "Unified operators" for arithmetic operators (`+`, `-`, `*`, `/`, `mod`). https://github.com/rescript-lang/rescript-compiler/pull/7057
 - Add remainder (`%`, aka modulus) operator. https://github.com/rescript-lang/rescript-compiler/pull/7152
 
-
 #### :bug: Bug fix
+
 - Fix and clean up boolean and/or optimizations. https://github.com/rescript-lang/rescript-compiler/pull/7134 https://github.com/rescript-lang/rescript-compiler/pull/7151
 
 #### :nail_care: Polish
+
 - Improve code generation for pattern matching of untagged variants. https://github.com/rescript-lang/rescript-compiler/pull/7128
 - Improve negation handling in combination with and/or to simplify generated code (especially coming out of pattern matching). https://github.com/rescript-lang/rescript-compiler/pull/7138
 - optimize JavaScript code generation by using x == null checks and improving type-based optimizations for string/number literals. https://github.com/rescript-lang/rescript-compiler/pull/7141
@@ -30,6 +31,9 @@
 - Further improve boolean optimizations. https://github.com/rescript-lang/rescript-compiler/pull/7149
 - Simplify code generated for conditionals. https://github.com/rescript-lang/rescript-compiler/pull/7151
 
+#### :house: Internal
+
+- Move rescript-editor-analysis and rescript-tools into compiler repo. https://github.com/rescript-lang/rescript-compiler/pull/7000
 
 # 12.0.0-alpha.4
 
@@ -61,6 +65,7 @@
 - Fix genType JSX component compilation. https://github.com/rescript-lang/rescript-compiler/pull/7107
 
 #### :nail_care: Polish
+
 - Add some context to error message for unused variables. https://github.com/rescript-lang/rescript-compiler/pull/7050
 - Improve error message when passing `children` prop to a component that doesn't accept it. https://github.com/rescript-lang/rescript-compiler/pull/7044
 - Improve error messages for pattern matching on option vs non-option, and vice versa. https://github.com/rescript-lang/rescript-compiler/pull/7035
@@ -71,7 +76,6 @@
 - Provide additional context in error message when `unit` is expected. https://github.com/rescript-lang/rescript-compiler/pull/7045
 - Improve error message when passing an object where a record is expected. https://github.com/rescript-lang/rescript-compiler/pull/7101
 
-
 #### :house: Internal
 
 - Remove uncurried flag from bsb. https://github.com/rescript-lang/rescript-compiler/pull/7049

analysis.opam

@@ -0,0 +1,27 @@
+# This file is generated by dune, edit dune-project instead
+opam-version: "2.0"
+synopsis: "ReScript Analysis"
+maintainer: ["Hongbo Zhang <bobzhang1988@gmail.com>" "Cristiano Calcagno"]
+authors: ["Hongbo Zhang <bobzhang1988@gmail.com>"]
+license: "LGPL-3.0-or-later"
+homepage: "https://github.com/rescript-lang/rescript-compiler"
+bug-reports: "https://github.com/rescript-lang/rescript-compiler/issues"
+depends: [
+  "ocaml" {>= "4.10"}
+  "cppo" {= "1.6.9"}
+  "dune"
+]
+build: [
+  ["dune" "subst"] {pinned}
+  [
+    "dune"
+    "build"
+    "-p"
+    name
+    "-j"
+    jobs
+    "@install"
+    "@runtest" {with-test}
+    "@doc" {with-doc}
+  ]
+]

analysis/README.md

@@ -0,0 +1,26 @@
+# Analysis Library and Binary
+
+This subfolder builds a private command line binary used by the plugin to power a few functionalities such as jump to definition, hover and autocomplete.
+
+The binary reads the `.cmt` and `.cmti` files and analyses them.
+
+For installation & build instructions, see the main CONTRIBUTING.md.
+
+## Overview
+
+See main CONTRIBUTING.md's repo structure. Additionally, `examples/` is a convenience debugging repo. Check out `test.sh` (invoked through `make test`) to see the snapshots testing workflow stored in `tests/`.
+
+## Usage
+
+At root:
+```sh
+./rescript-editor-analysis.exe --help
+
+# or
+
+dune exec -- rescript-editor-analysis --help
+```
+
+## History
+
+This project is based on a fork of [Reason Language Server](https://github.com/jaredly/reason-language-server).

analysis/bin/dune

@@ -0,0 +1,11 @@
+(env
+ (static
+  (flags
+   (:standard -ccopt -static))))
+
+(executable
+ (public_name rescript-editor-analysis)
+ (package analysis)
+ (modes byte exe)
+ (name main)
+ (libraries analysis))

analysis/bin/main.ml

@@ -0,0 +1,218 @@
+open Analysis
+
+let help =
+  {|
+**Private CLI For rescript-vscode usage only**
+
+API examples:
+  ./rescript-editor-analysis.exe completion src/MyFile.res 0 4 currentContent.res true
+  ./rescript-editor-analysis.exe definition src/MyFile.res 9 3
+  ./rescript-editor-analysis.exe typeDefinition src/MyFile.res 9 3
+  ./rescript-editor-analysis.exe documentSymbol src/Foo.res
+  ./rescript-editor-analysis.exe hover src/MyFile.res 10 2 true
+  ./rescript-editor-analysis.exe references src/MyFile.res 10 2
+  ./rescript-editor-analysis.exe rename src/MyFile.res 10 2 foo
+  ./rescript-editor-analysis.exe diagnosticSyntax src/MyFile.res
+  ./rescript-editor-analysis.exe inlayHint src/MyFile.res 0 3 25
+  ./rescript-editor-analysis.exe codeLens src/MyFile.res
+
+Dev-time examples:
+  ./rescript-editor-analysis.exe dump src/MyFile.res src/MyFile2.res
+  ./rescript-editor-analysis.exe test src/MyFile.res
+
+Note: positions are zero-indexed (start at 0 0), following LSP.
+https://microsoft.github.io/language-server-protocol/specification#position
+
+Options:
+  completion: compute autocomplete for MyFile.res at line 0 and column 4,
+    where MyFile.res is being edited and the editor content is in file current.res.
+
+    ./rescript-editor-analysis.exe completion src/MyFile.res 0 4 current.res
+
+  definition: get definition for item in MyFile.res at line 10 column 2:
+
+    ./rescript-editor-analysis.exe definition src/MyFile.res 10 2
+
+  typeDefinition: get type definition for item in MyFile.res at line 10 column 2:
+
+    ./rescript-editor-analysis.exe typeDefinition src/MyFile.res 10 2
+
+  documentSymbol: get all symbols declared in MyFile.res
+
+    ./rescript-editor-analysis.exe documentSymbol src/MyFile.res
+
+  hover: get inferred type for MyFile.res at line 10 column 2 (supporting markdown links):
+
+    ./rescript-editor-analysis.exe hover src/MyFile.res 10 2 true
+
+  references: get all references to item in MyFile.res at line 10 column 2:
+
+    ./rescript-editor-analysis.exe references src/MyFile.res 10 2
+
+  rename: rename all appearances of item in MyFile.res at line 10 column 2 with foo:
+
+    ./rescript-editor-analysis.exe rename src/MyFile.res 10 2 foo
+
+  semanticTokens: return token semantic highlighting info for MyFile.res
+
+    ./rescript-editor-analysis.exe semanticTokens src/MyFile.res
+
+  createInterface: print to stdout the interface file for src/MyFile.res
+
+    ./rescript-editor-analysis.exe createInterface src/MyFile.res lib/bs/src/MyFile.cmi
+
+  format: print to stdout the formatted version of the provided file
+
+    ./rescript-editor-analysis.exe format src/MyFile.res
+
+  diagnosticSyntax: print to stdout diagnostic for syntax
+
+    ./rescript-editor-analysis.exe diagnosticSyntax src/MyFile.res
+
+  inlayHint: get all inlay Hint between line 0 and 3 declared in MyFile.res. Last argument is maximum of character length for inlay hints
+
+    ./rescript-editor-analysis.exe inlayHint src/MyFile.res 0 3 25
+
+  codeLens: get all code lens entries for file src/MyFile.res
+
+    ./rescript-editor-analysis.exe codeLens src/MyFile.res
+
+  signatureHelp: get signature help if available for position at line 10 column 2 in src/MyFile.res
+
+    ./rescript-editor-analysis.exe signatureHelp src/MyFile.res 10 2
+
+  test: run tests specified by special comments in file src/MyFile.res
+
+    ./rescript-editor-analysis.exe test src/src/MyFile.res
+|}
+
+let main () =
+  let args = Array.to_list Sys.argv in
+  let debugLevel, args =
+    match args with
+    | _ :: "debug-dump" :: logLevel :: rest ->
+      ( (match logLevel with
+        | "verbose" -> Debug.Verbose
+        | "regular" -> Regular
+        | _ -> Off),
+        "dummy" :: rest )
+    | args -> (Off, args)
+  in
+  Debug.debugLevel := debugLevel;
+  let debug = debugLevel <> Debug.Off in
+  let printHeaderInfo path line col =
+    if debug then
+      Printf.printf "Debug level: %s\n%s:%s-%s\n\n"
+        (match debugLevel with
+        | Debug.Verbose -> "verbose"
+        | Regular -> "regular"
+        | Off -> "off")
+        path line col
+  in
+  match args with
+  | [_; "cache-project"; rootPath] -> (
+    Cfg.readProjectConfigCache := false;
+    let uri = Uri.fromPath rootPath in
+    match Packages.getPackage ~uri with
+    | Some package -> Cache.cacheProject package
+    | None -> print_endline "\"ERR\"")
+  | [_; "cache-delete"; rootPath] -> (
+    Cfg.readProjectConfigCache := false;
+    let uri = Uri.fromPath rootPath in
+    match Packages.findRoot ~uri (Hashtbl.create 0) with
+    | Some (`Bs rootPath) -> (
+      match BuildSystem.getLibBs rootPath with
+      | None -> print_endline "\"ERR\""
+      | Some libBs ->
+        Cache.deleteCache (Cache.targetFileFromLibBs libBs);
+        print_endline "\"OK\"")
+    | _ -> print_endline "\"ERR: Did not find root \"")
+  | [_; "completion"; path; line; col; currentFile] ->
+    printHeaderInfo path line col;
+    Commands.completion ~debug ~path
+      ~pos:(int_of_string line, int_of_string col)
+      ~currentFile
+  | [_; "completionResolve"; path; modulePath] ->
+    Commands.completionResolve ~path ~modulePath
+  | [_; "definition"; path; line; col] ->
+    Commands.definition ~path
+      ~pos:(int_of_string line, int_of_string col)
+      ~debug
+  | [_; "typeDefinition"; path; line; col] ->
+    Commands.typeDefinition ~path
+      ~pos:(int_of_string line, int_of_string col)
+      ~debug
+  | [_; "documentSymbol"; path] -> DocumentSymbol.command ~path
+  | [_; "hover"; path; line; col; currentFile; supportsMarkdownLinks] ->
+    Commands.hover ~path
+      ~pos:(int_of_string line, int_of_string col)
+      ~currentFile ~debug
+      ~supportsMarkdownLinks:
+        (match supportsMarkdownLinks with
+        | "true" -> true
+        | _ -> false)
+  | [
+   _; "signatureHelp"; path; line; col; currentFile; allowForConstructorPayloads;
+  ] ->
+    Commands.signatureHelp ~path
+      ~pos:(int_of_string line, int_of_string col)
+      ~currentFile ~debug
+      ~allowForConstructorPayloads:
+        (match allowForConstructorPayloads with
+        | "true" -> true
+        | _ -> false)
+  | [_; "inlayHint"; path; line_start; line_end; maxLength] ->
+    Commands.inlayhint ~path
+      ~pos:(int_of_string line_start, int_of_string line_end)
+      ~maxLength ~debug
+  | [_; "codeLens"; path] -> Commands.codeLens ~path ~debug
+  | [_; "codeAction"; path; startLine; startCol; endLine; endCol; currentFile]
+    ->
+    Commands.codeAction ~path
+      ~startPos:(int_of_string startLine, int_of_string startCol)
+      ~endPos:(int_of_string endLine, int_of_string endCol)
+      ~currentFile ~debug
+  | [_; "codemod"; path; line; col; typ; hint] ->
+    let typ =
+      match typ with
+      | "add-missing-cases" -> Codemod.AddMissingCases
+      | _ -> raise (Failure "unsupported type")
+    in
+    let res =
+      Codemod.transform ~path
+        ~pos:(int_of_string line, int_of_string col)
+        ~debug ~typ ~hint
+      |> Json.escape
+    in
+    Printf.printf "\"%s\"" res
+  | [_; "diagnosticSyntax"; path] -> Commands.diagnosticSyntax ~path
+  | _ :: "reanalyze" :: _ ->
+    let len = Array.length Sys.argv in
+    for i = 1 to len - 2 do
+      Sys.argv.(i) <- Sys.argv.(i + 1)
+    done;
+    Sys.argv.(len - 1) <- "";
+    Reanalyze.cli ()
+  | [_; "references"; path; line; col] ->
+    Commands.references ~path
+      ~pos:(int_of_string line, int_of_string col)
+      ~debug
+  | [_; "rename"; path; line; col; newName] ->
+    Commands.rename ~path
+      ~pos:(int_of_string line, int_of_string col)
+      ~newName ~debug
+  | [_; "semanticTokens"; currentFile] ->
+    SemanticTokens.semanticTokens ~currentFile
+  | [_; "createInterface"; path; cmiFile] ->
+    Printf.printf "\"%s\""
+      (Json.escape (CreateInterface.command ~path ~cmiFile))
+  | [_; "format"; path] ->
+    Printf.printf "\"%s\"" (Json.escape (Commands.format ~path))
+  | [_; "test"; path] -> Commands.test ~path
+  | args when List.mem "-h" args || List.mem "--help" args -> prerr_endline help
+  | _ ->
+    prerr_endline help;
+    exit 1
+;;
+
+main ()

analysis/dune

@@ -0,0 +1,16 @@
+(dirs bin src reanalyze vendor)
+
+(env
+ (dev
+  (env-vars
+   (CPPO_FLAGS -U=RELEASE)))
+ (release
+  (env-vars
+   (CPPO_FLAGS -D=RELEASE))
+  (ocamlopt_flags
+   (:standard -O3 -unbox-closures)))
+ (static
+  (env-vars
+   (CPPO_FLAGS -D=RELEASE))
+  (ocamlopt_flags
+   (:standard -O3 -unbox-closures))))

analysis/examples/example-project/.gitignore

@@ -0,0 +1,2 @@
+lib
+.merlin

analysis/examples/example-project/bsconfig.json

@@ -0,0 +1,14 @@
+{
+  "name": "tryit",
+  "sources": "src",
+  "bsc-flags": ["-bs-super-errors", "-open Belt"],
+  "warnings": {
+    "number": "-32-26-27-33"
+  },
+  "bs-dependencies": ["reason-react"],
+  "reason": { "react-jsx": 3 },
+  "namespace": "my-namespace",
+  "reanalyze": {
+    "analysis": ["dce", "exception"]
+  }
+}