use new runtime encoding, support lazy and recursive modules

Author: Hongbo Zhang

.gitignore

@@ -50,4 +50,5 @@ jscomp/bench/*.js
 .vscode
 *.jsx
 osc
-jscomp/pre_load.js
+jscomp/pre_load.js
+boot

jscomp/compiler.mllib

@@ -81,6 +81,8 @@ js_of_lam_tuple
 js_of_lam_string
 js_of_lam_block
 js_of_lam_array
+js_of_lam_module
+js_of_lam_exception
 js_implementation
 js_closure
 js_op
@@ -93,3 +95,4 @@ js_number
 js_helper
 js_cmj_datasets
 js_main
+

jscomp/j.ml

@@ -47,13 +47,15 @@ and ident_info = Js_op.ident_info
 
 and exports = Js_op.exports
 
+and tag_info = Js_op.tag_info 
+ 
 and required_modules = Js_op.required_modules
 
 (** object literal, if key is ident, in this case, it might be renamed by 
     Google Closure  optimizer,
     currently we always use quote
  *)
-and property_name =  string
+and property_name =  Js_op.property_name
 
 and ident = Ident.t 
 
@@ -101,8 +103,9 @@ and expression_desc =
      *)
   | Array_copy of expression (* shallow copy, like [x.slice] *)
   | Array_append of expression * expression (* For [caml_array_append]*)
-  | Tag_ml_obj of expression
+  (* | Tag_ml_obj of expression *)
   | String_append of expression * expression 
+
   | Int_of_boolean of expression 
   (* https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Operator_Precedence 
      [typeof] is an operator     
@@ -201,6 +204,28 @@ and expression_desc =
        which is better to leave it alone
      *)
   | Array of expression list * mutable_flag
+  | Caml_block of expression list * mutable_flag * expression * tag_info 
+
+  | Caml_uninitialized_obj of expression * expression
+  (* [tag] and [size] tailed  for [Obj.new_block] *)
+
+  (* For setter, it still return the value of expression, 
+     we can not use 
+     {[
+       type 'a access = Get | Set of 'a
+     ]}
+     in another module, since it will break our code generator
+     [Caml_block_tag] can return [undefined], 
+     you have to use [E.tag] in a safe way     
+  *)
+  | Caml_block_tag of expression
+  | Caml_block_set_tag of expression * expression
+  | Caml_block_length of expression
+  | Caml_block_set_length of expression * expression
+  (* It will just fetch tag, to make it safe, when creating it, 
+     we need apply "|0", we don't do it in the 
+     last step since "|0" can potentially be optimized
+  *)      
   | Number of number
   | Object of property_map
 

jscomp/js_analyzer.ml

@@ -79,15 +79,52 @@ let rec no_side_effect (x : J.expression)  =
   | Fun _ -> true
   | Number _ -> true (* Can be refined later *)
   | Array (xs,_mutable_flag)  
+  | Caml_block (xs, _mutable_flag, _, _)
     ->
       (** create [immutable] block,
           does not really mean that this opreation itself is [pure].
           
           the block is mutable does not mean this operation is non-pure
        *)
       List.for_all no_side_effect  xs 
+  | Array_append (a,b) 
+  | String_append (a,b)
   | Seq (a,b) -> no_side_effect a && no_side_effect b 
-  | _ -> false 
+  | Caml_block_length e
+  | Array_length e
+  | String_length e 
+  | Bytes_length e 
+  | Function_length e
+  | Char_of_int e 
+  | Char_to_int e 
+  | Caml_block_tag e 
+  | Typeof e
+    -> no_side_effect e 
+  | Bin (op, a, b) -> 
+    op != Eq && no_side_effect a && no_side_effect b     
+  | Math _ 
+  | Array_of_size _
+  | Array_copy _ 
+  (* | Tag_ml_obj _ *)
+  | Int_of_boolean _ 
+
+  | Not _ 
+  | String_of_small_int_array _ 
+  | Json_stringify _ 
+  | Anything_to_string _ 
+  | Dump _ 
+  | Cond _ 
+
+  | FlatCall _ 
+  | Call _ 
+  | Dot _ 
+  | New _ 
+  | Caml_uninitialized_obj _
+  | String_access _
+  | Object _
+  | Caml_block_set_tag _ 
+  | Caml_block_set_length _ (* actually true? *)
+    -> false 
 
 let no_side_effect_expression (x : J.expression) = no_side_effect x 
 
@@ -181,3 +218,13 @@ let rev_toplevel_flatten block =
 
     | x :: xs -> aux (x :: acc) xs  in
   aux [] block
+
+let rec is_constant (x : J.expression)  = 
+  match x.expression_desc with 
+  | Access (a,b) -> is_constant a && is_constant b 
+  | Str (b,_) -> b
+  | Number _ -> true (* Can be refined later *)
+  | Array (xs,_mutable_flag)  -> List.for_all is_constant  xs 
+  | Caml_block(xs, Immutable, tag, _) 
+    -> List.for_all is_constant xs && is_constant tag 
+  | _ -> false 

jscomp/js_analyzer.mli

@@ -60,3 +60,5 @@ val rev_flatten_seq : J.expression -> J.block
 
 val rev_toplevel_flatten : J.block -> J.block
 (** return the block in reverse order *)
+
+val is_constant : J.expression -> bool

jscomp/js_config.ml

@@ -85,7 +85,7 @@ let stdlib_set = String_set.of_list [
 let runtime_set = String_set.of_list [
     "caml_array.js";
     "caml_float.js";
-    "caml_obj_runtime.js";
+    "caml_obj.js";
     "caml_bigarray.js";
     "caml_format.js";		
     "caml_oo.js";		
@@ -98,6 +98,7 @@ let runtime_set = String_set.of_list [
     "caml_curry.js";
     "caml_file.js";
     "caml_lexer.js";
+    "caml_parser.js";
     "caml_string.js"
     (* "caml_sys.js"; *)
     (* "caml_unix.js"; *)
@@ -114,9 +115,9 @@ let io = "Caml_io"
 
 let sys = "Caml_sys"
 
-let lex_parse = "Caml_lexer"
-
-let obj_runtime = "Caml_obj_runtime"
+let lexer = "Caml_lexer"
+let parser = "Caml_parser"
+let obj_runtime = "Caml_obj"
 
 let array = "Caml_array"
 
@@ -128,3 +129,4 @@ let float = "Caml_float"
 
 let oo = "Caml_oo"
 let curry = "Caml_curry"
+let internalMod = "Caml_internalMod"

jscomp/js_config.mli

@@ -40,8 +40,8 @@ val oo : string
 
 val sys : string
 
-val lex_parse : string 
-
+val lexer : string 
+val parser : string
 val obj_runtime : string
 
 val array : string
@@ -54,5 +54,5 @@ val float : string
 
 val curry : string 
 
-
+val internalMod : string
 

jscomp/js_dump.ml

@@ -89,6 +89,7 @@ module L = struct
   let strict_directive = "'use strict';"
 
   let curry = "curry" (* curry arbitrary args *)
+  let tag = "tag"
 end
 let return_indent = (String.length L.return / Ext_pp.indent_length) 
 
@@ -420,17 +421,17 @@ and
     if l > 15 then P.paren_group f 1 action   
     else action ()
 
-  | Tag_ml_obj e -> 
-    P.group f 1 (fun _ -> 
-        P.string f "Object.defineProperty";
-        P.paren_group f 1 (fun _ ->
-            let cxt = expression 1 cxt f e in
-            P.string f L.comma;
-            P.space f ; 
-            P.string f {|"##ml"|};
-            P.string f L.comma;
-            P.string f {|{"value" : true, "writable" : false}|} ; 
-            cxt ))
+  (* | Tag_ml_obj e ->  *)
+  (*   P.group f 1 (fun _ ->  *)
+  (*       P.string f "Object.defineProperty"; *)
+  (*       P.paren_group f 1 (fun _ -> *)
+  (*           let cxt = expression 1 cxt f e in *)
+  (*           P.string f L.comma; *)
+  (*           P.space f ;  *)
+  (*           P.string f {|"##ml"|}; *)
+  (*           P.string f L.comma; *)
+  (*           P.string f {|{"value" : true, "writable" : false}|} ;  *)
+  (*           cxt )) *)
 
   | FlatCall(e,el) -> 
     P.group f 1 (fun _ -> 
@@ -580,6 +581,18 @@ and
     P.string f "typeof"; 
     P.space f;
     expression 13 cxt f e     
+  | Caml_block_set_tag(a,b) -> 
+    expression_desc cxt l f 
+      (Bin(Eq, 
+           {expression_desc = Caml_block_tag a; comment = None},
+           b
+          ))
+  | Caml_block_set_length(a,b) -> 
+    expression_desc cxt l f 
+      (Bin(Eq, 
+           {expression_desc = Caml_block_length a; comment = None},
+           b
+          ))
   | Bin (Eq, {expression_desc = Var i },
          {expression_desc = 
             (
@@ -744,7 +757,45 @@ and
       | []| [ _ ] -> P.bracket_group f 1 @@ fun _ -> array_element_list  cxt f el 
       | _ -> P.bracket_vgroup f 1 @@ fun _ -> array_element_list  cxt f el 
     end
+  | Caml_uninitialized_obj (tag, size) 
+    -> 
+    expression_desc cxt l f (Object [Length, size ; Tag, tag])    
+  | Caml_block( el, mutable_flag, tag, tag_info) 
+    -> 
+    (* Note that, if we ignore more than tag [0] we loose some information 
+       with regard tag  *)
+    begin match tag.expression_desc, tag_info with 
+
+    | Number (Int { i = 0 ; _})  , 
+      (Tuple | Array | Variant _ | Record | NA 
+      |  Constructor ("Some" | "::")) 
+      (* Hack to optimize option which is really pervasive in ocaml, 
+         we need concrete benchmark to support this
+      *) 
+      -> expression_desc cxt l f  (Array (el, mutable_flag))
+    (* TODO: for numbers like 248, 255 we can reverse engineer to make it 
+       [Obj.xx_flag], but we can not do this in runtime libraries
+    *)
 
+    | _, _
+      -> 
+      expression_desc cxt l f 
+        (J.Object (
+            let length, rev_list = 
+              List.fold_left (fun (i,acc) v -> 
+                (i+1, (Js_op.Int_key i, v) :: acc)                
+              ) (0, []) el in
+            List.rev_append rev_list 
+              [(Js_op.Length, E.int length) ;  (Js_op.Tag, tag)]
+          )
+        )
+    end
+  | Caml_block_tag e ->
+    P.group f 1 (fun _ ->  
+        let cxt = expression 15 cxt f  e in
+        P.string f L.dot ;
+        P.string f L.tag ;
+        cxt)
   | Access (e, e') 
 
   | String_access (e,e')
@@ -757,7 +808,8 @@ and
     in
     if l > 15 then P.paren_group f 1 action else action ()
 
-  | Array_length e | String_length e | Bytes_length e | Function_length e -> 
+  | Array_length e | String_length e | Bytes_length e 
+  | Function_length e | Caml_block_length e -> 
     let action () =  (** Todo: check parens *)
       let cxt = expression 15 cxt f e in
       P.string f L.dot;
@@ -840,24 +892,27 @@ and
     P.brace_vgroup f 1 @@ fun _ -> 
       property_name_and_value_list cxt f lst
 
-and property_name cxt f (s : J.property_name) : Ext_pp_scope.t =
-  pp_string f ~utf:true ~quote:(best_string_quote s) s; cxt 
+and property_name cxt f (s : J.property_name) : unit =
+  match s with
+  | Tag -> P.string f L.tag
+  | Length -> P.string f L.length
+  | Key s -> 
+    pp_string f ~utf:true ~quote:(best_string_quote s) s
+  | Int_key i -> P.string f (string_of_int i)
 
 and property_name_and_value_list cxt f l : Ext_pp_scope.t =
   match l with
   | [] -> cxt
   | [(pn, e)] ->
-    P.group f 0 @@ fun _ -> 
-      let cxt = property_name cxt  f pn in
-      P.string f L.colon;
-      P.space f;
-      expression 1 cxt f e 
+    property_name cxt  f pn ;
+    P.string f L.colon;
+    P.space f;
+    expression 1 cxt f e 
   | (pn, e) :: r ->
-    let cxt = P.group f 0 @@ fun _ -> 
-        let cxt = property_name cxt f pn in
-        P.string f L.colon;
-        P.space f;
-        expression 1 cxt f e in
+    property_name cxt f pn ; 
+    P.string f L.colon;
+    P.space f;
+    let cxt = expression 1 cxt f e in
     P.string f L.comma;
     P.newline f;
     property_name_and_value_list cxt f r
@@ -973,14 +1028,18 @@ and statement_desc top cxt f (s : J.statement_desc) : Ext_pp_scope.t =
     let rec need_paren  (e : J.expression) =
       match e.expression_desc with
       | Call ({expression_desc = Fun _; },_,_) -> true
-
+      | Caml_uninitialized_obj _ 
       | Fun _ | Object _ -> true
+      | Caml_block_set_tag _ 
+      | Caml_block_length _
+      | Caml_block_set_length _ 
       | Anything_to_string _ 
       | String_of_small_int_array _
       | Call _ 
       | Array_append _ 
       | Array_copy _ 
-      | Tag_ml_obj _
+      (* | Tag_ml_obj _ *)
+      | Caml_block_tag _ 
       | Seq _
       | Dot _
       | Cond _
@@ -1000,6 +1059,7 @@ and statement_desc top cxt f (s : J.statement_desc) : Ext_pp_scope.t =
       | Var _ 
       | Str _ 
       | Array _ 
+      | Caml_block  _ 
       | FlatCall _ 
       | Typeof _
       | Function_length _