new encoding for mlast, more efficient, readable

Author: bobzhang

jscomp/bsb_helper/bsb_helper_depfile_gen.ml

@@ -51,26 +51,28 @@ let write_file name  (buf : Ext_buffer.t) =
 (* Make sure it is the same as {!Binary_ast.magic_sep_char}*)
 let magic_sep_char = '\n'
 
-let deps_of_channel (ic : in_channel) : string array = 
+let deps_of_channel (ic : in_channel) : string list = 
   let size = input_binary_int ic in 
-  let s = really_input_string ic size in 
-  let first_tab  = String.index s magic_sep_char in 
-  let return_arr = Array.make (int_of_string (String.sub s 0 first_tab)) "" in 
-  let rec aux s ith (offset : int) : unit = 
+  let s = really_input_string ic size in   
+  let rec aux (s : string) acc (offset : int) size : string list = 
     if offset < size then
-      let next_tab = String.index_from s offset magic_sep_char  in 
-      return_arr.(ith) <- String.sub s offset (next_tab - offset) ; 
-      aux s (ith + 1) (next_tab + 1) 
+      let next_tab = String.index_from s offset magic_sep_char in        
+      aux s 
+        (String.sub s offset (next_tab - offset)::acc) (next_tab + 1) 
+        size
+    else acc    
   in 
-  aux s 0 (first_tab + 1) ; 
+  aux s [] 1 size 
+
+  
+
 
-  return_arr 
 
 (** Please refer to {!Binary_ast} for encoding format, we move it here 
     mostly for cutting the dependency so that [bsb_helper.exe] does
     not depend on compler-libs
 *)
-let read_deps (fn : string) : string array = 
+let read_deps (fn : string) : string list = 
   let ic = open_in_bin fn in 
   let v = deps_of_channel ic in 
   close_in ic;
@@ -123,7 +125,7 @@ let find_module db dependent_module is_not_lib_dir (index : Bsb_dir_index.t) =
       Bsb_db_decode.find_opt db (index :> int) dependent_module 
     else None 
 let oc_impl 
-    (dependent_module_set : string array)
+    (dependent_module_set : string list)
     (input_file : string)
     (index : Bsb_dir_index.t)
     (db : Bsb_db_decode.t)
@@ -145,7 +147,7 @@ let oc_impl
       Ext_buffer.add_string buf Literals.suffix_cmi;
     ) ; (* TODO: moved into static files*)
   let is_not_lib_dir = not (Bsb_dir_index.is_lib_dir index) in 
-  Ext_array.iter dependent_module_set (fun dependent_module ->
+  Ext_list.iter dependent_module_set (fun dependent_module ->
       match  
         find_module db dependent_module is_not_lib_dir index  
       with      
@@ -165,7 +167,7 @@ let oc_impl
     [.cmi] file
 *)
 let oc_intf
-    (dependent_module_set : string array)
+    (dependent_module_set : string list)
     input_file 
     (index : Bsb_dir_index.t)
     (db : Bsb_db_decode.t)
@@ -183,7 +185,7 @@ let oc_intf
       Ext_buffer.add_string buf Literals.suffix_cmi;
     ) ; 
   let is_not_lib_dir = not (Bsb_dir_index.is_lib_dir index)  in  
-  Ext_array.iter dependent_module_set begin fun dependent_module ->
+  Ext_list.iter dependent_module_set begin fun dependent_module ->
     match  find_module db dependent_module is_not_lib_dir index 
     with     
     | None -> ()

jscomp/bsb_helper/bsb_helper_depfile_gen.mli

@@ -27,7 +27,7 @@ type kind = Js | Bytecode | Native
 (** [deps_of_channel ic]
     given an input_channel dumps all modules it depend on, only used for debugging 
 *)
-val deps_of_channel : in_channel -> string array
+val deps_of_channel : in_channel -> string list
 
 (**
   [make compilation_kind filename index namespace]

jscomp/depends/binary_ast.ml

@@ -23,13 +23,11 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. *)
 
+ (** Synced up with module {!Bsb_helper_depfile_gen} *)
 module String_set = Ast_extract.String_set
 
 
 
-
-
-
 let read_ast (type t ) (kind : t  Ml_binary.kind) fn : t  =
   let ic = open_in_bin fn in
   try
@@ -50,14 +48,14 @@ let magic_sep_char = '\n'
 *)      
 let write_ast (type t) ~(fname : string) ~output (kind : t Ml_binary.kind) ( pt : t) : unit =
   let oc = open_out_bin output in 
-
   let output_set = Ast_extract.read_parse_and_extract kind pt in
-  let buf = Ext_buffer.create 64 in
-  let number = String_set.cardinal output_set in 
-  Ext_buffer.add_string buf (string_of_int number) ;
+  let buf = Ext_buffer.create 1000 in
+  
   Ext_buffer.add_char buf magic_sep_char;
-  String_set.iter (fun s -> Ext_buffer.add_string buf s ; Ext_buffer.add_char buf magic_sep_char) output_set ;  
-  output_binary_int oc (Ext_buffer.length buf);
+  String_set.iter (fun s -> Ext_buffer.add_string buf s ; 
+    Ext_buffer.add_char buf magic_sep_char) output_set ;  
+
+  output_binary_int oc (Ext_buffer.length buf);  
   Ext_buffer.output_buffer oc buf;
   Ml_binary.write_ast kind fname pt oc;
   close_out oc 

jscomp/main/astdump_main.ml

@@ -37,7 +37,7 @@ let () =
       (Format.pp_print_list ~pp_sep:(fun fmt () ->
            p fmt "@ ;"
          ) Format.pp_print_string) 
-      (Array.to_list arrs);
+       arrs;
     p Format.std_formatter "@.==============================@.";
     if Ext_path.check_suffix_case file ".mlast" then 
       begin

lib/4.02.3/bsb_helper.ml

@@ -4561,7 +4561,7 @@ type kind = Js | Bytecode | Native
 (** [deps_of_channel ic]
     given an input_channel dumps all modules it depend on, only used for debugging 
 *)
-val deps_of_channel : in_channel -> string array
+val deps_of_channel : in_channel -> string list
 
 (**
   [make compilation_kind filename index namespace]
@@ -4636,26 +4636,28 @@ let write_file name  (buf : Ext_buffer.t) =
 (* Make sure it is the same as {!Binary_ast.magic_sep_char}*)
 let magic_sep_char = '\n'
 
-let deps_of_channel (ic : in_channel) : string array = 
+let deps_of_channel (ic : in_channel) : string list = 
   let size = input_binary_int ic in 
-  let s = really_input_string ic size in 
-  let first_tab  = String.index s magic_sep_char in 
-  let return_arr = Array.make (int_of_string (String.sub s 0 first_tab)) "" in 
-  let rec aux s ith (offset : int) : unit = 
+  let s = really_input_string ic size in   
+  let rec aux (s : string) acc (offset : int) size : string list = 
     if offset < size then
-      let next_tab = String.index_from s offset magic_sep_char  in 
-      return_arr.(ith) <- String.sub s offset (next_tab - offset) ; 
-      aux s (ith + 1) (next_tab + 1) 
+      let next_tab = String.index_from s offset magic_sep_char in        
+      aux s 
+        (String.sub s offset (next_tab - offset)::acc) (next_tab + 1) 
+        size
+    else acc    
   in 
-  aux s 0 (first_tab + 1) ; 
+  aux s [] 1 size 
+
+  
+
 
-  return_arr 
 
 (** Please refer to {!Binary_ast} for encoding format, we move it here 
     mostly for cutting the dependency so that [bsb_helper.exe] does
     not depend on compler-libs
 *)
-let read_deps (fn : string) : string array = 
+let read_deps (fn : string) : string list = 
   let ic = open_in_bin fn in 
   let v = deps_of_channel ic in 
   close_in ic;
@@ -4708,7 +4710,7 @@ let find_module db dependent_module is_not_lib_dir (index : Bsb_dir_index.t) =
       Bsb_db_decode.find_opt db (index :> int) dependent_module 
     else None 
 let oc_impl 
-    (dependent_module_set : string array)
+    (dependent_module_set : string list)
     (input_file : string)
     (index : Bsb_dir_index.t)
     (db : Bsb_db_decode.t)
@@ -4730,7 +4732,7 @@ let oc_impl
       Ext_buffer.add_string buf Literals.suffix_cmi;
     ) ; (* TODO: moved into static files*)
   let is_not_lib_dir = not (Bsb_dir_index.is_lib_dir index) in 
-  Ext_array.iter dependent_module_set (fun dependent_module ->
+  Ext_list.iter dependent_module_set (fun dependent_module ->
       match  
         find_module db dependent_module is_not_lib_dir index  
       with      
@@ -4750,7 +4752,7 @@ let oc_impl
     [.cmi] file
 *)
 let oc_intf
-    (dependent_module_set : string array)
+    (dependent_module_set : string list)
     input_file 
     (index : Bsb_dir_index.t)
     (db : Bsb_db_decode.t)
@@ -4768,7 +4770,7 @@ let oc_intf
       Ext_buffer.add_string buf Literals.suffix_cmi;
     ) ; 
   let is_not_lib_dir = not (Bsb_dir_index.is_lib_dir index)  in  
-  Ext_array.iter dependent_module_set begin fun dependent_module ->
+  Ext_list.iter dependent_module_set begin fun dependent_module ->
     match  find_module db dependent_module is_not_lib_dir index 
     with     
     | None -> ()

lib/4.02.3/whole_compiler.ml

@@ -28738,13 +28738,11 @@ end = struct
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. *)
 
+ (** Synced up with module {!Bsb_helper_depfile_gen} *)
 module String_set = Ast_extract.String_set
 
 
 
-
-
-
 let read_ast (type t ) (kind : t  Ml_binary.kind) fn : t  =
   let ic = open_in_bin fn in
   try
@@ -28765,14 +28763,14 @@ let magic_sep_char = '\n'
 *)      
 let write_ast (type t) ~(fname : string) ~output (kind : t Ml_binary.kind) ( pt : t) : unit =
   let oc = open_out_bin output in 
-
   let output_set = Ast_extract.read_parse_and_extract kind pt in
-  let buf = Ext_buffer.create 64 in
-  let number = String_set.cardinal output_set in 
-  Ext_buffer.add_string buf (string_of_int number) ;
+  let buf = Ext_buffer.create 1000 in
+  
   Ext_buffer.add_char buf magic_sep_char;
-  String_set.iter (fun s -> Ext_buffer.add_string buf s ; Ext_buffer.add_char buf magic_sep_char) output_set ;  
-  output_binary_int oc (Ext_buffer.length buf);
+  String_set.iter (fun s -> Ext_buffer.add_string buf s ; 
+    Ext_buffer.add_char buf magic_sep_char) output_set ;  
+
+  output_binary_int oc (Ext_buffer.length buf);  
   Ext_buffer.output_buffer oc buf;
   Ml_binary.write_ast kind fname pt oc;
   close_out oc