diff --git a/emcc b/emcc index e6f3317d26ae8..bfbd61a6d4cee 100755 --- a/emcc +++ b/emcc @@ -415,6 +415,7 @@ try: proxy_to_worker = False default_object_extension = '.o' valid_abspaths = [] + separate_asm = False def is_valid_abspath(path_name): # Any path that is underneath the emscripten repository root must be ok. @@ -648,6 +649,9 @@ try: valid_abspaths.append(newargs[i+1]) newargs[i] = '' newargs[i+1] = '' + elif newargs[i] == '--separate-asm': + separate_asm = True + newargs[i] = '' elif newargs[i].startswith(('-I', '-L')): path_name = newargs[i][2:] if not absolute_warning_shown and os.path.isabs(path_name) and not is_valid_abspath(path_name): @@ -845,6 +849,8 @@ try: assert not (Compression.on and final_suffix != 'html'), 'Compression only works when generating HTML' + assert not (separate_asm and final_suffix != 'html'), '--separate-asm requires building to HTML' + # If we are using embind and generating JS, now is the time to link in bind.cpp if bind and final_suffix in JS_CONTAINING_SUFFIXES: input_files.append((next_arg_index, shared.path_from_root('system', 'lib', 'embind', 'bind.cpp'))) @@ -1333,6 +1339,8 @@ try: file_args.append('--no-heap-copy') if not use_closure_compiler: file_args.append('--no-closure') + if shared.Settings.LZ4: + file_args.append('--lz4') file_code = execute([shared.PYTHON, shared.FILE_PACKAGER, unsuffixed(target) + '.data'] + file_args, stdout=PIPE)[0] pre_js = file_code + pre_js @@ -1852,6 +1860,23 @@ try { })(); ''' % os.path.basename(memfile)) + script_inline + if separate_asm: + un_src() + asm_target = js_target[:-3] + '.asm.js' + temp_target = misc_temp_files.get(suffix='.js').name + execute([shared.PYTHON, shared.path_from_root('tools', 'separate_asm.py'), js_target, asm_target, temp_target]) + shutil.move(temp_target, js_target) + script_inline = ''' + var script = document.createElement('script'); + script.src = "%s"; + script.onload = function() { + setTimeout(function() { + %s + }, 1); // delaying even 1ms is enough to allow compilation memory to be reclaimed + }; + document.body.appendChild(script); +''' % (os.path.basename(asm_target), script_inline) + html = open(target, 'w') assert (script_src or script_inline) and not (script_src and script_inline) if script_src: diff --git a/emrun b/emrun index 48dd16eb36ca6..6cb3db8addd3b 100755 --- a/emrun +++ b/emrun @@ -206,16 +206,15 @@ user_pref("app.update.silent", false); user_pref("app.update.mode", 0); user_pref("app.update.service.enabled", false); // Don't check compatibility with add-ons, or (auto)update them -clearPref("extensions.lastAppVersion"); -lockPref("plugins.hide_infobar_for_outdated_plugin", true); -clearPref("plugins.update.url"); +user_pref("extensions.lastAppVersion", ''); +user_pref("plugins.hide_infobar_for_outdated_plugin", true); +user_pref("plugins.update.url", ''); // Disable health reporter -lockPref("datareporting.healthreport.service.enabled", false); +user_pref("datareporting.healthreport.service.enabled", false); // Disable crash reporter -lockPref("toolkit.crashreporter.enabled", false); -Components.classes["@mozilla.org/toolkit/crash-reporter;1"].getService(Components.interfaces.nsICrashReporter).submitReports = false; +user_pref("toolkit.crashreporter.enabled", false); // Don't show WhatsNew on first run after every update -pref("browser.startup.homepage_override.mstone","ignore"); +user_pref("browser.startup.homepage_override.mstone","ignore"); // Don't show 'know your rights' and a bunch of other nag windows at startup user_pref("browser.rights.3.shown", true); user_pref('devtools.devedition.promo.shown', true); @@ -226,6 +225,24 @@ user_pref('browser.customizemode.tip0.shown', true); user_pref("browser.toolbarbuttons.introduced.pocket-button", true); // Start in private browsing mode to not cache anything to disk (everything will be wiped anyway after this run) user_pref("browser.privatebrowsing.autostart", true); +// Don't ask the user if he wants to close the browser when there are multiple tabs. +user_pref("browser.tabs.warnOnClose", false); +// Allow the launched script window to close itself, so that we don't need to kill the browser process in order to move on. +user_pref("dom.allow_scripts_to_close_windows", true); +// Set various update timers to a large value in the future in order to not +// trigger a large mass of update HTTP traffic on each Firefox run on the clean profile. +// "01/01/2100" is 4102437600 as seconds since Unix epoch. +user_pref("app.update.lastUpdateTime.addon-background-update-timer", 4102437600); +user_pref("app.update.lastUpdateTime.background-update-timer", 4102437600); +user_pref("app.update.lastUpdateTime.blocklist-background-update-timer", 4102437600); +user_pref("app.update.lastUpdateTime.browser-cleanup-thumbnails", 4102437600); +user_pref("app.update.lastUpdateTime.experiments-update-timer", 4102437600); +user_pref("app.update.lastUpdateTime.search-engine-update-timer", 4102437600); +user_pref("app.update.lastUpdateTime.xpi-signature-verification", 4102437600); +user_pref("extensions.getAddons.cache.lastUpdate", 4102437600); +user_pref("media.gmp-eme-adobe.lastUpdate", 4102437600); +user_pref("media.gmp-gmpopenh264.lastUpdate", 4102437600); +user_pref("datareporting.healthreport.nextDataSubmissionTime", 4102437600439); ''') f.close() logv('create_emrun_safe_firefox_profile: Created new Firefox profile "' + temp_firefox_profile_dir + '"') @@ -1171,9 +1188,9 @@ def main(): # Create temporary Firefox profile to run the page with. This is important to run after kill_browser_process()/kill_on_start op above, since that # cleans up the temporary profile if one exists. - if processname_killed_atexit == 'firefox' and options.safe_firefox_profile: + if processname_killed_atexit == 'firefox' and options.safe_firefox_profile and not options.no_browser: profile_dir = create_emrun_safe_firefox_profile() - browser += ['-profile', profile_dir.replace('\\', '/')] + browser += ['-no-remote', '-profile', profile_dir.replace('\\', '/')] if options.system_info: logi('Time of run: ' + time.strftime("%x %X")) @@ -1240,12 +1257,23 @@ def main(): if not options.no_browser: if options.kill_on_exit: kill_browser_process() - elif is_browser_process_alive(): - logv('Not terminating browser process, pass --kill_exit to terminate the browser when it calls exit().') + else: + if is_browser_process_alive(): + logv('Not terminating browser process, pass --kill_exit to terminate the browser when it calls exit().') + # If we have created a temporary Firefox profile, we would really really like to wait until the browser closes, + # or otherwise we'll just have to litter temp files and keep the temporary profile alive. It is possible here + # that the browser is cooperatively shutting down, but has not yet had time to do so, so wait for a short while. + if temp_firefox_profile_dir != None: time.sleep(3) + + if not is_browser_process_alive(): + # Browser is no longer running, make sure to clean up the temp Firefox profile, if we created one. + delete_emrun_safe_firefox_profile() return page_exit_code if __name__ == '__main__': returncode = main() logv('emrun quitting with process exit code ' + str(returncode)) + if temp_firefox_profile_dir != None: + logi('Warning: Had to leave behind a temporary Firefox profile directory ' + temp_firefox_profile_dir + ' because --safe_firefox_profile was set and the browser did not quit before emrun did.') sys.exit(returncode) diff --git a/emscripten-version.txt b/emscripten-version.txt index edc3fbe22457d..edec5febbb738 100644 --- a/emscripten-version.txt +++ b/emscripten-version.txt @@ -1,2 +1,2 @@ -1.34.7 +1.34.8 diff --git a/emscripten.py b/emscripten.py index 7920aca2aa925..a70d500ac9c08 100755 --- a/emscripten.py +++ b/emscripten.py @@ -289,7 +289,7 @@ def save_settings(): return ASM_CONSTS[code](%s) | 0; }''' % (arity, ', '.join(all_args), ', '.join(args))) - pre = pre.replace('// === Body ===', '// === Body ===\n' + '\nvar ASM_CONSTS = [' + ', '.join(asm_consts) + '];\n' + '\n'.join(asm_const_funcs) + '\n') + pre = pre.replace('// === Body ===', '// === Body ===\n' + '\nvar ASM_CONSTS = [' + ',\n '.join(asm_consts) + '];\n' + '\n'.join(asm_const_funcs) + '\n') #if DEBUG: outfile.write('// pre\n') outfile.write(pre) diff --git a/site/build/text/docs/api_reference/emscripten.h.txt b/site/build/text/docs/api_reference/emscripten.h.txt index 4670d5b926665..c1b7aba58bdd1 100644 --- a/site/build/text/docs/api_reference/emscripten.h.txt +++ b/site/build/text/docs/api_reference/emscripten.h.txt @@ -418,12 +418,12 @@ int emscripten_set_main_loop_timing(int mode, int value) * **mode** (*int*) -- The timing mode to use. Allowed values are - EM_TIMING_SETTIMEOUT, EM_TIMING_RAF. + EM_TIMING_SETTIMEOUT, EM_TIMING_RAF and + EM_TIMING_SETIMMEDIATE. param int value: The timing value to activate for the main loop. This value - interpreted differently according to the - >>``<* for more + information. Note that this mode is **strongly not + recommended** to be used when deploying Emscripten output + to the web, since it depends on an unstable web extension + that is in draft status, browsers other than IE do not + currently support it, and its implementation has been + considered controversial in review. + rtype: int @@ -453,8 +465,8 @@ int emscripten_set_main_loop_timing(int mode, int value) loop active before calling this function. Note: Browsers heavily optimize towards using - "requestAnimationFrame" for animation instead of - "setTimeout". Because of that, for best experience across + "requestAnimationFrame" for animation instead of the other + provided modes. Because of that, for best experience across browsers, calling this function with "mode=EM_TIMING_RAF" and "value=1" will yield best results. Using the JavaScript "setTimeout" function is known to cause stutter and @@ -481,13 +493,11 @@ void emscripten_get_main_loop_timing(int *mode, int *value) "emscripten_set_main_loop()". Parameters: - * ***mode** (*int*) -- - - If not null, the used timing mode is returned here. - - * ***value** (*int*) -- + * **mode** (*int**) -- If not null, the used timing mode is + returned here. - If not null, the used timing value is returned here. + * **value** (*int**) -- If not null, the used timing value is + returned here. void emscripten_set_main_loop_expected_blockers(int num) @@ -1852,7 +1862,7 @@ void emscripten_idb_exists(const char *db_name, const char *file_id, int* pexist * **file_id** -- The name of the file to check * **pexists** -- An out parameter that will be filled with a - non- zero value if the file exists in that database. + non-zero value if the file exists in that database. * **perror** -- An out parameter that will be filled with a non- zero value if an error occurred. diff --git a/site/build/text/docs/api_reference/html5.h.txt b/site/build/text/docs/api_reference/html5.h.txt index 5e46e7b914400..426ed1130ec09 100644 --- a/site/build/text/docs/api_reference/html5.h.txt +++ b/site/build/text/docs/api_reference/html5.h.txt @@ -589,8 +589,8 @@ em_mouse_callback_func Parameters: * **eventType** (*int*) -- The type of "mouse event". - * **mouseEvent** (*const EmscriptenMouseEvent**) -- Information - about the mouse event that occurred. + * **mouseEvent** (*const EmscriptenMouseEvent**) -- + Information about the mouse event that occurred. * **userData** (*void**) -- The "userData" originally passed to the registration function. @@ -727,8 +727,8 @@ em_wheel_callback_func * **eventType** (*int*) -- The type of wheel event ("EMSCRIPTEN_EVENT_WHEEL"). - * **wheelEvent** (*const EmscriptenWheelEvent**) -- Information - about the wheel event that occurred. + * **wheelEvent** (*const EmscriptenWheelEvent**) -- + Information about the wheel event that occurred. * **userData** (*void**) -- The "userData" originally passed to the registration function. @@ -933,8 +933,8 @@ em_focus_callback_func * **eventType** (*int*) -- The type of focus event ("EMSCRIPTEN_EVENT_BLUR"). - * **focusEvent** (*const EmscriptenFocusEvent**) -- Information - about the focus event that occurred. + * **focusEvent** (*const EmscriptenFocusEvent**) -- + Information about the focus event that occurred. * **userData** (*void**) -- The "userData" originally passed to the registration function. @@ -1042,8 +1042,9 @@ em_deviceorientation_callback_func * **eventType** (*int*) -- The type of orientation event ("EMSCRIPTEN_EVENT_DEVICEORIENTATION"). - * **deviceOrientationEvent** (*const EmscriptenDeviceOrientationEvent**) -- - Information about the orientation event that occurred. + * **deviceOrientationEvent** (*const + EmscriptenDeviceOrientationEvent**) -- Information about the + orientation event that occurred. * **userData** (*void**) -- The "userData" originally passed to the registration function. @@ -1158,8 +1159,8 @@ em_devicemotion_callback_func * **eventType** (*int*) -- The type of devicemotion event ("EMSCRIPTEN_EVENT_DEVICEMOTION"). - * **deviceMotionEvent** (*const EmscriptenDeviceMotionEvent**) -- - Information about the devicemotion event that occurred. + * **deviceMotionEvent** (*const EmscriptenDeviceMotionEvent**) + -- Information about the devicemotion event that occurred. * **userData** (*void**) -- The "userData" originally passed to the registration function. @@ -1281,8 +1282,9 @@ em_orientationchange_callback_func * **eventType** (*int*) -- The type of orientationchange event ("EMSCRIPTEN_EVENT_ORIENTATIONCHANGE"). - * **orientationChangeEvent** (*const EmscriptenOrientationChangeEvent**) -- - Information about the orientationchange event that occurred. + * **orientationChangeEvent** (*const + EmscriptenOrientationChangeEvent**) -- Information about the + orientationchange event that occurred. * **userData** (*void**) -- The "userData" originally passed to the registration function. @@ -1397,8 +1399,8 @@ EMSCRIPTEN_FULLSCREEN_SCALE_STRETCH Specifies that the Emscripten runtime should explicitly stretch the CSS size of the target element to cover the whole screen when - transitioning to fullscreen mode. This will change the aspect - ratio of the displayed content. + transitioning to fullscreen mode. This will change the aspect ratio + of the displayed content. EMSCRIPTEN_FULLSCREEN_SCALE_ASPECT @@ -1567,8 +1569,9 @@ em_fullscreenchange_callback_func * **eventType** (*int*) -- The type of fullscreen event ("EMSCRIPTEN_EVENT_FULLSCREENCHANGE"). - * **fullscreenChangeEvent** (*const EmscriptenFullscreenChangeEvent**) -- - Information about the fullscreen event that occurred. + * **fullscreenChangeEvent** (*const + EmscriptenFullscreenChangeEvent**) -- Information about the + fullscreen event that occurred. * **userData** (*void**) -- The "userData" originally passed to the registration function. @@ -1681,11 +1684,10 @@ EMSCRIPTEN_RESULT emscripten_request_fullscreen_strategy(const char *target, EM fullscreen request without any modifications to the DOM. Parameters: - * **EmscriptenFullscreenStrategy *fullscreenStrategy** - (*const*) -- - - [in] Points to a configuration structure filled by the caller - which specifies display options for the fullscreen mode. + * **fullscreenStrategy** (*const + EmscriptenFullscreenStrategy**) -- [in] Points to a + configuration structure filled by the caller which specifies + display options for the fullscreen mode. EMSCRIPTEN_RESULT emscripten_exit_fullscreen(void) @@ -1775,8 +1777,9 @@ em_pointerlockchange_callback_func * **eventType** (*int*) -- The type of pointerlockchange event ("EMSCRIPTEN_EVENT_POINTERLOCKCHANGE"). - * **pointerlockChangeEvent** (*const EmscriptenPointerlockChangeEvent**) -- - Information about the pointerlockchange event that occurred. + * **pointerlockChangeEvent** (*const + EmscriptenPointerlockChangeEvent**) -- Information about the + pointerlockchange event that occurred. * **userData** (*void**) -- The "userData" originally passed to the registration function. @@ -1932,8 +1935,9 @@ em_visibilitychange_callback_func * **eventType** (*int*) -- The type of "visibilitychange" event ("EMSCRIPTEN_VISIBILITY_HIDDEN"). - * **visibilityChangeEvent** (*const EmscriptenVisibilityChangeEvent**) -- - Information about the "visibilitychange" event that occurred. + * **visibilityChangeEvent** (*const + EmscriptenVisibilityChangeEvent**) -- Information about the + "visibilitychange" event that occurred. * **userData** (*void**) -- The "userData" originally passed to the registration function. @@ -2086,8 +2090,8 @@ em_touch_callback_func * **eventType** (*int*) -- The type of touch event ("EMSCRIPTEN_EVENT_TOUCHSTART"). - * **touchEvent** (*const EmscriptenTouchEvent**) -- Information - about the touch event that occurred. + * **touchEvent** (*const EmscriptenTouchEvent**) -- + Information about the touch event that occurred. * **userData** (*void**) -- The "userData" originally passed to the registration function. diff --git a/site/build/text/docs/api_reference/preamble.js.txt b/site/build/text/docs/api_reference/preamble.js.txt index f65e889b64775..0899f71a9f755 100644 --- a/site/build/text/docs/api_reference/preamble.js.txt +++ b/site/build/text/docs/api_reference/preamble.js.txt @@ -228,9 +228,9 @@ getValue(ptr, type[, noSafe]) * **type** -- An LLVM IR type as a string (see "note" above). * **noSafe** (*bool*) -- Developers should ignore this - variable. It is only used in "SAFE_HEAP" compilation mode, where - it can help avoid infinite recursion in some specialist use - cases. + variable. It is only used in "SAFE_HEAP" compilation mode, + where it can help avoid infinite recursion in some specialist + use cases. Returns: The value stored at the specified memory address. diff --git a/site/build/text/docs/tools_reference/emcc.txt b/site/build/text/docs/tools_reference/emcc.txt index 01529284cb85d..ce80309bd898c 100644 --- a/site/build/text/docs/tools_reference/emcc.txt +++ b/site/build/text/docs/tools_reference/emcc.txt @@ -555,6 +555,12 @@ Options that are modified or new in *emcc* are listed below: with other bitcode files), instead of compiling all the way to JavaScript. +"--separate-asm" + Emits asm.js in one file, and the rest of the code in another, and + emits HTML that loads the asm.js first, in order to reduce memory + load during startup. See *Avoid memory spikes by separating out + asm.js*. + Environment variables ===================== diff --git a/site/source/docs/api_reference/Filesystem-API.rst b/site/source/docs/api_reference/Filesystem-API.rst index 8c22fcea6cfd1..f1aacf2983098 100644 --- a/site/source/docs/api_reference/Filesystem-API.rst +++ b/site/source/docs/api_reference/Filesystem-API.rst @@ -131,7 +131,7 @@ File system API Mounts the FS object specified by ``type`` to the directory specified by ``mountpoint``. The ``opts`` object is specific to each file system type. - :param type: The :ref:`file system type `: ``MEMFS``, ``NODEFS``, or ``IDBFS``. + :param type: The :ref:`file system type `: ``MEMFS``, ``NODEFS``, ``IDBFS`` or ``WORKERFS``. :param object opts: A generic settings object used by the underlying file system. ``NODFES`` uses the `root` parameter to map the Emscripten directory to the physical directory. For example, to mount the current folder as a NODEFS instance: diff --git a/site/source/docs/getting_started/FAQ.rst b/site/source/docs/getting_started/FAQ.rst index 883040f0fa3ff..16fa48c0666b6 100644 --- a/site/source/docs/getting_started/FAQ.rst +++ b/site/source/docs/getting_started/FAQ.rst @@ -255,9 +255,16 @@ To make sure a C function remains available to be called from normal JavaScript, ./emcc -s EXPORTED_FUNCTIONS="['_main', '_my_func']" ... +.. note:: + + `EXPORTED_FUNCTIONS` affects compilation to JavaScript. If you first compile to an object file, + then compile the object to JavaScript, you need that option on the second command. + If your function is used in other functions, LLVM may inline it and it will not appear as a unique function in the JavaScript. Prevent inlining by defining the function with :c:type:`EMSCRIPTEN_KEEPALIVE`: :: void EMSCRIPTEN_KEEPALIVE yourCfunc() {..} + +`EMSCRIPTEN_KEEPALIVE` also exports the function, as if it were on `EXPORTED_FUNCTIONS`. .. note:: diff --git a/site/source/docs/getting_started/downloads.rst b/site/source/docs/getting_started/downloads.rst index 8c2eced8790e3..ee981197ab09f 100644 --- a/site/source/docs/getting_started/downloads.rst +++ b/site/source/docs/getting_started/downloads.rst @@ -117,10 +117,7 @@ These instructions explain how to install **all** the :ref:`required tools `_. - - .. tip:: This specific version (2.8.10) is recommended — it has been tested and shown to work. Other versions may not correctly set up the PATH variables, with the result that running *cmake* gives you "not found" errors. - + - Download and install latest CMake from `Kitware CMake downloads `_. #. Install *node.js* from http://nodejs.org/ @@ -157,8 +154,6 @@ Linux # Install cmake sudo apt-get install cmake -.. note:: You will probably need CMake version 2.8.8 or later. - - *Python*, *node.js* or *Java* are not provided by *emsdk*. The user is expected to install these beforehand with the *system package manager*: :: diff --git a/site/source/docs/optimizing/Optimizing-Code.rst b/site/source/docs/optimizing/Optimizing-Code.rst index 3dc36b6e45378..13f6e40b4d4fc 100644 --- a/site/source/docs/optimizing/Optimizing-Code.rst +++ b/site/source/docs/optimizing/Optimizing-Code.rst @@ -91,12 +91,16 @@ Very large codebases The previous section on reducing code size can be helpful on very large codebases. In addition, here are some other topics that might be useful. +.. _optimizing-code-separating_asm: + Avoid memory spikes by separating out asm.js -------------------------------------------- By default Emscripten emits one JS file, containing the entire codebase: Both the asm.js code that was compiled, and the general code that sets up the environment, connects to browser APIs, etc. in a very large codebase, this can be inefficient in terms of memory usage, as having all of that in one script means the JS engine might use some memory to parse and compile the asm.js, and might not free it before starting to run the codebase. And in a large game, starting to run the code might allocate a large typed array for memory, so you might see a "spike" of memory, after which temporary compilation memory will be freed. And if big enough, that spike can cause the browser to run out of memory and fail to load the application. This is a known problem on `Chrome `_ (other browsers do not seem to have this issue). -A workaround is to separate out the asm.js into another file, and to make sure that the browser has a turn of the event loop between compiling the asm.js module and starting to run the application. This can be achieved as follows: +A workaround is to separate out the asm.js into another file, and to make sure that the browser has a turn of the event loop between compiling the asm.js module and starting to run the application. This can be achieved by running **emcc** with ``--separate-asm``. + +You can also do this manually, as follows: * Run ``tools/separate_asm.py``. This receives as inputs the filename of the full project, and two filenames to emit: the asm.js file and a file for everything else. * Load the asm.js script first, then after a turn of the event loop, the other one, for example using code like this in your HTML file: diff --git a/site/source/docs/porting/connecting_cpp_and_javascript/Interacting-with-code.rst b/site/source/docs/porting/connecting_cpp_and_javascript/Interacting-with-code.rst index 13a6622e73a90..e12f4a6f30dc5 100644 --- a/site/source/docs/porting/connecting_cpp_and_javascript/Interacting-with-code.rst +++ b/site/source/docs/porting/connecting_cpp_and_javascript/Interacting-with-code.rst @@ -65,6 +65,13 @@ home directory:: ./emcc tests/hello_function.cpp -o function.html -s EXPORTED_FUNCTIONS="['_int_sqrt']" +.. note:: + + `EXPORTED_FUNCTIONS` affects compilation to JavaScript. If you first compile to an object file, + then compile the object to JavaScript, you need that option on the second command. If you do + it all together as in the example here (source straight to JavaScript) then this just works, + of course. + After compiling, you can call this function with :js:func:`cwrap` using the following JavaScript:: @@ -73,7 +80,7 @@ following JavaScript:: int_sqrt(28) The first parameter is the name of the function to be wrapped, the second is -the return type of the function, and the third is an array of parameter +the return type of the function (or a JavaScript `null` value if there isn't one), and the third is an array of parameter types (which may be omitted if there are no parameters). The types are native JavaScript types, "number" (for a C integer, float, or general pointer) or "string" (for a C ``char*`` that represents a string). @@ -221,9 +228,18 @@ an alert, followed by an exception. (Note, however, that under the hood Emscripten still does a function call even in this case, which has some amount of overhead.) -You can also send values from C into JavaScript inside :c:macro:`EM_ASM_`, -as well as receive values back (see the :c:macro:`linked macro ` -for details. The following example will print out ``I received: 100`` +You can also send values from C into JavaScript inside :c:macro:`EM_ASM_` +(note the extra "_" at the end), for example + +.. code-block:: cpp + + EM_ASM_({ + Module.print('I received: ' + $0); + }, 100); + +This will show ``I received: 100``. + +You can also receive values back, for example the following will print out ``I received: 100`` and then ``101``. .. code-block:: cpp @@ -234,6 +250,8 @@ and then ``101``. }, 100); printf("%d\n", x); +See the :c:macro:`emscripten.h docs ` for more details. + .. note:: - You need to specify if the return value is an ``int`` or a ``double`` diff --git a/site/source/docs/tools_reference/emcc.rst b/site/source/docs/tools_reference/emcc.rst index dc5370b17054f..f0c12d6765e2d 100644 --- a/site/source/docs/tools_reference/emcc.rst +++ b/site/source/docs/tools_reference/emcc.rst @@ -438,6 +438,9 @@ Options that are modified or new in *emcc* are listed below: ``-c`` Tells *emcc* to generate LLVM bitcode (which can then be linked with other bitcode files), instead of compiling all the way to JavaScript. +``--separate-asm`` + Emits asm.js in one file, and the rest of the code in another, and emits HTML that loads the asm.js first, in order to reduce memory load during startup. See :ref:`optimizing-code-separating_asm`. + .. _emcc-environment-variables: diff --git a/src/emrun_postjs.js b/src/emrun_postjs.js index 63da3f8ece0a5..d34ff0b6ec2ec 100644 --- a/src/emrun_postjs.js +++ b/src/emrun_postjs.js @@ -1,7 +1,32 @@ if (typeof window === "object" && (typeof ENVIRONMENT_IS_PTHREAD === 'undefined' || !ENVIRONMENT_IS_PTHREAD)) { function emrun_register_handlers() { + // When C code exit()s, we may still have remaining stdout and stderr messages in flight. In that case, we can't close + // the browser until all those XHRs have finished, so the following state variables track that all communication is done, + // after which we can close. + var emrun_num_post_messages_in_flight = 0; + var emrun_should_close_itself = false; + function postExit(msg) { + var http = new XMLHttpRequest(); + http.onreadystatechange = function() { + if (http.readyState == 4 /*DONE*/) { + try { + // Try closing the current browser window, since it exit()ed itself. This can shut down the browser process + // and emrun does not need to kill the whole browser process. + if (typeof window !== 'undefined' && window.close) window.close(); + } catch(e) {} + } + } + http.open("POST", "stdio.html", true); + http.send(msg); + } function post(msg) { var http = new XMLHttpRequest(); + ++emrun_num_post_messages_in_flight; + http.onreadystatechange = function() { + if (http.readyState == 4 /*DONE*/) { + if (--emrun_num_post_messages_in_flight == 0 && emrun_should_close_itself) postExit('^exit^'+EXITSTATUS); + } + } http.open("POST", "stdio.html", true); http.send(msg); } @@ -10,7 +35,7 @@ if (typeof window === "object" && (typeof ENVIRONMENT_IS_PTHREAD === 'undefined' var emrun_http_sequence_number = 1; var prevPrint = Module['print']; var prevErr = Module['printErr']; - function emrun_exit() { post('^exit^'+EXITSTATUS); }; + function emrun_exit() { if (emrun_num_post_messages_in_flight == 0) postExit('^exit^'+EXITSTATUS); else emrun_should_close_itself = true; }; Module['addOnExit'](emrun_exit); Module['print'] = function emrun_print(text) { post('^out^'+(emrun_http_sequence_number++)+'^'+encodeURIComponent(text)); prevPrint(text); } Module['printErr'] = function emrun_printErr(text) { post('^err^'+(emrun_http_sequence_number++)+'^'+encodeURIComponent(text)); prevErr(text); } diff --git a/src/library_lz4.js b/src/library_lz4.js new file mode 100644 index 0000000000000..1c46c4601886d --- /dev/null +++ b/src/library_lz4.js @@ -0,0 +1,181 @@ +#if LZ4 +mergeInto(LibraryManager.library, { + $LZ4__deps: ['$FS'], + $LZ4: { + DIR_MODE: {{{ cDefine('S_IFDIR') }}} | 511 /* 0777 */, + FILE_MODE: {{{ cDefine('S_IFREG') }}} | 511 /* 0777 */, + CHUNK_SIZE: -1, + codec: null, + init: function() { + if (LZ4.codec) return; + LZ4.codec = (function() { + {{{ read('mini-lz4.js') }}}; + return MiniLZ4; + })(); + LZ4.CHUNK_SIZE = LZ4.codec.CHUNK_SIZE; + }, + loadPackage: function (pack) { + LZ4.init(); + var compressedData = pack['compressedData']; + if (!compressedData) compressedData = LZ4.codec.compressPackage(pack['data']); + assert(compressedData.cachedIndexes.length === compressedData.cachedChunks.length); + for (var i = 0; i < compressedData.cachedIndexes.length; i++) { + compressedData.cachedIndexes[i] = -1; + compressedData.cachedChunks[i] = compressedData.data.subarray(compressedData.cachedOffset + i*LZ4.CHUNK_SIZE, + compressedData.cachedOffset + (i+1)*LZ4.CHUNK_SIZE); + assert(compressedData.cachedChunks[i].length === LZ4.CHUNK_SIZE); + } + console.log('loading package'); + pack['metadata'].files.forEach(function(file) { + var dir = PATH.dirname(file.filename); + var name = PATH.basename(file.filename); + FS.createPath('', dir, true, true); + var parent = FS.analyzePath(dir).object; + LZ4.createNode(parent, name, LZ4.FILE_MODE, 0, { + compressedData: compressedData, + start: file.start, + end: file.end, + }); + }); + }, + createNode: function (parent, name, mode, dev, contents, mtime) { + var node = FS.createNode(parent, name, mode); + node.mode = mode; + node.node_ops = LZ4.node_ops; + node.stream_ops = LZ4.stream_ops; + node.timestamp = (mtime || new Date).getTime(); + assert(LZ4.FILE_MODE !== LZ4.DIR_MODE); + if (mode === LZ4.FILE_MODE) { + node.size = contents.end - contents.start; + node.contents = contents; + } else { + node.size = 4096; + node.contents = {}; + } + if (parent) { + parent.contents[name] = node; + } + return node; + }, + node_ops: { + getattr: function(node) { + return { + dev: 1, + ino: undefined, + mode: node.mode, + nlink: 1, + uid: 0, + gid: 0, + rdev: undefined, + size: node.size, + atime: new Date(node.timestamp), + mtime: new Date(node.timestamp), + ctime: new Date(node.timestamp), + blksize: 4096, + blocks: Math.ceil(node.size / 4096), + }; + }, + setattr: function(node, attr) { + if (attr.mode !== undefined) { + node.mode = attr.mode; + } + if (attr.timestamp !== undefined) { + node.timestamp = attr.timestamp; + } + }, + lookup: function(parent, name) { + throw new FS.ErrnoError(ERRNO_CODES.ENOENT); + }, + mknod: function (parent, name, mode, dev) { + throw new FS.ErrnoError(ERRNO_CODES.EPERM); + }, + rename: function (oldNode, newDir, newName) { + throw new FS.ErrnoError(ERRNO_CODES.EPERM); + }, + unlink: function(parent, name) { + throw new FS.ErrnoError(ERRNO_CODES.EPERM); + }, + rmdir: function(parent, name) { + throw new FS.ErrnoError(ERRNO_CODES.EPERM); + }, + readdir: function(node) { + throw new FS.ErrnoError(ERRNO_CODES.EPERM); + }, + symlink: function(parent, newName, oldPath) { + throw new FS.ErrnoError(ERRNO_CODES.EPERM); + }, + readlink: function(node) { + throw new FS.ErrnoError(ERRNO_CODES.EPERM); + }, + }, + stream_ops: { + read: function (stream, buffer, offset, length, position) { + //console.log('LZ4 read ' + [offset, length, position]); + length = Math.min(length, stream.node.size - position); + if (length <= 0) return 0; + var contents = stream.node.contents; + var compressedData = contents.compressedData; + var written = 0; + while (written < length) { + var start = contents.start + position + written; // start index in uncompressed data + var desired = length - written; + //console.log('current read: ' + ['start', start, 'desired', desired]); + var chunkIndex = Math.floor(start / LZ4.CHUNK_SIZE); + var compressedStart = compressedData.offsets[chunkIndex]; + var compressedSize = compressedData.sizes[chunkIndex]; + var currChunk; + if (compressedData.successes[chunkIndex]) { + var found = compressedData.cachedIndexes.indexOf(chunkIndex); + if (found >= 0) { + currChunk = compressedData.cachedChunks[found]; + } else { + // decompress the chunk + compressedData.cachedIndexes.pop(); + compressedData.cachedIndexes.unshift(chunkIndex); + currChunk = compressedData.cachedChunks.pop(); + compressedData.cachedChunks.unshift(currChunk); + if (compressedData.debug) { + console.log('decompressing chunk ' + chunkIndex); + Module['decompressedChunks'] = (Module['decompressedChunks'] || 0) + 1; + } + var compressed = compressedData.data.subarray(compressedStart, compressedStart + compressedSize); + //var t = Date.now(); + var originalSize = LZ4.codec.uncompress(compressed, currChunk); + //console.log('decompress time: ' + (Date.now() - t)); + if (chunkIndex < compressedData.successes.length-1) assert(originalSize === LZ4.CHUNK_SIZE); // all but the last chunk must be full-size + } + } else { + // uncompressed + currChunk = compressedData.data.subarray(compressedStart, compressedStart + LZ4.CHUNK_SIZE); + } + var startInChunk = start % LZ4.CHUNK_SIZE; + var endInChunk = Math.min(startInChunk + desired, LZ4.CHUNK_SIZE); + buffer.set(currChunk.subarray(startInChunk, endInChunk), offset + written); + var currWritten = endInChunk - startInChunk; + written += currWritten; + } + return written; + }, + write: function (stream, buffer, offset, length, position) { + throw new FS.ErrnoError(ERRNO_CODES.EIO); + }, + llseek: function (stream, offset, whence) { + var position = offset; + if (whence === 1) { // SEEK_CUR. + position += stream.position; + } else if (whence === 2) { // SEEK_END. + if (FS.isFile(stream.node.mode)) { + position += stream.node.size; + } + } + if (position < 0) { + throw new FS.ErrnoError(ERRNO_CODES.EINVAL); + } + return position; + }, + }, + }, +}); +LibraryManager.library['$FS__deps'].push('$LZ4'); // LZ4=1, so auto-include us +#endif + diff --git a/src/mini-lz4.js b/src/mini-lz4.js new file mode 100644 index 0000000000000..6cbb16b23cd59 --- /dev/null +++ b/src/mini-lz4.js @@ -0,0 +1,339 @@ +/* +MiniLZ4: Minimal LZ4 block decoding and encoding. + +based off of node-lz4, https://github.com/pierrec/node-lz4 + +==== +Copyright (c) 2012 Pierre Curto + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +==== + +changes have the same license +*/ + +var MiniLZ4 = (function() { + +var exports = {}; + +/** + * Decode a block. Assumptions: input contains all sequences of a + * chunk, output is large enough to receive the decoded data. + * If the output buffer is too small, an error will be thrown. + * If the returned value is negative, an error occured at the returned offset. + * + * @param input {Buffer} input data + * @param output {Buffer} output data + * @return {Number} number of decoded bytes + * @private + */ +exports.uncompress = function (input, output, sIdx, eIdx) { + sIdx = sIdx || 0 + eIdx = eIdx || (input.length - sIdx) + // Process each sequence in the incoming data + for (var i = sIdx, n = eIdx, j = 0; i < n;) { + var token = input[i++] + + // Literals + var literals_length = (token >> 4) + if (literals_length > 0) { + // length of literals + var l = literals_length + 240 + while (l === 255) { + l = input[i++] + literals_length += l + } + + // Copy the literals + var end = i + literals_length + while (i < end) output[j++] = input[i++] + + // End of buffer? + if (i === n) return j + } + + // Match copy + // 2 bytes offset (little endian) + var offset = input[i++] | (input[i++] << 8) + + // XXX 0 is an invalid offset value + if (offset === 0) return j + if (offset > j) return -(i-2) + + // length of match copy + var match_length = (token & 0xf) + var l = match_length + 240 + while (l === 255) { + l = input[i++] + match_length += l + } + + // Copy the match + var pos = j - offset // position of the match copy in the current output + var end = j + match_length + 4 // minmatch = 4 + while (j < end) output[j++] = output[pos++] + } + + return j +} + +var + maxInputSize = 0x7E000000 +, minMatch = 4 +// uint32() optimization +, hashLog = 16 +, hashShift = (minMatch * 8) - hashLog +, hashSize = 1 << hashLog + +, copyLength = 8 +, lastLiterals = 5 +, mfLimit = copyLength + minMatch +, skipStrength = 6 + +, mlBits = 4 +, mlMask = (1 << mlBits) - 1 +, runBits = 8 - mlBits +, runMask = (1 << runBits) - 1 + +, hasher = /* XXX uint32( */ 2654435761 /* ) */ + +// CompressBound returns the maximum length of a lz4 block, given it's uncompressed length +exports.compressBound = function (isize) { + return isize > maxInputSize + ? 0 + : (isize + (isize/255) + 16) | 0 +} + +exports.compress = function (src, dst, sIdx, eIdx) { + // V8 optimization: non sparse array with integers + var hashTable = new Array(hashSize) + for (var i = 0; i < hashSize; i++) { + hashTable[i] = 0 + } + return compressBlock(src, dst, 0, hashTable, sIdx || 0, eIdx || dst.length) +} + +function compressBlock (src, dst, pos, hashTable, sIdx, eIdx) { + // XXX var Hash = uint32() // Reusable unsigned 32 bits integer + var dpos = sIdx + var dlen = eIdx - sIdx + var anchor = 0 + + if (src.length >= maxInputSize) throw new Error("input too large") + + // Minimum of input bytes for compression (LZ4 specs) + if (src.length > mfLimit) { + var n = exports.compressBound(src.length) + if ( dlen < n ) throw Error("output too small: " + dlen + " < " + n) + + var + step = 1 + , findMatchAttempts = (1 << skipStrength) + 3 + // Keep last few bytes incompressible (LZ4 specs): + // last 5 bytes must be literals + , srcLength = src.length - mfLimit + + while (pos + minMatch < srcLength) { + // Find a match + // min match of 4 bytes aka sequence + var sequenceLowBits = src[pos+1]<<8 | src[pos] + var sequenceHighBits = src[pos+3]<<8 | src[pos+2] + // compute hash for the current sequence + var hash = Math.imul(sequenceLowBits | (sequenceHighBits << 16), hasher) >>> hashShift; + /* XXX Hash.fromBits(sequenceLowBits, sequenceHighBits) + .multiply(hasher) + .shiftr(hashShift) + .toNumber() */ + // get the position of the sequence matching the hash + // NB. since 2 different sequences may have the same hash + // it is double-checked below + // do -1 to distinguish between initialized and uninitialized values + var ref = hashTable[hash] - 1 + // save position of current sequence in hash table + hashTable[hash] = pos + 1 + + // first reference or within 64k limit or current sequence !== hashed one: no match + if ( ref < 0 || + ((pos - ref) >>> 16) > 0 || + ( + ((src[ref+3]<<8 | src[ref+2]) != sequenceHighBits) || + ((src[ref+1]<<8 | src[ref]) != sequenceLowBits ) + ) + ) { + // increase step if nothing found within limit + step = findMatchAttempts++ >> skipStrength + pos += step + continue + } + + findMatchAttempts = (1 << skipStrength) + 3 + + // got a match + var literals_length = pos - anchor + var offset = pos - ref + + // minMatch already verified + pos += minMatch + ref += minMatch + + // move to the end of the match (>=minMatch) + var match_length = pos + while (pos < srcLength && src[pos] == src[ref]) { + pos++ + ref++ + } + + // match length + match_length = pos - match_length + + // token + var token = match_length < mlMask ? match_length : mlMask + + // encode literals length + if (literals_length >= runMask) { + // add match length to the token + dst[dpos++] = (runMask << mlBits) + token + for (var len = literals_length - runMask; len > 254; len -= 255) { + dst[dpos++] = 255 + } + dst[dpos++] = len + } else { + // add match length to the token + dst[dpos++] = (literals_length << mlBits) + token + } + + // write literals + for (var i = 0; i < literals_length; i++) { + dst[dpos++] = src[anchor+i] + } + + // encode offset + dst[dpos++] = offset + dst[dpos++] = (offset >> 8) + + // encode match length + if (match_length >= mlMask) { + match_length -= mlMask + while (match_length >= 255) { + match_length -= 255 + dst[dpos++] = 255 + } + + dst[dpos++] = match_length + } + + anchor = pos + } + } + + // cannot compress input + if (anchor == 0) return 0 + + // Write last literals + // encode literals length + literals_length = src.length - anchor + if (literals_length >= runMask) { + // add match length to the token + dst[dpos++] = (runMask << mlBits) + for (var ln = literals_length - runMask; ln > 254; ln -= 255) { + dst[dpos++] = 255 + } + dst[dpos++] = ln + } else { + // add match length to the token + dst[dpos++] = (literals_length << mlBits) + } + + // write literals + pos = anchor + while (pos < src.length) { + dst[dpos++] = src[pos++] + } + + return dpos +} + +exports.CHUNK_SIZE = 2048; // musl libc does readaheads of 1024 bytes, so a multiple of that is a good idea + +exports.compressPackage = function(data, verify) { + if (verify) { + var temp = new Uint8Array(exports.CHUNK_SIZE); + } + // compress the data in chunks + assert(data instanceof ArrayBuffer); + data = new Uint8Array(data); + console.log('compressing package of size ' + data.length); + var compressedChunks = []; + var successes = []; + var offset = 0; + var total = 0; + while (offset < data.length) { + var chunk = data.subarray(offset, offset + exports.CHUNK_SIZE); + //console.log('compress a chunk ' + [offset, total, data.length]); + offset += exports.CHUNK_SIZE; + var bound = exports.compressBound(chunk.length); + var compressed = new Uint8Array(bound); + var compressedSize = exports.compress(chunk, compressed); + if (compressedSize > 0) { + assert(compressedSize <= bound); + compressed = compressed.subarray(0, compressedSize); + compressedChunks.push(compressed); + total += compressedSize; + successes.push(1); + if (verify) { + var back = exports.uncompress(compressed, temp); + assert(back === chunk.length, [back, chunk.length]); + for (var i = 0; i < chunk.length; i++) { + assert(chunk[i] === temp[i]); + } + } + } else { + assert(compressedSize === 0); + // failure to compress :( + compressedChunks.push(chunk); + total += chunk.length; // last chunk may not be the full exports.CHUNK_SIZE size + successes.push(0); + } + } + data = null; // XXX null out pack['data'] too? + var compressedData = { + data: new Uint8Array(total + exports.CHUNK_SIZE*2), // store all the compressed data, plus room for two cached decompressed chunk, in one fast array + cachedOffset: total, + cachedIndexes: [-1, -1], // cache last two blocks, so that reading 1,2,3 + preloading another block won't trigger decompress thrashing + cachedChunks: [null, null], + offsets: [], // chunk# => start in compressed data + sizes: [], + successes: successes, // 1 if chunk is compressed + }; + offset = 0; + for (var i = 0; i < compressedChunks.length; i++) { + compressedData.data.set(compressedChunks[i], offset); + compressedData.offsets[i] = offset; + compressedData.sizes[i] = compressedChunks[i].length + offset += compressedChunks[i].length; + } + console.log('compressed package into ' + [compressedData.data.length]); + assert(offset === total); + return compressedData; +}; + +return exports; + +})(); + diff --git a/src/modules.js b/src/modules.js index e5c85b9e50850..b2b833f440b1c 100644 --- a/src/modules.js +++ b/src/modules.js @@ -110,7 +110,8 @@ var LibraryManager = { 'library_nodefs.js', 'library_sockfs.js', 'library_workerfs.js', - 'library_tty.js' + 'library_tty.js', + 'library_lz4.js', ]); } if (!NO_BROWSER) { diff --git a/src/postamble.js b/src/postamble.js index 28cb673195bb5..fb32fa7e0ccb8 100644 --- a/src/postamble.js +++ b/src/postamble.js @@ -346,21 +346,25 @@ run(); var workerResponded = false, workerCallbackId = -1; (function() { - var messageBuffer = null; + var messageBuffer = null, buffer = 0, bufferSize = 0; - function messageResender() { + function flushMessages() { + if (!messageBuffer) return; if (runtimeInitialized) { - assert(messageBuffer && messageBuffer.length > 0); - messageBuffer.forEach(function(message) { + var temp = messageBuffer; + messageBuffer = null; + temp.forEach(function(message) { onmessage(message); }); - messageBuffer = null; - } else { - setTimeout(messageResender, 100); } } - var buffer = 0, bufferSize = 0; + function messageResender() { + flushMessages(); + if (messageBuffer) { + setTimeout(messageResender, 100); // still more to do + } + } onmessage = function onmessage(msg) { // if main has not yet been called (mem init file, other async things), buffer messages @@ -372,6 +376,7 @@ var workerResponded = false, workerCallbackId = -1; messageBuffer.push(msg); return; } + flushMessages(); var func = Module['_' + msg.data['funcName']]; if (!func) throw 'invalid worker function to call: ' + msg.data['funcName']; diff --git a/src/settings.js b/src/settings.js index 068852f458aee..d59637d273469 100644 --- a/src/settings.js +++ b/src/settings.js @@ -247,6 +247,18 @@ var STB_IMAGE = 0; // Enables building of stb-image, a tiny public-domain librar // When enabled, stb-image will be used automatically from IMG_Load and IMG_Load_RW. You // can also call the stbi_* functions directly yourself. +var LZ4 = 0; // Enable this to support lz4-compressed file packages. They are stored compressed in memory, and + // decompressed on the fly, avoiding storing the entire decompressed data in memory at once. + // If you run the file packager separately, you still need to build the main program with this flag, + // and also pass --lz4 to the file packager. + // (You can also manually compress one on the client, using LZ4.loadPackage(), but that is less + // recommended.) + // Limitations: + // * LZ4-compressed files are only decompressed when needed, so they are not available + // for special preloading operations like pre-decoding of images using browser codecs, + // preloadPlugin stuff, etc. + // * LZ4 files are read-only. + var DISABLE_EXCEPTION_CATCHING = 0; // Disables generating code to actually catch exceptions. If the code you // are compiling does not actually rely on catching exceptions (but the // compiler generates code for it, maybe because of stdlibc++ stuff), @@ -401,14 +413,16 @@ var EXPORTED_GLOBALS = []; // Global non-function variables that are explicitly // exported, so they are guaranteed to be // accessible outside of the generated code. -var INCLUDE_FULL_LIBRARY = 0; // Whether to include the whole library rather than just the - // functions used by the generated code. This is needed when - // dynamically loading modules that make use of runtime +var INCLUDE_FULL_LIBRARY = 0; // Include all JS library functions instead of the sum of + // DEFAULT_LIBRARY_FUNCS_TO_INCLUDE + any functions used + // by the generated code. This is needed when dynamically + // loading (i.e. dlopen) modules that make use of runtime // library functions that are not used in the main module. - // Note that this includes js libraries but *not* C. You will - // need the main file to include all needed C libraries. For - // example, if a library uses malloc or new, you will need - // to use those in the main file too to link in dlmalloc. + // Note that this only applies to js libraries, *not* C. You + // will need the main file to include all needed C libraries. + // For example, if a module uses malloc or new, you will + // need to use those in the main file too to pull in dlmalloc + // for use by the module. var SHELL_FILE = 0; // set this to a string to override the shell file used diff --git a/system/lib/libc/musl/src/stdlib/strtod.c b/system/lib/libc/musl/src/stdlib/strtod.c index 461dcf858e0c3..07cd599adac57 100644 --- a/system/lib/libc/musl/src/stdlib/strtod.c +++ b/system/lib/libc/musl/src/stdlib/strtod.c @@ -32,9 +32,28 @@ long double strtold(const char *restrict s, char **restrict p) return strtox(s, p, 2); } +#ifndef __EMSCRIPTEN__ weak_alias(strtof, strtof_l); weak_alias(strtod, strtod_l); weak_alias(strtold, strtold_l); weak_alias(strtof, __strtof_l); weak_alias(strtod, __strtod_l); weak_alias(strtold, __strtold_l); +#else +// can't just drop last parameter in emscripten, undefined behavior +float strtof_l(const char *restrict s, char **restrict p, struct __locale_struct *l) +{ + return strtof(s, p); +} + +double strtod_l(const char *restrict s, char **restrict p, struct __locale_struct *l) +{ + return strtod(s, p); +} + +long double strtold_l(const char *restrict s, char **restrict p, struct __locale_struct *l) +{ + return strtold(s, p); +} +#endif + diff --git a/tests/fs/test_lz4fs.cpp b/tests/fs/test_lz4fs.cpp new file mode 100644 index 0000000000000..d50efe345f474 --- /dev/null +++ b/tests/fs/test_lz4fs.cpp @@ -0,0 +1,181 @@ +#include +#include +#include +#include + +#include + +#define TOTAL_SIZE (10*1024*128) + +double before_it_all; + +extern "C" { + +void EMSCRIPTEN_KEEPALIVE finish() { + // load some file data, SYNCHRONOUSLY :) + char buffer[100]; + int num; + + printf("load files\n"); + FILE *f1 = fopen("file1.txt", "r"); + assert(f1); + FILE *f2 = fopen("subdir/file2.txt", "r"); + assert(f2); + FILE *f3 = fopen("file3.txt", "r"); + assert(f3); + FILE *files[] = { f1, f2, f3 }; + double before = emscripten_get_now(); + int counter = 0; + int i = 0; + printf("read from files\n"); + for (int i = 0; i < TOTAL_SIZE - 5; i += random() % 1000) { + int which = i % 3; + FILE *f = files[which]; + //printf("%d read %d: %d (%d)\n", counter, which, i, i % 10); + int off = i % 10; + int ret = fseek(f, i, SEEK_SET); + assert(ret == 0); + num = fread(buffer, 1, 5, f); + if (num != 5) { + printf("%d read %d: %d failed num\n", counter, which, i); + abort(); + } + if (which != 2) { + buffer[5] = 0; + char correct[] = "01234567890123456789"; + if (strncmp(buffer, correct + which + off, 5) != 0) { + printf("%d read %d: %d (%d) failed data\n", counter, which, i, i % 10); + abort(); + } + } + counter++; + } + assert(counter == 2657); + double after = emscripten_get_now(); + + printf("final test on random data\n"); + int ret = fseek(f3, 17, SEEK_SET); + assert(ret == 0); + num = fread(buffer, 1, 1, f3); + assert(num == 1); + assert(buffer[0] == 'X'); + + printf("read success. read IO time: %f (%d reads), total time: %f\n", after - before, counter, after - before_it_all); + +#if LOAD_MANUALLY + printf("caching tests\n"); + ret = fseek(f3, TOTAL_SIZE - 5, SEEK_SET); assert(ret == 0); + num = fread(buffer, 1, 1, f3); assert(num == 1); // read near the end + ret = fseek(f3, TOTAL_SIZE - 5000, SEEK_SET); assert(ret == 0); + num = fread(buffer, 1, 1, f3); assert(num == 1); // also near the end + EM_ASM({ + assert(!Module['decompressedChunks']); + Module.compressedData.debug = true; + console.log('last cached indexes ' + Module.compressedData.cachedIndexes); + assert(Module.compressedData.cachedIndexes.indexOf(0) < 0); // 0 is not cached + }); + printf("multiple reads of same byte\n"); + for (int i = 0; i < 100; i++) { + ret = fseek(f1, 0, SEEK_SET); // read near the start, should trigger one decompress, then all cache hits + assert(ret == 0); + num = fread(buffer, 1, 1, f1); + assert(num == 1); + } + EM_ASM({ + assert(Module['decompressedChunks'] == 1, ['seeing', Module['decompressedChunks'], 'decompressed chunks']); + }); + printf("multiple reads of adjoining byte\n"); + for (int i = 0; i < 100; i++) { + ret = fseek(f1, i, SEEK_SET); + assert(ret == 0); + num = fread(buffer, 1, 1, f1); + assert(num == 1); + } + EM_ASM({ + assert(Module['decompressedChunks'] == 1, ['seeing', Module['decompressedChunks'], 'decompressed chunks']); + }); + printf("multiple reads across two chunks\n"); + for (int i = 0; i < 2100; i++) { + ret = fseek(f1, i, SEEK_SET); + assert(ret == 0); + num = fread(buffer, 1, 1, f1); + assert(num == 1); + } + EM_ASM({ + assert(Module['decompressedChunks'] == 2, ['seeing', Module['decompressedChunks'], 'decompressed chunks']); + }); + printf("caching test ok\n"); +#endif + + fclose(f1); + fclose(f2); + fclose(f3); + + // all done + int result; +#if LOAD_MANUALLY + result = 1; +#else + result = 2; +#endif + REPORT_RESULT(); +} + +} + +int main() { + before_it_all = emscripten_get_now(); + +#if LOAD_MANUALLY + EM_ASM({ + var COMPLETE_SIZE = 10*1024*128*3; + + var meta, data; + function maybeReady() { + if (!(meta && data)) return; + + meta = JSON.parse(meta); + + Module.print('loading into filesystem'); + FS.mkdir('/files'); + LZ4.loadPackage({ 'metadata': meta, 'data': data }); + + Module.compressedData = FS.root.contents['file1.txt'].contents.compressedData; + var compressedSize = Module.compressedData.data.length; + var low = COMPLETE_SIZE/3; + var high = COMPLETE_SIZE/2; + console.log('seeing compressed size of ' + compressedSize + ', expect in ' + [low, high]); + assert(compressedSize > low && compressedSize < high); // more than 1/3, because 1/3 is uncompressible, but still, less than 1/2 + + Module['ccall']('finish'); + } + + var meta_xhr = new XMLHttpRequest(); + meta_xhr.open("GET", "files.js.metadata", true); + meta_xhr.responseType = "text"; + meta_xhr.onload = function() { + Module.print('got metadata'); + meta = meta_xhr.response; + maybeReady(); + }; + meta_xhr.send(); + + var data_xhr = new XMLHttpRequest(); + data_xhr.open("GET", "files.data", true); + data_xhr.responseType = "arraybuffer"; + data_xhr.onload = function() { + Module.print('got data'); + data = data_xhr.response; + maybeReady(); + }; + data_xhr.send(); + }); + + emscripten_exit_with_live_runtime(); +#else + finish(); +#endif + + return 1; +} + diff --git a/tests/test_browser.py b/tests/test_browser.py index 0bf1b0743d671..e80d72b2f7fad 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -1031,6 +1031,52 @@ def test_fs_workerfs_package(self): Popen([PYTHON, FILE_PACKAGER, 'files.data', '--preload', 'file1.txt', os.path.join('sub', 'file2.txt'), '--separate-metadata', '--js-output=files.js']).communicate() self.btest(os.path.join('fs', 'test_workerfs_package.cpp'), '1', args=['--proxy-to-worker']) + def test_fs_lz4fs_package(self): + # generate data + import random + try_delete('subdir') + os.mkdir('subdir') + open('file1.txt', 'w').write('0123456789' * (1024*128)) + open(os.path.join('subdir', 'file2.txt'), 'w').write('1234567890' * (1024*128)) + random_data = [chr(random.randint(0,255)) for x in range(1024*128*10 + 1)] + random_data[17] = 'X' + open('file3.txt', 'w').write(''.join(random_data)) + + # compress in emcc, -s LZ4=1 tells it to tell the file packager + print 'emcc-normal' + self.btest(os.path.join('fs', 'test_lz4fs.cpp'), '2', args=['-s', 'LZ4=1', '--preload-file', 'file1.txt', '--preload-file', 'subdir/file2.txt', '--preload-file', 'file3.txt'], timeout=60) + print ' emcc-opts' + self.btest(os.path.join('fs', 'test_lz4fs.cpp'), '2', args=['-s', 'LZ4=1', '--preload-file', 'file1.txt', '--preload-file', 'subdir/file2.txt', '--preload-file', 'file3.txt', '-O2'], timeout=60) + + # compress in the file packager, on the server. the client receives compressed data and can just use it. this is typical usage + print 'normal' + out = subprocess.check_output([PYTHON, FILE_PACKAGER, 'files.data', '--preload', 'file1.txt', 'subdir/file2.txt', 'file3.txt', '--lz4']) + open('files.js', 'w').write(out) + self.btest(os.path.join('fs', 'test_lz4fs.cpp'), '2', args=['--pre-js', 'files.js', '-s', 'LZ4=1'], timeout=60) + print ' opts' + self.btest(os.path.join('fs', 'test_lz4fs.cpp'), '2', args=['--pre-js', 'files.js', '-s', 'LZ4=1', '-O2'], timeout=60) + + # load the data into LZ4FS manually at runtime. This means we compress on the client. This is generally not recommended + print 'manual' + subprocess.check_output([PYTHON, FILE_PACKAGER, 'files.data', '--preload', 'file1.txt', 'subdir/file2.txt', 'file3.txt', '--separate-metadata', '--js-output=files.js']) + self.btest(os.path.join('fs', 'test_lz4fs.cpp'), '1', args=['-DLOAD_MANUALLY', '-s', 'LZ4=1'], timeout=60) + print ' opts' + self.btest(os.path.join('fs', 'test_lz4fs.cpp'), '1', args=['-DLOAD_MANUALLY', '-s', 'LZ4=1', '-O2'], timeout=60) + print ' opts+closure' + self.btest(os.path.join('fs', 'test_lz4fs.cpp'), '1', args=['-DLOAD_MANUALLY', '-s', 'LZ4=1', '-O2', '--closure', '1', '-g1'], timeout=60) + + '''# non-lz4 for comparison + try: + os.mkdir('files') + except: + pass + shutil.copyfile('file1.txt', os.path.join('files', 'file1.txt')) + shutil.copyfile('file2.txt', os.path.join('files', 'file2.txt')) + shutil.copyfile('file3.txt', os.path.join('files', 'file3.txt')) + out = subprocess.check_output([PYTHON, FILE_PACKAGER, 'files.data', '--preload', 'files/file1.txt', 'files/file2.txt', 'files/file3.txt']) + open('files.js', 'w').write(out) + self.btest(os.path.join('fs', 'test_lz4fs.cpp'), '2', args=['--pre-js', 'files.js'], timeout=60)''' + def test_idbstore(self): secret = str(time.time()) for stage in [0, 1, 2, 3, 0, 1, 2, 0, 0, 1, 4, 2, 5]: @@ -1856,7 +1902,7 @@ def test_emrun(self): # and the browser will not close as part of the test, pinning down the cwd on Windows and it wouldn't be possible to delete it. Therefore switch away from that directory # before launching. os.chdir(path_from_root()) - args = [PYTHON, path_from_root('emrun'), '--timeout', '30', '--verbose', '--log_stdout', os.path.join(outdir, 'stdout.txt'), '--log_stderr', os.path.join(outdir, 'stderr.txt')] + args = [PYTHON, path_from_root('emrun'), '--timeout', '30', '--safe_firefox_profile', '--verbose', '--log_stdout', os.path.join(outdir, 'stdout.txt'), '--log_stderr', os.path.join(outdir, 'stderr.txt')] if emscripten_browser is not None: args += ['--browser', emscripten_browser] args += [os.path.join(outdir, 'hello_world.html'), '1', '2', '--3'] @@ -2807,6 +2853,13 @@ def test_separate_asm(self): ''') self.run_browser('two.html', None, '/report_result?0') + self.clear() + assert not os.path.exists('tests.asm.js') + self.btest('browser_test_hello_world.c', expected='0', args=['-O' + str(opts), '--separate-asm']) + assert os.path.exists('test.asm.js') + os.unlink('test.asm.js') + self.run_browser('test.html', None, '[no http server activity]', timeout=5) # fail without the asm + def test_emterpretify_file(self): open('shell.html', 'w').write('''