From 7a4ddc9a7b786998bdcda52698e25fa642fcd1a2 Mon Sep 17 00:00:00 2001 From: Iurii Zakipnyi Date: Tue, 31 Jan 2017 17:00:03 +0100 Subject: [PATCH 01/72] GL_ACTIVE_UNIFORM_BLOCK_MAX_NAME_LENGTH and webgl2_ubos test fixed --- src/library_gl.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/library_gl.js b/src/library_gl.js index 868c99b3a6b00..c76926569e3ec 100644 --- a/src/library_gl.js +++ b/src/library_gl.js @@ -3368,7 +3368,7 @@ var LibraryGL = { ptable.maxUniformBlockNameLength = 0; for (var i = 0; i < numBlocks; ++i) { var activeBlockName = GLctx.getActiveUniformBlockName(program, i); - ptable.maxUniformBlockNameLength = Math.max(ptable.maxAttributeLength, activeBlockName.length+1); + ptable.maxUniformBlockNameLength = Math.max(ptable.maxUniformBlockNameLength, activeBlockName.length+1); } } {{{ makeSetValue('p', '0', 'ptable.maxUniformBlockNameLength', 'i32') }}}; From 6d2508a702c10bb420b22f896c2806b7c0d2a2cf Mon Sep 17 00:00:00 2001 From: Iurii Zakipnyi Date: Tue, 31 Jan 2017 17:01:37 +0100 Subject: [PATCH 02/72] =?UTF-8?q?webgl2=5Fubos=20test=20now=20covers=20?= =?UTF-8?q?=E2=80=98glGetUniformLocation=20for=20named=20uniform=20block?= =?UTF-8?q?=E2=80=99=20bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/webgl2_ubos.cpp | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/tests/webgl2_ubos.cpp b/tests/webgl2_ubos.cpp index 9f2e096a0e631..69c036309489b 100644 --- a/tests/webgl2_ubos.cpp +++ b/tests/webgl2_ubos.cpp @@ -43,6 +43,7 @@ int main() const char *vertexShader = "#version 300 es\n" + "uniform mat4 a;\n" "uniform Block1a {\n" " uniform mat4 var1;\n" " uniform vec4 variable2;\n" @@ -52,7 +53,7 @@ int main() " uniform vec4 variable2;\n" "} block2dddd;\n" "void main() {\n" - " gl_Position = block1bb.var1*block1bb.variable2 + block2dddd.var1*block2dddd.variable2;\n" + " gl_Position = a * block1bb.var1*block1bb.variable2 + block2dddd.var1*block2dddd.variable2;\n" "}\n"; const char *fragmentShader = @@ -112,6 +113,30 @@ int main() glGetProgramiv(program, GL_ACTIVE_UNIFORM_BLOCK_MAX_NAME_LENGTH, &maxLength); printf("GL_ACTIVE_UNIFORM_BLOCK_MAX_NAME_LENGTH: %d\n", maxLength); assert(maxLength == 12); + + GLint numActiveUniforms = -1; + glGetProgramiv(program, GL_ACTIVE_UNIFORMS, &numActiveUniforms); + printf("GL_ACTIVE_UNIFORMS: %d\n", numActiveUniforms); + assert(numActiveUniforms == 7); + + for(int i = 0; i < numActiveUniforms; ++i) + { + char str[256] = {}; + GLsizei length = -1; + GLint size = -1; + GLenum type = -1; + glGetActiveUniform(program, i, 255, &length, &size, &type, str); + + GLint loc = glGetUniformLocation(program, str); + + GLint indx = -1; + glGetActiveUniformsiv(program, 1, (GLuint*)&i, GL_UNIFORM_BLOCK_INDEX, &indx); + + printf("Active uniform at index %d: %s\n", i, str); + printf("glGetUniformLocation = %d \t GL_UNIFORM_BLOCK_INDEX = %d \t size = %d \t type = %d\n", loc, indx, size, type); + + assert((loc == -1) != (indx == -1)); // one of them must be true + } GLint numActiveUniformBlocks = -1; glGetProgramiv(program, GL_ACTIVE_UNIFORM_BLOCKS, &numActiveUniformBlocks); From 2e772d40b76c042a141dfda4e78d1d40ffb690a0 Mon Sep 17 00:00:00 2001 From: Iurii Zakipnyi Date: Tue, 31 Jan 2017 17:04:32 +0100 Subject: [PATCH 03/72] =?UTF-8?q?=E2=80=98glGetUniformLocation=20for=20nam?= =?UTF-8?q?ed=20uniform=20block=E2=80=99=20bug=20fixed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/library_gl.js | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/library_gl.js b/src/library_gl.js index c76926569e3ec..2cd62d7249ea0 100644 --- a/src/library_gl.js +++ b/src/library_gl.js @@ -690,16 +690,19 @@ var LibraryGL = { // only store the string 'colors' in utable, and 'colors[0]', 'colors[1]' and 'colors[2]' will be parsed as 'colors'+i. // Note that for the GL.uniforms table, we still need to fetch the all WebGLUniformLocations for all the indices. var loc = GLctx.getUniformLocation(p, name); - var id = GL.getNewId(GL.uniforms); - utable[name] = [u.size, id]; - GL.uniforms[id] = loc; + if (loc != null) + { + var id = GL.getNewId(GL.uniforms); + utable[name] = [u.size, id]; + GL.uniforms[id] = loc; - for (var j = 1; j < u.size; ++j) { - var n = name + '['+j+']'; - loc = GLctx.getUniformLocation(p, n); - id = GL.getNewId(GL.uniforms); + for (var j = 1; j < u.size; ++j) { + var n = name + '['+j+']'; + loc = GLctx.getUniformLocation(p, n); + id = GL.getNewId(GL.uniforms); - GL.uniforms[id] = loc; + GL.uniforms[id] = loc; + } } } } From 1009e1955573101e49799a92a8b92f6c9a3c04a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Sun, 5 Feb 2017 20:25:34 +0200 Subject: [PATCH 04/72] Optimize to avoid generating temporary garbage in Gamepad API when no gamepads are connected. --- site/source/docs/api_reference/html5.h.rst | 6 ++++++ src/library_html5.js | 13 +++++++++++++ 2 files changed, 19 insertions(+) diff --git a/site/source/docs/api_reference/html5.h.rst b/site/source/docs/api_reference/html5.h.rst index 129b187610580..f5161d07eddf9 100644 --- a/site/source/docs/api_reference/html5.h.rst +++ b/site/source/docs/api_reference/html5.h.rst @@ -1698,6 +1698,12 @@ Functions .. note:: A gamepad does not show up as connected until a button on it is pressed. + .. note:: Gamepad API uses an array of gamepad state objects to return the state of each device. The devices are identified via the index they are present in in + this array. Because of that, if one first connects gamepad A, then gamepad B, and then disconnects gamepad A, the gamepad B shall not take the place of gamepad A, + so in this scenario, this function will still keep returning two for the count of connected gamepads, even though gamepad A is no longer present. To find the actual + number of connected gamepads, listen for the gamepadconnected and gamepaddisconnected events. + Consider the return value of this function as the largest value (-1) that can be passed to the function emscripten_get_gamepad_status(). + :returns: :c:data:`EMSCRIPTEN_RESULT_SUCCESS`, or one of the other result values. :rtype: int diff --git a/src/library_html5.js b/src/library_html5.js index 851a5d648700d..7d4b045f31d0e 100644 --- a/src/library_html5.js +++ b/src/library_html5.js @@ -1,4 +1,5 @@ var LibraryJSEvents = { + $JSEvents__postset: 'JSEvents.staticInit();', $JSEvents: { // pointers to structs malloc()ed to Emscripten HEAP for JS->C interop. keyEvent: 0, @@ -19,6 +20,7 @@ var LibraryJSEvents = { // state. lastGamepadState: null, lastGamepadStateFrame: null, // The integer value of Browser.mainLoop.currentFrameNumber of when the last gamepad state was produced. + numGamepadsConnected: 0, // Keep track of how many gamepads are connected, to optimize to not poll gamepads when none are connected. // When we transition from fullscreen to windowed mode, we remember here the element that was just in fullscreen mode // so that we can report information about that element in the event message. @@ -33,6 +35,11 @@ var LibraryJSEvents = { // Track in this field whether we have yet registered that __ATEXIT__ handler. removeEventListenersRegistered: false, + staticInit: function() { + window.addEventListener("gamepadconnected", function() { ++JSEvents.numGamepadsConnected; }); + window.addEventListener("gamepaddisconnected", function() { --JSEvents.numGamepadsConnected; }); + }, + registerRemoveEventListeners: function() { if (!JSEvents.removeEventListenersRegistered) { __ATEXIT__.push(function() { @@ -1703,6 +1710,9 @@ var LibraryJSEvents = { }, _emscripten_sample_gamepad_data: function() { + // Polling gamepads generates garbage, so don't do it when we know there are no gamepads connected. + if (!JSEvents.numGamepadsConnected) return; + // Produce a new Gamepad API sample if we are ticking a new game frame, or if not using emscripten_set_main_loop() at all to drive animation. if (Browser.mainLoop.currentFrameNumber !== JSEvents.lastGamepadStateFrame || !Browser.mainLoop.currentFrameNumber) { JSEvents.lastGamepadState = navigator.getGamepads ? navigator.getGamepads() : (navigator.webkitGetGamepads ? navigator.webkitGetGamepads : null); @@ -1712,6 +1722,9 @@ var LibraryJSEvents = { emscripten_get_num_gamepads__deps: ['_emscripten_sample_gamepad_data'], emscripten_get_num_gamepads: function() { + // Polling gamepads generates garbage, so don't do it when we know there are no gamepads connected. + if (!JSEvents.numGamepadsConnected) return 0; + __emscripten_sample_gamepad_data(); if (!JSEvents.lastGamepadState) return {{{ cDefine('EMSCRIPTEN_RESULT_NOT_SUPPORTED') }}}; return JSEvents.lastGamepadState.length; From 9d7ec366d9db28f6c377d99474cfd7961068b731 Mon Sep 17 00:00:00 2001 From: Hiroaki GOTO as GORRY Date: Wed, 15 Feb 2017 02:57:12 +0900 Subject: [PATCH 05/72] fix for Closure Compiler optimization --- src/library_sdl.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/library_sdl.js b/src/library_sdl.js index 9232e3ac95561..0fd017eac70e9 100644 --- a/src/library_sdl.js +++ b/src/library_sdl.js @@ -1213,7 +1213,7 @@ var LibrarySDL = { if (typeof button === 'object') { // Current gamepad API editor's draft (Firefox Nightly) // https://dvcs.w3.org/hg/gamepad/raw-file/default/gamepad.html#idl-def-GamepadButton - return button.pressed; + return button['pressed']; } else { // Current gamepad API working draft (Firefox / Chrome Stable) // http://www.w3.org/TR/2012/WD-gamepad-20120529/#gamepad-interface From 9a912429bbb94e0bf522c5be13f2695a64b80c17 Mon Sep 17 00:00:00 2001 From: Hiroaki GOTO as GORRY Date: Wed, 15 Feb 2017 03:04:26 +0900 Subject: [PATCH 06/72] fix crashing on joystick detached --- src/library_sdl.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/library_sdl.js b/src/library_sdl.js index 0fd017eac70e9..0674ff26dbc01 100644 --- a/src/library_sdl.js +++ b/src/library_sdl.js @@ -1225,6 +1225,8 @@ var LibrarySDL = { for (var joystick in SDL.lastJoystickState) { var state = SDL.getGamepad(joystick - 1); var prevState = SDL.lastJoystickState[joystick]; + // PATCHED: If joystick was removed, state returns null. + if (typeof state == 'undefined') return; // Check only if the timestamp has differed. // NOTE: Timestamp is not available in Firefox. if (typeof state.timestamp !== 'number' || state.timestamp !== prevState.timestamp) { From 084c7b92b3f9fe9933fcea4f3b442fc3eb7cbcc7 Mon Sep 17 00:00:00 2001 From: Hiroaki GOTO as GORRY Date: Wed, 15 Feb 2017 03:08:44 +0900 Subject: [PATCH 07/72] initialize panning to center for Chrome --- src/library_sdl.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/library_sdl.js b/src/library_sdl.js index 0674ff26dbc01..b47884b9ba0fc 100644 --- a/src/library_sdl.js +++ b/src/library_sdl.js @@ -1101,6 +1101,7 @@ var LibrarySDL = { audio.webAudioNode['onended'] = function() { audio['onended'](); } // For element compatibility, route the onended signal to the instance. audio.webAudioPannerNode = SDL.audioContext['createPanner'](); + audio.webAudioPannerNode["setPosition"](0, 0, -.5); audio.webAudioPannerNode['panningModel'] = 'equalpower'; // Add an intermediate gain node to control volume. From 65c7a28b05242fef2801ffe4658b9ad50c87f43b Mon Sep 17 00:00:00 2001 From: Hiroaki GOTO as GORRY Date: Wed, 15 Feb 2017 16:10:32 +0900 Subject: [PATCH 08/72] add comment "why set posz = -.5" --- src/library_sdl.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/library_sdl.js b/src/library_sdl.js index b47884b9ba0fc..0086dad3c824f 100644 --- a/src/library_sdl.js +++ b/src/library_sdl.js @@ -1101,6 +1101,9 @@ var LibrarySDL = { audio.webAudioNode['onended'] = function() { audio['onended'](); } // For element compatibility, route the onended signal to the instance. audio.webAudioPannerNode = SDL.audioContext['createPanner'](); + // avoid Chrome bug + // If posz = 0, the sound will come from only the right. + // By posz = -0.5 (slightly ahead), the sound will come from right and left correctly. audio.webAudioPannerNode["setPosition"](0, 0, -.5); audio.webAudioPannerNode['panningModel'] = 'equalpower'; From 92910bf73c1965e1ddc67a7e0d985a8d966e992d Mon Sep 17 00:00:00 2001 From: Hiroaki GOTO as GORRY Date: Wed, 15 Feb 2017 16:12:37 +0900 Subject: [PATCH 09/72] fix compare operator to "===" --- src/library_sdl.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/library_sdl.js b/src/library_sdl.js index 0086dad3c824f..a8f18dae69808 100644 --- a/src/library_sdl.js +++ b/src/library_sdl.js @@ -1230,7 +1230,7 @@ var LibrarySDL = { var state = SDL.getGamepad(joystick - 1); var prevState = SDL.lastJoystickState[joystick]; // PATCHED: If joystick was removed, state returns null. - if (typeof state == 'undefined') return; + if (typeof state === 'undefined') return; // Check only if the timestamp has differed. // NOTE: Timestamp is not available in Firefox. if (typeof state.timestamp !== 'number' || state.timestamp !== prevState.timestamp) { From a69a0d571dd5dd3fefb4d118365d097e1d9a691f Mon Sep 17 00:00:00 2001 From: Hiroaki GOTO as GORRY Date: Wed, 15 Feb 2017 16:16:05 +0900 Subject: [PATCH 10/72] add my name to AUTHORS --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 0441963dded57..a341b9f5044e9 100644 --- a/AUTHORS +++ b/AUTHORS @@ -269,3 +269,4 @@ a license to everyone to use it as detailed in LICENSE.) * Vilibald Wanča * Alex Hixon * Vladimir Davidovich +* Hiroaki GOTO as "GORRY" From 345394a4db9d403bcb59e2c827e592acc9cdf375 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Wed, 21 Dec 2016 14:41:20 +0200 Subject: [PATCH 11/72] SharedArrayBuffer shouldn't be present in transfer list anymore. --- src/Fetch.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Fetch.js b/src/Fetch.js index bd42c39f70b25..1c9f752da3b2f 100644 --- a/src/Fetch.js +++ b/src/Fetch.js @@ -65,7 +65,7 @@ var Fetch = { initFetchWorker: function() { var stackSize = 128*1024; var stack = allocate(stackSize>>2, "i32*", ALLOC_DYNAMIC); - Fetch.worker.postMessage({cmd: 'init', TOTAL_MEMORY: TOTAL_MEMORY, DYNAMICTOP_PTR: DYNAMICTOP_PTR, STACKTOP: stack, STACK_MAX: stack + stackSize, queuePtr: _fetch_work_queue, buffer: HEAPU8.buffer}, [HEAPU8.buffer]); + Fetch.worker.postMessage({cmd: 'init', TOTAL_MEMORY: TOTAL_MEMORY, DYNAMICTOP_PTR: DYNAMICTOP_PTR, STACKTOP: stack, STACK_MAX: stack + stackSize, queuePtr: _fetch_work_queue, buffer: HEAPU8.buffer}); }, staticInit: function() { From 6beabbc6ff32be04a005d798285a58dcbdaea4d8 Mon Sep 17 00:00:00 2001 From: Derek Schuff Date: Fri, 17 Feb 2017 16:03:04 -0800 Subject: [PATCH 12/72] Expose Building method to get a new multiprocessing pool (#4957) Use it for Building.read_link_inputs to work around #4941 --- tools/shared.py | 65 ++++++++++++++++++++++++++++--------------------- 1 file changed, 37 insertions(+), 28 deletions(-) diff --git a/tools/shared.py b/tools/shared.py index 2abdc366d2658..c8194993cf799 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -1244,41 +1244,49 @@ class Building: @staticmethod def get_multiprocessing_pool(): if not Building.multiprocessing_pool: - cores = int(os.environ.get('EMCC_CORES') or multiprocessing.cpu_count()) - - # If running with one core only, create a mock instance of a pool that does not - # actually spawn any new subprocesses. Very useful for internal debugging. - if cores == 1: - class FakeMultiprocessor: - def map(self, func, tasks): - results = [] - for t in tasks: - results += [func(t)] - return results - Building.multiprocessing_pool = FakeMultiprocessor() - else: - child_env = [ + Building.multiprocessing_pool = Building.create_multiprocessing_pool() + return Building.multiprocessing_pool + + @staticmethod + def create_multiprocessing_pool(): + cores = int(os.environ.get('EMCC_CORES') or multiprocessing.cpu_count()) + + # If running with one core only, create a mock instance of a pool that does not + # actually spawn any new subprocesses. Very useful for internal debugging. + if cores == 1: + class FakeMultiprocessor: + def map(self, func, tasks): + results = [] + for t in tasks: + results += [func(t)] + return results + pool = FakeMultiprocessor() + else: + child_env = [ # Multiprocessing pool children need to avoid all calling check_vanilla() again and again, # otherwise the compiler can deadlock when building system libs, because the multiprocess parent can have the Emscripten cache directory locked for write # access, and the EMCC_WASM_BACKEND check also requires locked access to the cache, which the multiprocess children would not get. 'EMCC_WASM_BACKEND='+os.getenv('EMCC_WASM_BACKEND', '0'), 'EMCC_CORES=1' # Multiprocessing pool children can't spawn their own linear number of children, that could cause a quadratic amount of spawned processes. - ] - Building.multiprocessing_pool = multiprocessing.Pool(processes=cores, initializer=g_multiprocessing_initializer, initargs=child_env) + ] + pool = multiprocessing.Pool(processes=cores, initializer=g_multiprocessing_initializer, initargs=child_env) - def close_multiprocessing_pool(): - try: - # Shut down the pool explicitly, because leaving that for Python to do at process shutdown is buggy and can generate - # noisy "WindowsError: [Error 5] Access is denied" spam which is not fatal. - Building.multiprocessing_pool.terminate() - Building.multiprocessing_pool.join() + def close_multiprocessing_pool(): + try: + # Shut down the pool explicitly, because leaving that for Python to do at process shutdown is buggy and can generate + # noisy "WindowsError: [Error 5] Access is denied" spam which is not fatal. + pool.terminate() + pool.join() + if pool == Building.multiprocessing_pool: Building.multiprocessing_pool = None - except WindowsError, e: - # Mute the "WindowsError: [Error 5] Access is denied" errors, raise all others through - if e.winerror != 5: raise - atexit.register(close_multiprocessing_pool) + except WindowsError, e: + # Mute the "WindowsError: [Error 5] Access is denied" errors, raise all others through + if e.winerror != 5: raise + atexit.register(close_multiprocessing_pool) + + return pool + - return Building.multiprocessing_pool @staticmethod def get_building_env(native=False): @@ -1560,7 +1568,8 @@ def read_link_inputs(files): object_names.append(absolute_path_f) # Archives contain objects, so process all archives first in parallel to obtain the object files in them. - pool = Building.get_multiprocessing_pool() + # new_pool is a workaround for https://github.com/kripken/emscripten/issues/4941 TODO: a better fix + pool = Building.create_multiprocessing_pool() object_names_in_archives = pool.map(extract_archive_contents, archive_names) for n in range(len(archive_names)): From 1a1166f8f0d1cdb9af7bba98aae610969afdb276 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Sat, 18 Feb 2017 11:37:17 -0800 Subject: [PATCH 13/72] add --ignore-implicit-traps support for binaryen (#4954) --- emcc.py | 2 ++ src/settings.js | 5 +++++ tests/test_other.py | 24 ++++++++++++++++++++++++ 3 files changed, 31 insertions(+) diff --git a/emcc.py b/emcc.py index 102ae0ac3b7a1..b32f0c689fb7d 100755 --- a/emcc.py +++ b/emcc.py @@ -2110,6 +2110,8 @@ def do_minify(): # minifies the code. this is also when we do certain optimizati cmd = [os.path.join(binaryen_bin, 'asm2wasm'), asm_target, '--total-memory=' + str(shared.Settings.TOTAL_MEMORY)] if shared.Settings.BINARYEN_IMPRECISE: cmd += ['--imprecise'] + if shared.Settings.BINARYEN_IGNORE_IMPLICIT_TRAPS: + cmd += ['--ignore-implicit-traps'] # pass optimization level to asm2wasm (if not optimizing, or which passes we should run was overridden, do not optimize) if opt_level > 0 and not shared.Settings.BINARYEN_PASSES: cmd.append(shared.Building.opt_level_to_str(opt_level, shrink_level)) diff --git a/src/settings.js b/src/settings.js index 69d14b9706d2a..36c6763376729 100644 --- a/src/settings.js +++ b/src/settings.js @@ -687,6 +687,11 @@ var BINARYEN_SCRIPTS = ""; // An optional comma-separated list of script hooks t var BINARYEN_IMPRECISE = 0; // Whether to apply imprecise/unsafe binaryen optimizations. If enabled, // code will run faster, but some types of undefined behavior might // trap in wasm. +var BINARYEN_IGNORE_IMPLICIT_TRAPS = 0; // Whether to ignore implicit traps when optimizing in binaryen. + // Implicit traps are the unlikely traps that happen in a load that + // is out of bounds, or div/rem of 0, etc. We can reorder them, + // but we can't ignore that they have side effects, so turning on + // this flag lets us do a little more to reduce code size. var BINARYEN_PASSES = ""; // A comma-separated list of passes to run in the binaryen optimizer, // for example, "dce,precompute,vacuum". // When set, this overrides the default passes we would normally run. diff --git a/tests/test_other.py b/tests/test_other.py index 1020ea5850d48..18776c254e71f 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -7345,6 +7345,30 @@ def test_binaryen_debuginfo(self): finally: del os.environ['EMCC_DEBUG'] + def test_binaryen_ignore_implicit_traps(self): + with clean_write_access_to_canonical_temp_dir(): + if os.environ.get('EMCC_DEBUG'): return self.skip('cannot run in debug mode') + sizes = [] + try: + os.environ['EMCC_DEBUG'] = '1' + for args, expect in [ + ([], False), + (['-s', 'BINARYEN_IGNORE_IMPLICIT_TRAPS=1'], True), + ]: + print args, expect + cmd = [PYTHON, EMCC, path_from_root('tests', 'hello_world.cpp'), '-s', 'WASM=1', '-O3'] + args + print ' '.join(cmd) + output, err = Popen(cmd, stdout=PIPE, stderr=PIPE).communicate() + asm2wasm_line = filter(lambda x: 'asm2wasm' in x, err.split('\n'))[0] + asm2wasm_line = asm2wasm_line.strip() + ' ' # ensure it ends with a space, for simpler searches below + print '|' + asm2wasm_line + '|' + assert expect == (' --ignore-implicit-traps ' in asm2wasm_line) + sizes.append(os.stat('a.out.wasm').st_size) + finally: + del os.environ['EMCC_DEBUG'] + print 'sizes:', sizes + assert sizes[1] < sizes[0], 'ignoring implicit traps must reduce code size' + def test_wasm_targets(self): for f in ['a.wasm', 'a.wast']: process = Popen([PYTHON, EMCC, path_from_root('tests', 'hello_world.cpp'), '-o', f], stdout=PIPE, stderr=PIPE) From da2d623fc165749896207eb6bfc9053f81b71a5e Mon Sep 17 00:00:00 2001 From: Sam Clegg Date: Sat, 18 Feb 2017 16:29:51 -0800 Subject: [PATCH 14/72] Fix bug in extract_archive_contents when TMP=. (#4958) Remove redundant call to os.path.join(temp_dir). This code is running with CWD=temp_dir already. In most cases this is harmless since temp_dir is normally absolute and therefore the resulting path is absolute. However, setting TMP=. will break this as python's gettempdir() will then return '.'. To reproduce the problem simply run "TMP=. emcc hello.cc". The effect is that all the libraries apprar to be empty and you will get undefined symbols. --- tools/shared.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/shared.py b/tools/shared.py index c8194993cf799..d893d08a9c477 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -1203,7 +1203,6 @@ def extract_archive_contents(f): if dirname: safe_ensure_dirs(dirname) Popen([LLVM_AR, 'xo', f], stdout=PIPE).communicate() # if absolute paths, files will appear there. otherwise, in this directory - contents = map(lambda content: os.path.join(temp_dir, content), contents) contents = filter(os.path.exists, map(os.path.abspath, contents)) contents = filter(Building.is_bitcode, contents) return { From 481a4ea0535ebe8f7334bbee080c07905c605387 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Sun, 19 Feb 2017 14:33:43 +0200 Subject: [PATCH 15/72] Guard HTML5 API .staticInit() for the case when window object is not present. --- src/library_html5.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/library_html5.js b/src/library_html5.js index 7d4b045f31d0e..da5ded65c6c48 100644 --- a/src/library_html5.js +++ b/src/library_html5.js @@ -36,8 +36,10 @@ var LibraryJSEvents = { removeEventListenersRegistered: false, staticInit: function() { - window.addEventListener("gamepadconnected", function() { ++JSEvents.numGamepadsConnected; }); - window.addEventListener("gamepaddisconnected", function() { --JSEvents.numGamepadsConnected; }); + if (typeof window !== 'undefined') { + window.addEventListener("gamepadconnected", function() { ++JSEvents.numGamepadsConnected; }); + window.addEventListener("gamepaddisconnected", function() { --JSEvents.numGamepadsConnected; }); + } }, registerRemoveEventListeners: function() { From 51c509d22797f140b9f525e7db71a68bbfadcbb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Sat, 4 Feb 2017 05:17:34 +0200 Subject: [PATCH 16/72] Switch to 64KB page sizes from 4KB, since Wasm pages are 64KB. Revise the heap grow strategy to specifically cap to 2GB-64KB/2GB-16MB for wasm/asm.js. Only report memory growth being slow if running asm.js. Pass maximum WebAssembly Memory size when BINARYEN_MEM_MAX is specified. Fix memory growth to work in binaryen wasm+asm.js builds. (exports['__growWasmMemory'] does not exist in asm.js in that case) --- src/preamble.js | 84 ++++++++++++++++++++++++++++--------------------- 1 file changed, 48 insertions(+), 36 deletions(-) diff --git a/src/preamble.js b/src/preamble.js index 98ab56551265a..c79abbba9ff6c 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -930,11 +930,11 @@ function stackTrace() { // Memory management -var PAGE_SIZE = 4096; +var PAGE_SIZE = 65536; // Wasm pages are fixed to 64k. function alignMemoryPage(x) { - if (x % 4096 > 0) { - x += (4096 - (x % 4096)); + if (x % PAGE_SIZE > 0) { + x += PAGE_SIZE - (x % PAGE_SIZE); } return x; } @@ -1044,50 +1044,48 @@ function enlargeMemory() { // TOTAL_MEMORY is the current size of the actual array, and DYNAMICTOP is the new top. #if ASSERTIONS assert(HEAP32[DYNAMICTOP_PTR>>2] > TOTAL_MEMORY); // This function should only ever be called after the ceiling of the dynamic heap has already been bumped to exceed the current total size of the asm.js heap. - assert(TOTAL_MEMORY > 4); // So the loop below will not be infinite #endif - var OLD_TOTAL_MEMORY = TOTAL_MEMORY; - #if EMSCRIPTEN_TRACING // Report old layout one last time _emscripten_trace_report_memory_layout(); #endif - var LIMIT = Math.pow(2, 31); // 2GB is a practical maximum, as we use signed ints as pointers - // and JS engines seem unhappy to give us 2GB arrays currently - if (HEAP32[DYNAMICTOP_PTR>>2] >= LIMIT) return false; + if (Module["usingWasm"]) { + var LIMIT = 2147483648-PAGE_SIZE; // In Wasm, we can do one 64KB page short of 2GB as theoretical maximum. + } else { + var LIMIT = 2147483648-16777216; // In asm.js, the theoretical maximum is 16MB short of 2GB. + } + + if (HEAP32[DYNAMICTOP_PTR>>2] > LIMIT) { +#if ASSERTIONS + Module.printErr('Cannot enlarge memory, asked to go up to ' + HEAP32[DYNAMICTOP_PTR>>2] + ' bytes, but the limit is ' + LIMIT + ' bytes!'); +#endif + return false; + } + + var OLD_TOTAL_MEMORY = TOTAL_MEMORY; + TOTAL_MEMORY = Math.max(TOTAL_MEMORY, 16*1024*1024); // So the loop below will not be infinite, and minimum asm.js memory size is 16MB. while (TOTAL_MEMORY < HEAP32[DYNAMICTOP_PTR>>2]) { // Keep incrementing the heap size as long as it's less than what is requested. - if (TOTAL_MEMORY < LIMIT/2) { + if (TOTAL_MEMORY <= 536870912) { TOTAL_MEMORY = alignMemoryPage(2*TOTAL_MEMORY); // // Simple heuristic: double until 1GB... } else { - var last = TOTAL_MEMORY; - TOTAL_MEMORY = alignMemoryPage((3*TOTAL_MEMORY + LIMIT)/4); // ..., but after that, add smaller increments towards 2GB, which we cannot reach - if (TOTAL_MEMORY <= last) return false; + TOTAL_MEMORY = Math.min(alignMemoryPage((3*TOTAL_MEMORY + 2147483648)/4), LIMIT); // ..., but after that, add smaller increments towards 2GB, which we cannot reach } } - TOTAL_MEMORY = Math.max(TOTAL_MEMORY, 16*1024*1024); - - if (TOTAL_MEMORY >= LIMIT) return false; - -#if ASSERTIONS - Module.printErr('Warning: Enlarging memory arrays, this is not fast! ' + [OLD_TOTAL_MEMORY, TOTAL_MEMORY]); -#endif - -#if EMSCRIPTEN_TRACING - _emscripten_trace_js_log_message("Emscripten", "Enlarging memory arrays from " + OLD_TOTAL_MEMORY + " to " + TOTAL_MEMORY); - // And now report the new layout - _emscripten_trace_report_memory_layout(); -#endif - #if ASSERTIONS var start = Date.now(); #endif var replacement = Module['reallocBuffer'](TOTAL_MEMORY); - if (!replacement) return false; + if (!replacement) { +#if ASSERTIONS + Module.printErr('Failed to grow the heap from ' + OLD_TOTAL_MEMORY + ' bytes to ' + TOTAL_MEMORY + ' bytes, not enough memory!'); +#endif + return false; + } // everything worked @@ -1098,6 +1096,18 @@ function enlargeMemory() { Module.printErr('enlarged memory arrays from ' + OLD_TOTAL_MEMORY + ' to ' + TOTAL_MEMORY + ', took ' + (Date.now() - start) + ' ms (has ArrayBuffer.transfer? ' + (!!ArrayBuffer.transfer) + ')'); #endif +#if ASSERTIONS + if (!Module["usingWasm"]) { + Module.printErr('Warning: Enlarging memory arrays, this is not fast! ' + [OLD_TOTAL_MEMORY, TOTAL_MEMORY]); + } +#endif + +#if EMSCRIPTEN_TRACING + _emscripten_trace_js_log_message("Emscripten", "Enlarging memory arrays from " + OLD_TOTAL_MEMORY + " to " + TOTAL_MEMORY); + // And now report the new layout + _emscripten_trace_report_memory_layout(); +#endif + return true; #endif // ALLOW_MEMORY_GROWTH #endif // USE_PTHREADS @@ -1116,9 +1126,7 @@ try { var TOTAL_STACK = Module['TOTAL_STACK'] || {{{ TOTAL_STACK }}}; var TOTAL_MEMORY = Module['TOTAL_MEMORY'] || {{{ TOTAL_MEMORY }}}; -var WASM_PAGE_SIZE = 64 * 1024; - -var totalMemory = WASM_PAGE_SIZE; +var totalMemory = PAGE_SIZE; while (totalMemory < TOTAL_MEMORY || totalMemory < 2*TOTAL_STACK) { if (totalMemory < 16*1024*1024) { totalMemory *= 2; @@ -1221,12 +1229,16 @@ if (Module['buffer']) { #if BINARYEN if (typeof WebAssembly === 'object' && typeof WebAssembly.Memory === 'function') { #if ASSERTIONS - assert(TOTAL_MEMORY % WASM_PAGE_SIZE === 0); + assert(TOTAL_MEMORY % PAGE_SIZE === 0); #endif #if ALLOW_MEMORY_GROWTH - Module['wasmMemory'] = new WebAssembly.Memory({ initial: TOTAL_MEMORY / WASM_PAGE_SIZE }); +#if BINARYEN_MEM_MAX + Module['wasmMemory'] = new WebAssembly.Memory({ initial: TOTAL_MEMORY / PAGE_SIZE, maximum: {{{ BINARYEN_MEM_MAX }}} / PAGE_SIZE }); +#else + Module['wasmMemory'] = new WebAssembly.Memory({ initial: TOTAL_MEMORY / PAGE_SIZE }); +#endif #else - Module['wasmMemory'] = new WebAssembly.Memory({ initial: TOTAL_MEMORY / WASM_PAGE_SIZE, maximum: TOTAL_MEMORY / WASM_PAGE_SIZE }); + Module['wasmMemory'] = new WebAssembly.Memory({ initial: TOTAL_MEMORY / PAGE_SIZE, maximum: TOTAL_MEMORY / PAGE_SIZE }); #endif buffer = Module['wasmMemory'].buffer; } else @@ -2305,10 +2317,10 @@ function integrateWasmJS(Module) { // Memory growth integration code Module['reallocBuffer'] = function(size) { - size = Math.ceil(size / wasmPageSize) * wasmPageSize; // round up to wasm page size + size = alignMemoryPage(size); // round up to wasm page size var old = Module['buffer']; - var result = exports['__growWasmMemory'](size / wasmPageSize); // tiny wasm method that just does grow_memory if (Module["usingWasm"]) { + var result = exports['__growWasmMemory'](size / wasmPageSize); // tiny wasm method that just does grow_memory if (result !== (-1 | 0)) { // success in native wasm memory growth, get the buffer from the memory return Module['buffer'] = Module['wasmMemory'].buffer; From 5a581ff27c04bbb7901bdf5133e6380659415ba1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Sat, 4 Feb 2017 07:28:26 +0200 Subject: [PATCH 17/72] Growing memory in wasm can actually give a larger heap than what was asked, so prepare for that as well. --- src/preamble.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/preamble.js b/src/preamble.js index c79abbba9ff6c..955fbf27198a2 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -1080,12 +1080,13 @@ function enlargeMemory() { #endif var replacement = Module['reallocBuffer'](TOTAL_MEMORY); - if (!replacement) { + if (!replacement || replacement.byteLength < TOTAL_MEMORY) { #if ASSERTIONS Module.printErr('Failed to grow the heap from ' + OLD_TOTAL_MEMORY + ' bytes to ' + TOTAL_MEMORY + ' bytes, not enough memory!'); #endif return false; } + TOTAL_MEMORY = replacement.byteLength; // everything worked From ed7d2bd0f7d0216b0e94dd915cf00e96e8c3f6a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Sat, 4 Feb 2017 07:32:32 +0200 Subject: [PATCH 18/72] Use WebAssembly.Memory.grow() from JS side instead of the Wasm instruction, since it is a bit cleaner because Module['reallocBuffer']() is already JS code anyways. --- src/preamble.js | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/preamble.js b/src/preamble.js index 955fbf27198a2..f4f5c60724372 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -2320,12 +2320,20 @@ function integrateWasmJS(Module) { Module['reallocBuffer'] = function(size) { size = alignMemoryPage(size); // round up to wasm page size var old = Module['buffer']; + var oldSize = old.byteLength; if (Module["usingWasm"]) { - var result = exports['__growWasmMemory'](size / wasmPageSize); // tiny wasm method that just does grow_memory - if (result !== (-1 | 0)) { - // success in native wasm memory growth, get the buffer from the memory - return Module['buffer'] = Module['wasmMemory'].buffer; - } else { + try { + var result = Module['wasmMemory'].grow(size / wasmPageSize); // tiny wasm method that just does grow_memory + if (result !== (-1 | 0)) { + // success in native wasm memory growth, get the buffer from the memory + return Module['buffer'] = Module['wasmMemory'].buffer; + } else { + return null; + } + } catch(e) { +#if ASSERTIONS + console.error('Module.reallocBuffer: Attempted to grow from ' + oldSize + ' bytes to ' + size + ' bytes, but got error: ' + e); +#endif return null; } } else { From 9092db488b3a5e93407f62cda4f4d705c9479501 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Sun, 5 Feb 2017 03:20:14 +0200 Subject: [PATCH 19/72] WebAssembly.Memory.grow() API call takes in the delta of the amount of memory to grow, instead of an absolute value for the new buffer, which the previous code misassumed. --- src/preamble.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/preamble.js b/src/preamble.js index f4f5c60724372..3d4e5255b6bc6 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -1080,13 +1080,15 @@ function enlargeMemory() { #endif var replacement = Module['reallocBuffer'](TOTAL_MEMORY); - if (!replacement || replacement.byteLength < TOTAL_MEMORY) { + if (!replacement || replacement.byteLength != TOTAL_MEMORY) { #if ASSERTIONS Module.printErr('Failed to grow the heap from ' + OLD_TOTAL_MEMORY + ' bytes to ' + TOTAL_MEMORY + ' bytes, not enough memory!'); + if (replacement) { + Module.printErr('Expected to get back a buffer of size ' + TOTAL_MEMORY + ' bytes, but instead got back a buffer of size ' + replacement.byteLength); + } #endif return false; } - TOTAL_MEMORY = replacement.byteLength; // everything worked @@ -2323,7 +2325,7 @@ function integrateWasmJS(Module) { var oldSize = old.byteLength; if (Module["usingWasm"]) { try { - var result = Module['wasmMemory'].grow(size / wasmPageSize); // tiny wasm method that just does grow_memory + var result = Module['wasmMemory'].grow((size - oldSize) / wasmPageSize); // .grow() takes a delta compared to the previous size if (result !== (-1 | 0)) { // success in native wasm memory growth, get the buffer from the memory return Module['buffer'] = Module['wasmMemory'].buffer; From f9f29add43f78dfe78289e5024a47dbdced45550 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Tue, 14 Feb 2017 11:57:57 +0200 Subject: [PATCH 20/72] Apply code formatting. --- src/preamble.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/preamble.js b/src/preamble.js index 3d4e5255b6bc6..38ca8da8235b6 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -1052,9 +1052,9 @@ function enlargeMemory() { #endif if (Module["usingWasm"]) { - var LIMIT = 2147483648-PAGE_SIZE; // In Wasm, we can do one 64KB page short of 2GB as theoretical maximum. + var LIMIT = 2147483648 - PAGE_SIZE; // In Wasm, we can do one 64KB page short of 2GB as theoretical maximum. } else { - var LIMIT = 2147483648-16777216; // In asm.js, the theoretical maximum is 16MB short of 2GB. + var LIMIT = 2147483648 - 16777216; // In asm.js, the theoretical maximum is 16MB short of 2GB. } if (HEAP32[DYNAMICTOP_PTR>>2] > LIMIT) { @@ -1065,13 +1065,13 @@ function enlargeMemory() { } var OLD_TOTAL_MEMORY = TOTAL_MEMORY; - TOTAL_MEMORY = Math.max(TOTAL_MEMORY, 16*1024*1024); // So the loop below will not be infinite, and minimum asm.js memory size is 16MB. + TOTAL_MEMORY = Math.max(TOTAL_MEMORY, 16 * 1024 * 1024); // So the loop below will not be infinite, and minimum asm.js memory size is 16MB. while (TOTAL_MEMORY < HEAP32[DYNAMICTOP_PTR>>2]) { // Keep incrementing the heap size as long as it's less than what is requested. if (TOTAL_MEMORY <= 536870912) { - TOTAL_MEMORY = alignMemoryPage(2*TOTAL_MEMORY); // // Simple heuristic: double until 1GB... + TOTAL_MEMORY = alignMemoryPage(2 * TOTAL_MEMORY); // Simple heuristic: double until 1GB... } else { - TOTAL_MEMORY = Math.min(alignMemoryPage((3*TOTAL_MEMORY + 2147483648)/4), LIMIT); // ..., but after that, add smaller increments towards 2GB, which we cannot reach + TOTAL_MEMORY = Math.min(alignMemoryPage((3 * TOTAL_MEMORY + 2147483648) / 4), LIMIT); // ..., but after that, add smaller increments towards 2GB, which we cannot reach } } From 0a1f5cedad96ed485a0027f5f24726822e1cf700 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Tue, 14 Feb 2017 12:00:52 +0200 Subject: [PATCH 21/72] Assert that BINARYEN_MEM_MAX is divisible by page size. --- src/preamble.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/preamble.js b/src/preamble.js index 38ca8da8235b6..5ec1d75530e16 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -1236,6 +1236,9 @@ if (Module['buffer']) { #endif #if ALLOW_MEMORY_GROWTH #if BINARYEN_MEM_MAX +#if ASSERTIONS + assert({{{ BINARYEN_MEM_MAX }}} % PAGE_SIZE == 0); +#endif Module['wasmMemory'] = new WebAssembly.Memory({ initial: TOTAL_MEMORY / PAGE_SIZE, maximum: {{{ BINARYEN_MEM_MAX }}} / PAGE_SIZE }); #else Module['wasmMemory'] = new WebAssembly.Memory({ initial: TOTAL_MEMORY / PAGE_SIZE }); From 65ab9dbc7f24fdead4b6ef08b5e6618332c9206a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Tue, 14 Feb 2017 13:35:54 +0200 Subject: [PATCH 22/72] Stop automatically resizing TOTAL_MEMORY at startup if it is wrong, but require at compile time that the size is specified appropriately. Fixes #4850. --- emcc.py | 12 ++++++++++++ src/preamble.js | 21 +++++---------------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/emcc.py b/emcc.py index b32f0c689fb7d..2a3e138670750 100755 --- a/emcc.py +++ b/emcc.py @@ -1021,6 +1021,11 @@ def check(input_file): # Apply -s settings in newargs here (after optimization levels, so they can override them) for change in settings_changes: key, value = change.split('=') + + # In those settings fields that represent amount of memory, translate suffixes to multiples of 1024. + if key in ['TOTAL_STACK', 'TOTAL_MEMORY', 'GL_MAX_TEMP_BUFFER_SIZE', 'SPLIT_MEMORY', 'BINARYEN_MEM_MAX']: + value = value.lower().replace('tb', '*1024*1024*1024*1024').replace('gb', '*1024*1024*1024').replace('mb', '*1024*1024').replace('kb', '*1024').replace('b', '') + original_exported_response = False if value[0] == '@': @@ -1067,6 +1072,13 @@ def check(input_file): logging.error('Compiler settings are incompatible with fastcomp. You can fall back to the older compiler core, although that is not recommended, see http://kripken.github.io/emscripten-site/docs/building_from_source/LLVM-Backend.html') raise e + assert shared.Settings.TOTAL_MEMORY >= 16*1024*1024, 'TOTAL_MEMORY must be at least 16MB, was ' + str(shared.Settings.TOTAL_MEMORY) + if shared.Settings.WASM or shared.Settings.BINARYEN: + assert shared.Settings.TOTAL_MEMORY % 65536 == 0, 'For wasm, TOTAL_MEMORY must be a multiple of 64KB, was ' + str(shared.Settings.TOTAL_MEMORY) + else: + assert shared.Settings.TOTAL_MEMORY % (16*1024*1024) == 0, 'For asm.js, TOTAL_MEMORY must be a multiple of 16MB, was ' + str(shared.Settings.TOTAL_MEMORY) + assert shared.Settings.TOTAL_MEMORY >= shared.Settings.TOTAL_STACK, 'TOTAL_MEMORY must be larger than TOTAL_STACK, was ' + str(shared.Settings.TOTAL_MEMORY) + ' (TOTAL_STACK=' + str(shared.Settings.TOTAL_STACK) + ')' + assert not shared.Settings.PGO, 'cannot run PGO in ASM_JS mode' if shared.Settings.SAFE_HEAP: diff --git a/src/preamble.js b/src/preamble.js index 5ec1d75530e16..1e496ebd06235 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -1129,23 +1129,12 @@ try { var TOTAL_STACK = Module['TOTAL_STACK'] || {{{ TOTAL_STACK }}}; var TOTAL_MEMORY = Module['TOTAL_MEMORY'] || {{{ TOTAL_MEMORY }}}; -var totalMemory = PAGE_SIZE; -while (totalMemory < TOTAL_MEMORY || totalMemory < 2*TOTAL_STACK) { - if (totalMemory < 16*1024*1024) { - totalMemory *= 2; - } else { - totalMemory += 16*1024*1024; - } -} -#if ALLOW_MEMORY_GROWTH -totalMemory = Math.max(totalMemory, 16*1024*1024); +if (TOTAL_MEMORY < 16*1024*1024) Module.printErr('TOTAL_MEMORY should be at least 16MB, was ' + TOTAL_MEMORY + '!'); +if (TOTAL_MEMORY % 65536 != 0) Module.printErr('For wasm, TOTAL_MEMORY should be a multiple of 64KB, was ' + TOTAL_MEMORY + '!'); +#if !BINARYEN +if (TOTAL_MEMORY % (16*1024*1024) != 0) Module.printErr('For asm.js, TOTAL_MEMORY should be a multiple of 16MB, was ' + TOTAL_MEMORY + '!'); #endif -if (totalMemory !== TOTAL_MEMORY) { -#if ASSERTIONS - Module.printErr('increasing TOTAL_MEMORY to ' + totalMemory + ' to be compliant with the asm.js spec (and given that TOTAL_STACK=' + TOTAL_STACK + ')'); -#endif - TOTAL_MEMORY = totalMemory; -} +if (TOTAL_MEMORY < TOTAL_STACK) Module.printErr('TOTAL_MEMORY should be larger than TOTAL_STACK, was ' + TOTAL_MEMORY + '! (TOTAL_STACK=' + TOTAL_STACK + ')'); // Initialize the runtime's memory #if ASSERTIONS From 4e9bca97a7e8be5750c23fa3bcec3e39c6d7da37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Tue, 14 Feb 2017 13:47:00 +0200 Subject: [PATCH 23/72] Make page size 16KB. --- .../source/docs/api_reference/preamble.js.rst | 2 +- src/library_bootstrap_structInfo.js | 2 +- src/preamble.js | 32 +++++++++---------- .../libc/musl/arch/emscripten/bits/limits.h | 2 +- tools/shared.py | 2 +- 5 files changed, 19 insertions(+), 21 deletions(-) diff --git a/site/source/docs/api_reference/preamble.js.rst b/site/source/docs/api_reference/preamble.js.rst index 17c9105f65a92..501bcf7c5def1 100644 --- a/site/source/docs/api_reference/preamble.js.rst +++ b/site/source/docs/api_reference/preamble.js.rst @@ -409,7 +409,7 @@ The :ref:`emscripten-memory-model` uses a typed array buffer (``ArrayBuffer``) t Module['ALLOC_NONE'] = ALLOC_NONE; Module['HEAP'] = HEAP; Module['IHEAP'] = IHEAP; - function alignMemoryPage(x) + function alignUp(x, multiple) function enlargeMemory() function demangle(func) function demangleAll(text) diff --git a/src/library_bootstrap_structInfo.js b/src/library_bootstrap_structInfo.js index 535265576f6bc..2b6cf7ea8818d 100644 --- a/src/library_bootstrap_structInfo.js +++ b/src/library_bootstrap_structInfo.js @@ -20,7 +20,7 @@ LibraryManager.library = { // We control the "dynamic" memory - DYNAMIC_BASE to DYNAMICTOP var self = _sbrk; if (!self.called) { - HEAP32[DYNAMICTOP_PTR>>2] = alignMemoryPage(HEAP32[DYNAMICTOP_PTR>>2]); // make sure we start out aligned + HEAP32[DYNAMICTOP_PTR>>2] = alignUp(HEAP32[DYNAMICTOP_PTR>>2], 16777216); // make sure we start out aligned self.called = true; assert(Runtime.dynamicAlloc); self.alloc = Runtime.dynamicAlloc; diff --git a/src/preamble.js b/src/preamble.js index 1e496ebd06235..7588d42770412 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -930,11 +930,11 @@ function stackTrace() { // Memory management -var PAGE_SIZE = 65536; // Wasm pages are fixed to 64k. +var PAGE_SIZE = 16384; -function alignMemoryPage(x) { - if (x % PAGE_SIZE > 0) { - x += PAGE_SIZE - (x % PAGE_SIZE); +function alignUp(x, multiple) { + if (x % multiple > 0) { + x += multiple - (x % multiple); } return x; } @@ -1051,11 +1051,8 @@ function enlargeMemory() { _emscripten_trace_report_memory_layout(); #endif - if (Module["usingWasm"]) { - var LIMIT = 2147483648 - PAGE_SIZE; // In Wasm, we can do one 64KB page short of 2GB as theoretical maximum. - } else { - var LIMIT = 2147483648 - 16777216; // In asm.js, the theoretical maximum is 16MB short of 2GB. - } + var PAGE_MULTIPLE = Module["usingWasm"] ? 65536 : 16777216; // In wasm, heap size must be a multiple of 64KB. In asm.js, they need to be multiples of 16MB. + var LIMIT = 2147483648 - PAGE_MULTIPLE; // We can do one page short of 2GB as theoretical maximum. if (HEAP32[DYNAMICTOP_PTR>>2] > LIMIT) { #if ASSERTIONS @@ -1069,9 +1066,9 @@ function enlargeMemory() { while (TOTAL_MEMORY < HEAP32[DYNAMICTOP_PTR>>2]) { // Keep incrementing the heap size as long as it's less than what is requested. if (TOTAL_MEMORY <= 536870912) { - TOTAL_MEMORY = alignMemoryPage(2 * TOTAL_MEMORY); // Simple heuristic: double until 1GB... + TOTAL_MEMORY = alignUp(2 * TOTAL_MEMORY, PAGE_MULTIPLE); // Simple heuristic: double until 1GB... } else { - TOTAL_MEMORY = Math.min(alignMemoryPage((3 * TOTAL_MEMORY + 2147483648) / 4), LIMIT); // ..., but after that, add smaller increments towards 2GB, which we cannot reach + TOTAL_MEMORY = Math.min(alignUp((3 * TOTAL_MEMORY + 2147483648) / 4, PAGE_MULTIPLE), LIMIT); // ..., but after that, add smaller increments towards 2GB, which we cannot reach } } @@ -1221,19 +1218,19 @@ if (Module['buffer']) { #if BINARYEN if (typeof WebAssembly === 'object' && typeof WebAssembly.Memory === 'function') { #if ASSERTIONS - assert(TOTAL_MEMORY % PAGE_SIZE === 0); + assert(TOTAL_MEMORY % 65536 === 0); #endif #if ALLOW_MEMORY_GROWTH #if BINARYEN_MEM_MAX #if ASSERTIONS - assert({{{ BINARYEN_MEM_MAX }}} % PAGE_SIZE == 0); + assert({{{ BINARYEN_MEM_MAX }}} % 65536 == 0); #endif - Module['wasmMemory'] = new WebAssembly.Memory({ initial: TOTAL_MEMORY / PAGE_SIZE, maximum: {{{ BINARYEN_MEM_MAX }}} / PAGE_SIZE }); + Module['wasmMemory'] = new WebAssembly.Memory({ initial: TOTAL_MEMORY / 65536, maximum: {{{ BINARYEN_MEM_MAX }}} / 65536 }); #else - Module['wasmMemory'] = new WebAssembly.Memory({ initial: TOTAL_MEMORY / PAGE_SIZE }); + Module['wasmMemory'] = new WebAssembly.Memory({ initial: TOTAL_MEMORY / 65536 }); #endif #else - Module['wasmMemory'] = new WebAssembly.Memory({ initial: TOTAL_MEMORY / PAGE_SIZE, maximum: TOTAL_MEMORY / PAGE_SIZE }); + Module['wasmMemory'] = new WebAssembly.Memory({ initial: TOTAL_MEMORY / 65536, maximum: TOTAL_MEMORY / 65536 }); #endif buffer = Module['wasmMemory'].buffer; } else @@ -2312,7 +2309,8 @@ function integrateWasmJS(Module) { // Memory growth integration code Module['reallocBuffer'] = function(size) { - size = alignMemoryPage(size); // round up to wasm page size + var PAGE_MULTIPLE = Module["usingWasm"] ? 65536 : 16777216; // In wasm, heap size must be a multiple of 64KB. In asm.js, they need to be multiples of 16MB. + size = alignUp(size, PAGE_MULTIPLE); // round up to wasm page size var old = Module['buffer']; var oldSize = old.byteLength; if (Module["usingWasm"]) { diff --git a/system/lib/libc/musl/arch/emscripten/bits/limits.h b/system/lib/libc/musl/arch/emscripten/bits/limits.h index 65a3dd6477bce..3fd933110e657 100644 --- a/system/lib/libc/musl/arch/emscripten/bits/limits.h +++ b/system/lib/libc/musl/arch/emscripten/bits/limits.h @@ -1,6 +1,6 @@ #if defined(_POSIX_SOURCE) || defined(_POSIX_C_SOURCE) \ || defined(_XOPEN_SOURCE) || defined(_GNU_SOURCE) || defined(_BSD_SOURCE) -#define PAGE_SIZE 4096 +#define PAGE_SIZE 16384 #define LONG_BIT 32 #endif diff --git a/tools/shared.py b/tools/shared.py index d893d08a9c477..07ce4649a14a0 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -2512,7 +2512,7 @@ def read_and_preprocess(filename): # worker in -s ASMFS=1 mode. def make_fetch_worker(source_file, output_file): src = open(source_file, 'r').read() - funcs_to_import = ['alignMemoryPage', 'getTotalMemory', 'stringToUTF8', 'intArrayFromString', 'lengthBytesUTF8', 'stringToUTF8Array', '_emscripten_is_main_runtime_thread', '_emscripten_futex_wait'] + funcs_to_import = ['getTotalMemory', 'stringToUTF8', 'intArrayFromString', 'lengthBytesUTF8', 'stringToUTF8Array', '_emscripten_is_main_runtime_thread', '_emscripten_futex_wait'] asm_funcs_to_import = ['_malloc', '_free', '_sbrk', '_pthread_mutex_lock', '_pthread_mutex_unlock'] function_prologue = '''this.onerror = function(e) { console.error(e); From 8ee6142ee7e6a3d819d00a1961e0c5706964eb2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Thu, 16 Feb 2017 15:39:12 +0200 Subject: [PATCH 24/72] Remove redundant memory size tests in preamble.js. Add a test. --- emcc.py | 14 +++++++------- src/preamble.js | 25 +++++++++++-------------- tests/test_other.py | 11 +++++++++++ 3 files changed, 29 insertions(+), 21 deletions(-) diff --git a/emcc.py b/emcc.py index 2a3e138670750..c10166bd1aa63 100755 --- a/emcc.py +++ b/emcc.py @@ -1072,13 +1072,6 @@ def check(input_file): logging.error('Compiler settings are incompatible with fastcomp. You can fall back to the older compiler core, although that is not recommended, see http://kripken.github.io/emscripten-site/docs/building_from_source/LLVM-Backend.html') raise e - assert shared.Settings.TOTAL_MEMORY >= 16*1024*1024, 'TOTAL_MEMORY must be at least 16MB, was ' + str(shared.Settings.TOTAL_MEMORY) - if shared.Settings.WASM or shared.Settings.BINARYEN: - assert shared.Settings.TOTAL_MEMORY % 65536 == 0, 'For wasm, TOTAL_MEMORY must be a multiple of 64KB, was ' + str(shared.Settings.TOTAL_MEMORY) - else: - assert shared.Settings.TOTAL_MEMORY % (16*1024*1024) == 0, 'For asm.js, TOTAL_MEMORY must be a multiple of 16MB, was ' + str(shared.Settings.TOTAL_MEMORY) - assert shared.Settings.TOTAL_MEMORY >= shared.Settings.TOTAL_STACK, 'TOTAL_MEMORY must be larger than TOTAL_STACK, was ' + str(shared.Settings.TOTAL_MEMORY) + ' (TOTAL_STACK=' + str(shared.Settings.TOTAL_STACK) + ')' - assert not shared.Settings.PGO, 'cannot run PGO in ASM_JS mode' if shared.Settings.SAFE_HEAP: @@ -1254,6 +1247,13 @@ def check(input_file): if shared.Settings.WASM: shared.Settings.BINARYEN = 1 # these are synonyms + assert shared.Settings.TOTAL_MEMORY >= 16*1024*1024, 'TOTAL_MEMORY must be at least 16MB, was ' + str(shared.Settings.TOTAL_MEMORY) + if shared.Settings.BINARYEN: + assert shared.Settings.TOTAL_MEMORY % 65536 == 0, 'For wasm, TOTAL_MEMORY must be a multiple of 64KB, was ' + str(shared.Settings.TOTAL_MEMORY) + else: + assert shared.Settings.TOTAL_MEMORY % (16*1024*1024) == 0, 'For asm.js, TOTAL_MEMORY must be a multiple of 16MB, was ' + str(shared.Settings.TOTAL_MEMORY) + assert shared.Settings.TOTAL_MEMORY >= shared.Settings.TOTAL_STACK, 'TOTAL_MEMORY must be larger than TOTAL_STACK, was ' + str(shared.Settings.TOTAL_MEMORY) + ' (TOTAL_STACK=' + str(shared.Settings.TOTAL_STACK) + ')' + if shared.Settings.WASM_BACKEND: js_opts = None shared.Settings.BINARYEN = 1 diff --git a/src/preamble.js b/src/preamble.js index 7588d42770412..75e276214d987 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -931,6 +931,9 @@ function stackTrace() { // Memory management var PAGE_SIZE = 16384; +var WASM_PAGE_SIZE = 65536; +var ASMJS_PAGE_SIZE = 16777216; +var MIN_TOTAL_MEMORY = 16777216; function alignUp(x, multiple) { if (x % multiple > 0) { @@ -1051,7 +1054,7 @@ function enlargeMemory() { _emscripten_trace_report_memory_layout(); #endif - var PAGE_MULTIPLE = Module["usingWasm"] ? 65536 : 16777216; // In wasm, heap size must be a multiple of 64KB. In asm.js, they need to be multiples of 16MB. + var PAGE_MULTIPLE = Module["usingWasm"] ? WASM_PAGE_SIZE : ASMJS_PAGE_SIZE; // In wasm, heap size must be a multiple of 64KB. In asm.js, they need to be multiples of 16MB. var LIMIT = 2147483648 - PAGE_MULTIPLE; // We can do one page short of 2GB as theoretical maximum. if (HEAP32[DYNAMICTOP_PTR>>2] > LIMIT) { @@ -1062,7 +1065,7 @@ function enlargeMemory() { } var OLD_TOTAL_MEMORY = TOTAL_MEMORY; - TOTAL_MEMORY = Math.max(TOTAL_MEMORY, 16 * 1024 * 1024); // So the loop below will not be infinite, and minimum asm.js memory size is 16MB. + TOTAL_MEMORY = Math.max(TOTAL_MEMORY, MIN_TOTAL_MEMORY); // So the loop below will not be infinite, and minimum asm.js memory size is 16MB. while (TOTAL_MEMORY < HEAP32[DYNAMICTOP_PTR>>2]) { // Keep incrementing the heap size as long as it's less than what is requested. if (TOTAL_MEMORY <= 536870912) { @@ -1125,12 +1128,6 @@ try { var TOTAL_STACK = Module['TOTAL_STACK'] || {{{ TOTAL_STACK }}}; var TOTAL_MEMORY = Module['TOTAL_MEMORY'] || {{{ TOTAL_MEMORY }}}; - -if (TOTAL_MEMORY < 16*1024*1024) Module.printErr('TOTAL_MEMORY should be at least 16MB, was ' + TOTAL_MEMORY + '!'); -if (TOTAL_MEMORY % 65536 != 0) Module.printErr('For wasm, TOTAL_MEMORY should be a multiple of 64KB, was ' + TOTAL_MEMORY + '!'); -#if !BINARYEN -if (TOTAL_MEMORY % (16*1024*1024) != 0) Module.printErr('For asm.js, TOTAL_MEMORY should be a multiple of 16MB, was ' + TOTAL_MEMORY + '!'); -#endif if (TOTAL_MEMORY < TOTAL_STACK) Module.printErr('TOTAL_MEMORY should be larger than TOTAL_STACK, was ' + TOTAL_MEMORY + '! (TOTAL_STACK=' + TOTAL_STACK + ')'); // Initialize the runtime's memory @@ -1218,19 +1215,19 @@ if (Module['buffer']) { #if BINARYEN if (typeof WebAssembly === 'object' && typeof WebAssembly.Memory === 'function') { #if ASSERTIONS - assert(TOTAL_MEMORY % 65536 === 0); + assert(TOTAL_MEMORY % WASM_PAGE_SIZE === 0); #endif #if ALLOW_MEMORY_GROWTH #if BINARYEN_MEM_MAX #if ASSERTIONS - assert({{{ BINARYEN_MEM_MAX }}} % 65536 == 0); + assert({{{ BINARYEN_MEM_MAX }}} % WASM_PAGE_SIZE == 0); #endif - Module['wasmMemory'] = new WebAssembly.Memory({ initial: TOTAL_MEMORY / 65536, maximum: {{{ BINARYEN_MEM_MAX }}} / 65536 }); + Module['wasmMemory'] = new WebAssembly.Memory({ initial: TOTAL_MEMORY / WASM_PAGE_SIZE, maximum: {{{ BINARYEN_MEM_MAX }}} / WASM_PAGE_SIZE }); #else - Module['wasmMemory'] = new WebAssembly.Memory({ initial: TOTAL_MEMORY / 65536 }); + Module['wasmMemory'] = new WebAssembly.Memory({ initial: TOTAL_MEMORY / WASM_PAGE_SIZE }); #endif #else - Module['wasmMemory'] = new WebAssembly.Memory({ initial: TOTAL_MEMORY / 65536, maximum: TOTAL_MEMORY / 65536 }); + Module['wasmMemory'] = new WebAssembly.Memory({ initial: TOTAL_MEMORY / WASM_PAGE_SIZE, maximum: TOTAL_MEMORY / WASM_PAGE_SIZE }); #endif buffer = Module['wasmMemory'].buffer; } else @@ -2309,7 +2306,7 @@ function integrateWasmJS(Module) { // Memory growth integration code Module['reallocBuffer'] = function(size) { - var PAGE_MULTIPLE = Module["usingWasm"] ? 65536 : 16777216; // In wasm, heap size must be a multiple of 64KB. In asm.js, they need to be multiples of 16MB. + var PAGE_MULTIPLE = Module["usingWasm"] ? WASM_PAGE_SIZE : ASMJS_PAGE_SIZE; // In wasm, heap size must be a multiple of 64KB. In asm.js, they need to be multiples of 16MB. size = alignUp(size, PAGE_MULTIPLE); // round up to wasm page size var old = Module['buffer']; var oldSize = old.byteLength; diff --git a/tests/test_other.py b/tests/test_other.py index 18776c254e71f..8d7b45e089670 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -7292,6 +7292,17 @@ def test_binaryen_mem(self): else: assert parts[6] == str(expect_max) + def test_binaryen_invalid_mem(self): + ret = subprocess.check_call([PYTHON, EMCC, path_from_root('tests', 'hello_world.c'), '-s', 'WASM=1', '-s', 'TOTAL_MEMORY=33MB']) + + ret = subprocess.Popen([PYTHON, EMCC, path_from_root('tests', 'hello_world.c'), '-s', 'WASM=1', '-s', 'TOTAL_MEMORY=32MB+1'], stderr=subprocess.PIPE).communicate()[1] + assert 'TOTAL_MEMORY must be a multiple of 64KB' in ret, ret + + ret = subprocess.check_call([PYTHON, EMCC, path_from_root('tests', 'hello_world.c'), '-s', 'TOTAL_MEMORY=32MB']) + + ret = subprocess.Popen([PYTHON, EMCC, path_from_root('tests', 'hello_world.c'), '-s', 'TOTAL_MEMORY=33MB'], stderr=subprocess.PIPE).communicate()[1] + assert 'TOTAL_MEMORY must be a multiple of 16MB' in ret, ret + def test_binaryen_ctors(self): if SPIDERMONKEY_ENGINE not in JS_ENGINES: return self.skip('cannot run without spidermonkey') # ctor order must be identical to js builds, deterministically From 74acd7c2992fc4d86a5e1e791c41a9694bdc3cda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Fri, 17 Feb 2017 12:43:07 +0200 Subject: [PATCH 25/72] Add helper expand_byte_size_suffixes() to expand arithmetic and KB/MB suffixes to integer. --- emcc.py | 2 +- tools/shared.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/emcc.py b/emcc.py index c10166bd1aa63..99fffe090c8af 100755 --- a/emcc.py +++ b/emcc.py @@ -1024,7 +1024,7 @@ def check(input_file): # In those settings fields that represent amount of memory, translate suffixes to multiples of 1024. if key in ['TOTAL_STACK', 'TOTAL_MEMORY', 'GL_MAX_TEMP_BUFFER_SIZE', 'SPLIT_MEMORY', 'BINARYEN_MEM_MAX']: - value = value.lower().replace('tb', '*1024*1024*1024*1024').replace('gb', '*1024*1024*1024').replace('mb', '*1024*1024').replace('kb', '*1024').replace('b', '') + value = str(shared.expand_byte_size_suffixes(value)) original_exported_response = False diff --git a/tools/shared.py b/tools/shared.py index 07ce4649a14a0..46091e3d4533e 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -1072,6 +1072,11 @@ def expand_response(data): return json.loads(open(data[1:]).read()) return data +# Given a string with arithmetic and/or KB/MB size suffixes, such as "1024*1024" or "32MB", computes how many bytes that is and returns it as an integer. +def expand_byte_size_suffixes(value): + value = value.lower().replace('tb', '*1024*1024*1024*1024').replace('gb', '*1024*1024*1024').replace('mb', '*1024*1024').replace('kb', '*1024').replace('b', '') + return eval(value) + # Settings. A global singleton. Not pretty, but nicer than passing |, settings| everywhere class Settings2(type): From 11093c6599e81e650d5fc9dc5814a047df6ad5b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Fri, 17 Feb 2017 12:53:18 +0200 Subject: [PATCH 26/72] Import alignUp to fetch worker. --- tools/shared.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/shared.py b/tools/shared.py index 46091e3d4533e..bca985b0b9799 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -2517,7 +2517,7 @@ def read_and_preprocess(filename): # worker in -s ASMFS=1 mode. def make_fetch_worker(source_file, output_file): src = open(source_file, 'r').read() - funcs_to_import = ['getTotalMemory', 'stringToUTF8', 'intArrayFromString', 'lengthBytesUTF8', 'stringToUTF8Array', '_emscripten_is_main_runtime_thread', '_emscripten_futex_wait'] + funcs_to_import = ['alignUp', 'getTotalMemory', 'stringToUTF8', 'intArrayFromString', 'lengthBytesUTF8', 'stringToUTF8Array', '_emscripten_is_main_runtime_thread', '_emscripten_futex_wait'] asm_funcs_to_import = ['_malloc', '_free', '_sbrk', '_pthread_mutex_lock', '_pthread_mutex_unlock'] function_prologue = '''this.onerror = function(e) { console.error(e); From d771580f3a3c6f26ad037651bd95dcdb86d726c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Sun, 19 Feb 2017 14:40:26 +0200 Subject: [PATCH 27/72] Add check for BINARYEN_MEM_MAX being a multiple of 64KB at compile time. --- emcc.py | 1 + tests/test_other.py | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/emcc.py b/emcc.py index 99fffe090c8af..3a76bf2fe2e0a 100755 --- a/emcc.py +++ b/emcc.py @@ -1253,6 +1253,7 @@ def check(input_file): else: assert shared.Settings.TOTAL_MEMORY % (16*1024*1024) == 0, 'For asm.js, TOTAL_MEMORY must be a multiple of 16MB, was ' + str(shared.Settings.TOTAL_MEMORY) assert shared.Settings.TOTAL_MEMORY >= shared.Settings.TOTAL_STACK, 'TOTAL_MEMORY must be larger than TOTAL_STACK, was ' + str(shared.Settings.TOTAL_MEMORY) + ' (TOTAL_STACK=' + str(shared.Settings.TOTAL_STACK) + ')' + assert shared.Settings.BINARYEN_MEM_MAX == -1 or shared.Settings.BINARYEN_MEM_MAX % 65536 == 0, 'BINARYEN_MEM_MAX must be a multiple of 64KB, was ' + str(shared.Settings.BINARYEN_MEM_MAX) if shared.Settings.WASM_BACKEND: js_opts = None diff --git a/tests/test_other.py b/tests/test_other.py index 8d7b45e089670..484070ddff82d 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -7303,6 +7303,12 @@ def test_binaryen_invalid_mem(self): ret = subprocess.Popen([PYTHON, EMCC, path_from_root('tests', 'hello_world.c'), '-s', 'TOTAL_MEMORY=33MB'], stderr=subprocess.PIPE).communicate()[1] assert 'TOTAL_MEMORY must be a multiple of 16MB' in ret, ret + ret = subprocess.Popen([PYTHON, EMCC, path_from_root('tests', 'hello_world.c'), '-s', 'BINARYEN_MEM_MAX=33MB'], stderr=subprocess.PIPE).communicate()[1] + assert 'BINARYEN_MEM_MAX must be a multiple of 64KB' not in ret, ret + + ret = subprocess.Popen([PYTHON, EMCC, path_from_root('tests', 'hello_world.c'), '-s', 'BINARYEN_MEM_MAX=33MB+1'], stderr=subprocess.PIPE).communicate()[1] + assert 'BINARYEN_MEM_MAX must be a multiple of 64KB' in ret, ret + def test_binaryen_ctors(self): if SPIDERMONKEY_ENGINE not in JS_ENGINES: return self.skip('cannot run without spidermonkey') # ctor order must be identical to js builds, deterministically From 30867e8291e0a142fccddff94e3c8a0f023c3efb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Sun, 19 Feb 2017 15:23:52 +0200 Subject: [PATCH 28/72] Fix tests to allocate a valid amount of total memory. --- tests/test_browser.py | 14 +++++++------- tests/test_core.py | 2 +- tests/test_other.py | 12 ++++++------ 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/test_browser.py b/tests/test_browser.py index 017338d82f064..74abc247d636f 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -1246,7 +1246,7 @@ def test_idbstore_sync(self): def test_idbstore_sync_worker(self): secret = str(time.time()) self.clear() - self.btest(path_from_root('tests', 'idbstore_sync_worker.c'), '6', force_c=True, args=['-lidbstore.js', '-DSECRET=\"' + secret + '\"', '-s', 'EMTERPRETIFY=1', '-s', 'EMTERPRETIFY_ASYNC=1', '--memory-init-file', '1', '-O3', '-g2', '--proxy-to-worker', '-s', 'TOTAL_MEMORY=75000000']) + self.btest(path_from_root('tests', 'idbstore_sync_worker.c'), '6', force_c=True, args=['-lidbstore.js', '-DSECRET=\"' + secret + '\"', '-s', 'EMTERPRETIFY=1', '-s', 'EMTERPRETIFY_ASYNC=1', '--memory-init-file', '1', '-O3', '-g2', '--proxy-to-worker', '-s', 'TOTAL_MEMORY=80MB']) def test_force_exit(self): self.btest('force_exit.c', force_c=True, expected='17') @@ -1783,7 +1783,7 @@ def test_sdl_canvas_palette_2(self): self.btest('sdl_canvas_palette_2.c', reference='sdl_canvas_palette_b.png', args=['--pre-js', 'pre.js', '--pre-js', 'args-b.js', '-lSDL', '-lGL']) def test_sdl_alloctext(self): - self.btest('sdl_alloctext.c', expected='1', args=['-O2', '-s', 'TOTAL_MEMORY=' + str(1024*1024*8), '-lSDL', '-lGL']) + self.btest('sdl_alloctext.c', expected='1', args=['-O2', '-s', 'TOTAL_MEMORY=8MB', '-lSDL', '-lGL']) def test_sdl_surface_refcount(self): self.btest('sdl_surface_refcount.c', args=['-lSDL'], expected='1') @@ -2926,7 +2926,7 @@ def test_dynamic_link_glemu(self): def test_memory_growth_during_startup(self): open('data.dat', 'w').write('X' * (30*1024*1024)) - self.btest('browser_test_hello_world.c', '0', args=['-s', 'ASSERTIONS=1', '-s', 'ALLOW_MEMORY_GROWTH=1', '-s', 'TOTAL_MEMORY=10000', '-s', 'TOTAL_STACK=5000', '--preload-file', 'data.dat']) + self.btest('browser_test_hello_world.c', '0', args=['-s', 'ASSERTIONS=1', '-s', 'ALLOW_MEMORY_GROWTH=1', '-s', 'TOTAL_MEMORY=16MB', '-s', 'TOTAL_STACK=5000', '--preload-file', 'data.dat']) # pthreads tests @@ -3030,7 +3030,7 @@ def test_pthread_malloc(self): # Stress test pthreads allocating memory that will call to sbrk(), and main thread has to free up the data. def test_pthread_malloc_free(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_malloc_free.cpp'), expected='0', args=['-O3', '-s', 'USE_PTHREADS=2', '--separate-asm', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'TOTAL_MEMORY=268435456'], timeout=30) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_malloc_free.cpp'), expected='0', args=['-O3', '-s', 'USE_PTHREADS=2', '--separate-asm', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'TOTAL_MEMORY=256MB'], timeout=30) # Test that the pthread_barrier API works ok. def test_pthread_barrier(self): @@ -3141,7 +3141,7 @@ def test_pthread_sbrk(self): print 'aborting malloc=' + str(aborting_malloc) # With aborting malloc = 1, test allocating memory in threads # With aborting malloc = 0, allocate so much memory in threads that some of the allocations fail. - self.btest(path_from_root('tests', 'pthread', 'test_pthread_sbrk.cpp'), expected='0', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '--separate-asm', '-s', 'ABORTING_MALLOC=' + str(aborting_malloc), '-DABORTING_MALLOC=' + str(aborting_malloc), '-s', 'TOTAL_MEMORY=134217728'], timeout=30) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_sbrk.cpp'), expected='0', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '--separate-asm', '-s', 'ABORTING_MALLOC=' + str(aborting_malloc), '-DABORTING_MALLOC=' + str(aborting_malloc), '-s', 'TOTAL_MEMORY=128MB'], timeout=30) # Test that -s ABORTING_MALLOC=0 works in both pthreads and non-pthreads builds. (sbrk fails gracefully) def test_pthread_gauge_available_memory(self): @@ -3277,7 +3277,7 @@ def test_in_flight_memfile_request(self): def test_split_memory_large_file(self): size = 2*1024*1024 open('huge.dat', 'w').write(''.join([chr((x*x)&255) for x in range(size*2)])) # larger than a memory chunk - self.btest('split_memory_large_file.cpp', expected='1', args=['-s', 'SPLIT_MEMORY=' + str(size), '-s', 'TOTAL_MEMORY=100000000', '-s', 'TOTAL_STACK=10240', '--preload-file', 'huge.dat'], timeout=60) + self.btest('split_memory_large_file.cpp', expected='1', args=['-s', 'SPLIT_MEMORY=' + str(size), '-s', 'TOTAL_MEMORY=128MB', '-s', 'TOTAL_STACK=10240', '--preload-file', 'huge.dat'], timeout=60) def test_binaryen(self): self.btest('browser_test_hello_world.c', expected='0', args=['-s', 'BINARYEN=1', '-s', 'BINARYEN_METHOD="interpret-binary"']) @@ -3338,7 +3338,7 @@ def test_webgl_offscreen_canvas_in_mainthread_after_pthread(self): # Tests the feature that shell html page can preallocate the typed array and place it to Module.buffer before loading the script page. # In this build mode, the -s TOTAL_MEMORY=xxx option will be ignored. def test_preallocated_heap(self): - self.btest('test_preallocated_heap.cpp', expected='1', args=['-s', 'TOTAL_MEMORY='+str(16*1024*1024), '-s', 'ABORTING_MALLOC=0', '--shell-file', path_from_root('tests', 'test_preallocated_heap_shell.html')]) + self.btest('test_preallocated_heap.cpp', expected='1', args=['-s', 'TOTAL_MEMORY=16MB', '-s', 'ABORTING_MALLOC=0', '--shell-file', path_from_root('tests', 'test_preallocated_heap_shell.html')]) # Tests emscripten_fetch() usage to XHR data directly to memory without persisting results to IndexedDB. def test_fetch_to_memory(self): diff --git a/tests/test_core.py b/tests/test_core.py index 1779a4a1df48d..f62ca8f23abed 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -4857,7 +4857,7 @@ def test_dlmalloc(self): # emcc should build in dlmalloc automatically, and do all the sign correction etc. for it try_delete(os.path.join(self.get_dir(), 'src.cpp.o.js')) - output = Popen([PYTHON, EMCC, path_from_root('tests', 'dlmalloc_test.c'), '-s', 'TOTAL_MEMORY=' + str(128*1024*1024), + output = Popen([PYTHON, EMCC, path_from_root('tests', 'dlmalloc_test.c'), '-s', 'TOTAL_MEMORY=128MB', '-o', os.path.join(self.get_dir(), 'src.cpp.o.js')], stdout=PIPE, stderr=self.stderr_redirect).communicate() self.do_run('x', '*1,0*', ['200', '1'], no_build=True) diff --git a/tests/test_other.py b/tests/test_other.py index 484070ddff82d..9206d4fb49ba8 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -6311,7 +6311,7 @@ def test_split_memory(self): # make sure multiple split memory chunks get used ''') for opts in [0, 1, 2]: print opts - check_execute([PYTHON, EMCC, 'src.c', '-s', 'SPLIT_MEMORY=8388608', '-s', 'TOTAL_MEMORY=50000000', '-O' + str(opts)]) + check_execute([PYTHON, EMCC, 'src.c', '-s', 'SPLIT_MEMORY=8388608', '-s', 'TOTAL_MEMORY=64MB', '-O' + str(opts)]) self.assertContained('success.', run_js('a.out.js')) def test_split_memory_2(self): # make allocation starts in the first chunk, and moves forward properly @@ -6355,7 +6355,7 @@ def test_split_memory_2(self): # make allocation starts in the first chunk, and ''') for opts in [0, 1, 2]: print opts - check_execute([PYTHON, EMCC, 'src.c', '-s', 'SPLIT_MEMORY=8388608', '-s', 'TOTAL_MEMORY=50000000', '-O' + str(opts)]) + check_execute([PYTHON, EMCC, 'src.c', '-s', 'SPLIT_MEMORY=8388608', '-s', 'TOTAL_MEMORY=64MB', '-O' + str(opts)]) self.assertContained('success.', run_js('a.out.js')) def test_split_memory_sbrk(self): @@ -6393,7 +6393,7 @@ def test_split_memory_sbrk(self): ''') for opts in [0, 1, 2]: print opts - check_execute([PYTHON, EMCC, 'src.c', '-s', 'SPLIT_MEMORY=8388608', '-s', 'TOTAL_MEMORY=50000000', '-O' + str(opts)]) + check_execute([PYTHON, EMCC, 'src.c', '-s', 'SPLIT_MEMORY=8388608', '-s', 'TOTAL_MEMORY=64MB', '-O' + str(opts)]) self.assertContained('success.', run_js('a.out.js')) def test_split_memory_faking(self): # fake HEAP8 etc. objects have some faked fake method. they are fake @@ -6492,7 +6492,7 @@ def test_split_memory_faking(self): # fake HEAP8 etc. objects have some faked fa ''') for opts in [0, 1, 2]: print opts - check_execute([PYTHON, EMCC, 'src.c', '-s', 'SPLIT_MEMORY=8388608', '-s', 'TOTAL_MEMORY=50000000', '-O' + str(opts), '-s', 'ASSERTIONS=1']) + check_execute([PYTHON, EMCC, 'src.c', '-s', 'SPLIT_MEMORY=8388608', '-s', 'TOTAL_MEMORY=64MB', '-O' + str(opts), '-s', 'ASSERTIONS=1']) self.assertContained('success.', run_js('a.out.js', stderr=PIPE, assert_returncode=None)) def test_split_memory_release(self): @@ -6538,7 +6538,7 @@ def test_split_memory_release(self): ''') for opts in [0, 1, 2]: print opts - check_execute([PYTHON, EMCC, 'src.c', '-s', 'SPLIT_MEMORY=8388608', '-s', 'TOTAL_MEMORY=50000000', '-O' + str(opts)]) + check_execute([PYTHON, EMCC, 'src.c', '-s', 'SPLIT_MEMORY=8388608', '-s', 'TOTAL_MEMORY=64MB', '-O' + str(opts)]) self.assertContained('success.', run_js('a.out.js')) def test_split_memory_use_existing(self): @@ -6579,7 +6579,7 @@ def test_split_memory_use_existing(self): ''') for opts in [0, 1, 2]: print opts - check_execute([PYTHON, EMCC, 'src.c', '-s', 'SPLIT_MEMORY=8388608', '-s', 'TOTAL_MEMORY=50000000', '-O' + str(opts)]) + check_execute([PYTHON, EMCC, 'src.c', '-s', 'SPLIT_MEMORY=8388608', '-s', 'TOTAL_MEMORY=64MB', '-O' + str(opts)]) self.assertContained('success.', run_js('a.out.js')) def test_sixtyfour_bit_return_value(self): From 94cf240c253fc0cfa4467e73e2e4d6192b6a1941 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Sun, 12 Feb 2017 20:51:57 +0200 Subject: [PATCH 29/72] Optimize alBufferData() by tightening inner loops. Gives ~5x faster performance. Also fix an issue if page is on the background and a next loop of an audio source is due to start, it is possible that the audio buffer has not yet been set to play. In that case, alSourcef() does not need to adjust existing buffers when changing pitch. --- src/library_openal.js | 120 +++++++++++++++++++++++------------------- 1 file changed, 65 insertions(+), 55 deletions(-) diff --git a/src/library_openal.js b/src/library_openal.js index 6d71f723f5b20..9d79939b05366 100644 --- a/src/library_openal.js +++ b/src/library_openal.js @@ -573,6 +573,7 @@ var LibraryOpenAL = { if (src.state === 0x1012 /* AL_PLAYING */) { // update currently playing entry var entry = src.queue[src.buffersPlayed]; + if (!entry || !entry.src) return; // It is possible that AL.updateSources() has not yet fed the next buffer, if so, skip. var currentTime = AL.currentContext.ctx.currentTime; var oldrate = entry.src.playbackRate.value; var offset = currentTime - src.bufferPosition; @@ -838,65 +839,74 @@ var LibraryOpenAL = { #endif return; } - var channels, bytes; - switch (format) { - case 0x1100 /* AL_FORMAT_MONO8 */: - bytes = 1; - channels = 1; - break; - case 0x1101 /* AL_FORMAT_MONO16 */: - bytes = 2; - channels = 1; - break; - case 0x1102 /* AL_FORMAT_STEREO8 */: - bytes = 1; - channels = 2; - break; - case 0x1103 /* AL_FORMAT_STEREO16 */: - bytes = 2; - channels = 2; - break; - case 0x10010 /* AL_FORMAT_MONO_FLOAT32 */: - bytes = 4; - channels = 1; - break; - case 0x10011 /* AL_FORMAT_STEREO_FLOAT32 */: - bytes = 4; - channels = 2; - break; - default: + + try { + switch (format) { + case 0x1100 /* AL_FORMAT_MONO8 */: + var buf = AL.currentContext.ctx.createBuffer(1, size, freq); + buf.bytesPerSample = 1; + var channel0 = buf.getChannelData(0); + for (var i = 0; i < size; ++i) channel0[i] = HEAPU8[data++] * 0.0078125 /* 1/128 */ - 1.0; + break; + case 0x1101 /* AL_FORMAT_MONO16 */: + var buf = AL.currentContext.ctx.createBuffer(1, size>>1, freq); + buf.bytesPerSample = 2; + var channel0 = buf.getChannelData(0); + data >>= 1; + for (var i = 0; i < size>>1; ++i) channel0[i] = HEAP16[data++] * 0.000030517578125 /* 1/32768 */; + break; + case 0x1102 /* AL_FORMAT_STEREO8 */: + var buf = AL.currentContext.ctx.createBuffer(2, size>>1, freq); + buf.bytesPerSample = 1; + var channel0 = buf.getChannelData(0); + var channel1 = buf.getChannelData(1); + for (var i = 0; i < size>>1; ++i) { + channel0[i] = HEAPU8[data++] * 0.0078125 /* 1/128 */ - 1.0; + channel1[i] = HEAPU8[data++] * 0.0078125 /* 1/128 */ - 1.0; + } + break; + case 0x1103 /* AL_FORMAT_STEREO16 */: + var buf = AL.currentContext.ctx.createBuffer(2, size>>2, freq); + buf.bytesPerSample = 2; + var channel0 = buf.getChannelData(0); + var channel1 = buf.getChannelData(1); + data >>= 1; + for (var i = 0; i < size>>2; ++i) { + channel0[i] = HEAP16[data++] * 0.000030517578125 /* 1/32768 */; + channel1[i] = HEAP16[data++] * 0.000030517578125 /* 1/32768 */; + } + break; + case 0x10010 /* AL_FORMAT_MONO_FLOAT32 */: + var buf = AL.currentContext.ctx.createBuffer(1, size>>2, freq); + buf.bytesPerSample = 4; + var channel0 = buf.getChannelData(0); + data >>= 2; + for (var i = 0; i < size>>2; ++i) channel0[i] = HEAPF32[data++]; + break; + case 0x10011 /* AL_FORMAT_STEREO_FLOAT32 */: + var buf = AL.currentContext.ctx.createBuffer(2, size>>3, freq); + buf.bytesPerSample = 4; + var channel0 = buf.getChannelData(0); + var channel1 = buf.getChannelData(1); + data >>= 2; + for (var i = 0; i < size>>2; ++i) { + channel0[i] = HEAPF32[data++]; + channel1[i] = HEAPF32[data++]; + } + break; + default: #if OPENAL_DEBUG - console.error("alBufferData called with invalid format " + format); + console.error("alBufferData called with invalid format " + format); #endif - return; - } - try { - AL.currentContext.buf[buffer - 1] = AL.currentContext.ctx.createBuffer(channels, size / (bytes * channels), freq); - AL.currentContext.buf[buffer - 1].bytesPerSample = bytes; + AL.currentContext.err = 0xA003 /* AL_INVALID_VALUE */; + break; + } + AL.currentContext.buf[buffer - 1] = buf; } catch (e) { +#if OPENAL_DEBUG + console.error("alBufferData upload failed with an exception " + e); +#endif AL.currentContext.err = 0xA003 /* AL_INVALID_VALUE */; - return; - } - var buf = new Array(channels); - for (var i = 0; i < channels; ++i) { - buf[i] = AL.currentContext.buf[buffer - 1].getChannelData(i); - } - for (var i = 0; i < size / (bytes * channels); ++i) { - for (var j = 0; j < channels; ++j) { - switch (bytes) { - case 1: - var val = {{{ makeGetValue('data', 'i*channels+j', 'i8') }}} & 0xff; // unsigned - buf[j][i] = -1.0 + val * (2/256); - break; - case 2: - var val = {{{ makeGetValue('data', '2*(i*channels+j)', 'i16') }}}; - buf[j][i] = val/32768; - break; - case 4: - buf[j][i] = {{{ makeGetValue('data', '4*(i*channels+j)', 'float') }}}; - break; - } - } } }, From 55e0a58059db6dff128b18e3364fbe2700e179f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Mon, 13 Feb 2017 15:37:11 +0200 Subject: [PATCH 30/72] When outputting wasm only, don't generate .asm.js. Also, don't overwrite a pre-existing .asm.js file in that directory if one wants to deploy both asm.js and wasm. --- emcc.py | 7 +++++++ tests/test_other.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/emcc.py b/emcc.py index 3a76bf2fe2e0a..0faf777ff5682 100755 --- a/emcc.py +++ b/emcc.py @@ -1247,6 +1247,9 @@ def check(input_file): if shared.Settings.WASM: shared.Settings.BINARYEN = 1 # these are synonyms + if shared.Building.is_wasm_only(): + asm_target = asm_target.replace('.asm.js', '.temp.asm.js') + assert shared.Settings.TOTAL_MEMORY >= 16*1024*1024, 'TOTAL_MEMORY must be at least 16MB, was ' + str(shared.Settings.TOTAL_MEMORY) if shared.Settings.BINARYEN: assert shared.Settings.TOTAL_MEMORY % 65536 == 0, 'For wasm, TOTAL_MEMORY must be a multiple of 64KB, was ' + str(shared.Settings.TOTAL_MEMORY) @@ -2159,6 +2162,10 @@ def do_minify(): # minifies the code. this is also when we do certain optimizati logging.debug('asm2wasm (asm.js => WebAssembly): ' + ' '.join(cmd)) TimeLogger.update() subprocess.check_call(cmd) + # When only targeting wasm, the .asm.js file is not executable, so is treated as an intermediate build file that can be cleaned up. + if shared.Building.is_wasm_only(): + shared.try_delete(asm_target) + if not target_binary: cmd = [os.path.join(binaryen_bin, 'wasm-as'), wasm_text_target, '-o', wasm_binary_target] if debug_level >= 2 or profiling_funcs: diff --git a/tests/test_other.py b/tests/test_other.py index 9206d4fb49ba8..017ff62c3d193 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -16,6 +16,16 @@ def multiprocess_task(c_file, cache_dir_name): print '------' sys.exit(1 if 'generating system library: libc.bc' in output else 0) +class temp_directory: + def __enter__(self): + self.directory = tempfile.mkdtemp(prefix='emsripten_temp_', dir=TEMP_DIR) + self.prev_cwd = os.getcwd() + return self.directory + + def __exit__(self, type, value, traceback): + os.chdir(self.prev_cwd) # On Windows, we can't have CWD in the directory we're deleting + try_delete(self.directory) + class clean_write_access_to_canonical_temp_dir: def clean_emcc_files_in_temp_dir(self): for x in os.listdir(CANONICAL_TEMP_DIR): @@ -7270,6 +7280,32 @@ def test_binaryen_default_method(self): self.assertContained('trying binaryen method: native-wasm', out) # native is the default assert out.count('trying binaryen method') == 1, 'must not try any other method' + def test_binaryen_asmjs_outputs(self): + # Test that an .asm.js file is outputted exactly when it is requested. + for args, output_asmjs in [ + ([], False), + (['-s', 'BINARYEN_METHOD="native-wasm"'], False), + (['-s', 'BINARYEN_METHOD="native-wasm,asmjs"'], True) + ]: + with temp_directory() as temp_dir: + cmd = [PYTHON, EMCC, path_from_root('tests', 'hello_world.c'), '-s', 'WASM=1', '-o', os.path.join(temp_dir, 'a.js')] + args + print ' '.join(cmd) + subprocess.check_call(cmd) + assert os.path.exists(os.path.join(temp_dir, 'a.asm.js')) == output_asmjs + + # Test that outputting to .wasm does not nuke an existing .asm.js file, if user wants to manually dual-deploy both to same directory. + with temp_directory() as temp_dir: + cmd = [PYTHON, EMCC, path_from_root('tests', 'hello_world.c'), '-o', os.path.join(temp_dir, 'a.js'), '--separate-asm'] + print ' '.join(cmd) + subprocess.check_call(cmd) + assert os.path.exists(os.path.join(temp_dir, 'a.asm.js')) + + cmd = [PYTHON, EMCC, path_from_root('tests', 'hello_world.c'), '-o', os.path.join(temp_dir, 'a.js'), '-s', 'WASM=1'] + print ' '.join(cmd) + subprocess.check_call(cmd) + assert os.path.exists(os.path.join(temp_dir, 'a.asm.js')) + assert os.path.exists(os.path.join(temp_dir, 'a.wasm')) + def test_binaryen_mem(self): for args, expect_initial, expect_max in [ (['-s', 'TOTAL_MEMORY=20971520'], 320, 320), From 6acd3a612f10b7aec105078a0c5b40bbcf65fba2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Tue, 14 Feb 2017 13:54:24 +0200 Subject: [PATCH 31/72] use temp_files.note() to clean up the temporary .asm.js file. --- emcc.py | 6 +++--- tests/test_other.py | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/emcc.py b/emcc.py index 0faf777ff5682..6690c3fce237c 100755 --- a/emcc.py +++ b/emcc.py @@ -1247,8 +1247,11 @@ def check(input_file): if shared.Settings.WASM: shared.Settings.BINARYEN = 1 # these are synonyms + # When only targeting wasm, the .asm.js file is not executable, so is treated as an intermediate build file that can be cleaned up. if shared.Building.is_wasm_only(): asm_target = asm_target.replace('.asm.js', '.temp.asm.js') + if not DEBUG: + misc_temp_files.note(asm_target) assert shared.Settings.TOTAL_MEMORY >= 16*1024*1024, 'TOTAL_MEMORY must be at least 16MB, was ' + str(shared.Settings.TOTAL_MEMORY) if shared.Settings.BINARYEN: @@ -2162,9 +2165,6 @@ def do_minify(): # minifies the code. this is also when we do certain optimizati logging.debug('asm2wasm (asm.js => WebAssembly): ' + ' '.join(cmd)) TimeLogger.update() subprocess.check_call(cmd) - # When only targeting wasm, the .asm.js file is not executable, so is treated as an intermediate build file that can be cleaned up. - if shared.Building.is_wasm_only(): - shared.try_delete(asm_target) if not target_binary: cmd = [os.path.join(binaryen_bin, 'wasm-as'), wasm_text_target, '-o', wasm_binary_target] diff --git a/tests/test_other.py b/tests/test_other.py index 017ff62c3d193..54690d9758a66 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -7292,6 +7292,7 @@ def test_binaryen_asmjs_outputs(self): print ' '.join(cmd) subprocess.check_call(cmd) assert os.path.exists(os.path.join(temp_dir, 'a.asm.js')) == output_asmjs + assert not os.path.exists(os.path.join(temp_dir, 'a.temp.asm.js')) # Test that outputting to .wasm does not nuke an existing .asm.js file, if user wants to manually dual-deploy both to same directory. with temp_directory() as temp_dir: @@ -7306,6 +7307,8 @@ def test_binaryen_asmjs_outputs(self): assert os.path.exists(os.path.join(temp_dir, 'a.asm.js')) assert os.path.exists(os.path.join(temp_dir, 'a.wasm')) + assert not os.path.exists(os.path.join(temp_dir, 'a.temp.asm.js')) + def test_binaryen_mem(self): for args, expect_initial, expect_max in [ (['-s', 'TOTAL_MEMORY=20971520'], 320, 320), From ee866d10b8fc13814231d3991293e02001bb1978 Mon Sep 17 00:00:00 2001 From: Charlie Birks Date: Mon, 20 Feb 2017 21:10:07 +0000 Subject: [PATCH 32/72] Add a test for SDL2 custom cursors (#4955) * Add a test for SDL2 custom cursors * Update SDL2 port to version 13 --- tests/cursor.bmp | Bin 0 -> 4234 bytes tests/sdl2_custom_cursor.c | 50 +++++++++++++++++++++++++++++++++++++ tests/test_browser.py | 4 +++ tools/ports/sdl.py | 2 +- 4 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 tests/cursor.bmp create mode 100644 tests/sdl2_custom_cursor.c diff --git a/tests/cursor.bmp b/tests/cursor.bmp new file mode 100644 index 0000000000000000000000000000000000000000..2273f79c96c75d3a47eb6743b0c04b2abe4726a5 GIT binary patch literal 4234 zcmeH}Jx;?w5QPV5kZ2H8NJ#h&Tws-kiiS&Y06IiVlZKkhU>|@xmj2A z9edZoitV8wqrB{o$Nt{VZmdjBr@K_f6yH}+QfULdE%XW3_jkUbSqHO4ujpSx2Zknut=oI>oskF4ZT2$*A2sGB_5$x* zJva8AXjXwikwPebZ60-ahQ>G>H4gJdDf-8ukD<>-E1!KlLcAw?p^pspa8Tcf@fXb2 z5+5U+jTrw!vvog(FBjJ0wnq_S{9ygs1NU!;X80o^_o6&2hdiTf`PsiIMNZ<`Zy=wq zO!v>}#BZ>3a9s~J-&%5|#XzrruEz5_qQ#NV&vB1~`UVi*gW_o){7!n>_IX3G*?SxR zQ@nsZJM?)&vX}A8^@FX?+R*H!d;@kX--sRM8?pnwF+1QV +#include +#include + +#include + +int result = 0; + +int main(int argc, char *argv[]) +{ + SDL_Window *window; + SDL_Surface *surface; + SDL_Cursor *cursor; + + if ( SDL_Init(SDL_INIT_VIDEO) != 0 ) { + printf("Unable to initialize SDL: %s\n", SDL_GetError()); + return 1; + } + + window = SDL_CreateWindow( + "sdl2_custom_cursor", + SDL_WINDOWPOS_UNDEFINED, + SDL_WINDOWPOS_UNDEFINED, + 800, + 600, + 0 + ); + + surface = SDL_LoadBMP("cursor.bmp"); + cursor = SDL_CreateColorCursor(surface, 0, 0); + + assert(cursor != NULL); + + SDL_SetCursor(cursor); + + int cursor_updated = EM_ASM_INT_V( + return Module['canvas'].style['cursor'].startsWith("url("); + ); + + assert(cursor_updated != 0); + + SDL_DestroyWindow(window); + SDL_Quit(); + result = 1; + REPORT_RESULT(); + + return 0; +} diff --git a/tests/test_browser.py b/tests/test_browser.py index 74abc247d636f..61c192f549a9b 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -2721,6 +2721,10 @@ def test_sdl2_ttf(self): message='You should see colorful "hello" and "world" in the window', timeout=30) + def test_sdl2_custom_cursor(self): + shutil.copyfile(path_from_root('tests', 'cursor.bmp'), os.path.join(self.get_dir(), 'cursor.bmp')) + self.btest('sdl2_custom_cursor.c', expected='1', args=['--preload-file', 'cursor.bmp', '-s', 'USE_SDL=2']) + def test_emterpreter_async(self): for opts in [0, 1, 2, 3]: print opts diff --git a/tools/ports/sdl.py b/tools/ports/sdl.py index 8b8584abc4a18..a52879f4689a8 100644 --- a/tools/ports/sdl.py +++ b/tools/ports/sdl.py @@ -1,6 +1,6 @@ import os, shutil, logging -TAG = 'version_12' +TAG = 'version_13' def get_with_configure(ports, settings, shared): # not currently used; no real need for configure on emscripten users' machines! if settings.USE_SDL == 2: From 91e56cbc2b7068fdb9b635e7e133cb26a1a103b9 Mon Sep 17 00:00:00 2001 From: "Alon Zakai (kripken)" Date: Mon, 20 Feb 2017 21:14:33 -0800 Subject: [PATCH 33/72] fix comment --- src/library_sdl.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/library_sdl.js b/src/library_sdl.js index cb774f0432949..f3c19947eb5fc 100644 --- a/src/library_sdl.js +++ b/src/library_sdl.js @@ -1229,7 +1229,7 @@ var LibrarySDL = { for (var joystick in SDL.lastJoystickState) { var state = SDL.getGamepad(joystick - 1); var prevState = SDL.lastJoystickState[joystick]; - // PATCHED: If joystick was removed, state returns null. + // If joystick was removed, state returns null. if (typeof state === 'undefined') return; // Check only if the timestamp has differed. // NOTE: Timestamp is not available in Firefox. From 86e321c174669b7cc5b22ffd00a867629e043175 Mon Sep 17 00:00:00 2001 From: "Alon Zakai (kripken)" Date: Tue, 21 Feb 2017 13:02:20 -0800 Subject: [PATCH 34/72] add some preamble docs #4964 --- site/source/docs/api_reference/preamble.js.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/site/source/docs/api_reference/preamble.js.rst b/site/source/docs/api_reference/preamble.js.rst index 501bcf7c5def1..1a44e41d434d5 100644 --- a/site/source/docs/api_reference/preamble.js.rst +++ b/site/source/docs/api_reference/preamble.js.rst @@ -6,6 +6,8 @@ preamble.js The JavaScript APIs in `preamble.js `_ provide programmatic access for interacting with the compiled C code, including: calling compiled C functions, accessing memory, converting pointers to JavaScript ``Strings`` and ``Strings`` to pointers (with different encodings/formats), and other convenience functions. +We call this "``preamble.js``" because Emscripten's output JS, at a high level, contains the preamble (from ``src/preamble.js``), then the compiled code, then the postamble. (In slightly more detail, the preamble contains utility functions and setup, while the postamble connects things and handles running the application.) Thus, the preamble code is included in the output JS, which means you can use the APIs described in this document without needing to do anything special. + .. note:: All functions should be called though the :ref:`Module ` object (for example: ``Module.functionName``). At optimisation ``-O2`` (and higher) function names are minified by the closure compiler, and calling them directly will fail. From b70a251643c2ab94df2a5f3b0baa3d38a1eb1a8c Mon Sep 17 00:00:00 2001 From: Bailey Hayes Date: Wed, 22 Feb 2017 18:16:03 -0500 Subject: [PATCH 35/72] add WarningManager (#4775) Use it to allow suppression of default warnings for absolute include/link paths (-Wno-absolute-paths/-Wno-warn-absolute-paths), separating asm (-Wno-separate-asm), and missing optimizations due to allowing memory growth (-Wno-almost-asm). From #4473 --- emcc.py | 27 +++++++++--------------- tests/test_other.py | 21 +++++++++++++++++++ tools/shared.py | 50 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 17 deletions(-) diff --git a/emcc.py b/emcc.py index 6690c3fce237c..b7d9aa959a2ac 100755 --- a/emcc.py +++ b/emcc.py @@ -478,18 +478,12 @@ def is_valid_abspath(path_name): def check_bad_eq(arg): assert '=' not in arg, 'Invalid parameter (do not use "=" with "--" options)' - # Defaults to not showing absolute path warnings - absolute_warning_shown = True + # Scan and strip emscripten specific cmdline warning flags + # This needs to run before other cmdline flags have been parsed, so that warnings are properly printed during arg parse + newargs = shared.WarningManager.capture_warnings(newargs) for i in range(len(newargs)): - # Scan for path warning flag in advance from other cmdline flags, so that it works even if -I or -L directives are present before this. - if newargs[i] == '-Wwarn-absolute-paths': - newargs[i] = '' - absolute_warning_shown = False - elif newargs[i] == '-Wno-warn-absolute-paths': - newargs[i] = '' - absolute_warning_shown = True - elif newargs[i] in ['-l', '-L', '-I']: + if newargs[i] in ['-l', '-L', '-I']: # Scan for individual -l/-L/-I arguments and concatenate the next arg on if there is no suffix newargs[i] += newargs[i+1] newargs[i+1] = '' @@ -703,9 +697,8 @@ def detect_fixed_language_mode(args): newargs[i] = '' elif newargs[i].startswith(('-I', '-L')): path_name = newargs[i][2:] - if not absolute_warning_shown and os.path.isabs(path_name) and not is_valid_abspath(path_name): - logging.warning('-I or -L of an absolute path "' + newargs[i] + '" encountered. If this is to a local system header/library, it may cause problems (local system files make sense for compiling natively on your system, but not necessarily to JavaScript).') # Of course an absolute path to a non-system-specific library or header is fine, and you can ignore this warning. The danger are system headers that are e.g. x86 specific and nonportable. The emscripten bundled headers are modified to be portable, local system ones are generally not - absolute_warning_shown = True + if os.path.isabs(path_name) and not is_valid_abspath(path_name): + shared.WarningManager.warn('ABSOLUTE_PATHS', '-I or -L of an absolute path "' + newargs[i] + '" encountered. If this is to a local system header/library, it may cause problems (local system files make sense for compiling natively on your system, but not necessarily to JavaScript).') # Of course an absolute path to a non-system-specific library or header is fine, and you can ignore this warning. The danger are system headers that are e.g. x86 specific and nonportable. The emscripten bundled headers are modified to be portable, local system ones are generally not elif newargs[i] == '--emrun': emrun = True newargs[i] = '' @@ -1001,7 +994,7 @@ def check(input_file): newargs = CC_ADDITIONAL_ARGS + newargs if separate_asm and final_suffix != 'html': - logging.warning("--separate-asm works best when compiling to HTML. otherwise, you must yourself load the '.asm.js' file that is emitted separately, and must do so before loading the main '.js` file") + shared.WarningManager.warn('SEPARATE_ASM') # If we are using embind and generating JS, now is the time to link in bind.cpp if bind and final_suffix in JS_CONTAINING_SUFFIXES: @@ -1092,7 +1085,7 @@ def check(input_file): logging.error('fatal: closure compiler is not configured correctly') sys.exit(1) if use_closure_compiler == 2 and shared.Settings.ASM_JS == 1: - logging.warning('not all asm.js optimizations are possible with --closure 2, disabling those - your code will be run more slowly') + shared.WarningManager.warn('ALMOST_ASM', 'not all asm.js optimizations are possible with --closure 2, disabling those - your code will be run more slowly') shared.Settings.ASM_JS = 2 if shared.Settings.MAIN_MODULE: @@ -1119,7 +1112,7 @@ def check(input_file): assert shared.Settings.SPLIT_MEMORY > shared.Settings.TOTAL_STACK, 'SPLIT_MEMORY must be at least TOTAL_STACK (stack must fit in first chunk)' assert shared.Settings.SPLIT_MEMORY & (shared.Settings.SPLIT_MEMORY-1) == 0, 'SPLIT_MEMORY must be a power of 2' if shared.Settings.ASM_JS == 1: - logging.warning('not all asm.js optimizations are possible with SPLIT_MEMORY, disabling those') + shared.WarningManager.warn('ALMOST_ASM', "not all asm.js optimizations are possible with SPLIT_MEMORY, disabling those.") shared.Settings.ASM_JS = 2 if shared.Settings.SAFE_HEAP: shared.Settings.SAFE_HEAP = 0 @@ -1335,7 +1328,7 @@ def check(input_file): if shared.Settings.ALLOW_MEMORY_GROWTH and shared.Settings.ASM_JS == 1: # this is an issue in asm.js, but not wasm if not shared.Settings.WASM or 'asmjs' in shared.Settings.BINARYEN_METHOD: - logging.warning('not all asm.js optimizations are possible with ALLOW_MEMORY_GROWTH, disabling those') + shared.WarningManager.warn('ALMOST_ASM') shared.Settings.ASM_JS = 2 # memory growth does not validate as asm.js http://discourse.wicg.io/t/request-for-comments-switching-resizing-heaps-in-asm-js/641/23 if js_opts: diff --git a/tests/test_other.py b/tests/test_other.py index 54690d9758a66..33f3279899aba 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -6048,6 +6048,23 @@ def test_no_warn_exported_jslibfunc(self): out, err = Popen([PYTHON, EMCC, path_from_root('tests', 'hello_world.c'), '-s', 'DEFAULT_LIBRARY_FUNCS_TO_INCLUDE=["alGetError"]', '-s', 'EXPORTED_FUNCTIONS=["_main", "_alGetError"]'], stdout=PIPE, stderr=PIPE).communicate() self.assertNotContained('''function requested to be exported, but not implemented: "_alGetError"''', err) + def test_almost_asm_warning(self): + warning = "[-Walmost-asm]" + for args, expected in [(['-O1', '-s', 'SPLIT_MEMORY=8388608', '-s', 'TOTAL_MEMORY=' + str(16*1024*1024)], True), # default + # suppress almost-asm warning when building with ALLOW_MEMORY_GROWTH + (['-O1', '-s', 'ALLOW_MEMORY_GROWTH=1', '-Wno-almost-asm'], False), + # suppress almost-asm warning when building with SPLIT_MEMORY + (['-O1', '-s', 'SPLIT_MEMORY=8388608', '-s', 'TOTAL_MEMORY=' + str(16*1024*1024), '-Wno-almost-asm'], False), + # last warning flag should "win" + (['-O1', '-s', 'ALLOW_MEMORY_GROWTH=1', '-Wno-almost-asm', '-Walmost-asm'], True)]: + print args, expected + proc = Popen([PYTHON, EMCC, path_from_root('tests', 'hello_world.c')] + args, stderr=PIPE) + err = proc.communicate()[1] + assert proc.returncode is 0 + assert (warning in err) == expected, err + if not expected: + assert err == '', err + def test_static_syscalls(self): Popen([PYTHON, EMCC, path_from_root('tests', 'hello_world.c')]).communicate() src = open('a.out.js').read() @@ -6672,6 +6689,10 @@ def test_separate_asm_warning(self): stdout, stderr = Popen([PYTHON, EMCC, path_from_root('tests', 'hello_world.c'), '--separate-asm', '-o', 'a.html'], stderr=PIPE).communicate() self.assertNotContained(warning, stderr) + # test that the warning can be suppressed + stdout, stderr = Popen([PYTHON, EMCC, path_from_root('tests', 'hello_world.c'), '--separate-asm', '-Wno-separate-asm'], stderr=PIPE).communicate() + self.assertNotContained(warning, stderr) + def test_canonicalize_nan_warning(self): open('src.cpp', 'w').write(r''' #include diff --git a/tools/shared.py b/tools/shared.py index bca985b0b9799..247390385e283 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -772,6 +772,56 @@ def clean_temp(): prepare_to_clean_temp(EMSCRIPTEN_TEMP_DIR) # this global var might change later return EMSCRIPTEN_TEMP_DIR +class WarningManager: + warnings = { + 'ABSOLUTE_PATHS': { + 'enabled': False, # warning about absolute-paths is disabled by default + 'printed': False, + 'message': '-I or -L of an absolute path encountered. If this is to a local system header/library, it may cause problems (local system files make sense for compiling natively on your system, but not necessarily to JavaScript).', + }, + 'SEPARATE_ASM': { + 'enabled': True, + 'printed': False, + 'message': "--separate-asm works best when compiling to HTML. Otherwise, you must yourself load the '.asm.js' file that is emitted separately, and must do so before loading the main '.js' file.", + }, + 'ALMOST_ASM': { + 'enabled': True, + 'printed': False, + 'message': 'not all asm.js optimizations are possible with ALLOW_MEMORY_GROWTH, disabling those.', + }, + } + + @staticmethod + def capture_warnings(cmd_args): + for i in range(len(cmd_args)): + if not cmd_args[i].startswith('-W'): + continue + + # special case pre-existing warn-absolute-paths + if cmd_args[i] == '-Wwarn-absolute-paths': + cmd_args[i] = '' + WarningManager.warnings['ABSOLUTE_PATHS']['enabled'] = True + elif cmd_args[i] == '-Wno-warn-absolute-paths': + cmd_args[i] = '' + WarningManager.warnings['ABSOLUTE_PATHS']['enabled'] = False + else: + # convert to string representation of Warning + warning_enum = cmd_args[i].replace('-Wno-', '').replace('-W', '') + warning_enum = warning_enum.upper().replace('-', '_') + + if warning_enum in WarningManager.warnings: + WarningManager.warnings[warning_enum]['enabled'] = not cmd_args[i].startswith('-Wno-') + cmd_args[i] = '' + + return cmd_args + + @staticmethod + def warn(warning_type, message=None): + warning = WarningManager.warnings[warning_type] + if warning['enabled'] and not warning['printed']: + warning['printed'] = True + logging.warning((message or warning['message']) + ' [-W' + warning_type.lower().replace('_', '-') + ']') + class Configuration: def __init__(self, environ=os.environ): self.DEBUG = environ.get('EMCC_DEBUG') From 816b90bfac47c36c03d468bfe6ebce0c4c61c2ba Mon Sep 17 00:00:00 2001 From: jgravelle-google Date: Thu, 23 Feb 2017 07:27:24 -0800 Subject: [PATCH 36/72] Fix EM_ASM for wasm backend (#4949) * Change EM_ASM implementation to not use varargs for wasm_backend * Move test_em_asm_parameter_pack out to a file * Add test for several combinations of args and return types for EM_ASM * Enable all-but-one of the no_wasm_backend tests that needed EM_ASM * Add explanation for EM_ASM declarations. Reorder C++ asm/wasm cases for readability * asmjs always expects vararg em_asms, always leave them as vararg --- system/include/emscripten/em_asm.h | 50 +++++++++++++++ system/include/emscripten/emscripten.h | 16 +---- tests/core/test_em_asm.cpp | 43 +++++++++++++ tests/core/test_em_asm.out | 75 +++++++++++++++++++++++ tests/core/test_em_asm_parameter_pack.cpp | 17 +++++ tests/core/test_em_asm_parameter_pack.out | 3 + tests/test_core.py | 37 ++--------- 7 files changed, 196 insertions(+), 45 deletions(-) create mode 100644 system/include/emscripten/em_asm.h create mode 100644 tests/core/test_em_asm.cpp create mode 100644 tests/core/test_em_asm.out create mode 100644 tests/core/test_em_asm_parameter_pack.cpp create mode 100644 tests/core/test_em_asm_parameter_pack.out diff --git a/system/include/emscripten/em_asm.h b/system/include/emscripten/em_asm.h new file mode 100644 index 0000000000000..61e491b243f08 --- /dev/null +++ b/system/include/emscripten/em_asm.h @@ -0,0 +1,50 @@ +#ifndef __em_asm_h__ +#define __em_asm_h__ + +// The wasm backend calls vararg functions by passing the variadic arguments as +// an array on the stack. For EM_ASM's underlying functions, s2wasm needs to +// translate them to emscripten_asm_const_[signature], but the expected +// signature and arguments has been lost as part of the vararg buffer. +// Therefore, we declare EM_ASM's implementing functions as non-variadic. + +#ifndef __asmjs +#ifndef __cplusplus +// In C, declare these as non-prototype declarations. This is obsolete K&R C +// (that is still supported) that causes the C frontend to consider any calls +// to them as valid, and avoids using the vararg calling convention. +void emscripten_asm_const(); +int emscripten_asm_const_int(); +double emscripten_asm_const_double(); +#else +// C++ interprets an empty parameter list as a function taking no arguments, +// instead of a K&R C function declaration. Variadic templates are lowered as +// non-vararg calls to the instantiated templated function, which we then +// replace in s2wasm. +template void emscripten_asm_const(const char* code, Args...); +template int emscripten_asm_const_int(const char* code, Args...); +template double emscripten_asm_const_double(const char* code, Args...); +#endif // __cplusplus +#else // __asmjs +// asmjs expects these to be vararg, so let them be vararg. +#ifdef __cplusplus +extern "C" { +#endif +void emscripten_asm_const(const char* code); +int emscripten_asm_const_int(const char* code, ...); +double emscripten_asm_const_double(const char* code, ...); +#ifdef __cplusplus +} +#endif +#endif // __asmjs + +void emscripten_asm_const(const char* code); + +#define EM_ASM(code) emscripten_asm_const(#code) +#define EM_ASM_(code, ...) emscripten_asm_const_int(#code, __VA_ARGS__) +#define EM_ASM_ARGS(code, ...) emscripten_asm_const_int(#code, __VA_ARGS__) +#define EM_ASM_INT(code, ...) emscripten_asm_const_int(#code, __VA_ARGS__) +#define EM_ASM_DOUBLE(code, ...) emscripten_asm_const_double(#code, __VA_ARGS__) +#define EM_ASM_INT_V(code) emscripten_asm_const_int(#code) +#define EM_ASM_DOUBLE_V(code) emscripten_asm_const_double(#code) + +#endif // __em_asm_h__ diff --git a/system/include/emscripten/emscripten.h b/system/include/emscripten/emscripten.h index 56aa0e48ef044..d1ab165fd5088 100644 --- a/system/include/emscripten/emscripten.h +++ b/system/include/emscripten/emscripten.h @@ -14,6 +14,8 @@ * is up at http://kripken.github.io/emscripten-site/docs/api_reference/emscripten.h.html */ +#include "em_asm.h" + #ifdef __cplusplus extern "C" { #endif @@ -52,15 +54,6 @@ typedef void (*em_arg_callback_func)(void*); typedef void (*em_str_callback_func)(const char *); -#define EM_ASM(...) emscripten_asm_const(#__VA_ARGS__) -#define EM_ASM_(code, ...) emscripten_asm_const_int(#code, __VA_ARGS__) -#define EM_ASM_ARGS(code, ...) emscripten_asm_const_int(#code, __VA_ARGS__) -#define EM_ASM_INT(code, ...) emscripten_asm_const_int(#code, __VA_ARGS__) -#define EM_ASM_DOUBLE(code, ...) emscripten_asm_const_double(#code, __VA_ARGS__) -#define EM_ASM_INT_V(code) emscripten_asm_const_int(#code) -#define EM_ASM_DOUBLE_V(code) emscripten_asm_const_double(#code) - - #define EMSCRIPTEN_KEEPALIVE __attribute__((used)) extern void emscripten_run_script(const char *script); @@ -254,11 +247,6 @@ int emscripten_print_double(double x, char *to, signed max); /* Internal APIs. Be careful with these. */ /* ===================================== */ -/* Helper API for EM_ASM - do not call this yourself */ -void emscripten_asm_const(const char *code); -int emscripten_asm_const_int(const char *code, ...); -double emscripten_asm_const_double(const char *code, ...); - #if __EMSCRIPTEN__ void emscripten_sleep(unsigned int ms); void emscripten_sleep_with_yield(unsigned int ms); diff --git a/tests/core/test_em_asm.cpp b/tests/core/test_em_asm.cpp new file mode 100644 index 0000000000000..b33f14e3aeca6 --- /dev/null +++ b/tests/core/test_em_asm.cpp @@ -0,0 +1,43 @@ +#include +#include + +int main() { + printf("BEGIN\n"); + EM_ASM({ Module['print']("no args works"); }); + printf(" EM_ASM_INT_V returned: %d\n", EM_ASM_INT_V({ Module['print']("no args returning int"); return 12; })); + printf(" EM_ASM_DOUBLE_V returned: %f\n", EM_ASM_DOUBLE_V({ Module['print']("no args returning double"); return 12.25; })); + +#define TEST() \ + FUNC({ Module['print'](" takes ints: " + $0);}, 5); \ + FUNC({ Module['print'](" takes doubles: " + $0);}, 5.0675); \ + FUNC({ Module['print'](" takes strings: " + Pointer_stringify($0)); return 7.75; }, "string arg"); \ + FUNC({ Module['print'](" takes multiple ints: " + $0 + ", " + $1); return 6; }, 5, 7); \ + FUNC({ Module['print'](" mixed arg types: " + $0 + ", " + Pointer_stringify($1) + ", " + $2); return 8.125; }, 3, "hello", 4.75); \ + FUNC({ Module['print'](" ignores unused args"); return 5.5; }, 0); \ + FUNC({ Module['print'](" skips unused args: " + $1); return 6; }, 5, 7); \ + FUNC({ Module['print'](" " + $0 + " + " + $2); return $0 + $2; }, 5.5, 7.0, 14.375); + +#define FUNC_WITH(macro, format, ...) printf(" returned: " format "\n", macro(__VA_ARGS__)); +#define FUNC(...) FUNC_WITH(EM_ASM_, "%d", __VA_ARGS__) + printf("EM_ASM_ :\n"); + TEST() +#undef FUNC + +#define FUNC(...) FUNC_WITH(EM_ASM_INT, "%d", __VA_ARGS__) + printf("EM_ASM_INT :\n"); + TEST() +#undef FUNC + +#define FUNC(...) FUNC_WITH(EM_ASM_ARGS, "%d", __VA_ARGS__) + printf("EM_ASM_ARGS :\n"); + TEST() +#undef FUNC + +#define FUNC(...) FUNC_WITH(EM_ASM_DOUBLE, "%f", __VA_ARGS__) + printf("EM_ASM_DOUBLE :\n"); + TEST() +#undef FUNC + + printf("END\n"); + return 0; +} diff --git a/tests/core/test_em_asm.out b/tests/core/test_em_asm.out new file mode 100644 index 0000000000000..ae1a6594b01b2 --- /dev/null +++ b/tests/core/test_em_asm.out @@ -0,0 +1,75 @@ +BEGIN +no args works +no args returning int + EM_ASM_INT_V returned: 12 +no args returning double + EM_ASM_DOUBLE_V returned: 12.250000 +EM_ASM_ : + takes ints: 5 + returned: 0 + takes doubles: 5.0675 + returned: 0 + takes strings: string arg + returned: 7 + takes multiple ints: 5, 7 + returned: 6 + mixed arg types: 3, hello, 4.75 + returned: 8 + ignores unused args + returned: 5 + skips unused args: 7 + returned: 6 + 5.5 + 14.375 + returned: 19 +EM_ASM_INT : + takes ints: 5 + returned: 0 + takes doubles: 5.0675 + returned: 0 + takes strings: string arg + returned: 7 + takes multiple ints: 5, 7 + returned: 6 + mixed arg types: 3, hello, 4.75 + returned: 8 + ignores unused args + returned: 5 + skips unused args: 7 + returned: 6 + 5.5 + 14.375 + returned: 19 +EM_ASM_ARGS : + takes ints: 5 + returned: 0 + takes doubles: 5.0675 + returned: 0 + takes strings: string arg + returned: 7 + takes multiple ints: 5, 7 + returned: 6 + mixed arg types: 3, hello, 4.75 + returned: 8 + ignores unused args + returned: 5 + skips unused args: 7 + returned: 6 + 5.5 + 14.375 + returned: 19 +EM_ASM_DOUBLE : + takes ints: 5 + returned: nan + takes doubles: 5.0675 + returned: nan + takes strings: string arg + returned: 7.750000 + takes multiple ints: 5, 7 + returned: 6.000000 + mixed arg types: 3, hello, 4.75 + returned: 8.125000 + ignores unused args + returned: 5.500000 + skips unused args: 7 + returned: 6.000000 + 5.5 + 14.375 + returned: 19.875000 +END \ No newline at end of file diff --git a/tests/core/test_em_asm_parameter_pack.cpp b/tests/core/test_em_asm_parameter_pack.cpp new file mode 100644 index 0000000000000..db9d3c6318c6b --- /dev/null +++ b/tests/core/test_em_asm_parameter_pack.cpp @@ -0,0 +1,17 @@ +#include +template +int call(Args... args) { + return(EM_ASM_INT( + { + Module.print(Array.prototype.join.call(arguments, ',')); + }, + args... + )); +} + +int main(int argc, char **argv) { + call(1); + call(1, 2); + call(1, 2, 3); + return 0; +} diff --git a/tests/core/test_em_asm_parameter_pack.out b/tests/core/test_em_asm_parameter_pack.out new file mode 100644 index 0000000000000..941a57bc6ab33 --- /dev/null +++ b/tests/core/test_em_asm_parameter_pack.out @@ -0,0 +1,3 @@ +1 +1,2 +1,2,3 diff --git a/tests/test_core.py b/tests/test_core.py index f62ca8f23abed..5cba53d87631e 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1585,6 +1585,10 @@ def test_inlinejs4(self): } ''', 'false') + def test_em_asm(self): + self.do_run_in_out_file_test('tests', 'core', 'test_em_asm') + self.do_run_in_out_file_test('tests', 'core', 'test_em_asm', force_c=True) + def test_em_asm_unicode(self): self.do_run(r''' #include @@ -1594,7 +1598,6 @@ def test_em_asm_unicode(self): } ''', 'hello world…') - @no_wasm_backend('requires EM_ASM args support in wasm backend') def test_em_asm_unused_arguments(self): src = r''' #include @@ -1614,30 +1617,9 @@ def test_em_asm_unused_arguments(self): # Maybe tests will later be joined into larger compilation units? # Then this must still be compiled separately from other code using EM_ASM # macros with arities 1-3. Otherwise this may incorrectly report a success. - @no_wasm_backend('requires EM_ASM args support in wasm backend') def test_em_asm_parameter_pack(self): Building.COMPILER_TEST_OPTS += ['-std=c++11'] - src = r''' - #include - - template - int call(Args... args) { - return(EM_ASM_INT( - { - Module.print(Array.prototype.join.call(arguments, ',')); - }, - args... - )); - } - - int main(int argc, char **argv) { - call(1); - call(1, 2); - call(1, 2, 3); - return 0; - } - ''' - self.do_run(src, '''1\n1,2\n1,2,3''') + self.do_run_in_out_file_test('tests', 'core', 'test_em_asm_parameter_pack') def test_memorygrowth(self): self.emcc_args += ['-s', 'ALLOW_MEMORY_GROWTH=0'] # start with 0 @@ -4368,23 +4350,19 @@ def test_utf(self): self.do_run_in_out_file_test('tests', 'core', 'test_utf') - @no_wasm_backend('requires EM_ASM args support in wasm backend') def test_utf32(self): Settings.EXTRA_EXPORTED_RUNTIME_METHODS = ['UTF32ToString', 'stringToUTF32', 'lengthBytesUTF32'] self.do_run(open(path_from_root('tests', 'utf32.cpp')).read(), 'OK.') self.do_run(open(path_from_root('tests', 'utf32.cpp')).read(), 'OK.', args=['-fshort-wchar']) - @no_wasm_backend('requires EM_ASM args support in wasm backend') def test_utf8(self): Building.COMPILER_TEST_OPTS += ['-std=c++11'] self.do_run(open(path_from_root('tests', 'utf8.cpp')).read(), 'OK.') - @no_wasm_backend('requires EM_ASM args support in wasm backend') def test_utf8_textdecoder(self): Building.COMPILER_TEST_OPTS += ['--embed-file', path_from_root('tests/utf8_corpus.txt')+ '@/utf8_corpus.txt'] self.do_run(open(path_from_root('tests', 'benchmark_utf8.cpp')).read(), 'OK.') - @no_wasm_backend('requires EM_ASM args support in wasm backend') def test_utf16_textdecoder(self): Settings.EXTRA_EXPORTED_RUNTIME_METHODS = ['UTF16ToString', 'stringToUTF16', 'lengthBytesUTF16'] Building.COMPILER_TEST_OPTS += ['--embed-file', path_from_root('tests/utf16_corpus.txt')+ '@/utf16_corpus.txt'] @@ -4888,7 +4866,6 @@ def test_dlmalloc_partial(self): ''' self.do_run(src, 'new 4!\n*1,0*') - @no_wasm_backend('requires EM_ASM args support in wasm backend') def test_dlmalloc_partial_2(self): if 'SAFE_HEAP' in str(self.emcc_args): return self.skip('we do unsafe stuff here') # present part of the symbols of dlmalloc, not all. malloc is harder to link than new which is weak. @@ -6040,7 +6017,7 @@ def test_large_exported_response(self): self.do_run(src, '''waka 4999!''') assert '_exported_func_from_response_file_1' in open('src.cpp.o.js').read() - @no_wasm_backend('requires EM_ASM args support in wasm backend') + @no_wasm_backend('RuntimeError: function signature mismatch') def test_add_function(self): Settings.INVOKE_RUN = 0 Settings.RESERVED_FUNCTION_POINTERS = 1 @@ -6075,7 +6052,6 @@ def test_add_function(self): Settings.EMULATED_FUNCTION_POINTERS = 1 # with emulation, we don't need to reserve self.do_run_from_file(src, expected) - @no_wasm_backend('requires EM_ASM args support in wasm backend') def test_getFuncWrapper_sig_alias(self): src = r''' #include @@ -7247,7 +7223,6 @@ def test_brk(self): self.do_run(open(path_from_root('tests', 'sbrk_brk.cpp')).read(), 'OK.') # Tests that we can use the dlmalloc mallinfo() function to obtain information about malloc()ed blocks and compute how much memory is used/freed. - @no_wasm_backend('requires EM_ASM args support in wasm backend') def test_mallinfo(self): self.do_run(open(path_from_root('tests', 'mallinfo.cpp')).read(), 'OK.') From 44c9b724017009499a9d6f46e05e388565944b63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Thu, 23 Feb 2017 18:32:38 +0200 Subject: [PATCH 37/72] Some applications unconditionally perform pitch and volume changes in OpenAL, which shows up in profilers as a few % overhead, so avoid calling out to Web Audio API if it's not necessary. --- src/library_openal.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/library_openal.js b/src/library_openal.js index 9d79939b05366..cbb3f921cacaa 100644 --- a/src/library_openal.js +++ b/src/library_openal.js @@ -86,7 +86,7 @@ var LibraryOpenAL = { entry.src = AL.currentContext.ctx.createBufferSource(); entry.src.buffer = entry.buffer; entry.src.connect(src.gain); - entry.src.playbackRate.value = src.playbackRate; + if (src.playbackRate != 1.0) entry.src.playbackRate.value = src.playbackRate; entry.src.duration = entry.buffer.duration / src.playbackRate; if (typeof(entry.src.start) !== 'undefined') { entry.src.start(startTime, offset); @@ -580,7 +580,7 @@ var LibraryOpenAL = { // entry.src.duration is expressed after factoring in playbackRate, so when changing playback rate, need // to recompute/rescale the rate to the new playback speed. entry.src.duration = (entry.src.duration - offset) * oldrate / src.playbackRate; - entry.src.playbackRate.value = src.playbackRate; + if (entry.src.playbackRate.value != src.playbackRate) entry.src.playbackRate.value = src.playbackRate; src.bufferPosition = currentTime; // stop other buffers @@ -596,7 +596,7 @@ var LibraryOpenAL = { } break; case 0x100A /* AL_GAIN */: - src.gain.gain.value = value; + if (src.gain.gain.value != value) src.gain.gain.value = value; break; // case 0x100D /* AL_MIN_GAIN */: // break; @@ -1336,7 +1336,7 @@ var LibraryOpenAL = { } switch (param) { case 0x100A /* AL_GAIN */: - AL.currentContext.gain.gain.value = value; + if (AL.currentContext.gain.gain.value != value) AL.currentContext.gain.gain.value = value; break; default: #if OPENAL_DEBUG From 84c9c651195333626d01940331881f2388b46cae Mon Sep 17 00:00:00 2001 From: juj Date: Thu, 23 Feb 2017 21:49:56 +0200 Subject: [PATCH 38/72] Multiprocessing pool cwd fix (#4959) The issue with reusing the Multiprocessing pool in the test harness was that the pool was created to a CWD that was deleted in between tests, which would cause all later subprocess spawns from the pool workers to fail because the worker processes had CWD pointing to a nonexisting directory. To fix, root the pool subprocesses to a known safe directory (EMSCRIPTEN_TEMP) that is very unlikely to die throughout the main process run. Reverts PR #4957 because that addressed a workaround only and had a bug that close_multiprocessing_pool() would not bind to closing the right pool each time if multiple ones had been created. Also reverts PR #4958 because I don't think TMP=. is a feature we should support. Though tested that TMP=. emcc tests/hello_world.c does build fine without errors after this PR. --- tools/shared.py | 84 ++++++++++++++++++++++++++----------------------- 1 file changed, 44 insertions(+), 40 deletions(-) diff --git a/tools/shared.py b/tools/shared.py index 247390385e283..c3f1cc246b555 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -1237,8 +1237,8 @@ def warn_if_duplicate_entries(archive_contents, archive_filename_hint=''): warned.add(curr) def extract_archive_contents(f): - cwd = os.getcwd() try: + cwd = os.getcwd() temp_dir = os.path.join(tempfile.gettempdir(), f.replace('/', '_').replace('\\', '_').replace(':', '_') + '.archive_contents') # TODO: Make sure this is nice and sane safe_ensure_dirs(temp_dir) os.chdir(temp_dir) @@ -1264,9 +1264,16 @@ def extract_archive_contents(f): 'dir': temp_dir, 'files': contents } + except Exception, e: + print >> sys.stderr, 'extract archive contents('+str(f)+') failed with error: ' + str(e) finally: os.chdir(cwd) + return { + 'dir': None, + 'files': [] + } + class ObjectFileInfo: def __init__(self, defs, undefs, commons): self.defs = defs @@ -1281,7 +1288,10 @@ def g_llvm_nm_uncached(filename): def g_multiprocessing_initializer(*args): for item in args: (key, value) = item.split('=') - os.environ[key] = value + if key == 'EMCC_POOL_CWD': + os.chdir(value) + else: + os.environ[key] = value # Building @@ -1298,49 +1308,44 @@ class Building: @staticmethod def get_multiprocessing_pool(): if not Building.multiprocessing_pool: - Building.multiprocessing_pool = Building.create_multiprocessing_pool() - return Building.multiprocessing_pool - - @staticmethod - def create_multiprocessing_pool(): - cores = int(os.environ.get('EMCC_CORES') or multiprocessing.cpu_count()) - - # If running with one core only, create a mock instance of a pool that does not - # actually spawn any new subprocesses. Very useful for internal debugging. - if cores == 1: - class FakeMultiprocessor: - def map(self, func, tasks): - results = [] - for t in tasks: - results += [func(t)] - return results - pool = FakeMultiprocessor() - else: - child_env = [ + cores = int(os.environ.get('EMCC_CORES') or multiprocessing.cpu_count()) + + # If running with one core only, create a mock instance of a pool that does not + # actually spawn any new subprocesses. Very useful for internal debugging. + if cores == 1: + class FakeMultiprocessor: + def map(self, func, tasks): + results = [] + for t in tasks: + results += [func(t)] + return results + Building.multiprocessing_pool = FakeMultiprocessor() + else: + child_env = [ + # Multiprocessing pool children must have their current working directory set to a safe path that is guaranteed not to die in between of + # executing commands, or otherwise the pool children will have trouble spawning subprocesses of their own. + 'EMCC_POOL_CWD=' + path_from_root(), # Multiprocessing pool children need to avoid all calling check_vanilla() again and again, # otherwise the compiler can deadlock when building system libs, because the multiprocess parent can have the Emscripten cache directory locked for write # access, and the EMCC_WASM_BACKEND check also requires locked access to the cache, which the multiprocess children would not get. - 'EMCC_WASM_BACKEND='+os.getenv('EMCC_WASM_BACKEND', '0'), + 'EMCC_WASM_BACKEND=' + os.getenv('EMCC_WASM_BACKEND', '0'), 'EMCC_CORES=1' # Multiprocessing pool children can't spawn their own linear number of children, that could cause a quadratic amount of spawned processes. - ] - pool = multiprocessing.Pool(processes=cores, initializer=g_multiprocessing_initializer, initargs=child_env) + ] + Building.multiprocessing_pool = multiprocessing.Pool(processes=cores, initializer=g_multiprocessing_initializer, initargs=child_env) - def close_multiprocessing_pool(): - try: - # Shut down the pool explicitly, because leaving that for Python to do at process shutdown is buggy and can generate - # noisy "WindowsError: [Error 5] Access is denied" spam which is not fatal. - pool.terminate() - pool.join() - if pool == Building.multiprocessing_pool: + def close_multiprocessing_pool(): + try: + # Shut down the pool explicitly, because leaving that for Python to do at process shutdown is buggy and can generate + # noisy "WindowsError: [Error 5] Access is denied" spam which is not fatal. + Building.multiprocessing_pool.terminate() + Building.multiprocessing_pool.join() Building.multiprocessing_pool = None - except WindowsError, e: - # Mute the "WindowsError: [Error 5] Access is denied" errors, raise all others through - if e.winerror != 5: raise - atexit.register(close_multiprocessing_pool) - - return pool - + except WindowsError, e: + # Mute the "WindowsError: [Error 5] Access is denied" errors, raise all others through + if e.winerror != 5: raise + atexit.register(close_multiprocessing_pool) + return Building.multiprocessing_pool @staticmethod def get_building_env(native=False): @@ -1622,8 +1627,7 @@ def read_link_inputs(files): object_names.append(absolute_path_f) # Archives contain objects, so process all archives first in parallel to obtain the object files in them. - # new_pool is a workaround for https://github.com/kripken/emscripten/issues/4941 TODO: a better fix - pool = Building.create_multiprocessing_pool() + pool = Building.get_multiprocessing_pool() object_names_in_archives = pool.map(extract_archive_contents, archive_names) for n in range(len(archive_names)): From 4f5676132be06d56dd908c4c356887eb9aaa9c05 Mon Sep 17 00:00:00 2001 From: jgravelle-google Date: Thu, 23 Feb 2017 14:17:45 -0800 Subject: [PATCH 39/72] Fix EM_ASM test regression (other.test_split_memory_faking) (#4970) --- system/include/emscripten/em_asm.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system/include/emscripten/em_asm.h b/system/include/emscripten/em_asm.h index 61e491b243f08..676758e588564 100644 --- a/system/include/emscripten/em_asm.h +++ b/system/include/emscripten/em_asm.h @@ -39,7 +39,7 @@ double emscripten_asm_const_double(const char* code, ...); void emscripten_asm_const(const char* code); -#define EM_ASM(code) emscripten_asm_const(#code) +#define EM_ASM(...) emscripten_asm_const(#__VA_ARGS__) #define EM_ASM_(code, ...) emscripten_asm_const_int(#code, __VA_ARGS__) #define EM_ASM_ARGS(code, ...) emscripten_asm_const_int(#code, __VA_ARGS__) #define EM_ASM_INT(code, ...) emscripten_asm_const_int(#code, __VA_ARGS__) From c4c9d4b68ce8ada83c8caafe2a39ba352409ed98 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 23 Feb 2017 14:38:35 -0800 Subject: [PATCH 40/72] Fix Runtime.stackSave etc. in wasm async compilation mode (#4966) * fix wasm async compilation error, we were exporting stackSave etc. in a weird way, that didn't get the benefits of the wrapper that handles async compilation. standardize those exports, add a test, and move binaryen2 to test async compilation, which matches how the feature is actually used * fix makeStructuralReturn * fix setThrew * use a decorator * embind is not yet compatible with async compilation --- emcc.py | 6 +++++- emscripten.py | 23 ++++++++++++++++------- src/library.js | 4 ++-- src/parseTools.js | 6 +----- tests/test_core.py | 35 +++++++++++++++++++++++++++++++++-- tools/shared.py | 2 +- 6 files changed, 58 insertions(+), 18 deletions(-) diff --git a/emcc.py b/emcc.py index b7d9aa959a2ac..ac4f30c55d52e 100755 --- a/emcc.py +++ b/emcc.py @@ -1304,7 +1304,11 @@ def check(input_file): if 'BINARYEN_ASYNC_COMPILATION=0' not in settings_changes: shared.Settings.BINARYEN_ASYNC_COMPILATION = 1 if shared.Settings.BINARYEN_ASYNC_COMPILATION == 1: - if shared.Building.is_wasm_only(): + if bind: + shared.Settings.BINARYEN_ASYNC_COMPILATION = 0 + if 'BINARYEN_ASYNC_COMPILATION=1' in settings_changes: + logging.warning('BINARYEN_ASYNC_COMPILATION requested, but disabled since embind is not compatible with it yet') + elif shared.Building.is_wasm_only(): # async compilation requires a swappable module - we swap it in when it's ready shared.Settings.SWAPPABLE_ASM_MODULE = 1 else: diff --git a/emscripten.py b/emscripten.py index c555dbbfc90f3..534a094e08ecb 100755 --- a/emscripten.py +++ b/emscripten.py @@ -823,6 +823,15 @@ def table_size(table): exported_implemented_functions.append('runPostSets') if settings['ALLOW_MEMORY_GROWTH']: exported_implemented_functions.append('_emscripten_replace_memory') + if not settings.get('SIDE_MODULE'): + exported_implemented_functions += ['stackAlloc', 'stackSave', 'stackRestore', 'establishStackSpace'] + if settings['SAFE_HEAP']: + exported_implemented_functions += ['setDynamicTop'] + if not settings['RELOCATABLE']: + exported_implemented_functions += ['setTempRet0', 'getTempRet0'] + if not (settings['BINARYEN'] and settings['SIDE_MODULE']): + exported_implemented_functions += ['setThrew'] + all_exported = exported_implemented_functions + asm_runtime_funcs + function_tables exported_implemented_functions = list(set(exported_implemented_functions)) if settings['EMULATED_FUNCTION_POINTERS']: @@ -1313,20 +1322,20 @@ def finalize_output(metadata, post, funcs_js, need_asyncify, provide_fround, asm if not settings.get('SIDE_MODULE'): funcs_js.append(''' -Runtime.stackAlloc = asm['stackAlloc']; -Runtime.stackSave = asm['stackSave']; -Runtime.stackRestore = asm['stackRestore']; -Runtime.establishStackSpace = asm['establishStackSpace']; +Runtime.stackAlloc = Module['stackAlloc']; +Runtime.stackSave = Module['stackSave']; +Runtime.stackRestore = Module['stackRestore']; +Runtime.establishStackSpace = Module['establishStackSpace']; ''') if settings['SAFE_HEAP']: funcs_js.append(''' -Runtime.setDynamicTop = asm['setDynamicTop']; +Runtime.setDynamicTop = Module['setDynamicTop']; ''') if not settings['RELOCATABLE']: funcs_js.append(''' -Runtime.setTempRet0 = asm['setTempRet0']; -Runtime.getTempRet0 = asm['getTempRet0']; +Runtime.setTempRet0 = Module['setTempRet0']; +Runtime.getTempRet0 = Module['getTempRet0']; ''') # Set function table masks diff --git a/src/library.js b/src/library.js index f00e5cd8ee66e..086c6753c3436 100644 --- a/src/library.js +++ b/src/library.js @@ -1302,7 +1302,7 @@ LibraryManager.library = { __cxa_end_catch__deps: ['__cxa_free_exception', '$EXCEPTIONS'], __cxa_end_catch: function() { // Clear state flag. - asm['setThrew'](0); + Module['setThrew'](0); // Call destructor if one is registered then clear it. var ptr = EXCEPTIONS.caught.pop(); #if EXCEPTION_DEBUG @@ -2875,7 +2875,7 @@ LibraryManager.library = { longjmp__deps: ['saveSetjmp', 'testSetjmp'], longjmp: function(env, value) { - asm['setThrew'](env, value || 1); + Module['setThrew'](env, value || 1); throw 'longjmp'; }, emscripten_longjmp__deps: ['longjmp'], diff --git a/src/parseTools.js b/src/parseTools.js index c7871f5afa5f4..436623b32df72 100644 --- a/src/parseTools.js +++ b/src/parseTools.js @@ -1304,11 +1304,7 @@ function makeStructuralReturn(values, inAsm) { return 'return ' + asmCoercion(values.slice(1).map(function(value) { i++; if (!inAsm) { - if (!RELOCATABLE) { - return 'asm["setTempRet' + i + '"](' + value + ')'; - } else { - return 'Runtime.setTempRet' + i + '(' + value + ')'; - } + return 'Runtime.setTempRet' + i + '(' + value + ')'; } if (i === 0) { return makeSetTempRet0(value) diff --git a/tests/test_core.py b/tests/test_core.py index 5cba53d87631e..8dd618d558f8d 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -38,6 +38,12 @@ def decorated(f): return skip_if(f, 'is_wasm_backend', note) return decorated +# Async wasm compilation can't work in some tests, they are set up synchronously +def sync(f): + def decorated(self): + self.emcc_args += ['-s', 'BINARYEN_ASYNC_COMPILATION=0'] # test is set up synchronously + f(self) + return decorated class T(RunnerCore): # Short name, to make it more fun to use manually on the commandline def is_emterpreter(self): @@ -1621,6 +1627,20 @@ def test_em_asm_parameter_pack(self): Building.COMPILER_TEST_OPTS += ['-std=c++11'] self.do_run_in_out_file_test('tests', 'core', 'test_em_asm_parameter_pack') + def test_runtime_stacksave(self): + Settings.BINARYEN_ASYNC_COMPILATION = 1 + self.do_run(r''' +#include +#include + +int main() { + int x = EM_ASM_INT_V({ return Runtime.stackSave(); }); + int y = EM_ASM_INT_V({ return Runtime.stackSave(); }); + assert(x == y); + EM_ASM({ Module.print('success'); }); +} +''', 'success') + def test_memorygrowth(self): self.emcc_args += ['-s', 'ALLOW_MEMORY_GROWTH=0'] # start with 0 @@ -2147,6 +2167,7 @@ def test_flexarray_struct(self): def test_bsearch(self): self.do_run_in_out_file_test('tests', 'core', 'test_bsearch') + @sync @no_wasm_backend("wasm backend has no support for fastcomp's -emscripten-assertions flag") def test_stack_overflow(self): Settings.ASSERTIONS = 1 @@ -4077,13 +4098,14 @@ def process(filename): try_delete(mem_file) def clean(out, err): - return '\n'.join(filter(lambda line: 'binaryen' not in line, (out + err).split('\n'))) + return '\n'.join(filter(lambda line: 'binaryen' not in line and 'wasm' not in line, (out + err).split('\n'))) self.do_run(src, map(lambda x: x if 'SYSCALL_DEBUG=1' not in mode else ('syscall! 146,SYS_writev' if self.run_name == 'default' else 'syscall! 146'), ('size: 7\ndata: 100,-56,50,25,10,77,123\nloop: 100 -56 50 25 10 77 123 \ninput:hi there!\ntexto\n$\n5 : 10,30,20,11,88\nother=some data.\nseeked=me da.\nseeked=ata.\nseeked=ta.\nfscanfed: 10 - hello\n5 bytes to dev/null: 5\nok.\ntexte\n', 'size: 7\ndata: 100,-56,50,25,10,77,123\nloop: 100 -56 50 25 10 77 123 \ninput:hi there!\ntexto\ntexte\n$\n5 : 10,30,20,11,88\nother=some data.\nseeked=me da.\nseeked=ata.\nseeked=ta.\nfscanfed: 10 - hello\n5 bytes to dev/null: 5\nok.\n')), post_build=post, extra_emscripten_args=['-H', 'libc/fcntl.h'], output_nicerizer=clean) if self.uses_memory_init_file(): assert os.path.exists(mem_file), 'File %s does not exist' % mem_file + @sync def test_files_m(self): # Test for Module.stdin etc. @@ -5374,6 +5396,7 @@ def test(): assert len(old) > len(new) assert old.count('tempBigInt') > new.count('tempBigInt') + @sync @no_wasm_backend() def test_poppler(self): if WINDOWS: return self.skip('test_poppler depends on freetype, which uses a ./configure script to build and therefore currently only runs on Linux and OS X.') @@ -5446,6 +5469,7 @@ def process(filename): assert (num_original_funcs - self.count_funcs('src.cpp.o.js')) > 200 break + @sync def test_openjpeg(self): Building.COMPILER_TEST_OPTS = filter(lambda x: x != '-g', Building.COMPILER_TEST_OPTS) # remove -g, so we have one test without it by default @@ -5756,6 +5780,7 @@ def test_autodebug(self): ### Integration tests + @sync @no_wasm_backend() def test_ccall(self): post = ''' @@ -6017,6 +6042,7 @@ def test_large_exported_response(self): self.do_run(src, '''waka 4999!''') assert '_exported_func_from_response_file_1' in open('src.cpp.o.js').read() + @sync @no_wasm_backend('RuntimeError: function signature mismatch') def test_add_function(self): Settings.INVOKE_RUN = 0 @@ -6317,6 +6343,7 @@ def test_embind_4(self): ''' self.do_run(src, '107') + @sync @no_wasm_backend() def test_scriptaclass(self): Settings.EXPORT_BINDINGS = 1 @@ -6564,6 +6591,7 @@ def post3(filename): *ok* ''', post_build=(post2, post3)) + @sync @no_wasm_backend() def test_scriptaclass_2(self): Settings.EXPORT_BINDINGS = 1 @@ -6609,6 +6637,7 @@ def process(filename): ''' self.do_run(src, '|hello|43|world|41|', post_build=post) + @sync @no_wasm_backend() def test_webidl(self): assert 'asm2' in test_modes @@ -6880,6 +6909,7 @@ def test_emscripten_log(self): def test_float_literals(self): self.do_run_in_out_file_test('tests', 'test_float_literals') + @sync def test_exit_status(self): src = r''' #include @@ -7198,6 +7228,7 @@ def test_fs_dict(self): self.emcc_args += ['--pre-js', 'pre.js'] self.do_run('', 'object\nobject\nobject') + @sync @no_wasm_backend("wasm backend has no support for fastcomp's -emscripten-assertions flag") def test_stack_overflow_check(self): args = self.emcc_args + ['-s', 'TOTAL_STACK=1048576'] @@ -7300,7 +7331,7 @@ def setUp(self): binaryen0 = make_run("binaryen0", compiler=CLANG, emcc_args=['-O0', '-s', 'BINARYEN=1', '-s', 'BINARYEN_METHOD="native-wasm"']) binaryen1 = make_run("binaryen1", compiler=CLANG, emcc_args=['-O1', '-s', 'BINARYEN=1', '-s', 'BINARYEN_METHOD="native-wasm"']) -binaryen2 = make_run("binaryen2", compiler=CLANG, emcc_args=['-O2', '-s', 'BINARYEN=1', '-s', 'BINARYEN_METHOD="native-wasm"']) +binaryen2 = make_run("binaryen2", compiler=CLANG, emcc_args=['-O2', '-s', 'BINARYEN=1', '-s', 'BINARYEN_METHOD="native-wasm"', '-s', 'BINARYEN_ASYNC_COMPILATION=1']) binaryen3 = make_run("binaryen3", compiler=CLANG, emcc_args=['-O3', '-s', 'BINARYEN=1', '-s', 'BINARYEN_METHOD="native-wasm"', '-s', 'ASSERTIONS=1', "-s", "PRECISE_F32=1"]) binaryen2jo = make_run("binaryen2jo", compiler=CLANG, emcc_args=['-O2', '-s', 'BINARYEN=1', '-s', 'BINARYEN_METHOD="native-wasm,asmjs"']) diff --git a/tools/shared.py b/tools/shared.py index c3f1cc246b555..96a49fa75b190 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -2340,7 +2340,7 @@ def make_invoke(sig, named=True): %sModule["dynCall_%s"](%s); } catch(e) { if (typeof e !== 'number' && e !== 'longjmp') throw e; - asm["setThrew"](1, 0); + Module["setThrew"](1, 0); } }''' % ((' invoke_' + sig) if named else '', args, 'return ' if sig[0] != 'v' else '', sig, args) return ret From 52ee9cddd636081482b1a88b7349c126919b088e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Fri, 24 Feb 2017 13:45:34 +0200 Subject: [PATCH 41/72] Fix sysconf(_SC_PHYS_PAGES) test after changing page size from 4096 bytes to 16384 bytes. Fix sysconf(_SC_PHYS_PAGES) to return correct size when ALLOW_MEMORY_GROWTH=1 is specified or if BINARYEN_MEM_MAX=xxx is specified. --- src/library.js | 14 +++++++++++++- tests/test_core.py | 6 ++++++ tests/test_other.py | 16 ++++++++++++++++ tests/unistd/sysconf.c | 6 ++---- tests/unistd/sysconf.out | 7 ++----- tests/unistd/sysconf_phys_pages.c | 9 +++++++++ 6 files changed, 48 insertions(+), 10 deletions(-) create mode 100644 tests/unistd/sysconf_phys_pages.c diff --git a/src/library.js b/src/library.js index 086c6753c3436..79d98a4580e06 100644 --- a/src/library.js +++ b/src/library.js @@ -289,7 +289,19 @@ LibraryManager.library = { // http://pubs.opengroup.org/onlinepubs/009695399/functions/sysconf.html switch(name) { case {{{ cDefine('_SC_PAGE_SIZE') }}}: return PAGE_SIZE; - case {{{ cDefine('_SC_PHYS_PAGES') }}}: return totalMemory / PAGE_SIZE; + case {{{ cDefine('_SC_PHYS_PAGES') }}}: +#if BINARYEN + var maxHeapSize = 2*1024*1024*1024 - 65536; +#else + var maxHeapSize = 2*1024*1024*1024 - 16777216; +#endif +#if BINARYEN_MEM_MAX != -1 + maxHeapSize = {{{ BINARYEN_MEM_MAX }}}; +#endif +#if !ALLOW_MEMORY_GROWTH + maxHeapSize = HEAPU8.length; +#endif + return maxHeapSize / PAGE_SIZE; case {{{ cDefine('_SC_ADVISORY_INFO') }}}: case {{{ cDefine('_SC_BARRIERS') }}}: case {{{ cDefine('_SC_ASYNCHRONOUS_IO') }}}: diff --git a/tests/test_core.py b/tests/test_core.py index 8dd618d558f8d..0d20c210de281 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -4540,6 +4540,12 @@ def test_unistd_sysconf(self): expected = open(path_from_root('tests', 'unistd', 'sysconf.out'), 'r').read() self.do_run(src, expected) + def test_unistd_sysconf_phys_pages(self): + src = open(path_from_root('tests', 'unistd', 'sysconf_phys_pages.c'), 'r').read() + if Settings.ALLOW_MEMORY_GROWTH: expected = (2*1024*1024*1024-16777216) / 16384 + else: expected = 16*1024*1024 / 16384 + self.do_run(src, str(expected) + ', errno: 0') + def test_unistd_login(self): src = open(path_from_root('tests', 'unistd', 'login.c'), 'r').read() expected = open(path_from_root('tests', 'unistd', 'login.out'), 'r').read() diff --git a/tests/test_other.py b/tests/test_other.py index 33f3279899aba..06a2a94db6472 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -7446,6 +7446,22 @@ def test_binaryen_ignore_implicit_traps(self): print 'sizes:', sizes assert sizes[1] < sizes[0], 'ignoring implicit traps must reduce code size' + def test_sysconf_phys_pages(self): + for args, expected in [ + ([], 1024), + (['-s', 'TOTAL_MEMORY=32MB'], 2048), + (['-s', 'TOTAL_MEMORY=32MB', '-s', 'ALLOW_MEMORY_GROWTH=1'], (2*1024*1024*1024 - 16777216) / 16384), + (['-s', 'TOTAL_MEMORY=32MB', '-s', 'BINARYEN=1', '-s', 'BINARYEN_METHOD="interpret-asm2wasm"'], 2048), + (['-s', 'TOTAL_MEMORY=32MB', '-s', 'ALLOW_MEMORY_GROWTH=1', '-s', 'BINARYEN=1', '-s', 'BINARYEN_METHOD="interpret-asm2wasm"'], (2*1024*1024*1024 - 65536) / 16384), + (['-s', 'TOTAL_MEMORY=32MB', '-s', 'ALLOW_MEMORY_GROWTH=1', '-s', 'BINARYEN=1', '-s', 'BINARYEN_METHOD="interpret-asm2wasm"', '-s', 'BINARYEN_MEM_MAX=128MB'], 2048*4) + ]: + cmd = [PYTHON, EMCC, path_from_root('tests', 'unistd', 'sysconf_phys_pages.c')] + args + print str(cmd) + subprocess.check_call(cmd) + result = run_js('a.out.js').strip() + print result + assert result == str(expected) + ', errno: 0', expected + def test_wasm_targets(self): for f in ['a.wasm', 'a.wast']: process = Popen([PYTHON, EMCC, path_from_root('tests', 'hello_world.cpp'), '-o', f], stdout=PIPE, stderr=PIPE) diff --git a/tests/unistd/sysconf.c b/tests/unistd/sysconf.c index 370a3253a201c..3211608e1cd5d 100644 --- a/tests/unistd/sysconf.c +++ b/tests/unistd/sysconf.c @@ -125,8 +125,7 @@ int main() { _SC_AIO_PRIO_DELTA_MAX, _SC_STREAM_MAX, _SC_TZNAME_MAX, - _SC_THREAD_DESTRUCTOR_ITERATIONS, - _SC_PHYS_PAGES + _SC_THREAD_DESTRUCTOR_ITERATIONS }; char* names[] = { "_SC_PAGE_SIZE", @@ -250,8 +249,7 @@ int main() { "_SC_AIO_PRIO_DELTA_MAX", "_SC_STREAM_MAX", "_SC_TZNAME_MAX", - "_SC_THREAD_DESTRUCTOR_ITERATIONS", - "_SC_PHYS_PAGES" + "_SC_THREAD_DESTRUCTOR_ITERATIONS" }; char buffer[256]; diff --git a/tests/unistd/sysconf.out b/tests/unistd/sysconf.out index fa7bb8fe3919a..7da7101352563 100644 --- a/tests/unistd/sysconf.out +++ b/tests/unistd/sysconf.out @@ -1,7 +1,7 @@ -_SC_PAGE_SIZE: 4096 +_SC_PAGE_SIZE: 16384 errno: 0 -_SC_PAGESIZE: 4096 +_SC_PAGESIZE: 16384 errno: 0 _SC_ADVISORY_INFO: 200809 @@ -364,8 +364,5 @@ errno: 0 _SC_THREAD_DESTRUCTOR_ITERATIONS: 4 errno: 0 -_SC_PHYS_PAGES: 4096 -errno: 0 - (invalid): -1 errno: 22 diff --git a/tests/unistd/sysconf_phys_pages.c b/tests/unistd/sysconf_phys_pages.c new file mode 100644 index 0000000000000..327150457c4cf --- /dev/null +++ b/tests/unistd/sysconf_phys_pages.c @@ -0,0 +1,9 @@ +#include +#include +#include + +int main() { + printf("%ld, ", sysconf(_SC_PHYS_PAGES)); + printf("errno: %d\n", errno); + return 0; +} From 86cd04502464d642e73727d7dcbd39f213cec2f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Fri, 24 Feb 2017 13:50:30 +0200 Subject: [PATCH 42/72] Update test_mmap after changing page size from 4k to 16k. --- tests/core/test_mmap.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/core/test_mmap.c b/tests/core/test_mmap.c index 96f85327946cc..0307fa10eb2d1 100644 --- a/tests/core/test_mmap.c +++ b/tests/core/test_mmap.c @@ -5,15 +5,15 @@ int main(int argc, char* argv[]) { // Alignment check for mmap below should be consistent with the reported page size - // For now, just require it to be 4096 - assert(getpagesize() == 4096); - assert(sysconf(_SC_PAGESIZE) == 4096); + // For now, just require it to be 16384 + assert(getpagesize() == 16384); + assert(sysconf(_SC_PAGESIZE) == 16384); for (int i = 0; i < 10; i++) { int* map = (int*)mmap(0, 5000, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANON, -1, 0); assert(map != MAP_FAILED); - assert(((int)map) % 4096 == 0); // aligned + assert(((int)map) % 16384 == 0); // aligned assert(munmap(map, 5000) == 0); } From e7e32169e351fdbd2f8e2cfaec4945887b372d82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Fri, 24 Feb 2017 13:51:47 +0200 Subject: [PATCH 43/72] Fix browser.test_sdl_alloctext to allocate a valid heap size. --- tests/test_browser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_browser.py b/tests/test_browser.py index 61c192f549a9b..8e76423d628fd 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -1783,7 +1783,7 @@ def test_sdl_canvas_palette_2(self): self.btest('sdl_canvas_palette_2.c', reference='sdl_canvas_palette_b.png', args=['--pre-js', 'pre.js', '--pre-js', 'args-b.js', '-lSDL', '-lGL']) def test_sdl_alloctext(self): - self.btest('sdl_alloctext.c', expected='1', args=['-O2', '-s', 'TOTAL_MEMORY=8MB', '-lSDL', '-lGL']) + self.btest('sdl_alloctext.c', expected='1', args=['-O2', '-s', 'TOTAL_MEMORY=16MB', '-lSDL', '-lGL']) def test_sdl_surface_refcount(self): self.btest('sdl_surface_refcount.c', args=['-lSDL'], expected='1') From 288f8d1a8704ecc62716a8e2b47d133a2094b2c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Fri, 24 Feb 2017 14:47:23 +0200 Subject: [PATCH 44/72] Fix typo in testing whether garbage free WebGL 2 entry points exist. --- src/library_gl.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/library_gl.js b/src/library_gl.js index 9b607f2cf4469..328862d243b42 100644 --- a/src/library_gl.js +++ b/src/library_gl.js @@ -1326,7 +1326,7 @@ var LibraryGL = { } } #endif - if (GL.currentContext.supportsWebGL2EntryPoints >= 2) { + if (GL.currentContext.supportsWebGL2EntryPoints) { // WebGL 2 provides new garbage-free entry points to call to WebGL. Use those always when possible. if (GLctx.currentPixelUnpackBufferBinding) { GLctx.texImage2D(target, level, internalFormat, width, height, border, format, type, pixels); @@ -1360,7 +1360,7 @@ var LibraryGL = { if (type == 0x8d61/*GL_HALF_FLOAT_OES*/) type = 0x140B /*GL_HALF_FLOAT*/; } #endif - if (GL.currentContext.supportsWebGL2EntryPoints >= 2) { + if (GL.currentContext.supportsWebGL2EntryPoints) { // WebGL 2 provides new garbage-free entry points to call to WebGL. Use those always when possible. if (GLctx.currentPixelUnpackBufferBinding) { GLctx.texSubImage2D(target, level, internalFormat, width, height, border, format, type, pixels); From b39bb50eb9f2c8c43d69aca8582c7c57634a0f04 Mon Sep 17 00:00:00 2001 From: Derek Schuff Date: Fri, 24 Feb 2017 11:05:00 -0800 Subject: [PATCH 45/72] Always use async compile unless overridden (#4971) We want to avoid blocking the main thread whenever possible, even when debugging or not-optimizing. Having different behavior here between optimization levels is really weird and surprising. To enforce this preference to avoid blocking the main thread, Chrome will limit synchronous compiles to a pretty small module size; small enough that even emscripten's unoptimized hello world won't compile. --- emcc.py | 4 ---- src/settings.js | 7 +++---- tests/test_browser.py | 2 +- tests/test_core.py | 3 +-- 4 files changed, 5 insertions(+), 11 deletions(-) diff --git a/emcc.py b/emcc.py index ac4f30c55d52e..be82cb98cd48c 100755 --- a/emcc.py +++ b/emcc.py @@ -1299,10 +1299,6 @@ def check(input_file): if shared.Building.is_wasm_only() and shared.Settings.EVAL_CTORS: logging.debug('disabling EVAL_CTORS, as in wasm-only mode it hurts more than it helps. TODO: a wasm version of it') shared.Settings.EVAL_CTORS = 0 - # enable async compilation if optimizing and not turned off manually - if opt_level > 0: - if 'BINARYEN_ASYNC_COMPILATION=0' not in settings_changes: - shared.Settings.BINARYEN_ASYNC_COMPILATION = 1 if shared.Settings.BINARYEN_ASYNC_COMPILATION == 1: if bind: shared.Settings.BINARYEN_ASYNC_COMPILATION = 0 diff --git a/src/settings.js b/src/settings.js index 36c6763376729..1023ad7b81a5a 100644 --- a/src/settings.js +++ b/src/settings.js @@ -698,9 +698,9 @@ var BINARYEN_PASSES = ""; // A comma-separated list of passes to run in the bina var BINARYEN_MEM_MAX = -1; // Set the maximum size of memory in the wasm module (in bytes). // Without this, TOTAL_MEMORY is used (as it is used for the initial value), // or if memory growth is enabled, no limit is set. This overrides both of those. -var BINARYEN_ASYNC_COMPILATION = 0; // Whether to compile the wasm asynchronously, which is more - // efficient. This is off by default in unoptimized builds and - // on by default in optimized ones. +var BINARYEN_ASYNC_COMPILATION = 1; // Whether to compile the wasm asynchronously, which is more + // efficient and does not block the main thread. This is currently + // required for all but the smallest modules to run in V8 var BINARYEN_ROOT = ""; // Directory where we can find Binaryen. Will be automatically set for you, // but you can set it to override if you are a Binaryen developer. @@ -834,4 +834,3 @@ var ASMFS = 0; // If set to 1, uses the multithreaded filesystem that is impleme var WASM_TEXT_FILE = ''; // name of the file containing wasm text, if relevant var WASM_BINARY_FILE = ''; // name of the file containing wasm binary, if relevant var ASMJS_CODE_FILE = ''; // name of the file containing asm.js, if relevant - diff --git a/tests/test_browser.py b/tests/test_browser.py index 8e76423d628fd..f6a79ced06ba5 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -3314,7 +3314,7 @@ def test_binaryen_async(self): ''', )) for opts, expect in [ - ([], 0), + ([], 1), (['-O1'], 1), (['-O2'], 1), (['-O3'], 1), diff --git a/tests/test_core.py b/tests/test_core.py index 0d20c210de281..f504ed1c4ad46 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1628,7 +1628,6 @@ def test_em_asm_parameter_pack(self): self.do_run_in_out_file_test('tests', 'core', 'test_em_asm_parameter_pack') def test_runtime_stacksave(self): - Settings.BINARYEN_ASYNC_COMPILATION = 1 self.do_run(r''' #include #include @@ -7337,7 +7336,7 @@ def setUp(self): binaryen0 = make_run("binaryen0", compiler=CLANG, emcc_args=['-O0', '-s', 'BINARYEN=1', '-s', 'BINARYEN_METHOD="native-wasm"']) binaryen1 = make_run("binaryen1", compiler=CLANG, emcc_args=['-O1', '-s', 'BINARYEN=1', '-s', 'BINARYEN_METHOD="native-wasm"']) -binaryen2 = make_run("binaryen2", compiler=CLANG, emcc_args=['-O2', '-s', 'BINARYEN=1', '-s', 'BINARYEN_METHOD="native-wasm"', '-s', 'BINARYEN_ASYNC_COMPILATION=1']) +binaryen2 = make_run("binaryen2", compiler=CLANG, emcc_args=['-O2', '-s', 'BINARYEN=1', '-s', 'BINARYEN_METHOD="native-wasm"']) binaryen3 = make_run("binaryen3", compiler=CLANG, emcc_args=['-O3', '-s', 'BINARYEN=1', '-s', 'BINARYEN_METHOD="native-wasm"', '-s', 'ASSERTIONS=1', "-s", "PRECISE_F32=1"]) binaryen2jo = make_run("binaryen2jo", compiler=CLANG, emcc_args=['-O2', '-s', 'BINARYEN=1', '-s', 'BINARYEN_METHOD="native-wasm,asmjs"']) From 59d0ec8c8fca12d1207e1bac9bd357ed9e5abf57 Mon Sep 17 00:00:00 2001 From: ImplOfAnImpl Date: Sat, 25 Feb 2017 02:45:28 +0200 Subject: [PATCH 46/72] Fix for issue 4926 (#4965) * Fix for issue #4926: ELIMINATE_DUPLICATE_FUNCTIONS=1 produces invalid asm.js In function eliminator literals 0.0 are replaced with ZERO$DOT$ZERO before running Uglify * Added myself to authors --- AUTHORS | 2 +- ...nction-eliminator-double-parsed-correctly-output.js | 1 + ...test-function-eliminator-double-parsed-correctly.js | 1 + tests/test_other.py | 2 +- tools/duplicate_function_eliminator.py | 10 +++++++--- tools/validate_asmjs.py | 2 +- 6 files changed, 12 insertions(+), 6 deletions(-) diff --git a/AUTHORS b/AUTHORS index 2f8ff46dcda76..de5584db86f34 100644 --- a/AUTHORS +++ b/AUTHORS @@ -276,4 +276,4 @@ a license to everyone to use it as detailed in LICENSE.) * Sam Clegg (copyright owned by Google, Inc.) * Joshua Lind * Hiroaki GOTO as "GORRY" - +* Mikhail Kremnyov (copyright owned by XCDS International) diff --git a/tests/optimizer/test-function-eliminator-double-parsed-correctly-output.js b/tests/optimizer/test-function-eliminator-double-parsed-correctly-output.js index adab68feb486a..0359ee4ccfd67 100644 --- a/tests/optimizer/test-function-eliminator-double-parsed-correctly-output.js +++ b/tests/optimizer/test-function-eliminator-double-parsed-correctly-output.js @@ -1,6 +1,7 @@ // EMSCRIPTEN_START_ASM var asm = (function(global, env, buffer) { "use asm"; + var tempDouble = 0.0; var e = 0; // EMSCRIPTEN_START_FUNCS diff --git a/tests/optimizer/test-function-eliminator-double-parsed-correctly.js b/tests/optimizer/test-function-eliminator-double-parsed-correctly.js index 3ebb47215ab05..f8c40b88c1336 100644 --- a/tests/optimizer/test-function-eliminator-double-parsed-correctly.js +++ b/tests/optimizer/test-function-eliminator-double-parsed-correctly.js @@ -1,6 +1,7 @@ // EMSCRIPTEN_START_ASM var asm = (function(global, env, buffer) { "use asm"; + var tempDouble = 0.0; var e = 0; // EMSCRIPTEN_START_FUNCS diff --git a/tests/test_other.py b/tests/test_other.py index 06a2a94db6472..55e8fc50c18ea 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -7072,7 +7072,7 @@ def test_function_eliminator_double_parsed_correctly(self): # Compare expected_file_contents = self.get_file_contents(path_from_root('tests', 'optimizer', 'test-function-eliminator-double-parsed-correctly-output.js')) - self.assertIdentical(output_file_contents, expected_file_contents) + self.assertIdentical(expected_file_contents, output_file_contents) finally: tools.tempfiles.try_delete(output_file) diff --git a/tools/duplicate_function_eliminator.py b/tools/duplicate_function_eliminator.py index 2a788f1d6c9a4..c8b28da988cb8 100644 --- a/tools/duplicate_function_eliminator.py +++ b/tools/duplicate_function_eliminator.py @@ -184,13 +184,17 @@ def run_on_js(filename, gen_hash_info=False): # We need to split out the asm shell as well, for minification pre = js[:start_asm + len(start_asm_marker)] post = js[end_asm:] - asm_shell = js[start_asm + len(start_asm_marker):start_funcs + len(start_funcs_marker)] + ''' -EMSCRIPTEN_FUNCS(); -''' + js[end_funcs + len(end_funcs_marker):end_asm + len(end_asm_marker)] + asm_shell_pre = js[start_asm + len(start_asm_marker):start_funcs + len(start_funcs_marker)] + # Prevent "uglify" from turning 0.0 into 0 in variables' initialization. To do this we first replace 0.0 with + # ZERO$DOT$ZERO and then replace it back. + asm_shell_pre = re.sub(r'(\S+\s*=\s*)0\.0', r'\1ZERO$DOT$ZERO', asm_shell_pre) + asm_shell_post = js[end_funcs + len(end_funcs_marker):end_asm + len(end_asm_marker)] + asm_shell = asm_shell_pre + '\nEMSCRIPTEN_FUNCS();\n' + asm_shell_post js = js[start_funcs + len(start_funcs_marker):end_funcs] # we assume there is a maximum of one new name per line asm_shell_pre, asm_shell_post = process_shell(js, js_engine, asm_shell, equivalentfn_hash_info).split('EMSCRIPTEN_FUNCS();'); + asm_shell_pre = re.sub(r'(\S+\s*=\s*)ZERO\$DOT\$ZERO', r'\g<1>0.0', asm_shell_pre) asm_shell_post = asm_shell_post.replace('});', '})'); pre += asm_shell_pre + '\n' + start_funcs_marker post = end_funcs_marker + asm_shell_post + post diff --git a/tools/validate_asmjs.py b/tools/validate_asmjs.py index ab810a7d3e585..8adef753ba497 100755 --- a/tools/validate_asmjs.py +++ b/tools/validate_asmjs.py @@ -18,7 +18,7 @@ def validate_asmjs_jsfile(filename, muteOutput): cmd = shared.SPIDERMONKEY_ENGINE + ['-c', filename] if not shared.SPIDERMONKEY_ENGINE or cmd[0] == 'js-not-found' or len(cmd[0].strip()) == 0: - print >> sys.stderr, 'Could not find SpiderMonkey engine! Please set tis location to SPIDERMONKEY_ENGINE in your ' + shared.hint_config_file_location() + ' configuration file!' + print >> sys.stderr, 'Could not find SpiderMonkey engine! Please set its location to SPIDERMONKEY_ENGINE in your ' + shared.hint_config_file_location() + ' configuration file!' return False try: process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) From 47d4b6fff558f493a7bdddfa297560cd39125b22 Mon Sep 17 00:00:00 2001 From: "Alon Zakai (kripken)" Date: Fri, 24 Feb 2017 19:05:43 -0800 Subject: [PATCH 47/72] fix memory growth in binaryen interpreter, broken in #4918 --- src/preamble.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/preamble.js b/src/preamble.js index 75e276214d987..a397632085780 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -2326,6 +2326,7 @@ function integrateWasmJS(Module) { return null; } } else { + exports['__growWasmMemory']((size - oldSize) / wasmPageSize); // tiny wasm method that just does grow_memory // in interpreter, we replace Module.buffer if we allocate return Module['buffer'] !== old ? Module['buffer'] : null; // if it was reallocated, it changed } From f920c92779d3e10eae710f474c9af0699143621b Mon Sep 17 00:00:00 2001 From: "Alon Zakai (kripken)" Date: Sat, 25 Feb 2017 15:45:18 -0800 Subject: [PATCH 48/72] fix asm2.test_webidl, work around a closure compiler issue where a singleton object's methods were called without the instance --- tools/webidl_binder.py | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/tools/webidl_binder.py b/tools/webidl_binder.py index 72171ee5bf241..8b4f9e849f604 100644 --- a/tools/webidl_binder.py +++ b/tools/webidl_binder.py @@ -148,42 +148,42 @@ def emit_constructor(name): needed: 0, // the total size we need next time prepare: function() { - if (this.needed) { + if (ensureCache.needed) { // clear the temps - for (var i = 0; i < this.temps.length; i++) { - Module['_free'](this.temps[i]); + for (var i = 0; i < ensureCache.temps.length; i++) { + Module['_free'](ensureCache.temps[i]); } - this.temps.length = 0; + ensureCache.temps.length = 0; // prepare to allocate a bigger buffer - Module['_free'](this.buffer); - this.buffer = 0; - this.size += this.needed; + Module['_free'](ensureCache.buffer); + ensureCache.buffer = 0; + ensureCache.size += ensureCache.needed; // clean up - this.needed = 0; + ensureCache.needed = 0; } - if (!this.buffer) { // happens first time, or when we need to grow - this.size += 128; // heuristic, avoid many small grow events - this.buffer = Module['_malloc'](this.size); - assert(this.buffer); + if (!ensureCache.buffer) { // happens first time, or when we need to grow + ensureCache.size += 128; // heuristic, avoid many small grow events + ensureCache.buffer = Module['_malloc'](ensureCache.size); + assert(ensureCache.buffer); } - this.pos = 0; + ensureCache.pos = 0; }, alloc: function(array, view) { - assert(this.buffer); + assert(ensureCache.buffer); var bytes = view.BYTES_PER_ELEMENT; var len = array.length * bytes; len = (len + 7) & -8; // keep things aligned to 8 byte boundaries var ret; - if (this.pos + len >= this.size) { - // we failed to allocate in the buffer, this time around :( + if (ensureCache.pos + len >= ensureCache.size) { + // we failed to allocate in the buffer, ensureCache time around :( assert(len > 0); // null terminator, at least - this.needed += len; + ensureCache.needed += len; ret = Module['_malloc'](len); - this.temps.push(ret); + ensureCache.temps.push(ret); } else { // we can allocate in the buffer - ret = this.buffer + this.pos; - this.pos += len; + ret = ensureCache.buffer + ensureCache.pos; + ensureCache.pos += len; } var retShifted = ret; switch (bytes) { From 142adaf30f0a729b2cd7f8da8569557a98cd8b65 Mon Sep 17 00:00:00 2001 From: "Tasuku SUENAGA a.k.a. gunyarakun" Date: Mon, 27 Feb 2017 16:08:05 -0800 Subject: [PATCH 49/72] Fix handling node types on JSDCE() in js-optimizer.js (#4695) * Add a test case of js-optimizer for special object keys * Check variable names and object keys strictly on JSDCE * Move a variable definition and add comments to clarify what's done * Add Tasuku SUENAGA to AUTHORS --- AUTHORS | 1 + .../JSDCE-uglifyjsNodeTypes-output.js | 52 +++++++++++++++++++ tests/optimizer/JSDCE-uglifyjsNodeTypes.js | 45 ++++++++++++++++ tests/test_other.py | 2 + tools/js-optimizer.js | 19 +++++++ 5 files changed, 119 insertions(+) create mode 100644 tests/optimizer/JSDCE-uglifyjsNodeTypes-output.js create mode 100644 tests/optimizer/JSDCE-uglifyjsNodeTypes.js diff --git a/AUTHORS b/AUTHORS index de5584db86f34..8e0b1e57aa1da 100644 --- a/AUTHORS +++ b/AUTHORS @@ -277,3 +277,4 @@ a license to everyone to use it as detailed in LICENSE.) * Joshua Lind * Hiroaki GOTO as "GORRY" * Mikhail Kremnyov (copyright owned by XCDS International) +* Tasuku SUENAGA a.k.a. gunyarakun diff --git a/tests/optimizer/JSDCE-uglifyjsNodeTypes-output.js b/tests/optimizer/JSDCE-uglifyjsNodeTypes-output.js new file mode 100644 index 0000000000000..8de047f5af3d9 --- /dev/null +++ b/tests/optimizer/JSDCE-uglifyjsNodeTypes-output.js @@ -0,0 +1,52 @@ +var defun = (function() { +}); +var name = (function() { +}); +var object = (function() { +}); +var non_reserved = (function() { +}); +function func_1() { +} +function func_2() { +} +function func_3() { +} +function func_4() { +} +function func_5() { +} +function func_6() { +} +function func_7() { +} +function func_8() { +} +function func_9() { +} +function func_10() { +} +var quotedObject = { + "var": func_1, + "defun": func_2, + "function": func_3, + "name": func_4, + "non_reserved": func_5 +}; +var unquotedObject = { + "var": func_6, + defun: func_7, + "function": func_8, + name: func_9, + non_reserved: func_10 +}; +var recursiveObject = { + object: { + func: (function() { + }), + object: { + func: (function() { + }) + } + } +}; diff --git a/tests/optimizer/JSDCE-uglifyjsNodeTypes.js b/tests/optimizer/JSDCE-uglifyjsNodeTypes.js new file mode 100644 index 0000000000000..516eb840652b7 --- /dev/null +++ b/tests/optimizer/JSDCE-uglifyjsNodeTypes.js @@ -0,0 +1,45 @@ +var defun = function () { var a = 1; }; +var name = function () { var a = 1; }; +var object = function () { var a = 1; }; +var non_reserved = function () { var a = 1; }; + +function func_1() { var a = 1; } +function func_2() { var a = 1; } +function func_3() { var a = 1; } +function func_4() { var a = 1; } +function func_5() { var a = 1; } +function func_6() { var a = 1; } +function func_7() { var a = 1; } +function func_8() { var a = 1; } +function func_9() { var a = 1; } +function func_10() { var a = 1; } +function func_deleted() { var a = 1; } + +var quotedObject = { + "var": func_1, + "defun": func_2, + "function": func_3, + "name": func_4, + "non_reserved": func_5 +}; + +var unquotedObject = { + var: func_6, + defun: func_7, + function: func_8, + name: func_9, + non_reserved: func_10 +}; + +var recursiveObject = { + object: { + func: function () { + var a = 1; + }, + object: { + func: function () { + var b = 1; + } + } + } +}; diff --git a/tests/test_other.py b/tests/test_other.py index 55e8fc50c18ea..391b5877f7cf4 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -2075,6 +2075,8 @@ def test_js_optimizer(self): ['splitMemory']), (path_from_root('tests', 'optimizer', 'JSDCE.js'), open(path_from_root('tests', 'optimizer', 'JSDCE-output.js')).read(), ['JSDCE']), + (path_from_root('tests', 'optimizer', 'JSDCE-uglifyjsNodeTypes.js'), open(path_from_root('tests', 'optimizer', 'JSDCE-uglifyjsNodeTypes-output.js')).read(), + ['JSDCE']), ]: print input, passes diff --git a/tools/js-optimizer.js b/tools/js-optimizer.js index 6fc4cc446e974..4a04e7333e35d 100644 --- a/tools/js-optimizer.js +++ b/tools/js-optimizer.js @@ -7800,14 +7800,31 @@ function JSDCE(ast) { }); return ast; } + var isVarNameOrObjectKeys = []; + // isVarNameOrObjectKeys is a stack which saves the state the node is defining a variable or in an object literal. + // the second argument `type` passed into the callback function called by traverse() could be a variable name or object key name. + // You cannot distinguish the `type` is a real type or not without isVarNameOrObjectKeys. + // ex.) var name = true; // `type` can be 'name' + // var obj = { defun: true } // `type` can be 'defun' traverse(ast, function(node, type) { + if (isVarNameOrObjectKeys[isVarNameOrObjectKeys.length - 1]) { // check parent node defines a variable or is an object literal + // `type` is a variable name or an object key name + isVarNameOrObjectKeys.push(false); // doesn't define a variable nor be an object literal + return; + } if (type === 'var') { node[1].forEach(function(varItem, j) { var name = varItem[0]; ensureData(scopes[scopes.length-1], name).def = 1; }); + isVarNameOrObjectKeys.push(true); // this `node` defines a varible + return; + } + if (type === 'object') { + isVarNameOrObjectKeys.push(true); // this `node` is an object literal return; } + isVarNameOrObjectKeys.push(false); // doesn't define a variable nor be an object literal if (type === 'defun' || type === 'function') { if (node[1]) ensureData(scopes[scopes.length-1], node[1]).def = 1; var scope = {}; @@ -7822,6 +7839,8 @@ function JSDCE(ast) { ensureData(scopes[scopes.length-1], node[1]).use = 1; } }, function(node, type) { + isVarNameOrObjectKeys.pop(); + if (isVarNameOrObjectKeys[isVarNameOrObjectKeys.length - 1]) return; // `type` is a variable name or an object key name if (type === 'defun' || type === 'function') { var scope = scopes.pop(); var names = set(); From 94ac79b119ff72b2b23ef33066da5939c2177101 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 28 Feb 2017 11:01:34 -0800 Subject: [PATCH 50/72] setTempRet0's return value is received in some cases, even though it is ignored. JS engines no longer accept an undefined as a valid value now, so we must return an int properly (#4976) --- src/runtime.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/runtime.js b/src/runtime.js index ac293bb2e32ad..7bfe9f8292bee 100644 --- a/src/runtime.js +++ b/src/runtime.js @@ -132,6 +132,7 @@ var Runtime = { // word is placed in tempRet0. This provides an accessor for that value. setTempRet0: function(value) { tempRet0 = value; + return value; }, getTempRet0: function() { return tempRet0; @@ -565,6 +566,7 @@ if (MAIN_MODULE || SIDE_MODULE) { }; Runtime.setTempRet0 = function(x) { Runtime.tempRet0 = x; + return x; }; } From cee49eca79079716e77ebc86a2876e11457206e9 Mon Sep 17 00:00:00 2001 From: jgravelle-google Date: Tue, 28 Feb 2017 13:18:16 -0800 Subject: [PATCH 51/72] Add expected wasm_backend alignment to align_moar test (#4989) --- tests/test_core.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index f504ed1c4ad46..f164b837c1589 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -355,7 +355,6 @@ def test_align64(self): 0.00,10,123.46,0.00 : 0.00,10,123.46,0.00 ''') - @no_wasm_backend('specific alignment semantics may be asmjs-specific?') def test_align_moar(self): self.emcc_args = self.emcc_args + ['-msse'] def test(): @@ -382,7 +381,9 @@ def test(): printf("Alignment: %d addr: 0x%x\n", ((int)&v) % 16, (int)&v); printf("Alignment: %d addr: 0x%x\n", ((int)&m) % 16, (int)&m); } - ''', ('Alignment: 0 addr: 0xa20\nAlignment: 0 addr: 0xa60\n', 'Alignment: 0 addr: 0xe10\nAlignment: 0 addr: 0xe50\n')) + ''', ('Alignment: 0 addr: 0xa20\nAlignment: 0 addr: 0xa60\n', # asmjs + 'Alignment: 0 addr: 0xe10\nAlignment: 0 addr: 0xe50\n', # asm2wasm + 'Alignment: 0 addr: 0x410\nAlignment: 0 addr: 0x450\n',)) # wasm_backend test() print 'relocatable' From 88dba7bab608056a4defa98e752fe0ab79034d49 Mon Sep 17 00:00:00 2001 From: jgravelle-google Date: Tue, 28 Feb 2017 13:18:37 -0800 Subject: [PATCH 52/72] Enable test_math_lgamma for wasm_backend (fixed in clang) (#4988) --- tests/test_core.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_core.py b/tests/test_core.py index f164b837c1589..ead3040948b9a 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -512,7 +512,6 @@ def test_math_hyperbolic(self): expected = open(path_from_root('tests', 'hyperbolic', 'output.txt'), 'r').read() self.do_run(src, expected) - @no_wasm_backend() def test_math_lgamma(self): test_path = path_from_root('tests', 'math', 'lgamma') src, output = (test_path + s for s in ('.c', '.out')) From 47c2c823995b9f4491dc102b1a45206f94737801 Mon Sep 17 00:00:00 2001 From: jgravelle-google Date: Tue, 28 Feb 2017 13:18:54 -0800 Subject: [PATCH 53/72] Generify message for test_python, add message for test_cases (#4987) --- tests/test_core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index ead3040948b9a..b45be68eb8fe6 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -5601,7 +5601,7 @@ def clean(text): Settings.ALLOW_MEMORY_GROWTH = 0 do_test() - @no_wasm_backend("python.bc was compiled with asmjs, and we don't have unified triples") + @no_wasm_backend("uses bitcode compiled with asmjs, and we don't have unified triples") def test_python(self): Settings.EMULATE_FUNCTION_POINTER_CASTS = 1 @@ -5632,7 +5632,7 @@ def test_lifetime(self): # Test cases in separate files. Note that these files may contain invalid .ll! # They are only valid enough for us to read for test purposes, not for llvm-as # to process. - @no_wasm_backend() + @no_wasm_backend("uses bitcode compiled with asmjs, and we don't have unified triples") def test_cases(self): if Building.LLVM_OPTS: return self.skip("Our code is not exactly 'normal' llvm assembly") From 04255b4ada1983301cb35cafbc5483db036149fe Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 28 Feb 2017 15:30:27 -0800 Subject: [PATCH 54/72] ignore the 'no running' warning in test_files --- tests/test_core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_core.py b/tests/test_core.py index b45be68eb8fe6..fce9c6d5e2915 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -4097,7 +4097,7 @@ def process(filename): try_delete(mem_file) def clean(out, err): - return '\n'.join(filter(lambda line: 'binaryen' not in line and 'wasm' not in line, (out + err).split('\n'))) + return '\n'.join(filter(lambda line: 'binaryen' not in line and 'wasm' not in line and 'so not running' not in line, (out + err).split('\n'))) self.do_run(src, map(lambda x: x if 'SYSCALL_DEBUG=1' not in mode else ('syscall! 146,SYS_writev' if self.run_name == 'default' else 'syscall! 146'), ('size: 7\ndata: 100,-56,50,25,10,77,123\nloop: 100 -56 50 25 10 77 123 \ninput:hi there!\ntexto\n$\n5 : 10,30,20,11,88\nother=some data.\nseeked=me da.\nseeked=ata.\nseeked=ta.\nfscanfed: 10 - hello\n5 bytes to dev/null: 5\nok.\ntexte\n', 'size: 7\ndata: 100,-56,50,25,10,77,123\nloop: 100 -56 50 25 10 77 123 \ninput:hi there!\ntexto\ntexte\n$\n5 : 10,30,20,11,88\nother=some data.\nseeked=me da.\nseeked=ata.\nseeked=ta.\nfscanfed: 10 - hello\n5 bytes to dev/null: 5\nok.\n')), post_build=post, extra_emscripten_args=['-H', 'libc/fcntl.h'], output_nicerizer=clean) From 593306483e21c36e9f7c5def8a4bd481f20a8de4 Mon Sep 17 00:00:00 2001 From: "Alon Zakai (kripken)" Date: Thu, 2 Mar 2017 13:16:13 -0800 Subject: [PATCH 55/72] fix --output_eol docs #4972 --- site/source/docs/tools_reference/emcc.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/site/source/docs/tools_reference/emcc.rst b/site/source/docs/tools_reference/emcc.rst index 157733343ce83..f53b3ff8932a2 100644 --- a/site/source/docs/tools_reference/emcc.rst +++ b/site/source/docs/tools_reference/emcc.rst @@ -447,8 +447,8 @@ Options that are modified or new in *emcc* are listed below: ``--separate-asm`` Emits asm.js in one file, and the rest of the code in another, and emits HTML that loads the asm.js first, in order to reduce memory load during startup. See :ref:`optimizing-code-separating_asm`. -``--output-eol windows|linux`` - Specifies the line ending to generate for the text files that are outputted. If "--output-eol windows" is passed, the final output files will have Windows \r\n line endings in them. With "--output-eol linux", the final generated files will be written with Unix \n line endings. +``--output_eol windows|linux`` + Specifies the line ending to generate for the text files that are outputted. If "--output_eol windows" is passed, the final output files will have Windows \r\n line endings in them. With "--output_eol linux", the final generated files will be written with Unix \n line endings. ``--cflags`` Prints out the flags ``emcc`` would pass to ``clang`` to compile source code to object/bitcode form. You can use this to invoke clang yourself, and then run ``emcc`` on those outputs just for the final linking+conversion to JS. From 59b8f23087dbdf97161f24013eb32622a770eebe Mon Sep 17 00:00:00 2001 From: "Alon Zakai (kripken)" Date: Thu, 2 Mar 2017 21:50:06 -0800 Subject: [PATCH 56/72] update binaryen to version_29 --- tools/ports/binaryen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ports/binaryen.py b/tools/ports/binaryen.py index 0734f748d68e2..07d8d08632530 100644 --- a/tools/ports/binaryen.py +++ b/tools/ports/binaryen.py @@ -1,6 +1,6 @@ import os, shutil, logging -TAG = 'version_28' +TAG = 'version_29' def needed(settings, shared, ports): if not settings.BINARYEN: return False From cb1e78019c1e381b7464c2489804ec5219fb9cdb Mon Sep 17 00:00:00 2001 From: "Alon Zakai (kripken)" Date: Fri, 3 Mar 2017 11:48:38 -0800 Subject: [PATCH 57/72] csmith tweaks --- tests/fuzz/csmith_driver.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/fuzz/csmith_driver.py b/tests/fuzz/csmith_driver.py index c9e21cb47b925..676c3b7ee2b6e 100755 --- a/tests/fuzz/csmith_driver.py +++ b/tests/fuzz/csmith_driver.py @@ -103,6 +103,8 @@ def try_js(args=[]): js_args = [shared.PYTHON, shared.EMCC, fullname, '-o', filename + '.js'] + [opts] + llvm_opts + CSMITH_CFLAGS + args + ['-w'] if TEST_BINARYEN: js_args += ['-s', 'BINARYEN=1'] + if random.random() < 0.5: + js_args += ['-s', 'BINARYEN_IGNORE_IMPLICIT_TRAPS=1'] if random.random() < 0.1: if random.random() < 0.5: js_args += ['--js-opts', '0'] @@ -139,7 +141,7 @@ def try_js(args=[]): js_args += ['-s', 'MAIN_MODULE=1'] if random.random() < 0.25: js_args += ['-s', 'INLINING_LIMIT=1'] # inline nothing, for more call interaction - if random.random() < 0.1: + if random.random() < 0.01: js_args += ['-s', 'EMTERPRETIFY=1'] if random.random() < 0.5: if random.random() < 0.5: From e4ade2d8de73fba836ed6e08f1930c390324c749 Mon Sep 17 00:00:00 2001 From: AlexPerrot Date: Sun, 5 Mar 2017 03:09:05 +0100 Subject: [PATCH 58/72] Allow embind to work with async wasm compilation (#5001) * Use Module["asm"] instead of asm in embind to enable use of embind in wasm. * change test_core.test_embind_2 to work asynchronously. This allows the test to work with wasm async compilation. * reactivate embind when using BINARYEN_ASYNC_COMPILATION --- emcc.py | 23 +++++++++-------------- src/embind/embind.js | 4 ++-- tests/test_core.py | 6 +++++- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/emcc.py b/emcc.py index be82cb98cd48c..0871fa8cd74ad 100755 --- a/emcc.py +++ b/emcc.py @@ -1299,20 +1299,15 @@ def check(input_file): if shared.Building.is_wasm_only() and shared.Settings.EVAL_CTORS: logging.debug('disabling EVAL_CTORS, as in wasm-only mode it hurts more than it helps. TODO: a wasm version of it') shared.Settings.EVAL_CTORS = 0 - if shared.Settings.BINARYEN_ASYNC_COMPILATION == 1: - if bind: - shared.Settings.BINARYEN_ASYNC_COMPILATION = 0 - if 'BINARYEN_ASYNC_COMPILATION=1' in settings_changes: - logging.warning('BINARYEN_ASYNC_COMPILATION requested, but disabled since embind is not compatible with it yet') - elif shared.Building.is_wasm_only(): - # async compilation requires a swappable module - we swap it in when it's ready - shared.Settings.SWAPPABLE_ASM_MODULE = 1 - else: - # if not wasm-only, we can't do async compilation as the build can run in other - # modes than wasm (like asm.js) which may not support an async step - shared.Settings.BINARYEN_ASYNC_COMPILATION = 0 - if 'BINARYEN_ASYNC_COMPILATION=1' in settings_changes: - logging.warning('BINARYEN_ASYNC_COMPILATION requested, but disabled since not in wasm-only mode') + if shared.Settings.BINARYEN_ASYNC_COMPILATION == 1 and shared.Building.is_wasm_only(): + # async compilation requires a swappable module - we swap it in when it's ready + shared.Settings.SWAPPABLE_ASM_MODULE = 1 + else: + # if not wasm-only, we can't do async compilation as the build can run in other + # modes than wasm (like asm.js) which may not support an async step + shared.Settings.BINARYEN_ASYNC_COMPILATION = 0 + if 'BINARYEN_ASYNC_COMPILATION=1' in settings_changes: + logging.warning('BINARYEN_ASYNC_COMPILATION requested, but disabled since not in wasm-only mode') # wasm outputs are only possible with a side wasm if target.endswith(WASM_ENDINGS): diff --git a/src/embind/embind.js b/src/embind/embind.js index e39ac1a912fb6..b096de93c42d7 100644 --- a/src/embind/embind.js +++ b/src/embind/embind.js @@ -946,13 +946,13 @@ var LibraryEmbind = { // This has three main penalties: // - dynCall is another function call in the path from JavaScript to C++. // - JITs may not predict through the function table indirection at runtime. - var dc = asm['dynCall_' + signature]; + var dc = Module["asm"]['dynCall_' + signature]; if (dc === undefined) { // We will always enter this branch if the signature // contains 'f' and PRECISE_F32 is not enabled. // // Try again, replacing 'f' with 'd'. - dc = asm['dynCall_' + signature.replace(/f/g, 'd')]; + dc = Module["asm"]['dynCall_' + signature.replace(/f/g, 'd')]; if (dc === undefined) { throwBindingError("No dynCall invoker for signature: " + signature); } diff --git a/tests/test_core.py b/tests/test_core.py index fce9c6d5e2915..a22d790833048 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -6268,10 +6268,13 @@ def test_embind_2(self): Settings.NO_EXIT_RUNTIME = 1 # we emit some post.js that we need to see Building.COMPILER_TEST_OPTS += ['--bind', '--post-js', 'post.js'] open('post.js', 'w').write(''' - Module.print('lerp ' + Module.lerp(100, 200, 66) + '.'); + function printLerp() { + Module.print('lerp ' + Module.lerp(100, 200, 66) + '.'); + } ''') src = r''' #include + #include #include using namespace emscripten; int lerp(int a, int b, int t) { @@ -6281,6 +6284,7 @@ def test_embind_2(self): function("lerp", &lerp); } int main(int argc, char **argv) { + EM_ASM(printLerp()); return 0; } ''' From c8305b21cd9aa2175cbd38120f3efb81b10ea73e Mon Sep 17 00:00:00 2001 From: AlexPerrot Date: Mon, 6 Mar 2017 19:25:46 +0100 Subject: [PATCH 59/72] Add support for policies when calling val::as. (#4992) * Add support for policies when calling val::as. This allows to use allow_raw_pointers() with val::as to get a pointer. * Add test for support of val::as(allow_raw_pointers()). --- system/include/emscripten/val.h | 9 +++++---- tests/embind/embind.test.js | 11 +++++++++++ tests/embind/embind_test.cpp | 21 +++++++++++++++++++++ 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/system/include/emscripten/val.h b/system/include/emscripten/val.h index 836f4880346ee..2af53bdd1e900 100644 --- a/system/include/emscripten/val.h +++ b/system/include/emscripten/val.h @@ -87,7 +87,7 @@ namespace emscripten { EM_VAL _emval_typeof(EM_VAL value); } - template + template struct symbol_registrar { symbol_registrar() { internal::_emval_register_symbol(address); @@ -419,16 +419,17 @@ namespace emscripten { return MethodCaller::call(handle, name, std::forward(args)...); } - template - T as() const { + template + T as(Policies...) const { using namespace internal; typedef BindingType BT; + typename WithPolicies::template ArgTypeList targetType; EM_DESTRUCTORS destructors; EM_GENERIC_WIRE_TYPE result = _emval_as( handle, - TypeID::get(), + targetType.getTypes()[0], &destructors); DestructorsRunner dr(destructors); return fromGenericWireType(result); diff --git a/tests/embind/embind.test.js b/tests/embind/embind.test.js index 4bff00813aa74..a11d08463cf97 100644 --- a/tests/embind/embind.test.js +++ b/tests/embind/embind.test.js @@ -478,6 +478,17 @@ module({ assert.equal(3, cm.const_ref_adder(1, 2)); }); + test("get instance pointer as value", function() { + var v = cm.emval_test_instance_pointer(); + assert.instanceof(v, cm.DummyForPointer); + }); + + test("cast value to instance pointer using as", function() { + var v = cm.emval_test_instance_pointer(); + var p_value = cm.emval_test_value_from_instance_pointer(v); + assert.equal(42, p_value); + }); + test("passthrough", function() { var a = {foo: 'bar'}; var b = cm.emval_test_passthrough(a); diff --git a/tests/embind/embind_test.cpp b/tests/embind/embind_test.cpp index 0c03ca54d188f..f1ae74d10c866 100644 --- a/tests/embind/embind_test.cpp +++ b/tests/embind/embind_test.cpp @@ -39,6 +39,23 @@ val emval_test_new_object() { return rv; } +struct DummyForPointer { + int value; + DummyForPointer(const int v) : value(v) {} +}; + +static DummyForPointer emval_pointer_dummy(42); + +val emval_test_instance_pointer() { + DummyForPointer* p = &emval_pointer_dummy; + return val(p); +} + +int emval_test_value_from_instance_pointer(val v) { + DummyForPointer * p = v.as(allow_raw_pointers()); + return p->value; +} + unsigned emval_test_passthrough_unsigned(unsigned v) { return v; } @@ -1653,11 +1670,15 @@ EMSCRIPTEN_BINDINGS(tests) { register_vector("FloatVector"); register_vector>("IntegerVectorVector"); + class_("DummyForPointer"); + function("mallinfo", &emval_test_mallinfo); function("emval_test_new_integer", &emval_test_new_integer); function("emval_test_new_string", &emval_test_new_string); function("emval_test_get_string_from_val", &emval_test_get_string_from_val); function("emval_test_new_object", &emval_test_new_object); + function("emval_test_instance_pointer", &emval_test_instance_pointer); + function("emval_test_value_from_instance_pointer", &emval_test_value_from_instance_pointer); function("emval_test_passthrough_unsigned", &emval_test_passthrough_unsigned); function("emval_test_passthrough", &emval_test_passthrough); function("emval_test_return_void", &emval_test_return_void); From ebf098091e945e4493a5779ed42edc2133ebeb74 Mon Sep 17 00:00:00 2001 From: "Alon Zakai (kripken)" Date: Mon, 6 Mar 2017 14:25:20 -0800 Subject: [PATCH 60/72] make other.test_binaryen_warn_mem synchronous, as it is not compatible with async compilation yet, see #4971 --- tests/test_other.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_other.py b/tests/test_other.py index 391b5877f7cf4..14731da72417f 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -7281,13 +7281,13 @@ def test_binaryen_warn_mem(self): if SPIDERMONKEY_ENGINE not in JS_ENGINES: return self.skip('cannot run without spidermonkey') # if user changes TOTAL_MEMORY at runtime, the wasm module may not accept the memory import if it is too big/small open('pre.js', 'w').write('var Module = { TOTAL_MEMORY: 50*1024*1024 };\n') - subprocess.check_call([PYTHON, EMCC, path_from_root('tests', 'hello_world.cpp'), '-s', 'WASM=1', '-s', 'BINARYEN_METHOD="native-wasm"', '-s', 'TOTAL_MEMORY=' + str(16*1024*1024), '--pre-js', 'pre.js']) + subprocess.check_call([PYTHON, EMCC, path_from_root('tests', 'hello_world.cpp'), '-s', 'WASM=1', '-s', 'BINARYEN_METHOD="native-wasm"', '-s', 'TOTAL_MEMORY=' + str(16*1024*1024), '--pre-js', 'pre.js', '-s', 'BINARYEN_ASYNC_COMPILATION=0']) out = run_js('a.out.js', engine=SPIDERMONKEY_ENGINE, full_output=True, stderr=PIPE, assert_returncode=None) self.assertContained('imported Memory with incompatible size', out) self.assertContained('Memory size incompatibility issues may be due to changing TOTAL_MEMORY at runtime to something too large. Use ALLOW_MEMORY_GROWTH to allow any size memory (and also make sure not to set TOTAL_MEMORY at runtime to something smaller than it was at compile time).', out) self.assertNotContained('hello, world!', out) # and with memory growth, all should be good - subprocess.check_call([PYTHON, EMCC, path_from_root('tests', 'hello_world.cpp'), '-s', 'WASM=1', '-s', 'BINARYEN_METHOD="native-wasm"', '-s', 'TOTAL_MEMORY=' + str(16*1024*1024), '--pre-js', 'pre.js', '-s', 'ALLOW_MEMORY_GROWTH=1']) + subprocess.check_call([PYTHON, EMCC, path_from_root('tests', 'hello_world.cpp'), '-s', 'WASM=1', '-s', 'BINARYEN_METHOD="native-wasm"', '-s', 'TOTAL_MEMORY=' + str(16*1024*1024), '--pre-js', 'pre.js', '-s', 'ALLOW_MEMORY_GROWTH=1', '-s', 'BINARYEN_ASYNC_COMPILATION=0']) self.assertContained('hello, world!', run_js('a.out.js', engine=SPIDERMONKEY_ENGINE)) def test_binaryen_invalid_method(self): From eac8e0b0142ae5466a79b8aec070521dfd75ea76 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 6 Mar 2017 20:07:52 -0800 Subject: [PATCH 61/72] Minify JS in wasm mode too (#4999) * minify the JS in wasm mode when possible. even if the compiled code is in a side wasm file, in small projects the JS can be of significant size --- emcc.py | 16 +++++++++++++++- tests/test_other.py | 32 ++++++++++++++++++++++---------- tools/shared.py | 7 +++++++ 3 files changed, 44 insertions(+), 11 deletions(-) diff --git a/emcc.py b/emcc.py index 0871fa8cd74ad..76989d74295b8 100755 --- a/emcc.py +++ b/emcc.py @@ -1273,7 +1273,6 @@ def check(input_file): shared.Settings.ASMJS_CODE_FILE = os.path.basename(asm_target) shared.Settings.ASM_JS = 2 # when targeting wasm, we use a wasm Memory, but that is not compatible with asm.js opts - debug_level = max(1, debug_level) # keep whitespace readable, for asm.js parser simplicity shared.Settings.GLOBAL_BASE = 1024 # leave some room for mapping global vars assert not shared.Settings.SPLIT_MEMORY, 'WebAssembly does not support split memory' assert not shared.Settings.USE_PTHREADS, 'WebAssembly does not support pthreads' @@ -2195,6 +2194,21 @@ def do_minify(): # minifies the code. this is also when we do certain optimizati os.unlink(js_target) # we don't need the js, it can just confuse os.unlink(asm_target) # we don't need the asm.js, it can just confuse sys.exit(0) # and we are done. + if opt_level >= 2: + # minify the JS + do_minify() # calculate how to minify + if JSOptimizer.cleanup_shell or JSOptimizer.minify_whitespace or use_closure_compiler: + misc_temp_files.note(final) + shutil.move(js_target, final) + if DEBUG: save_intermediate('preclean', 'js') + if use_closure_compiler: + logging.debug('running closure on shell code') + final = shared.Building.closure_compiler(final, pretty=not JSOptimizer.minify_whitespace) + else: + assert JSOptimizer.cleanup_shell + logging.debug('running cleanup on shell code') + final = shared.Building.js_optimizer_no_asmjs(final, ['noPrintMetadata', 'JSDCE', 'last'] + (['minifyWhitespace'] if JSOptimizer.minify_whitespace else [])) + shutil.move(final, js_target) # If we were asked to also generate HTML, do that if final_suffix == 'html': diff --git a/tests/test_other.py b/tests/test_other.py index 14731da72417f..a4e0c7b9cd3e3 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -7394,24 +7394,32 @@ def test_binaryen_ctors(self): seen = run_js('b.out.js', engine=SPIDERMONKEY_ENGINE) assert correct == seen, correct + '\n vs \n' + seen - def test_binaryen_debuginfo(self): - with clean_write_access_to_canonical_temp_dir(): + # test debug info and debuggability of JS output + def test_binaryen_debug(self): if os.environ.get('EMCC_DEBUG'): return self.skip('cannot run in debug mode') try: os.environ['EMCC_DEBUG'] = '1' - for args, expect_dash_g, expect_emit_text in [ - (['-O0'], False, False), - (['-O0', '-g1'], False, False), - (['-O0', '-g2'], True, False), # in -g2+, we emit -g to asm2wasm so function names are saved - (['-O0', '-g'], True, True), - (['-O0', '--profiling-funcs'], True, False), - (['-O1'], False, False), + for args, expect_dash_g, expect_emit_text, expect_clean_js, expect_whitespace_js, expect_closured in [ + (['-O0'], False, False, False, True, False), + (['-O0', '-g1'], False, False, False, True, False), + (['-O0', '-g2'], True, False, False, True, False), # in -g2+, we emit -g to asm2wasm so function names are saved + (['-O0', '-g'], True, True, False, True, False), + (['-O0', '--profiling-funcs'], True, False, False, True, False), + (['-O1'], False, False, False, True, False), + (['-O2'], False, False, True, False, False), + (['-O2', '-g1'], False, False, True, True, False), + (['-O2', '-g'], True, True, False, True, False), + (['-O2', '--closure', '1'], False, False, True, False, True), + (['-O2', '--closure', '1', '-g1'], False, False, True, True, True), + (['-O2', '--js-opts', '1'], False, False, True, False, False), ]: print args, expect_dash_g, expect_emit_text try_delete('a.out.wast') cmd = [PYTHON, EMCC, path_from_root('tests', 'hello_world.cpp'), '-s', 'WASM=1'] + args print ' '.join(cmd) - output, err = Popen(cmd, stdout=PIPE, stderr=PIPE).communicate() + proc = Popen(cmd, stdout=PIPE, stderr=PIPE) + output, err = proc.communicate() + assert proc.returncode == 0 asm2wasm_line = filter(lambda x: 'asm2wasm' in x, err.split('\n'))[0] asm2wasm_line = asm2wasm_line.strip() + ' ' # ensure it ends with a space, for simpler searches below print '|' + asm2wasm_line + '|' @@ -7421,6 +7429,10 @@ def test_binaryen_debuginfo(self): text = open('a.out.wast').read() assert ';;' in text, 'must see debug info comment' assert 'hello_world.cpp:6' in text, 'must be file:line info' + js = open('a.out.js').read() + assert expect_clean_js == ('// ' not in js), 'cleaned-up js must not have comments' + assert expect_whitespace_js == ('{\n ' in js), 'whitespace-minified js must not have excess spacing' + assert expect_closured == ('var a;' in js), 'closured js must have tiny variable names' finally: del os.environ['EMCC_DEBUG'] diff --git a/tools/shared.py b/tools/shared.py index 96a49fa75b190..67a23ed71e63a 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -2018,6 +2018,13 @@ def js_optimizer(filename, passes, debug=False, extra_info=None, output_filename ret = output_filename return ret + # run JS optimizer on some JS, ignoring asm.js contents if any - just run on it all + @staticmethod + def js_optimizer_no_asmjs(filename, passes): + next = filename + '.jso.js' + subprocess.check_call(NODE_JS + [js_optimizer.JS_OPTIMIZER, filename] + passes, stdout=open(next, 'w')) + return next + @staticmethod def eval_ctors(js_file, mem_init_file): subprocess.check_call([PYTHON, path_from_root('tools', 'ctor_evaller.py'), js_file, mem_init_file, str(Settings.TOTAL_MEMORY), str(Settings.TOTAL_STACK), str(Settings.GLOBAL_BASE)]) From 972fb4e2f90f3a05ed14742438ac7c2aa38257c2 Mon Sep 17 00:00:00 2001 From: Jukka Jylanki Date: Tue, 7 Mar 2017 14:35:02 +0200 Subject: [PATCH 62/72] Propagate errors from llvm-nm and llvm-ar calls back up to main process from subprocesses. Clean up temporary directory created for llvm-ar at process exit. Fix race condition in llvm-ar temporary directory creation. Fixes #4960. --- tools/shared.py | 49 ++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/tools/shared.py b/tools/shared.py index 67a23ed71e63a..e5aa04999835a 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -1236,10 +1236,12 @@ def warn_if_duplicate_entries(archive_contents, archive_filename_hint=''): logging.warning(' duplicate: %s' % curr) warned.add(curr) +# N.B. This function creates a temporary directory specified by the 'dir' field in the returned dictionary. Caller +# is responsible for cleaning up those files after done. def extract_archive_contents(f): try: cwd = os.getcwd() - temp_dir = os.path.join(tempfile.gettempdir(), f.replace('/', '_').replace('\\', '_').replace(':', '_') + '.archive_contents') # TODO: Make sure this is nice and sane + temp_dir = tempfile.mkdtemp('_archive_contents', 'emscripten_temp_') safe_ensure_dirs(temp_dir) os.chdir(temp_dir) contents = filter(lambda x: len(x) > 0, Popen([LLVM_AR, 't', f], stdout=PIPE).communicate()[0].split('\n')) @@ -1247,6 +1249,7 @@ def extract_archive_contents(f): if len(contents) == 0: logging.debug('Archive %s appears to be empty (recommendation: link an .so instead of .a)' % f) return { + 'returncode': 0, 'dir': temp_dir, 'files': [] } @@ -1257,10 +1260,15 @@ def extract_archive_contents(f): dirname = os.path.dirname(content) if dirname: safe_ensure_dirs(dirname) - Popen([LLVM_AR, 'xo', f], stdout=PIPE).communicate() # if absolute paths, files will appear there. otherwise, in this directory - contents = filter(os.path.exists, map(os.path.abspath, contents)) - contents = filter(Building.is_bitcode, contents) + proc = Popen([LLVM_AR, 'xo', f], stdout=PIPE, stderr=PIPE) + stdout, stderr = proc.communicate() # if absolute paths, files will appear there. otherwise, in this directory + contents = map(os.path.abspath, contents) + nonexisting_contents = filter(lambda x: not os.path.exists(x), contents) + if len(nonexisting_contents) != 0: + raise Exception('llvm-ar failed to extract file(s) ' + str(nonexisting_contents) + ' from archive file ' + f + '! Error:' + str(stdout) + str(stderr)) + return { + 'returncode': proc.returncode, 'dir': temp_dir, 'files': contents } @@ -1270,12 +1278,15 @@ def extract_archive_contents(f): os.chdir(cwd) return { + 'returncode': 1, 'dir': None, 'files': [] } class ObjectFileInfo: - def __init__(self, defs, undefs, commons): + def __init__(self, returncode, output, defs=set(), undefs=set(), commons=set()): + self.returncode = returncode + self.output = output self.defs = defs self.undefs = undefs self.commons = commons @@ -1283,7 +1294,7 @@ def __init__(self, defs, undefs, commons): # Due to a python pickling issue, the following two functions must be at top level, or multiprocessing pool spawn won't find them. def g_llvm_nm_uncached(filename): - return Building.llvm_nm_uncached(filename, stdout=PIPE, stderr=None) + return Building.llvm_nm_uncached(filename) def g_multiprocessing_initializer(*args): for item in args: @@ -1608,6 +1619,8 @@ def parallel_llvm_nm(files): object_contents = pool.map(g_llvm_nm_uncached, files) for i in range(len(files)): + if object_contents[i].returncode != 0: + raise Exception('llvm-nm failed on file ' + files[i] + ': return code ' + str(object_contents[i].returncode) + ', error: ' + object_contents[i].output) Building.uninternal_nm_cache[files[i]] = object_contents[i] return object_contents @@ -1629,9 +1642,16 @@ def read_link_inputs(files): # Archives contain objects, so process all archives first in parallel to obtain the object files in them. pool = Building.get_multiprocessing_pool() object_names_in_archives = pool.map(extract_archive_contents, archive_names) + def clean_temporary_archive_contents_directory(directory): + def clean_at_exit(): + try_delete(directory) + if directory: atexit.register(clean_at_exit) for n in range(len(archive_names)): + if object_names_in_archives[n]['returncode'] != 0: + raise Exception('llvm-ar failed on archive ' + archive_names[n] + '!') Building.ar_contents[archive_names[n]] = object_names_in_archives[n]['files'] + clean_temporary_archive_contents_directory(object_names_in_archives[n]['dir']) for o in object_names_in_archives: for f in o['files']: @@ -1873,20 +1893,24 @@ def parse_symbols(output, include_internal=False): ( include_internal and status in ['W', 't', 'T', 'd', 'D']): # FIXME: using WTD in the previous line fails due to llvm-nm behavior on OS X, # so for now we assume all uppercase are normally defined external symbols defs.append(symbol) - return ObjectFileInfo(set(defs), set(undefs), set(commons)) + return ObjectFileInfo(0, None, set(defs), set(undefs), set(commons)) internal_nm_cache = {} # cache results of nm - it can be slow to run uninternal_nm_cache = {} ar_contents = {} # Stores the object files contained in different archive files passed as input @staticmethod - def llvm_nm_uncached(filename, stdout=PIPE, stderr=None, include_internal=False): + def llvm_nm_uncached(filename, stdout=PIPE, stderr=PIPE, include_internal=False): # LLVM binary ==> list of symbols - output = Popen([LLVM_NM, filename], stdout=stdout, stderr=stderr).communicate()[0] - return Building.parse_symbols(output, include_internal) + proc = Popen([LLVM_NM, filename], stdout=stdout, stderr=stderr) + stdout, stderr = proc.communicate() + if proc.returncode == 0: + return Building.parse_symbols(stdout, include_internal) + else: + return ObjectFileInfo(proc.returncode, str(stdout) + str(stderr)) @staticmethod - def llvm_nm(filename, stdout=PIPE, stderr=None, include_internal=False): + def llvm_nm(filename, stdout=PIPE, stderr=PIPE, include_internal=False): if include_internal and filename in Building.internal_nm_cache: return Building.internal_nm_cache[filename] elif not include_internal and filename in Building.uninternal_nm_cache: @@ -1894,6 +1918,9 @@ def llvm_nm(filename, stdout=PIPE, stderr=None, include_internal=False): ret = Building.llvm_nm_uncached(filename, stdout, stderr, include_internal) + if ret.returncode != 0: + raise Exception('llvm-nm failed on file ' + filename + ': return code ' + str(ret.returncode) + ', error: ' + ret.output) + if include_internal: Building.internal_nm_cache[filename] = ret else: Building.uninternal_nm_cache[filename] = ret From 28d43cc84b739bd3011f4768f9e2fed8c8a2000a Mon Sep 17 00:00:00 2001 From: Jukka Jylanki Date: Wed, 8 Mar 2017 05:39:49 +0200 Subject: [PATCH 63/72] When feeding and updating queued audio buffers in OpenAL, the buffers might have been created with different OpenAL audio contexts, so audio update code should not rely on a single active AL context, but update each source with the audio context that was used to create that audio source. Also fixes the issue where code un-makecurrents OpenAL audio context in a thread when there are audio sources playing. --- src/library_openal.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/library_openal.js b/src/library_openal.js index cbb3f921cacaa..7f16555e35eb9 100644 --- a/src/library_openal.js +++ b/src/library_openal.js @@ -52,7 +52,7 @@ var LibraryOpenAL = { return; } - var currentTime = AL.currentContext.ctx.currentTime; + var currentTime = src.context.ctx.currentTime; var startTime = src.bufferPosition; for (var i = src.buffersPlayed; i < src.queue.length; i++) { @@ -83,7 +83,7 @@ var LibraryOpenAL = { // If the start offset is negative, we need to offset the actual buffer. var offset = Math.abs(Math.min(startOffset, 0)); - entry.src = AL.currentContext.ctx.createBufferSource(); + entry.src = src.context.ctx.createBufferSource(); entry.src.buffer = entry.buffer; entry.src.connect(src.gain); if (src.playbackRate != 1.0) entry.src.playbackRate.value = src.playbackRate; @@ -372,6 +372,7 @@ var LibraryOpenAL = { var gain = AL.currentContext.ctx.createGain(); gain.connect(AL.currentContext.gain); AL.currentContext.src[AL.newSrcId] = { + context: AL.currentContext, state: 0x1011 /* AL_INITIAL */, queue: [], loop: false, From 93a082bc6dc39107e933ddf1b0d99df3df6e84be Mon Sep 17 00:00:00 2001 From: Jukka Jylanki Date: Wed, 8 Mar 2017 07:43:06 +0200 Subject: [PATCH 64/72] Demote llvm-nm failures from exceptions to logged debug messages, because it can be expected that llvm-nm might fail, and processing that failure is done proper by examining the returncode of the run later, so eagerly stopping the build here is not needed. Fixes #5007. --- tools/shared.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/shared.py b/tools/shared.py index e5aa04999835a..64af117e4216f 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -1620,7 +1620,7 @@ def parallel_llvm_nm(files): for i in range(len(files)): if object_contents[i].returncode != 0: - raise Exception('llvm-nm failed on file ' + files[i] + ': return code ' + str(object_contents[i].returncode) + ', error: ' + object_contents[i].output) + logging.debug('llvm-nm failed on file ' + files[i] + ': return code ' + str(object_contents[i].returncode) + ', error: ' + object_contents[i].output) Building.uninternal_nm_cache[files[i]] = object_contents[i] return object_contents @@ -1919,8 +1919,9 @@ def llvm_nm(filename, stdout=PIPE, stderr=PIPE, include_internal=False): ret = Building.llvm_nm_uncached(filename, stdout, stderr, include_internal) if ret.returncode != 0: - raise Exception('llvm-nm failed on file ' + filename + ': return code ' + str(ret.returncode) + ', error: ' + ret.output) + logging.debug('llvm-nm failed on file ' + filename + ': return code ' + str(ret.returncode) + ', error: ' + ret.output) + # Even if we fail, write the results to the NM cache so that we don't keep trying to llvm-nm the failing file again later. if include_internal: Building.internal_nm_cache[filename] = ret else: Building.uninternal_nm_cache[filename] = ret From 6c728c9df130fe8a53ef1a3511b3f3b0f2c7eb18 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 8 Mar 2017 15:18:57 -0800 Subject: [PATCH 65/72] clean up temp files in other.test_binaryen_debug --- tests/test_other.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_other.py b/tests/test_other.py index a4e0c7b9cd3e3..1b7310d156847 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -7396,6 +7396,7 @@ def test_binaryen_ctors(self): # test debug info and debuggability of JS output def test_binaryen_debug(self): + with clean_write_access_to_canonical_temp_dir(): if os.environ.get('EMCC_DEBUG'): return self.skip('cannot run in debug mode') try: os.environ['EMCC_DEBUG'] = '1' From 67fabbcf79bb16b0c3d3be8a2aafdb6e025290bd Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 24 Feb 2017 12:25:53 -0800 Subject: [PATCH 66/72] fix regression from c4c9d4b68ce8ada83c8caafe2a39ba352409ed98 in ONLY_MY_CODE, in that mode we should not export anything that is not the user's code --- emscripten.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/emscripten.py b/emscripten.py index 534a094e08ecb..15e5880d068fe 100755 --- a/emscripten.py +++ b/emscripten.py @@ -821,16 +821,16 @@ def table_size(table): exported_implemented_functions = list(exported_implemented_functions) + metadata['initializers'] if not settings['ONLY_MY_CODE']: exported_implemented_functions.append('runPostSets') - if settings['ALLOW_MEMORY_GROWTH']: - exported_implemented_functions.append('_emscripten_replace_memory') - if not settings.get('SIDE_MODULE'): - exported_implemented_functions += ['stackAlloc', 'stackSave', 'stackRestore', 'establishStackSpace'] - if settings['SAFE_HEAP']: - exported_implemented_functions += ['setDynamicTop'] - if not settings['RELOCATABLE']: - exported_implemented_functions += ['setTempRet0', 'getTempRet0'] - if not (settings['BINARYEN'] and settings['SIDE_MODULE']): - exported_implemented_functions += ['setThrew'] + if settings['ALLOW_MEMORY_GROWTH']: + exported_implemented_functions.append('_emscripten_replace_memory') + if not settings['SIDE_MODULE']: + exported_implemented_functions += ['stackAlloc', 'stackSave', 'stackRestore', 'establishStackSpace'] + if settings['SAFE_HEAP']: + exported_implemented_functions += ['setDynamicTop'] + if not settings['RELOCATABLE']: + exported_implemented_functions += ['setTempRet0', 'getTempRet0'] + if not (settings['BINARYEN'] and settings['SIDE_MODULE']): + exported_implemented_functions += ['setThrew'] all_exported = exported_implemented_functions + asm_runtime_funcs + function_tables exported_implemented_functions = list(set(exported_implemented_functions)) From e6e11c01709f082c7a6f7ed1d46c004ad7011c2f Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 8 Mar 2017 16:49:39 -0800 Subject: [PATCH 67/72] increase a test timeout --- tests/test_browser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_browser.py b/tests/test_browser.py index f6a79ced06ba5..12ce0c08daa1b 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -3026,7 +3026,7 @@ def test_pthread_cleanup(self): # Tests the pthread mutex api. def test_pthread_mutex(self): for arg in [[], ['-DSPINLOCK_TEST']]: - self.btest(path_from_root('tests', 'pthread', 'test_pthread_mutex.cpp'), expected='50', args=['-O3', '-s', 'USE_PTHREADS=2', '--separate-asm', '-s', 'PTHREAD_POOL_SIZE=8'] + arg, timeout=20) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_mutex.cpp'), expected='50', args=['-O3', '-s', 'USE_PTHREADS=2', '--separate-asm', '-s', 'PTHREAD_POOL_SIZE=8'] + arg, timeout=30) # Test that memory allocation is thread-safe. def test_pthread_malloc(self): From 5da32ace5d7831bac078b32df754deb0e54d7bdc Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 9 Mar 2017 13:31:26 -0800 Subject: [PATCH 68/72] Support Binaryen's 3 trap modes (#5006) * support binaryen's 3 trap modes: js compatibility, clamp, and allow trapping --- emcc.py | 11 +++++++++-- src/settings.js | 11 ++++++++--- tests/test_benchmark.py | 4 ++-- tests/test_core.py | 35 ++++++++++++++++++++++++++++++----- tests/wasm/trap-f2i.cpp | 9 +++++++++ tests/wasm/trap-idiv.cpp | 10 ++++++++++ 6 files changed, 68 insertions(+), 12 deletions(-) create mode 100644 tests/wasm/trap-f2i.cpp create mode 100644 tests/wasm/trap-idiv.cpp diff --git a/emcc.py b/emcc.py index 76989d74295b8..bc36c6addc5f1 100755 --- a/emcc.py +++ b/emcc.py @@ -2114,8 +2114,15 @@ def do_minify(): # minifies the code. this is also when we do certain optimizati # finish compiling to WebAssembly, using asm2wasm, if we didn't already emit WebAssembly directly using the wasm backend. if not shared.Settings.WASM_BACKEND: cmd = [os.path.join(binaryen_bin, 'asm2wasm'), asm_target, '--total-memory=' + str(shared.Settings.TOTAL_MEMORY)] - if shared.Settings.BINARYEN_IMPRECISE: - cmd += ['--imprecise'] + if shared.Settings.BINARYEN_TRAP_MODE == 'js': + cmd += ['--emit-jsified-potential-traps'] + elif shared.Settings.BINARYEN_TRAP_MODE == 'clamp': + cmd += ['--emit-clamped-potential-traps'] + elif shared.Settings.BINARYEN_TRAP_MODE == 'allow': + cmd += ['--emit-potential-traps'] + else: + logging.error('invalid BINARYEN_TRAP_MODE value: ' + shared.Settings.BINARYEN_TRAP_MODE + ' (should be js/clamp/allow)') + sys.exit(1) if shared.Settings.BINARYEN_IGNORE_IMPLICIT_TRAPS: cmd += ['--ignore-implicit-traps'] # pass optimization level to asm2wasm (if not optimizing, or which passes we should run was overridden, do not optimize) diff --git a/src/settings.js b/src/settings.js index 1023ad7b81a5a..268c56db07fa5 100644 --- a/src/settings.js +++ b/src/settings.js @@ -684,14 +684,19 @@ var BINARYEN_METHOD = "native-wasm"; // How we should run WebAssembly code. By d // See binaryen's src/js/wasm.js-post.js for more details and options. var BINARYEN_SCRIPTS = ""; // An optional comma-separated list of script hooks to run after binaryen, // in binaryen's /scripts dir. -var BINARYEN_IMPRECISE = 0; // Whether to apply imprecise/unsafe binaryen optimizations. If enabled, - // code will run faster, but some types of undefined behavior might - // trap in wasm. var BINARYEN_IGNORE_IMPLICIT_TRAPS = 0; // Whether to ignore implicit traps when optimizing in binaryen. // Implicit traps are the unlikely traps that happen in a load that // is out of bounds, or div/rem of 0, etc. We can reorder them, // but we can't ignore that they have side effects, so turning on // this flag lets us do a little more to reduce code size. +var BINARYEN_TRAP_MODE = "js"; // How we handle wasm operations that may trap, which includes integer + // div/rem of 0 and float-to-int of values too large to fit in an int. + // js: do exactly what js does. this can be slower. + // clamp: avoid traps by clamping to a reasonable value. this can be + // faster than "js". + // allow: allow creating operations that can trap. this is the most + // compact, as we just emit a single wasm operation, with no + // guards to trapping values, and also often the fastest. var BINARYEN_PASSES = ""; // A comma-separated list of passes to run in the binaryen optimizer, // for example, "dce,precompute,vacuum". // When set, this overrides the default passes we would normally run. diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index 5416d477d9e1f..5a47094e7c77d 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -137,11 +137,11 @@ def run(self, args): benchmarkers += [ JSBenchmarker('sm-asmjs', SPIDERMONKEY_ENGINE, ['-s', 'PRECISE_F32=2']), JSBenchmarker('sm-simd', SPIDERMONKEY_ENGINE, ['-s', 'SIMD=1']), - JSBenchmarker('sm-wasm', SPIDERMONKEY_ENGINE, ['-s', 'BINARYEN=1', '-s', 'BINARYEN_METHOD="native-wasm"', '-s', 'BINARYEN_IMPRECISE=1']) + JSBenchmarker('sm-wasm', SPIDERMONKEY_ENGINE, ['-s', 'BINARYEN=1', '-s', 'BINARYEN_METHOD="native-wasm"', '-s', 'BINARYEN_TRAP_MODE="allow"']) ] if V8_ENGINE and Building.which(V8_ENGINE[0]): benchmarkers += [ - JSBenchmarker('v8-wasm', V8_ENGINE, ['-s', 'BINARYEN=1', '-s', 'BINARYEN_METHOD="native-wasm"', '-s', 'BINARYEN_IMPRECISE=1']) + JSBenchmarker('v8-wasm', V8_ENGINE, ['-s', 'BINARYEN=1', '-s', 'BINARYEN_METHOD="native-wasm"', '-s', 'BINARYEN_TRAP_MODE="allow"']) ] except Exception, e: benchmarkers_error = str(e) diff --git a/tests/test_core.py b/tests/test_core.py index a22d790833048..0eb8454001c76 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -2342,7 +2342,7 @@ class Bar { post_build=self.dlfcn_post_build) def test_dlfcn_i64(self): - Settings.BINARYEN_IMPRECISE = 1 # avoid using asm2wasm imports, which don't work in side modules yet (should they?) + Settings.BINARYEN_TRAP_MODE = 'clamp' # avoid using asm2wasm imports, which don't work in side modules yet (should they?) if not self.can_dlfcn(): return self.prep_dlfcn_lib() @@ -3245,7 +3245,7 @@ def test_dylink_funcpointers_wrapper(self): ''') def test_dylink_funcpointers_float(self): - Settings.BINARYEN_IMPRECISE = 1 # avoid using asm2wasm imports, which don't work in side modules yet (should they?) + Settings.BINARYEN_TRAP_MODE = 'clamp' # avoid using asm2wasm imports, which don't work in side modules yet (should they?) self.dylink_test(r''' #include #include "header.h" @@ -3424,7 +3424,7 @@ def test_dylink_mallocs(self): ''', expected=['hello through side\n']) def test_dylink_jslib(self): - Settings.BINARYEN_IMPRECISE = 1 # avoid using asm2wasm imports, which don't work in side modules yet (should they?) + Settings.BINARYEN_TRAP_MODE = 'clamp' # avoid using asm2wasm imports, which don't work in side modules yet (should they?) open('lib.js', 'w').write(r''' mergeInto(LibraryManager.library, { test_lib_func: function(x) { @@ -3722,7 +3722,7 @@ def test_dylink_spaghetti(self): 'main init sees -524, -534, 72.\nside init sees 82, 72, -534.\nmain main sees -524, -534, 72.']) def test_dylink_zlib(self): - Settings.BINARYEN_IMPRECISE = 1 # avoid using asm2wasm imports, which don't work in side modules yet (should they?) + Settings.BINARYEN_TRAP_MODE = 'clamp' # avoid using asm2wasm imports, which don't work in side modules yet (should they?) Building.COMPILER_TEST_OPTS += ['-I' + path_from_root('tests', 'zlib')] Popen([PYTHON, path_from_root('embuilder.py'), 'build' ,'zlib']).communicate() @@ -5700,7 +5700,7 @@ def test_cases(self): def test_fuzz(self): Building.COMPILER_TEST_OPTS += ['-I' + path_from_root('tests', 'fuzz', 'include'), '-w'] - Settings.BINARYEN_IMPRECISE = 0 # some of these tests - 2.c', '9.c', '19.c', '21.c', '20.cpp' - div or rem i32 by 0, which traps in wasm + Settings.BINARYEN_TRAP_MODE = 'clamp' # some of these tests - 2.c', '9.c', '19.c', '21.c', '20.cpp' - div or rem i32 by 0, which traps in wasm def run_all(x): print x @@ -7255,6 +7255,31 @@ def test_binaryen(self): self.emcc_args += ['-s', 'BINARYEN=1', '-s', 'BINARYEN_METHOD="interpret-binary"'] self.do_run(open(path_from_root('tests', 'hello_world.c')).read(), 'hello, world!') + @no_wasm_backend("trap mode support") + def test_binaryen_trap_mode(self): + if not self.is_wasm(): return self.skip('wasm test') + TRAP_OUTPUTS = ('trap', 'RuntimeError') + default = Settings.BINARYEN_TRAP_MODE + print 'default is', default + for mode in ['js', 'clamp', 'allow', '']: + print 'mode:', mode + Settings.BINARYEN_TRAP_MODE = mode or default + if not mode: mode = default + print ' idiv' + self.do_run(open(path_from_root('tests', 'wasm', 'trap-idiv.cpp')).read(), + { + 'js': '|0|', + 'clamp': '|0|', + 'allow': TRAP_OUTPUTS + }[mode]) + print ' f2i' + self.do_run(open(path_from_root('tests', 'wasm', 'trap-f2i.cpp')).read(), + { + 'js': '|1337|', # JS did an fmod 2^32 + 'clamp': '|-2147483648|', + 'allow': TRAP_OUTPUTS + }[mode]) + def test_sbrk(self): self.do_run(open(path_from_root('tests', 'sbrk_brk.cpp')).read(), 'OK.') diff --git a/tests/wasm/trap-f2i.cpp b/tests/wasm/trap-f2i.cpp new file mode 100644 index 0000000000000..b352c5e6e2fed --- /dev/null +++ b/tests/wasm/trap-f2i.cpp @@ -0,0 +1,9 @@ +#include + +int main() { + volatile double d = 17179870521; + EM_ASM_({ + Module.print('|' + $0 + '|') + }, int(d)); +} + diff --git a/tests/wasm/trap-idiv.cpp b/tests/wasm/trap-idiv.cpp new file mode 100644 index 0000000000000..932bdf25e2b10 --- /dev/null +++ b/tests/wasm/trap-idiv.cpp @@ -0,0 +1,10 @@ +#include + +int main() { + volatile int i = 1; + volatile int j = 0; + EM_ASM_({ + Module.print('|' + $0 + '|') + }, i / j); +} + From df993f90ee4431e1290923fb26977e3e40dc3b62 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 9 Mar 2017 15:37:22 -0800 Subject: [PATCH 69/72] Fix outlining + growth + wasm (#5015) Minify JS at the end when building to wasm, as in intermediate steps we don't want minified code (can interfere with asm2wasm parsing) --- emcc.py | 3 ++- tests/test_other.py | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/emcc.py b/emcc.py index bc36c6addc5f1..091f673512648 100755 --- a/emcc.py +++ b/emcc.py @@ -1958,7 +1958,8 @@ def do_minify(): # minifies the code. this is also when we do certain optimizati if DEBUG: save_intermediate('eval-ctors', 'js') if js_opts: - if not shared.Settings.EMTERPRETIFY: + # some compilation modes require us to minify later or not at all + if not shared.Settings.EMTERPRETIFY and not shared.Settings.BINARYEN: do_minify() if opt_level >= 2: diff --git a/tests/test_other.py b/tests/test_other.py index 1b7310d156847..2f9297dd96ca3 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -7197,6 +7197,7 @@ def test_binaryen_opts(self): (['-O2', '-s', 'EMTERPRETIFY=1'], True, False), # option forced (['-O2', '-s', 'EVAL_CTORS=1'], False, True), # ctor evaller turned off since only-wasm (['-O2', '-s', 'OUTLINING_LIMIT=1000'], True, False), # option forced + (['-O2', '-s', 'OUTLINING_LIMIT=1000', '-s', 'ALLOW_MEMORY_GROWTH=1'], True, False), # option forced, and also check growth does not interfere (['-O2', '-s', "BINARYEN_METHOD='interpret-s-expr,asmjs'"], True, False), # asmjs in methods means we need good asm.js (['-O3'], False, True), (['-Os'], False, True), @@ -7207,7 +7208,9 @@ def test_binaryen_opts(self): try_delete('a.out.wast') cmd = [PYTHON, EMCC, path_from_root('tests', 'core', 'test_i64.c'), '-s', option + '=1', '-s', 'BINARYEN_METHOD="interpret-s-expr"'] + args print args, 'js opts:', expect_js_opts, 'only-wasm:', expect_only_wasm, ' ', ' '.join(cmd) - output, err = Popen(cmd, stdout=PIPE, stderr=PIPE).communicate() + proc = Popen(cmd, stdout=PIPE, stderr=PIPE) + output, err = proc.communicate() + assert proc.returncode == 0 assert expect_js_opts == ('applying js optimization passes:' in err), err assert expect_only_wasm == ('-emscripten-only-wasm' in err and '--wasm-only' in err), err # check both flag to fastcomp and to asm2wasm wast = open('a.out.wast').read() From 51350b0af3ae99c7cb3abaa0c37f659c9f475199 Mon Sep 17 00:00:00 2001 From: "Alon Zakai (kripken)" Date: Thu, 9 Mar 2017 15:38:10 -0800 Subject: [PATCH 70/72] update binaryen to version_30 --- tools/ports/binaryen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ports/binaryen.py b/tools/ports/binaryen.py index 07d8d08632530..b80e57613d171 100644 --- a/tools/ports/binaryen.py +++ b/tools/ports/binaryen.py @@ -1,6 +1,6 @@ import os, shutil, logging -TAG = 'version_29' +TAG = 'version_30' def needed(settings, shared, ports): if not settings.BINARYEN: return False From d50899404131fa89a1771f6e7c787c7b8f78d2ee Mon Sep 17 00:00:00 2001 From: jgravelle-google Date: Fri, 10 Mar 2017 15:04:02 -0800 Subject: [PATCH 71/72] Test runner main factoring (#5003) * Break some overlong lines * Scope main test runner logic into a function * Replace exec with equivalent setattr Non-global exec assignment isn't allowed due to variable scoping shenanigans, this avoids that and is a bit more direct with intent * Move get modules closer to first use * Start splitting main runner into sections * Continue factoring into functions * Refactor random test selection * Split long import to multiple lines * Split loading suites and running tests * Add missing modules argument to skip_requested_tests * Actually skip tests when requesting skipping Note: this brings skipme in line with the state of test skipping on incoming, namely that it doesn't work, but it does correctly replace the test case with the skipme method. --- tests/runner.py | 279 +++++++++++++++++++++++++++++++----------------- 1 file changed, 179 insertions(+), 100 deletions(-) diff --git a/tests/runner.py b/tests/runner.py index 42ff482978e5d..ff8bf1163ae98 100755 --- a/tests/runner.py +++ b/tests/runner.py @@ -9,7 +9,9 @@ from subprocess import Popen, PIPE, STDOUT -import os, unittest, tempfile, shutil, time, inspect, sys, math, glob, re, difflib, webbrowser, hashlib, threading, platform, BaseHTTPServer, SimpleHTTPServer, multiprocessing, functools, stat, string, random, operator, fnmatch, httplib +import os, unittest, tempfile, shutil, time, inspect, sys, math, glob, re, difflib +import webbrowser, hashlib, threading, platform, BaseHTTPServer, SimpleHTTPServer +import multiprocessing, functools, stat, string, random, fnmatch, httplib from urllib import unquote # Setup @@ -617,14 +619,23 @@ def filtered_js_engines(self, js_engines=None): js_engines = filter(lambda engine: engine == SPIDERMONKEY_ENGINE or engine == V8_ENGINE, js_engines) return js_engines - def do_run_from_file(self, src, expected_output, args=[], output_nicerizer=None, output_processor=None, no_build=False, main_file=None, additional_files=[], js_engines=None, post_build=None, basename='src.cpp', libraries=[], includes=[], force_c=False, build_ll_hook=None, extra_emscripten_args=[]): + def do_run_from_file(self, src, expected_output, + args=[], output_nicerizer=None, output_processor=None, + no_build=False, main_file=None, additional_files=[], + js_engines=None, post_build=None, basename='src.cpp', + libraries=[], includes=[], force_c=False, build_ll_hook=None, + extra_emscripten_args=[]): self.do_run(open(src).read(), open(expected_output).read(), args, output_nicerizer, output_processor, no_build, main_file, additional_files, js_engines, post_build, basename, libraries, includes, force_c, build_ll_hook, extra_emscripten_args) ## Does a complete test - builds, runs, checks output, etc. - def do_run(self, src, expected_output, args=[], output_nicerizer=None, output_processor=None, no_build=False, main_file=None, additional_files=[], js_engines=None, post_build=None, basename='src.cpp', libraries=[], includes=[], force_c=False, build_ll_hook=None, extra_emscripten_args=[], assert_returncode=None): + def do_run(self, src, expected_output, args=[], output_nicerizer=None, + output_processor=None, no_build=False, main_file=None, additional_files=[], + js_engines=None, post_build=None, basename='src.cpp', libraries=[], + includes=[], force_c=False, build_ll_hook=None, extra_emscripten_args=[], + assert_returncode=None): if Settings.ASYNCIFY == 1 and self.is_wasm_backend(): return self.skip("wasm backend doesn't support ASYNCIFY yet") if force_c or (main_file is not None and main_file[-2:]) == '.c': @@ -664,7 +675,9 @@ def do_run(self, src, expected_output, args=[], output_nicerizer=None, output_pr test_index += 1 # No building - just process an existing .ll file (or .bc, which we turn into .ll) - def do_ll_run(self, ll_file, expected_output=None, args=[], js_engines=None, output_nicerizer=None, post_build=None, force_recompile=False, build_ll_hook=None, extra_emscripten_args=[], assert_returncode=None): + def do_ll_run(self, ll_file, expected_output=None, args=[], js_engines=None, + output_nicerizer=None, post_build=None, force_recompile=False, + build_ll_hook=None, extra_emscripten_args=[], assert_returncode=None): filename = os.path.join(self.get_dir(), 'src.cpp') self.prep_ll_run(filename, ll_file, force_recompile, build_ll_hook) @@ -938,13 +951,18 @@ def btest(self, filename, expected=None, reference=None, force_c=False, referenc assert not post_build post_build = self.post_manual_reftest # run proxied - self.btest(filename, expected, reference, force_c, reference_slack, manual_reference, post_build, original_args + ['--proxy-to-worker', '-s', 'GL_TESTING=1'], outfile, message, timeout=timeout) + self.btest(filename, expected, reference, force_c, reference_slack, manual_reference, post_build, + original_args + ['--proxy-to-worker', '-s', 'GL_TESTING=1'], outfile, message, timeout=timeout) ################################################################################################### def get_zlib_library(runner_core): if WINDOWS: - return runner_core.get_library('zlib', os.path.join('libz.a'), configure=[path_from_root('emconfigure.bat')], configure_args=['cmake', '.', '-DBUILD_SHARED_LIBS=OFF'], make=['mingw32-make'], make_args=[]) + return runner_core.get_library('zlib', os.path.join('libz.a'), + configure=[path_from_root('emconfigure.bat')], + configure_args=['cmake', '.', '-DBUILD_SHARED_LIBS=OFF'], + make=['mingw32-make'], + make_args=[]) else: return runner_core.get_library('zlib', os.path.join('libz.a'), make_args=['libz.a']) @@ -966,25 +984,49 @@ def get_bullet_library(runner_core, use_cmake): os.path.join('src', '.libs', 'libBulletCollision.a'), os.path.join('src', '.libs', 'libLinearMath.a')] - return runner_core.get_library('bullet', generated_libs, configure=configure_commands, configure_args=configure_args, cache_name_extra=configure_commands[0]) - -if __name__ == '__main__': - if len(sys.argv) == 2 and sys.argv[1] in ['--help', '-h']: + return runner_core.get_library('bullet', generated_libs, + configure=configure_commands, + configure_args=configure_args, + cache_name_extra=configure_commands[0]) + + +def main(args): + print_help_if_args_empty(args) + args = get_default_args(args) + print_js_engine_message() + sanity_checks() + args = args_with_extracted_js_engine_override(args) + args = args_with_default_suite_prepended(args) + args = args_with_expanded_all_suite(args) + modules = get_and_import_modules() + all_tests = get_all_tests(modules) + args = args_with_expanded_wildcards(args, all_tests) + args = skip_requested_tests(args, modules) + args = args_for_random_tests(args, modules) + suites, unmatched_tests = load_test_suites(args, modules) + run_tests(suites, unmatched_tests) + +def print_help_if_args_empty(args): + if len(args) == 2 and args[1] in ['--help', '-h']: print HELP_TEXT sys.exit(0) +def get_default_args(args): # If no tests were specified, run the core suite - if len(sys.argv) == 1: - sys.argv = [sys.argv[0]] + map(lambda mode: mode, test_modes) + if len(args) == 1: print HELP_TEXT time.sleep(2) + return [args[0]] + map(lambda mode: mode, test_modes) + return args +def print_js_engine_message(): if use_all_engines: print '(using ALL js engines)' else: logging.warning('use EM_ALL_ENGINES=1 in the env to run against all JS engines, which is slower but provides more coverage') - # Sanity checks +def sanity_checks(): + global JS_ENGINES total_engines = len(JS_ENGINES) JS_ENGINES = filter(jsrun.check_engine, JS_ENGINES) if len(JS_ENGINES) == 0: @@ -992,38 +1034,46 @@ def get_bullet_library(runner_core, use_cmake): elif len(JS_ENGINES) < total_engines: print 'WARNING: Not all the JS engines in JS_ENGINES appears to work, ignoring those.' - # Create a list of modules to load tests from - modules = [] - for filename in glob.glob(os.path.join(os.path.dirname(__file__), 'test*.py')): - module_dir, module_file = os.path.split(filename) - module_name, module_ext = os.path.splitext(module_file) - __import__(module_name) - modules.append(sys.modules[module_name]) - - # Extract the JS engine override from the arguments (used by benchmarks) - for i in range(1, len(sys.argv)): - arg = sys.argv[i] +def args_with_extracted_js_engine_override(args): + # used by benchmarks + for i in range(1, len(args)): + arg = args[i] if arg.isupper(): print 'Interpreting all capital argument "%s" as JS_ENGINE override' % arg Building.JS_ENGINE_OVERRIDE = eval(arg) - sys.argv[i] = None - sys.argv = filter(lambda arg: arg is not None, sys.argv) + args[i] = None + return filter(lambda arg: arg is not None, args) - # If an argument comes in as test_*, treat it as a test of the default suite - sys.argv = map(lambda arg: arg if not arg.startswith('test_') else 'default.' + arg, sys.argv) +def args_with_default_suite_prepended(args): + def prepend_default(arg): + if arg.startswith('test_'): + return 'default.' + arg + return arg + return map(prepend_default, args) +def args_with_expanded_all_suite(args): # If a test (e.g. test_html) is specified as ALL.test_html, add an entry for each test_mode - new_args = [sys.argv[0]] - for i in range(1, len(sys.argv)): - arg = sys.argv[i] + new_args = [args[0]] + for i in range(1, len(args)): + arg = args[i] if arg.startswith('ALL.'): ignore, test = arg.split('.') print 'Running all test modes on test "%s"' % test new_args += map(lambda mode: mode+'.'+test, test_modes) else: new_args += [arg] - sys.argv = new_args + return new_args + +def get_and_import_modules(): + modules = [] + for filename in glob.glob(os.path.join(os.path.dirname(__file__), 'test*.py')): + module_dir, module_file = os.path.split(filename) + module_name, module_ext = os.path.splitext(module_file) + __import__(module_name) + modules.append(sys.modules[module_name]) + return modules +def get_all_tests(modules): # Create a list of all known tests so that we can choose from them based on a wildcard search all_tests = [] suites = test_modes + nondefault_test_modes + \ @@ -1033,11 +1083,13 @@ def get_bullet_library(runner_core, use_cmake): if hasattr(m, s): tests = filter(lambda t: t.startswith('test_'), dir(getattr(m, s))) all_tests += map(lambda t: s + '.' + t, tests) + return all_tests +def args_with_expanded_wildcards(args, all_tests): # Process wildcards, e.g. "browser.test_pthread_*" should expand to list all pthread tests - new_args = [sys.argv[0]] - for i in range(1, len(sys.argv)): - arg = sys.argv[i] + new_args = [args[0]] + for i in range(1, len(args)): + arg = args[i] if '*' in arg: if arg.startswith('skip:'): arg = arg[5:] @@ -1047,14 +1099,14 @@ def get_bullet_library(runner_core, use_cmake): new_args += fnmatch.filter(all_tests, arg) else: new_args += [arg] - if len(new_args) == 1 and len(sys.argv) > 1: - print 'No tests found to run in set ' + str(sys.argv[1:]) + if len(new_args) == 1 and len(args) > 1: + print 'No tests found to run in set ' + str(args[1:]) sys.exit(0) - sys.argv = new_args + return new_args - # Skip requested tests - for i in range(len(sys.argv)): - arg = sys.argv[i] +def skip_requested_tests(args, modules): + for i in range(len(args)): + arg = args[i] if arg.startswith('skip:'): which = arg.split('skip:')[1] if which.startswith('ALL.'): @@ -1066,85 +1118,108 @@ def get_bullet_library(runner_core, use_cmake): print >> sys.stderr, ','.join(which) for test in which: print >> sys.stderr, 'will skip "%s"' % test + suite_name, test_name = test.split('.') for m in modules: try: - exec('m.' + test + ' = RunnerCore("skipme")') + suite = getattr(m, suite_name) + setattr(suite, test_name, RunnerCore("skipme")) break except: pass - sys.argv[i] = None - sys.argv = filter(lambda arg: arg is not None, sys.argv) + args[i] = None + return filter(lambda arg: arg is not None, args) - # If we were asked to run random tests, do that - first = sys.argv[1] +def args_for_random_tests(args, modules): + first = args[1] if first.startswith('random'): - num = 1 - first = first[6:] - base_module = 'default' - relevant_modes = test_modes - if len(first) > 0: - if first.startswith('other'): - base_module = 'other' - relevant_modes = ['other'] - first = first.replace('other', '') - elif first.startswith('browser'): - base_module = 'browser' - relevant_modes = ['browser'] - first = first.replace('browser', '') - num = int(first) + random_arg = first[6:] + num_tests, base_module, relevant_modes = get_random_test_parameters(random_arg) for m in modules: if hasattr(m, base_module): - sys.argv = [sys.argv[0]] - tests = filter(lambda t: t.startswith('test_'), dir(getattr(m, base_module))) - print - chosen = set() - while len(chosen) < num: - test = random.choice(tests) - mode = random.choice(relevant_modes) - new_test = mode + '.' + test - before = len(chosen) - chosen.add(new_test) - if len(chosen) > before: - print '* ' + new_test - else: - # we may have hit the limit - if len(chosen) == len(tests)*len(relevant_modes): - print '(all possible tests chosen! %d = %d*%d)' % (len(chosen), len(tests), len(relevant_modes)) - break - sys.argv += list(chosen) - std = 0.5/math.sqrt(num) - print - print 'running those %d randomly-selected tests. if they all pass, then there is a greater than 95%% chance that at least %.2f%% of the test suite will pass' % (num, 100.0-100.0*std) - print - - import atexit - def show(): - print 'if all tests passed then there is a greater than 95%% chance that at least %.2f%% of the test suite will pass' % (100.0-100.0*std) - atexit.register(show) - - # Filter and load tests from the discovered modules + base = getattr(m, base_module) + new_args = [args[0]] + choose_random_tests(base, num_tests, relevant_modes) + print_random_test_statistics(num_tests) + return new_args + return args + +def get_random_test_parameters(arg): + num_tests = 1 + base_module = 'default' + relevant_modes = test_modes + if len(arg) > 0: + num_str = arg + if arg.startswith('other'): + base_module = 'other' + relevant_modes = ['other'] + num_str = arg.replace('other', '') + elif arg.startswith('browser'): + base_module = 'browser' + relevant_modes = ['browser'] + num_str = arg.replace('browser', '') + num_tests = int(num_str) + return num_tests, base_module, relevant_modes + +def choose_random_tests(base, num_tests, relevant_modes): + tests = filter(lambda t: t.startswith('test_'), dir(base)) + print + chosen = set() + while len(chosen) < num_tests: + test = random.choice(tests) + mode = random.choice(relevant_modes) + new_test = mode + '.' + test + before = len(chosen) + chosen.add(new_test) + if len(chosen) > before: + print '* ' + new_test + else: + # we may have hit the limit + if len(chosen) == len(tests)*len(relevant_modes): + print '(all possible tests chosen! %d = %d*%d)' % (len(chosen), len(tests), len(relevant_modes)) + break + return list(chosen) + +def print_random_test_statistics(num_tests): + std = 0.5/math.sqrt(num_tests) + expected = 100.0 * (1.0 - std) + print + print ('running those %d randomly-selected tests. if they all pass, then there is a ' + 'greater than 95%% chance that at least %.2f%% of the test suite will pass' + % (num_tests, expected)) + print + + import atexit + def show(): + print ('if all tests passed then there is a greater than 95%% chance that at least ' + '%.2f%% of the test suite will pass' + % (expected)) + atexit.register(show) + +def load_test_suites(args, modules): + import operator loader = unittest.TestLoader() - names = set(sys.argv[1:]) + unmatched_test_names = set(args[1:]) suites = [] for m in modules: - mnames = [] - for name in list(names): + names_in_module = [] + for name in list(unmatched_test_names): try: operator.attrgetter(name)(m) - mnames.append(name) - names.remove(name) + names_in_module.append(name) + unmatched_test_names.remove(name) except AttributeError: pass - if len(mnames) > 0: - suites.append((m.__name__, loader.loadTestsFromNames(sorted(mnames), m))) + if len(names_in_module) > 0: + suites.append((m.__name__, loader.loadTestsFromNames(sorted(names_in_module), m))) + return suites, unmatched_test_names +def run_tests(suites, unmatched_test_names): resultMessages = [] numFailures = 0 - if len(names) > 0: - print 'WARNING: could not find the following tests: ' + ' '.join(names) - numFailures += len(names) - resultMessages.append('Could not find %s tests' % (len(names),)) + if len(unmatched_test_names) > 0: + print 'WARNING: could not find the following tests: ' + ' '.join(unmatched_test_names) + numFailures += len(unmatched_test_names) + resultMessages.append('Could not find %s tests' % (len(unmatched_test_names),)) print 'Test suites:' print [s[0] for s in suites] @@ -1169,3 +1244,7 @@ def show(): # Return the number of failures as the process exit code for automating success/failure reporting. exitcode = min(numFailures, 255) sys.exit(exitcode) + + +if __name__ == '__main__': + main(sys.argv) From bda1131f8ce73fd67e761f2d772b16a9dc2eed3c Mon Sep 17 00:00:00 2001 From: Jacob Gravelle Date: Mon, 13 Mar 2017 08:54:36 -0700 Subject: [PATCH 72/72] 1.37.4 --- emscripten-version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/emscripten-version.txt b/emscripten-version.txt index 6d91342e8388c..c7e079dda6941 100644 --- a/emscripten-version.txt +++ b/emscripten-version.txt @@ -1 +1 @@ -"1.37.3" +"1.37.4"