From 425a791bbf2e56bb82e5724ad2023e2205179b03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Sat, 20 Aug 2016 15:08:19 +0300 Subject: [PATCH 01/26] Improve documentation on stringToUTF8/16/32. Simplify call to stringToUTF8 in writeStringToMemory(). --- site/source/docs/api_reference/preamble.js.rst | 12 ++++++------ src/preamble.js | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/site/source/docs/api_reference/preamble.js.rst b/site/source/docs/api_reference/preamble.js.rst index 0830506e8f4a2..ac37350a017ba 100644 --- a/site/source/docs/api_reference/preamble.js.rst +++ b/site/source/docs/api_reference/preamble.js.rst @@ -166,14 +166,14 @@ Conversion functions — strings, pointers and arrays -.. js:function:: stringToUTF8(str, outPtr[, maxBytesToWrite]) +.. js:function:: stringToUTF8(str, outPtr, maxBytesToWrite) Copies the given JavaScript ``String`` object ``str`` to the Emscripten HEAP at address ``outPtr``, null-terminated and encoded in UTF8 form. :param str: A JavaScript ``String`` object. :type str: String :param outPtr: Pointer to data copied from ``str``, encoded in UTF8 format and null-terminated. - :param maxBytesToWrite: A limit on the number of bytes to write out. + :param maxBytesToWrite: A limit on the number of bytes that this function can at most write out. If the string is longer than this, the output is truncated. The outputted string will always be null terminated, even if truncation occurred. .. js:function:: UTF16ToString(ptr) @@ -185,7 +185,7 @@ Conversion functions — strings, pointers and arrays -.. js:function:: stringToUTF16(str, outPtr[, maxBytesToWrite]) +.. js:function:: stringToUTF16(str, outPtr, maxBytesToWrite) Copies the given JavaScript ``String`` object ``str`` to the Emscripten HEAP at address ``outPtr``, null-terminated and encoded in UTF16LE form. @@ -194,7 +194,7 @@ Conversion functions — strings, pointers and arrays :param str: A JavaScript ``String`` object. :type str: String :param outPtr: Pointer to data copied from ``str``, encoded in UTF16LE format and null-terminated. - :param maxBytesToWrite: A limit on the number of bytes to write out. + :param maxBytesToWrite: A limit on the number of bytes that this function can at most write out. If the string is longer than this, the output is truncated. The outputted string will always be null terminated, even if truncation occurred. @@ -206,7 +206,7 @@ Conversion functions — strings, pointers and arrays :returns: A JavaScript ``String`` object. -.. js:function:: stringToUTF32(str, outPtr[, maxBytesToWrite]) +.. js:function:: stringToUTF32(str, outPtr, maxBytesToWrite) Copies the given JavaScript ``String`` object ``str`` to the Emscripten HEAP at address ``outPtr``, null-terminated and encoded in UTF32LE form. @@ -215,7 +215,7 @@ Conversion functions — strings, pointers and arrays :param str: A JavaScript ``String`` object. :type str: String :param outPtr: Pointer to data copied from ``str``, encoded in encoded in UTF32LE format and null-terminated. - :param maxBytesToWrite: A limit on the number of bytes to write out. + :param maxBytesToWrite: A limit on the number of bytes that this function can at most write out. If the string is longer than this, the output is truncated. The outputted string will always be null terminated, even if truncation occurred. diff --git a/src/preamble.js b/src/preamble.js index 77a8d8dc1350f..f5e9cb30eef98 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -1661,7 +1661,7 @@ function writeStringToMemory(string, buffer, dontAddNull) { end = buffer + lengthBytesUTF8(string); lastChar = HEAP8[end]; } - stringToUTF8Array(string, HEAP8, buffer, Infinity); + stringToUTF8(string, buffer, Infinity); if (dontAddNull) HEAP8[end] = lastChar; // Restore the value under the null character. } {{{ maybeExport('writeStringToMemory') }}} From 7bd0dd2f125c0d2eee246ed455488ca8ce7f775f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Sat, 20 Aug 2016 16:07:26 +0300 Subject: [PATCH 02/26] Clarify comment in GLUT touch->mouse emulation. Related to #4493. --- src/library_glut.js | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/library_glut.js b/src/library_glut.js index 3a0f19486fb08..672af2c465144 100644 --- a/src/library_glut.js +++ b/src/library_glut.js @@ -337,12 +337,15 @@ var LibraryGLUT = { var isTouchDevice = 'ontouchstart' in document.documentElement; if (isTouchDevice) { - // Historically handling of touch events was made in completely wrong way. - // onMouseButtonDown, onMouseButtonUp, onMousemove handlers depends - // on Browser.mouseX / Browser.mouseY fields. That fields - // doesn`t update by touch events. BTW, in touch mode we interested only - // in left-click (one finger), so we can use workaround and convert - // all touch events in mouse events. See touchHandler. + // onMouseButtonDown, onMouseButtonUp and onMousemove handlers + // depend on Browser.mouseX / Browser.mouseY fields. Those fields + // don't get updated by touch events. So register a touchHandler + // function that translates the touch events to mouse events. + + // GLUT doesn't support touch, mouse only, so from touch events we + // are only looking at single finger touches to emulate left click, + // so we can use workaround and convert all touch events in mouse + // events. See touchHandler. window.addEventListener("touchmove", GLUT.touchHandler, true); window.addEventListener("touchstart", GLUT.touchHandler, true); window.addEventListener("touchend", GLUT.touchHandler, true); From 00fe8a0f089ccce6a165fc41083f88fd06079a29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Sat, 20 Aug 2016 18:26:06 +0300 Subject: [PATCH 03/26] Fix glGetVertexAttribiv(_, GL_VERTEX_ATTRIB_ARRAY_BUFFER_BINDING, _) to correctly return the VBO that is bound to the requested index. Closes #1330. --- src/library_gl.js | 4 +++- tests/webgl_create_context.cpp | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/library_gl.js b/src/library_gl.js index 3bbb3af8a40d8..ca94fc916b509 100644 --- a/src/library_gl.js +++ b/src/library_gl.js @@ -2383,7 +2383,9 @@ var LibraryGL = { } #endif var data = GLctx.getVertexAttrib(index, pname); - if (typeof data == 'number' || typeof data == 'boolean') { + if (pname == 0x889F/*VERTEX_ATTRIB_ARRAY_BUFFER_BINDING*/) { + {{{ makeSetValue('params', '0', 'data["name"]', 'i32') }}}; + } else if (typeof data == 'number' || typeof data == 'boolean') { switch (type) { case 'Integer': {{{ makeSetValue('params', '0', 'data', 'i32') }}}; break; case 'Float': {{{ makeSetValue('params', '0', 'data', 'float') }}}; break; diff --git a/tests/webgl_create_context.cpp b/tests/webgl_create_context.cpp index 147202b961cac..e63f202f66500 100644 --- a/tests/webgl_create_context.cpp +++ b/tests/webgl_create_context.cpp @@ -136,6 +136,27 @@ int main() assert(!!numSamples == !!antialias); printf("\n"); + // Test bug https://github.com/kripken/emscripten/issues/1330: + unsigned vb; + glGenBuffers(1, &vb); + glBindBuffer(GL_ARRAY_BUFFER, vb); + glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, 0, 0); + + unsigned vb2; + glGenBuffers(1, &vb2); + glBindBuffer(GL_ARRAY_BUFFER, vb2); + glVertexAttribPointer(1, 3, GL_FLOAT, GL_FALSE, 0, 0); + + int vb3; + glGetVertexAttribiv(0, GL_VERTEX_ATTRIB_ARRAY_BUFFER_BINDING, &vb3); + if (vb != vb3) printf("Index 0: Generated VB: %d, read back VB: %d\n", vb, vb3); + assert(vb == vb3); + + int vb4; + glGetVertexAttribiv(1, GL_VERTEX_ATTRIB_ARRAY_BUFFER_BINDING, &vb4); + if (vb2 != vb4) printf("Index 1: Generated VB: %d, read back VB: %d\n", vb2, vb4); + assert(vb2 == vb4); + // Test that deleting the context works. res = emscripten_webgl_destroy_context(context); assert(res == 0); From be88032e900236b6eaca01fbbe2f7552090745a2 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 22 Aug 2016 10:52:21 -0700 Subject: [PATCH 04/26] tolerate the lack of a .mappedGlobals file, as binaryen is moving towards --- emcc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/emcc.py b/emcc.py index 3708ac8ee95e0..3a6564ad3a924 100755 --- a/emcc.py +++ b/emcc.py @@ -1967,7 +1967,8 @@ def do_minify(): # minifies the code. this is also when we do certain optimizati if 'native-wasm' in shared.Settings.BINARYEN_METHOD or 'interpret-binary' in shared.Settings.BINARYEN_METHOD: logging.debug('wasm-as (wasm => binary)') subprocess.check_call([os.path.join(binaryen_bin, 'wasm-as'), wasm_text_target, '-o', wasm_binary_target]) - shutil.copyfile(wasm_text_target + '.mappedGlobals', wasm_binary_target + '.mappedGlobals') + if os.path.exists(wasm_text_target + '.mappedGlobals'): # TODO: remove once we no longer use .mappedGlobals files at all, as binaryen is moving to https://github.com/WebAssembly/binaryen/issues/675 + shutil.copyfile(wasm_text_target + '.mappedGlobals', wasm_binary_target + '.mappedGlobals') # If we were asked to also generate HTML, do that if final_suffix == 'html': From ddc312a32eef3c2ad495721dfa507c1453fdc4c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Mon, 22 Aug 2016 22:18:35 +0300 Subject: [PATCH 05/26] Fix regression in PR #4492 which broke --proxy-to-worker build mode. Improve test --- emcc.py | 1 - tests/test_other.py | 20 ++++++++++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/emcc.py b/emcc.py index 3a6564ad3a924..0f8b5df41e8ae 100755 --- a/emcc.py +++ b/emcc.py @@ -2111,7 +2111,6 @@ def un_src(): # use this if you want to modify the script and need it to be inli shutil.move(js_target, js_target[:-3] + '.worker.js') # compiler output goes in .worker.js file worker_target_basename = target_basename + '.worker' target_contents = open(shared.path_from_root('src', 'webGLClient.js')).read() + '\n' + open(shared.path_from_root('src', 'proxyClient.js')).read().replace('{{{ filename }}}', shared.Settings.PROXY_TO_WORKER_FILENAME or worker_target_basename).replace('{{{ IDBStore.js }}}', open(shared.path_from_root('src', 'IDBStore.js')).read()) - target_contents = tools.line_endings.convert_line_endings(convert_line_endings, '\n', output_eol) open(target, 'wb').write(target_contents) for f in generated_text_files_with_native_eols: diff --git a/tests/test_other.py b/tests/test_other.py index 70756dcffc1e4..ddfcff9d2863c 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -6518,12 +6518,20 @@ def test_disable_inlining(self): try_delete('test.bc') def test_output_eol(self): - for params in [[], ['--separate-asm']]: - for eol in ['windows', 'linux']: - Popen([PYTHON, EMCC, path_from_root('tests', 'hello_world.c'), '-o', 'a.html', '--output_eol', eol] + params).communicate() - for f in ['a.html', 'a.js', 'a.asm.js']: - if os.path.isfile(f): - print str(params) + ' ' + eol + ' ' + f + for params in [[], ['--separate-asm'], ['--proxy-to-worker'], ['--proxy-to-worker', '--separate-asm']]: + for output_suffix in ['html', 'js']: + for eol in ['windows', 'linux']: + files = ['a.js'] + if '--separate-asm' in params: files += ['a.asm.js'] + if output_suffix == 'html': files += ['a.html'] + cmd = [PYTHON, EMCC, path_from_root('tests', 'hello_world.c'), '-o', 'a.' + output_suffix, '--output_eol', eol] + params + Popen(cmd).communicate() + for f in files: + print str(cmd) + ' ' + str(params) + ' ' + eol + ' ' + f + assert os.path.isfile(f) ret = tools.line_endings.check_line_endings(f, expect_only_specific_line_endings='\n' if eol == 'linux' else '\r\n') assert ret == 0 + for f in files: + try_delete(f) + From 80bd37a8a2fbd3bf73bf8825dba820358a206351 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Fri, 10 Jun 2016 12:19:31 +0300 Subject: [PATCH 06/26] Remove variable DYNAMICTOP and replace it with DYNAMICTOP_PTR which points to a memory location on the Emscripten HEAP that contains the ceiling of the dynamically allocated memory area. Remove proxying of sbrk. Relates to https://github.com/kripken/emscripten/issues/4391. --- emscripten.py | 20 ++++----- src/deterministic.js | 2 +- src/jsifier.js | 6 ++- src/library.js | 64 ++++++++++++++++++--------- src/library_bootstrap_structInfo.js | 4 +- src/library_trace.js | 2 +- src/memoryprofiler.js | 1 + src/preamble.js | 22 +++++---- src/runtime.js | 40 ++++++++++------- system/include/emscripten/threading.h | 1 - system/lib/pthread/library_pthread.c | 1 - system/lib/split_malloc.cpp | 2 +- tests/test_core.py | 2 +- tests/test_other.py | 2 +- tools/ctor_evaller.py | 6 +-- 15 files changed, 106 insertions(+), 69 deletions(-) diff --git a/emscripten.py b/emscripten.py index e0caff8dc90fd..98f54ffc5571e 100755 --- a/emscripten.py +++ b/emscripten.py @@ -651,7 +651,7 @@ def keyfunc(other): '(it is worth building your source files with -Werror (warnings are errors), as warnings can indicate undefined behavior which can cause this)' + \ '"); ' + extra - basic_funcs = ['abort', 'assert'] + [m.replace('.', '_') for m in math_envs] + basic_funcs = ['abort', 'assert', 'enlargeMemory', 'getTotalMemory'] + [m.replace('.', '_') for m in math_envs] if settings['STACK_OVERFLOW_CHECK']: basic_funcs += ['abortStackOverflow'] asm_safe_heap = settings['SAFE_HEAP'] and not settings['SAFE_HEAP_LOG'] and not settings['RELOCATABLE'] # optimized safe heap in asm, when we can @@ -667,11 +667,9 @@ def keyfunc(other): basic_funcs += ['nullFunc_' + sig] asm_setup += '\nfunction nullFunc_' + sig + '(x) { ' + get_function_pointer_error(sig) + 'abort(x) }\n' - basic_vars = ['STACKTOP', 'STACK_MAX', 'tempDoublePtr', 'ABORT'] + basic_vars = ['STACKTOP', 'STACK_MAX', 'DYNAMICTOP_PTR', 'tempDoublePtr', 'ABORT'] basic_float_vars = [] - if settings['SAFE_HEAP']: basic_vars += ['DYNAMICTOP'] - if metadata.get('preciseI64MathUsed'): basic_vars += ['cttz_i8'] else: @@ -1063,7 +1061,7 @@ def finalize_output(metadata, post, funcs_js, need_asyncify, provide_fround, asm '''] + ['' if not settings['SAFE_HEAP'] else ''' function setDynamicTop(value) { value = value | 0; - DYNAMICTOP = value; + HEAPU32[DYNAMICTOP_PTR>>2] = value; } '''] + ['' if not asm_safe_heap else ''' function SAFE_HEAP_STORE(dest, value, bytes) { @@ -1071,7 +1069,7 @@ def finalize_output(metadata, post, funcs_js, need_asyncify, provide_fround, asm value = value | 0; bytes = bytes | 0; if ((dest|0) <= 0) segfault(); - if (((dest + bytes)|0) > (DYNAMICTOP|0)) segfault(); + if (((dest + bytes)|0) > (HEAPU32[DYNAMICTOP_PTR>>2]|0)) segfault(); if ((bytes|0) == 4) { if ((dest&3)) alignfault(); HEAP32[dest>>2] = value; @@ -1087,7 +1085,7 @@ def finalize_output(metadata, post, funcs_js, need_asyncify, provide_fround, asm value = +value; bytes = bytes | 0; if ((dest|0) <= 0) segfault(); - if (((dest + bytes)|0) > (DYNAMICTOP|0)) segfault(); + if (((dest + bytes)|0) > (HEAPU32[DYNAMICTOP_PTR>>2]|0)) segfault(); if ((bytes|0) == 8) { if ((dest&7)) alignfault(); HEAPF64[dest>>3] = value; @@ -1101,7 +1099,7 @@ def finalize_output(metadata, post, funcs_js, need_asyncify, provide_fround, asm bytes = bytes | 0; unsigned = unsigned | 0; if ((dest|0) <= 0) segfault(); - if ((dest + bytes|0) > (DYNAMICTOP|0)) segfault(); + if ((dest + bytes|0) > (HEAPU32[DYNAMICTOP_PTR>>2]|0)) segfault(); if ((bytes|0) == 4) { if ((dest&3)) alignfault(); return HEAP32[dest>>2] | 0; @@ -1120,7 +1118,7 @@ def finalize_output(metadata, post, funcs_js, need_asyncify, provide_fround, asm dest = dest | 0; bytes = bytes | 0; if ((dest|0) <= 0) segfault(); - if ((dest + bytes|0) > (DYNAMICTOP|0)) segfault(); + if ((dest + bytes|0) > (HEAPU32[DYNAMICTOP_PTR>>2]|0)) segfault(); if ((bytes|0) == 8) { if ((dest&7)) alignfault(); return +HEAPF64[dest>>3]; @@ -1530,7 +1528,7 @@ def save_settings(): outfile.write(pre) pre = None - basic_funcs = ['abort', 'assert'] + basic_funcs = ['abort', 'assert', 'enlargeMemory', 'getTotalMemory'] access_quote = access_quoter(settings) @@ -1547,7 +1545,7 @@ def save_settings(): def math_fix(g): return g if not g.startswith('Math_') else g.split('_')[1] - basic_vars = ['STACKTOP', 'STACK_MAX', 'ABORT'] + basic_vars = ['STACKTOP', 'STACK_MAX', 'DYNAMICTOP_PTR', 'ABORT'] basic_float_vars = [] # Asm.js-style exception handling: invoke wrapper generation diff --git a/src/deterministic.js b/src/deterministic.js index da95835217d3e..5364f6f0769a4 100644 --- a/src/deterministic.js +++ b/src/deterministic.js @@ -15,7 +15,7 @@ Module['thisProgram'] = 'thisProgram'; // for consistency between different buil function hashMemory(id) { var ret = 0; - var len = Math.max(DYNAMICTOP, STATICTOP); + var len = Math.max(HEAPU32[DYNAMICTOP_PTR>>2], STATICTOP); for (var i = 0; i < len; i++) { ret = (ret*17 + HEAPU8[i])|0; } diff --git a/src/jsifier.js b/src/jsifier.js index 4d6fcf36a9539..f1aadef884ae5 100644 --- a/src/jsifier.js +++ b/src/jsifier.js @@ -381,10 +381,12 @@ function JSify(data, functionsOnly) { legalizedI64s = legalizedI64sDefault; if (!BUILD_AS_SHARED_LIB && !SIDE_MODULE) { + print('DYNAMICTOP_PTR = allocate(1, "i32", ALLOC_STATIC);\n'); print('STACK_BASE = STACKTOP = Runtime.alignMemory(STATICTOP);\n'); - print('staticSealed = true; // seal the static portion of memory\n'); print('STACK_MAX = STACK_BASE + TOTAL_STACK;\n'); - print('DYNAMIC_BASE = DYNAMICTOP = Runtime.alignMemory(STACK_MAX);\n'); + print('DYNAMIC_BASE = Runtime.alignMemory(STACK_MAX);\n'); + print('HEAPU32[DYNAMICTOP_PTR>>2] = DYNAMIC_BASE;\n'); + print('staticSealed = true; // seal the static portion of memory\n'); if (ASSERTIONS) print('assert(DYNAMIC_BASE < TOTAL_MEMORY, "TOTAL_MEMORY not big enough for stack");\n'); } if (SPLIT_MEMORY) { diff --git a/src/library.js b/src/library.js index ba81120a83ac8..d540a7a642a59 100644 --- a/src/library.js +++ b/src/library.js @@ -428,28 +428,52 @@ LibraryManager.library = { ___setErrNo(ERRNO_CODES.EINVAL); return -1; }, - sbrk: function(bytes) { + + // Implement a Linux-like 'memory area' for our 'process'. + // Changes the size of the memory area by |bytes|; returns the + // address of the previous top ('break') of the memory area + // We control the "dynamic" memory - DYNAMIC_BASE to DYNAMICTOP + sbrk__asm: true, + sbrk__sig: ['ii'], + sbrk__deps: ['__setErrNo'], + sbrk: function(increment) { + increment = increment|0; + var oldDynamicTop = 0; + var newDynamicTop = 0; + var totalMemory = 0; + increment = ((increment + 15) & -16)|0; #if USE_PTHREADS - if (ENVIRONMENT_IS_PTHREAD) return _emscripten_sync_run_in_main_thread_1({{{ cDefine('EM_PROXIED_SBRK') }}}, bytes); -#endif - // Implement a Linux-like 'memory area' for our 'process'. - // Changes the size of the memory area by |bytes|; returns the - // address of the previous top ('break') of the memory area - // We control the "dynamic" memory - DYNAMIC_BASE to DYNAMICTOP - var self = _sbrk; - if (!self.called) { - DYNAMICTOP = alignMemoryPage(DYNAMICTOP); // make sure we start out aligned - self.called = true; - assert(Runtime.dynamicAlloc); - self.alloc = Runtime.dynamicAlloc; - Runtime.dynamicAlloc = function() { abort('cannot dynamically allocate, sbrk now has control') }; - } - var ret = DYNAMICTOP; - if (bytes != 0) { - var success = self.alloc(bytes); - if (!success) return -1 >>> 0; // sbrk failure code + oldDynamicTop = Atomics_add(HEAPU32, DYNAMICTOP_PTR>>2, increment); +#else + oldDynamicTop = HEAPU32[DYNAMICTOP_PTR>>2]|0; + newDynamicTop = oldDynamicTop + increment | 0; + HEAPU32[DYNAMICTOP_PTR>>2] = newDynamicTop; + totalMemory = getTotalMemory()|0; + if ((newDynamicTop|0) > (totalMemory|0)) { + // Warning: when -s USE_PTHREADS=1, this code block is called in a multithreaded context, + // so enlargeMemory() will be set to abort. If enlarging with pthreads is enabled at some point, + // sbrk will need to get mutexes to ensure that incrementing the heap size above and this enlarge + // operation is performed as a single transaction. + if ((enlargeMemory()|0) == 0) { + ___setErrNo({{{ cDefine('ENOMEM') }}}); + HEAPU32[DYNAMICTOP_PTR>>2] = oldDynamicTop; + return -1; + } } - return ret; // Previous break location. +#endif + return oldDynamicTop|0; + }, + + brk__asm: true, + brk__sig: ['ii'], + brk: function(addr) { + addr = addr|0; +#if USE_PTHREADS + Atomics_set(HEAPU32, DYNAMICTOP_PTR>>2, addr); +#else + HEAPU32[DYNAMICTOP_PTR>>2] = addr; +#endif + return 0; // TODO: error checking, return -1 on error and set errno. }, system__deps: ['__setErrNo', '$ERRNO_CODES'], diff --git a/src/library_bootstrap_structInfo.js b/src/library_bootstrap_structInfo.js index a0808a7bebf5f..b0069c35e7874 100644 --- a/src/library_bootstrap_structInfo.js +++ b/src/library_bootstrap_structInfo.js @@ -20,13 +20,13 @@ LibraryManager.library = { // We control the "dynamic" memory - DYNAMIC_BASE to DYNAMICTOP var self = _sbrk; if (!self.called) { - DYNAMICTOP = alignMemoryPage(DYNAMICTOP); // make sure we start out aligned + HEAPU32[DYNAMICTOP_PTR>>2] = alignMemoryPage(HEAPU32[DYNAMICTOP_PTR>>2]); // make sure we start out aligned self.called = true; assert(Runtime.dynamicAlloc); self.alloc = Runtime.dynamicAlloc; Runtime.dynamicAlloc = function() { abort('cannot dynamically allocate, sbrk now has control') }; } - var ret = DYNAMICTOP; + var ret = HEAPU32[DYNAMICTOP_PTR>>2]; if (bytes != 0) self.alloc(bytes); return ret; // Previous break location. }, diff --git a/src/library_trace.js b/src/library_trace.js index 689922191d3b8..957935f30e616 100644 --- a/src/library_trace.js +++ b/src/library_trace.js @@ -262,7 +262,7 @@ var LibraryTracing = { 'stack_top': STACKTOP, 'stack_max': STACK_MAX, 'dynamic_base': DYNAMIC_BASE, - 'dynamic_top': DYNAMICTOP, + 'dynamic_top': HEAPU32[DYNAMICTOP_PTR>>2], 'total_memory': TOTAL_MEMORY }; var now = EmscriptenTrace.now(); diff --git a/src/memoryprofiler.js b/src/memoryprofiler.js index 118aa550c82af..d0547e6ffd829 100644 --- a/src/memoryprofiler.js +++ b/src/memoryprofiler.js @@ -302,6 +302,7 @@ var emscriptenMemoryProfiler = { html += '. STACK_MAX: ' + toHex(STACK_MAX, width) + '.'; html += '
STACK memory area used now (should be zero): ' + this.formatBytes(STACKTOP - STACK_BASE) + '.' + colorBar('#FFFF00') + ' STACK watermark highest seen usage (approximate lower-bound!): ' + this.formatBytes(this.stackTopWatermark - STACK_BASE); + var DYNAMICTOP = HEAPU32[DYNAMICTOP_PTR>>2]; html += '
' + colorBar('#70FF70') + 'DYNAMIC memory area size: ' + this.formatBytes(DYNAMICTOP-DYNAMIC_BASE); html += '. DYNAMIC_BASE: ' + toHex(DYNAMIC_BASE, width); html += '. DYNAMICTOP: ' + toHex(DYNAMICTOP, width) + '.'; diff --git a/src/preamble.js b/src/preamble.js index f5e9cb30eef98..61b94daa3ef50 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -52,8 +52,9 @@ function SAFE_HEAP_STORE(dest, value, bytes, isFloat) { #endif if (dest <= 0) abort('segmentation fault storing ' + bytes + ' bytes to address ' + dest); if (dest % bytes !== 0) abort('alignment error storing to address ' + dest + ', which was expected to be aligned to a multiple of ' + bytes); - if (dest + bytes > Math.max(DYNAMICTOP, STATICTOP)) abort('segmentation fault, exceeded the top of the available heap when storing ' + bytes + ' bytes to address ' + dest + '. STATICTOP=' + STATICTOP + ', DYNAMICTOP=' + DYNAMICTOP); - assert(DYNAMICTOP <= TOTAL_MEMORY); + if (dest + bytes > Math.max(HEAPU32[DYNAMICTOP_PTR>>2], STATICTOP)) abort('segmentation fault, exceeded the top of the available heap when storing ' + bytes + ' bytes to address ' + dest + '. STATICTOP=' + STATICTOP + ', DYNAMICTOP=' + HEAPU32[DYNAMICTOP_PTR>>2]); + assert(DYNAMICTOP_PTR); + assert(HEAPU32[DYNAMICTOP_PTR>>2] <= TOTAL_MEMORY); setValue(dest, value, getSafeHeapType(bytes, isFloat), 1); } function SAFE_HEAP_STORE_D(dest, value, bytes) { @@ -63,8 +64,9 @@ function SAFE_HEAP_STORE_D(dest, value, bytes) { function SAFE_HEAP_LOAD(dest, bytes, unsigned, isFloat) { if (dest <= 0) abort('segmentation fault loading ' + bytes + ' bytes from address ' + dest); if (dest % bytes !== 0) abort('alignment error loading from address ' + dest + ', which was expected to be aligned to a multiple of ' + bytes); - if (dest + bytes > Math.max(DYNAMICTOP, STATICTOP)) abort('segmentation fault, exceeded the top of the available heap when loading ' + bytes + ' bytes from address ' + dest + '. STATICTOP=' + STATICTOP + ', DYNAMICTOP=' + DYNAMICTOP); - assert(DYNAMICTOP <= TOTAL_MEMORY); + if (dest + bytes > Math.max(HEAPU32[DYNAMICTOP_PTR>>2], STATICTOP)) abort('segmentation fault, exceeded the top of the available heap when loading ' + bytes + ' bytes from address ' + dest + '. STATICTOP=' + STATICTOP + ', DYNAMICTOP=' + HEAPU32[DYNAMICTOP_PTR>>2]); + assert(DYNAMICTOP_PTR); + assert(HEAPU32[DYNAMICTOP_PTR>>2] <= TOTAL_MEMORY); var type = getSafeHeapType(bytes, isFloat); var ret = getValue(dest, type, 1); if (unsigned) ret = unSign(ret, parseInt(type.substr(1)), 1); @@ -935,7 +937,7 @@ function updateGlobalBufferViews() { var STATIC_BASE = 0, STATICTOP = 0, staticSealed = false; // static area var STACK_BASE = 0, STACKTOP = 0, STACK_MAX = 0; // stack area -var DYNAMIC_BASE = 0, DYNAMICTOP = 0; // dynamic area handled by sbrk +var DYNAMIC_BASE = 0, DYNAMICTOP_PTR = 0; // dynamic area handled by sbrk #if USE_PTHREADS if (ENVIRONMENT_IS_PTHREAD) { @@ -1007,7 +1009,7 @@ function enlargeMemory() { #else // TOTAL_MEMORY is the current size of the actual array, and DYNAMICTOP is the new top. #if ASSERTIONS - assert(DYNAMICTOP >= TOTAL_MEMORY); + assert(HEAPU32[DYNAMICTOP_PTR>>2] >= TOTAL_MEMORY); assert(TOTAL_MEMORY > 4); // So the loop below will not be infinite #endif @@ -1020,9 +1022,9 @@ function enlargeMemory() { var LIMIT = Math.pow(2, 31); // 2GB is a practical maximum, as we use signed ints as pointers // and JS engines seem unhappy to give us 2GB arrays currently - if (DYNAMICTOP >= LIMIT) return false; + if (HEAPU32[DYNAMICTOP_PTR>>2] >= LIMIT) return false; - while (TOTAL_MEMORY <= DYNAMICTOP) { // Simple heuristic. + while (TOTAL_MEMORY <= HEAPU32[DYNAMICTOP_PTR>>2]) { // Simple heuristic. if (TOTAL_MEMORY < LIMIT/2) { TOTAL_MEMORY = alignMemoryPage(2*TOTAL_MEMORY); // double until 1GB } else { @@ -1470,6 +1472,10 @@ function setF64(ptr, value) { #endif // USE_PTHREADS +function getTotalMemory() { + return TOTAL_MEMORY; +} + // Endianness check (note: assumes compiler arch was little-endian) #if SAFE_SPLIT_MEMORY == 0 #if USE_PTHREADS diff --git a/src/runtime.js b/src/runtime.js index cabeffa850745..db2d729985933 100644 --- a/src/runtime.js +++ b/src/runtime.js @@ -59,15 +59,19 @@ var RuntimeGenerator = { // allocation on the top of memory, adjusted dynamically by sbrk dynamicAlloc: function(size) { - if (ASSERTIONS) size = '(assert(DYNAMICTOP > 0),' + size + ')'; // dynamic area must be ready -#if USE_PTHREADS - if (typeof ENVIRONMENT_IS_PTHREAD !== 'undefined' && ENVIRONMENT_IS_PTHREAD) throw 'Runtime.dynamicAlloc is not available in pthreads!'; // This is because each worker has its own copy of DYNAMICTOP, of which main thread is authoritative. -#endif - var ret = RuntimeGenerator.alloc(size, 'DYNAMIC'); - if (SAFE_HEAP) ret += '; if (asm) { Runtime.setDynamicTop(DYNAMICTOP); }'; - ret += '; if (DYNAMICTOP >= TOTAL_MEMORY) { var success = enlargeMemory(); if (!success) { DYNAMICTOP = ret; '; - if (SAFE_HEAP) ret += 'if (asm) { Runtime.setDynamicTop(DYNAMICTOP); }'; - ret += ' return 0; } }' + var ret = ''; + if (ASSERTIONS) ret += 'assert(DYNAMICTOP_PTR);'; // dynamic area must be ready + ret += 'var ret = HEAPU32[DYNAMICTOP_PTR>>2];' + + 'var end = (((ret + size + 15)|0) & -16);' + + 'HEAPU32[DYNAMICTOP_PTR>>2] = end;' + + 'if (end >= TOTAL_MEMORY) {' + + 'var success = enlargeMemory();' + + 'if (!success) {' + + 'HEAPU32[DYNAMICTOP_PTR>>2] = ret;' + + 'return 0;' + + '}' + + '}' + + 'return ret;'; return ret; }, @@ -100,8 +104,12 @@ var RuntimeGenerator = { } }; -function unInline(name_, params) { - var src = '(function(' + params + ') { var ret = ' + RuntimeGenerator[name_].apply(null, params) + '; return ret; })'; +function unInline(name_, params, isExpression) { + if (isExpression) { + var src = '(function(' + params + ') { var ret = ' + RuntimeGenerator[name_].apply(null, params) + '; return ret; })'; + } else { + var src = '(function(' + params + ') { ' + RuntimeGenerator[name_].apply(null, params) + '})'; + } var ret = eval(src); return ret; } @@ -429,11 +437,11 @@ var Runtime = { #endif }; -Runtime.stackAlloc = unInline('stackAlloc', ['size']); -Runtime.staticAlloc = unInline('staticAlloc', ['size']); -Runtime.dynamicAlloc = unInline('dynamicAlloc', ['size']); -Runtime.alignMemory = unInline('alignMemory', ['size', 'quantum']); -Runtime.makeBigInt = unInline('makeBigInt', ['low', 'high', 'unsigned']); +Runtime.stackAlloc = unInline('stackAlloc', ['size'], true); +Runtime.staticAlloc = unInline('staticAlloc', ['size'], true); +Runtime.dynamicAlloc = unInline('dynamicAlloc', ['size'], false); +Runtime.alignMemory = unInline('alignMemory', ['size', 'quantum'], true); +Runtime.makeBigInt = unInline('makeBigInt', ['low', 'high', 'unsigned'], true); if (MAIN_MODULE || SIDE_MODULE) { Runtime.tempRet0 = 0; diff --git a/system/include/emscripten/threading.h b/system/include/emscripten/threading.h index cc01d2fcab6fb..560c367428e6e 100644 --- a/system/include/emscripten/threading.h +++ b/system/include/emscripten/threading.h @@ -175,7 +175,6 @@ struct thread_profiler_block #define EM_PROXIED_FPATHCONF 46 #define EM_PROXIED_CONFSTR 68 #define EM_PROXIED_SYSCONF 72 -#define EM_PROXIED_SBRK 73 #define EM_PROXIED_ATEXIT 110 #define EM_PROXIED_GETENV 111 #define EM_PROXIED_CLEARENV 112 diff --git a/system/lib/pthread/library_pthread.c b/system/lib/pthread/library_pthread.c index c630fcdc46682..43465baf615f9 100644 --- a/system/lib/pthread/library_pthread.c +++ b/system/lib/pthread/library_pthread.c @@ -282,7 +282,6 @@ static void _do_call(em_queued_call *q) case EM_PROXIED_FPATHCONF: q->returnValue.i = fpathconf(q->args[0].i, q->args[1].i); break; case EM_PROXIED_CONFSTR: q->returnValue.i = confstr(q->args[0].i, q->args[1].cp, q->args[2].i); break; case EM_PROXIED_SYSCONF: q->returnValue.i = sysconf(q->args[0].i); break; - case EM_PROXIED_SBRK: q->returnValue.vp = sbrk(q->args[0].i); break; case EM_PROXIED_ATEXIT: q->returnValue.i = atexit(q->args[0].vp); break; case EM_PROXIED_GETENV: q->returnValue.cp = getenv(q->args[0].cp); break; case EM_PROXIED_CLEARENV: q->returnValue.i = clearenv(); break; diff --git a/system/lib/split_malloc.cpp b/system/lib/split_malloc.cpp index 8a697da69fbbb..da8f2f6865552 100644 --- a/system/lib/split_malloc.cpp +++ b/system/lib/split_malloc.cpp @@ -71,7 +71,7 @@ struct Space { start = split_memory*index; } else { // small area in existing chunk 0 - start = EM_ASM_INT_V({ return (DYNAMICTOP+3)&-4; }); + start = EM_ASM_INT_V({ return (HEAPU32[DYNAMICTOP_PTR>>2]+3)&-4; }); assert(start < split_memory); } int size = (split_memory*(index+1)) - start; diff --git a/tests/test_core.py b/tests/test_core.py index 77c5da3005164..6261ca601561e 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -8160,7 +8160,7 @@ def test_memprof_requirements(self): typeof STACK_MAX === 'number' && typeof STACKTOP === 'number' && typeof DYNAMIC_BASE === 'number' && - typeof DYNAMICTOP === 'number') { + typeof DYNAMICTOP_PTR === 'number') { Module.print('able to run memprof'); } else { Module.print('missing the required variables to run memprof'); diff --git a/tests/test_other.py b/tests/test_other.py index ddfcff9d2863c..a2be83a99c8ad 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -5135,7 +5135,7 @@ def test_massive_alloc(self): def test_failing_alloc(self): for pre_fail, post_fail, opts in [ ('', '', []), - ('EM_ASM( Module.temp = DYNAMICTOP );', 'EM_ASM( assert(Module.temp === DYNAMICTOP, "must not adjust DYNAMICTOP when an alloc fails!") );', []), + ('EM_ASM( Module.temp = HEAPU32[DYNAMICTOP_PTR>>2] );', 'EM_ASM( assert(Module.temp === HEAPU32[DYNAMICTOP_PTR>>2], "must not adjust DYNAMICTOP when an alloc fails!") );', []), ('', '', ['-s', 'SPLIT_MEMORY=' + str(16*1024*1024)]), ]: for growth in [0, 1]: diff --git a/tools/ctor_evaller.py b/tools/ctor_evaller.py index b4eb1916deee7..775a0db549573 100644 --- a/tools/ctor_evaller.py +++ b/tools/ctor_evaller.py @@ -75,7 +75,7 @@ def add_func(asm, func): for bit in bits: name, value = map(lambda x: x.strip(), bit.split('=')) if value in ['0', '+0', '0.0'] or name in [ - 'STACKTOP', 'STACK_MAX', 'DYNAMICTOP', + 'STACKTOP', 'STACK_MAX', 'DYNAMICTOP_PTR', 'HEAP8', 'HEAP16', 'HEAP32', 'HEAPU8', 'HEAPU16', 'HEAPU32', 'HEAPF32', 'HEAPF64', @@ -125,7 +125,7 @@ def add_func(asm, func): var stackMax = stackTop + totalStack; if (stackMax >= totalMemory) throw 'not enough room for stack'; -var dynamicTop = stackMax; +var dynamicTopPtr = stackMax; if (!Math.imul) { Math.imul = Math.imul || function(a, b) { @@ -162,7 +162,7 @@ def add_func(asm, func): var libraryArg = { STACKTOP: stackTop, STACK_MAX: stackMax, - DYNAMICTOP: dynamicTop, + DYNAMICTOP_PTR: dynamicTopPtr, ___dso_handle: 0, // used by atexit, value doesn't matter _emscripten_memcpy_big: function(dest, src, num) { heap.set(heap.subarray(src, src+num), dest); From c8499dd0cb317d3842c8829266ca577e9fa20e72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Fri, 19 Aug 2016 22:12:09 +0300 Subject: [PATCH 07/26] Fix SAFE_HEAP_LOAD() and SAFE_HEAP_STORE() to be possible to be called in Emscripten JS library static postSet time, when the dynamic heap has not yet been established and DYNAMICTOP_PTR still points to address 0. --- src/preamble.js | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/preamble.js b/src/preamble.js index 61b94daa3ef50..b28ed6af74293 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -52,9 +52,13 @@ function SAFE_HEAP_STORE(dest, value, bytes, isFloat) { #endif if (dest <= 0) abort('segmentation fault storing ' + bytes + ' bytes to address ' + dest); if (dest % bytes !== 0) abort('alignment error storing to address ' + dest + ', which was expected to be aligned to a multiple of ' + bytes); - if (dest + bytes > Math.max(HEAPU32[DYNAMICTOP_PTR>>2], STATICTOP)) abort('segmentation fault, exceeded the top of the available heap when storing ' + bytes + ' bytes to address ' + dest + '. STATICTOP=' + STATICTOP + ', DYNAMICTOP=' + HEAPU32[DYNAMICTOP_PTR>>2]); - assert(DYNAMICTOP_PTR); - assert(HEAPU32[DYNAMICTOP_PTR>>2] <= TOTAL_MEMORY); + if (staticSealed) { + if (dest + bytes > HEAPU32[DYNAMICTOP_PTR>>2]) abort('segmentation fault, exceeded the top of the available dynamic heap when storing ' + bytes + ' bytes to address ' + dest + '. STATICTOP=' + STATICTOP + ', DYNAMICTOP=' + HEAPU32[DYNAMICTOP_PTR>>2]); + assert(DYNAMICTOP_PTR); + assert(HEAPU32[DYNAMICTOP_PTR>>2] <= TOTAL_MEMORY); + } else { + if (dest + bytes > STATICTOP) abort('segmentation fault, exceeded the top of the available static heap when storing ' + bytes + ' bytes to address ' + dest + '. STATICTOP=' + STATICTOP); + } setValue(dest, value, getSafeHeapType(bytes, isFloat), 1); } function SAFE_HEAP_STORE_D(dest, value, bytes) { @@ -64,9 +68,13 @@ function SAFE_HEAP_STORE_D(dest, value, bytes) { function SAFE_HEAP_LOAD(dest, bytes, unsigned, isFloat) { if (dest <= 0) abort('segmentation fault loading ' + bytes + ' bytes from address ' + dest); if (dest % bytes !== 0) abort('alignment error loading from address ' + dest + ', which was expected to be aligned to a multiple of ' + bytes); - if (dest + bytes > Math.max(HEAPU32[DYNAMICTOP_PTR>>2], STATICTOP)) abort('segmentation fault, exceeded the top of the available heap when loading ' + bytes + ' bytes from address ' + dest + '. STATICTOP=' + STATICTOP + ', DYNAMICTOP=' + HEAPU32[DYNAMICTOP_PTR>>2]); - assert(DYNAMICTOP_PTR); - assert(HEAPU32[DYNAMICTOP_PTR>>2] <= TOTAL_MEMORY); + if (staticSealed) { + if (dest + bytes > HEAPU32[DYNAMICTOP_PTR>>2]) abort('segmentation fault, exceeded the top of the available dynamic heap when loading ' + bytes + ' bytes from address ' + dest + '. STATICTOP=' + STATICTOP + ', DYNAMICTOP=' + HEAPU32[DYNAMICTOP_PTR>>2]); + assert(DYNAMICTOP_PTR); + assert(HEAPU32[DYNAMICTOP_PTR>>2] <= TOTAL_MEMORY); + } else { + if (dest + bytes > STATICTOP) abort('segmentation fault, exceeded the top of the available static heap when loading ' + bytes + ' bytes from address ' + dest + '. STATICTOP=' + STATICTOP); + } var type = getSafeHeapType(bytes, isFloat); var ret = getValue(dest, type, 1); if (unsigned) ret = unSign(ret, parseInt(type.substr(1)), 1); From 066cb842301fb6e28a1ad1d38209973866e9f93d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Fri, 19 Aug 2016 22:23:08 +0300 Subject: [PATCH 08/26] Be more precise about inequality limits in enlargeMemory(). --- src/preamble.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/preamble.js b/src/preamble.js index b28ed6af74293..2fd5117bf5ebc 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -1017,7 +1017,7 @@ function enlargeMemory() { #else // TOTAL_MEMORY is the current size of the actual array, and DYNAMICTOP is the new top. #if ASSERTIONS - assert(HEAPU32[DYNAMICTOP_PTR>>2] >= TOTAL_MEMORY); + assert(HEAPU32[DYNAMICTOP_PTR>>2] > TOTAL_MEMORY); // This function should only ever be called after the ceiling of the dynamic heap has already been bumped to exceed the current total size of the asm.js heap. assert(TOTAL_MEMORY > 4); // So the loop below will not be infinite #endif @@ -1032,12 +1032,12 @@ function enlargeMemory() { // and JS engines seem unhappy to give us 2GB arrays currently if (HEAPU32[DYNAMICTOP_PTR>>2] >= LIMIT) return false; - while (TOTAL_MEMORY <= HEAPU32[DYNAMICTOP_PTR>>2]) { // Simple heuristic. + while (TOTAL_MEMORY < HEAPU32[DYNAMICTOP_PTR>>2]) { // Keep incrementing the heap size as long as it's less than what is requested. if (TOTAL_MEMORY < LIMIT/2) { - TOTAL_MEMORY = alignMemoryPage(2*TOTAL_MEMORY); // double until 1GB + TOTAL_MEMORY = alignMemoryPage(2*TOTAL_MEMORY); // // Simple heuristic: double until 1GB... } else { var last = TOTAL_MEMORY; - TOTAL_MEMORY = alignMemoryPage((3*TOTAL_MEMORY + LIMIT)/4); // add smaller increments towards 2GB, which we cannot reach + TOTAL_MEMORY = alignMemoryPage((3*TOTAL_MEMORY + LIMIT)/4); // ..., but after that, add smaller increments towards 2GB, which we cannot reach if (TOTAL_MEMORY <= last) return false; } } From bb23567760d59852e585433653c709c1d8ea6816 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Sat, 20 Aug 2016 00:09:22 +0300 Subject: [PATCH 09/26] Restrict ctor evaller from accessing sbrk(). --- tools/ctor_evaller.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/ctor_evaller.py b/tools/ctor_evaller.py index 775a0db549573..6d116828a3b7b 100644 --- a/tools/ctor_evaller.py +++ b/tools/ctor_evaller.py @@ -61,6 +61,8 @@ def add_func(asm, func): asm = get_asm(js) assert len(asm) > 0 asm = asm.replace('use asm', 'not asm') # don't try to validate this + # Substitute sbrk with a failing stub: the dynamic heap memory area shouldn't get increased during static ctor initialization. + asm = asm.replace('function _sbrk(', 'function _sbrk(increment) { throw "no sbrk when evalling ctors!"; } function KILLED_sbrk(', 1) # find all global vars, and provide only safe ones. Also add dumping for those. pre_funcs_start = asm.find(';') + 1 pre_funcs_end = asm.find('function ', pre_funcs_start) @@ -81,7 +83,7 @@ def add_func(asm, func): 'HEAPF32', 'HEAPF64', 'Int8View', 'Int16View', 'Int32View', 'Uint8View', 'Uint16View', 'Uint32View', 'Float32View', 'Float64View', 'nan', 'inf', - '_emscripten_memcpy_big', '_sbrk', '___dso_handle', + '_emscripten_memcpy_big', '___dso_handle', '_atexit', '___cxa_atexit', ] or name.startswith('Math_'): if 'new ' not in value: @@ -125,7 +127,7 @@ def add_func(asm, func): var stackMax = stackTop + totalStack; if (stackMax >= totalMemory) throw 'not enough room for stack'; -var dynamicTopPtr = stackMax; +var dynamicTopPtr = 0; if (!Math.imul) { Math.imul = Math.imul || function(a, b) { @@ -238,7 +240,7 @@ def read_and_delete(filename): out_result = read_and_delete(out_file) err_result = read_and_delete(err_file) if proc.returncode != 0: - shared.logging.debug('unexpected error while trying to eval ctors:\n' + out_result) + shared.logging.debug('unexpected error while trying to eval ctors:\n' + out_result + '\n' + err_result) return (0, 0, 0, 0) # out contains the new mem init and other info From e1394879839e2fd9f6d49c93d222a71a5a59f422 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Sat, 20 Aug 2016 11:47:54 +0300 Subject: [PATCH 10/26] Ensure that Runtime.dynamicAlloc function is correctly called in getMemory() exactly when the C runtime has not yet been initialized but the static heap has already been sealed. --- src/preamble.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/preamble.js b/src/preamble.js index 2fd5117bf5ebc..a0859bdf55c46 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -461,7 +461,7 @@ function allocate(slab, types, allocator, ptr) { // Allocate memory during any stage of startup - static memory early on, dynamic memory later, malloc when ready function getMemory(size) { if (!staticSealed) return Runtime.staticAlloc(size); - if ((typeof _sbrk !== 'undefined' && !_sbrk.called) || !runtimeInitialized) return Runtime.dynamicAlloc(size); + if (!runtimeInitialized) return Runtime.dynamicAlloc(size); return _malloc(size); } {{{ maybeExport('getMemory') }}} From 41d3e93fb75d0b1ee8ce530384205e6275571df4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Sat, 20 Aug 2016 19:52:12 +0300 Subject: [PATCH 11/26] Update version to 1.36.9. --- emscripten-version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/emscripten-version.txt b/emscripten-version.txt index 2a4be5f6e8e25..d873f1145a203 100644 --- a/emscripten-version.txt +++ b/emscripten-version.txt @@ -1,2 +1,2 @@ -"1.36.8" +"1.36.9" From bf3ca57423b344974be33f529a2b04e80eed32ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Mon, 22 Aug 2016 15:10:37 +0300 Subject: [PATCH 12/26] Remove Firefox workaround for console logging in pthreads (bug 1049091), which was fixed quite some time ago on 2015-04-04. --- src/pthread-main.js | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/pthread-main.js b/src/pthread-main.js index b695b770df0b7..7ecf3dbb5737c 100644 --- a/src/pthread-main.js +++ b/src/pthread-main.js @@ -26,11 +26,11 @@ var Module = {}; function threadPrint() { var text = Array.prototype.slice.call(arguments).join(' '); - postMessage({cmd: 'print', text: text, threadId: selfThreadId}); + console.log(text); } function threadPrintErr() { var text = Array.prototype.slice.call(arguments).join(' '); - postMessage({cmd: 'printErr', text: text, threadId: selfThreadId}); + console.error(text); } function threadAlert() { var text = Array.prototype.slice.call(arguments).join(' '); @@ -38,13 +38,6 @@ function threadAlert() { } Module['print'] = threadPrint; Module['printErr'] = threadPrintErr; - -// Work around https://bugzilla.mozilla.org/show_bug.cgi?id=1049091 -console = { - log: threadPrint, - error: threadPrintErr -}; - this.alert = threadAlert; this.onmessage = function(e) { From f8dce4883911b0273663e8d36611fc72708de776 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Mon, 22 Aug 2016 19:13:28 +0300 Subject: [PATCH 13/26] Fix and test support of sbrk() in pthreads. --- src/jsifier.js | 2 + src/library.js | 35 +++++++- src/library_pthread.js | 4 + src/preamble.js | 6 +- src/pthread-main.js | 27 ++++-- tests/gauge_available_memory.cpp | 27 ++++++ tests/pthread/test_pthread_sbrk.cpp | 133 ++++++++++++++++++++++++++++ tests/test_browser.py | 14 +++ 8 files changed, 236 insertions(+), 12 deletions(-) create mode 100644 tests/gauge_available_memory.cpp create mode 100644 tests/pthread/test_pthread_sbrk.cpp diff --git a/src/jsifier.js b/src/jsifier.js index f1aadef884ae5..06ee30bf8121d 100644 --- a/src/jsifier.js +++ b/src/jsifier.js @@ -381,6 +381,7 @@ function JSify(data, functionsOnly) { legalizedI64s = legalizedI64sDefault; if (!BUILD_AS_SHARED_LIB && !SIDE_MODULE) { + if (USE_PTHREADS) print('if (!ENVIRONMENT_IS_PTHREAD) {\n // Only main thread initializes these, pthreads copy them over at thread worker init time (in pthread-main.js)'); print('DYNAMICTOP_PTR = allocate(1, "i32", ALLOC_STATIC);\n'); print('STACK_BASE = STACKTOP = Runtime.alignMemory(STATICTOP);\n'); print('STACK_MAX = STACK_BASE + TOTAL_STACK;\n'); @@ -388,6 +389,7 @@ function JSify(data, functionsOnly) { print('HEAPU32[DYNAMICTOP_PTR>>2] = DYNAMIC_BASE;\n'); print('staticSealed = true; // seal the static portion of memory\n'); if (ASSERTIONS) print('assert(DYNAMIC_BASE < TOTAL_MEMORY, "TOTAL_MEMORY not big enough for stack");\n'); + if (USE_PTHREADS) print('}\n'); } if (SPLIT_MEMORY) { print('assert(STACK_MAX < SPLIT_MEMORY, "SPLIT_MEMORY size must be big enough so the entire static memory + stack can fit in one chunk, need " + STACK_MAX);\n'); diff --git a/src/library.js b/src/library.js index d540a7a642a59..104ca46a5b06c 100644 --- a/src/library.js +++ b/src/library.js @@ -439,14 +439,47 @@ LibraryManager.library = { sbrk: function(increment) { increment = increment|0; var oldDynamicTop = 0; + var oldDynamicTopOnChange = 0; var newDynamicTop = 0; var totalMemory = 0; increment = ((increment + 15) & -16)|0; #if USE_PTHREADS - oldDynamicTop = Atomics_add(HEAPU32, DYNAMICTOP_PTR>>2, increment); + totalMemory = getTotalMemory()|0; + + // Perform a compare-and-swap loop to update the new dynamic top value. This is because + // this function can becalled simultaneously in multiple threads. + do { + oldDynamicTop = Atomics_load(HEAPU32, DYNAMICTOP_PTR>>2); + newDynamicTop = oldDynamicTop + increment | 0; + // Asking to increase dynamic top to a too high value? In pthreads builds we cannot + // enlarge memory, so this needs to fail. + if (((increment|0) > 0 & (newDynamicTop|0) < (oldDynamicTop|0)) // Detect and fail if we would wrap around signed 32-bit int. + | (newDynamicTop|0) < 0 // Also underflow, sbrk() should be able to be used to subtract. + | (newDynamicTop|0) > (totalMemory|0)) { +#if ABORTING_MALLOC + abortOnCannotGrowMemory()|0; #else + ___setErrNo({{{ cDefine('ENOMEM') }}}); + return -1; +#endif + } + // Attempt to update the dynamic top to new value. Another thread may have beat this thread to the update, + // in which case we will need to start over by iterating the loop body again. + oldDynamicTopOnChange = Atomics_compareExchange(HEAPU32, DYNAMICTOP_PTR>>2, oldDynamicTop|0, newDynamicTop|0); + } while((oldDynamicTopOnChange|0) != (oldDynamicTop|0)); +#else // singlethreaded build: (-s USE_PTHREADS=0) oldDynamicTop = HEAPU32[DYNAMICTOP_PTR>>2]|0; newDynamicTop = oldDynamicTop + increment | 0; + + if (((increment|0) > 0 & (newDynamicTop|0) < (oldDynamicTop|0)) // Detect and fail if we would wrap around signed 32-bit int. + | (newDynamicTop|0) < 0) { // Also underflow, sbrk() should be able to be used to subtract. +#if ABORTING_MALLOC + abortOnCannotGrowMemory()|0; +#endif + ___setErrNo({{{ cDefine('ENOMEM') }}}); + return -1; + } + HEAPU32[DYNAMICTOP_PTR>>2] = newDynamicTop; totalMemory = getTotalMemory()|0; if ((newDynamicTop|0) > (totalMemory|0)) { diff --git a/src/library_pthread.js b/src/library_pthread.js index 3400113b6550c..2548870dd4266 100644 --- a/src/library_pthread.js +++ b/src/library_pthread.js @@ -321,6 +321,10 @@ var LibraryPThread = { url: currentScriptUrl, buffer: HEAPU8.buffer, tempDoublePtr: tempDoublePtr, + TOTAL_MEMORY: TOTAL_MEMORY, + STATICTOP: STATICTOP, + DYNAMIC_BASE: DYNAMIC_BASE, + DYNAMICTOP_PTR: DYNAMICTOP_PTR, PthreadWorkerInit: PthreadWorkerInit }, [HEAPU8.buffer]); PThread.unusedWorkerPool.push(worker); diff --git a/src/preamble.js b/src/preamble.js index a0859bdf55c46..45e3cb87b75c1 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -943,9 +943,9 @@ function updateGlobalBufferViews() { Module['HEAPF64'] = HEAPF64 = new Float64Array(buffer); } -var STATIC_BASE = 0, STATICTOP = 0, staticSealed = false; // static area -var STACK_BASE = 0, STACKTOP = 0, STACK_MAX = 0; // stack area -var DYNAMIC_BASE = 0, DYNAMICTOP_PTR = 0; // dynamic area handled by sbrk +var STATIC_BASE, STATICTOP, staticSealed; // static area +var STACK_BASE, STACKTOP, STACK_MAX; // stack area +var DYNAMIC_BASE, DYNAMICTOP_PTR; // dynamic area handled by sbrk #if USE_PTHREADS if (ENVIRONMENT_IS_PTHREAD) { diff --git a/src/pthread-main.js b/src/pthread-main.js index 7ecf3dbb5737c..d3db0be12a186 100644 --- a/src/pthread-main.js +++ b/src/pthread-main.js @@ -2,22 +2,25 @@ // This is the entry point file that is loaded first by each Web Worker // that executes pthreads on the Emscripten application. -// All pthreads share the same Emscripten HEAP as SharedArrayBuffer -// with the main execution thread. -var buffer; - +// Thread-local: var threadInfoStruct = 0; // Info area for this thread in Emscripten HEAP (shared). If zero, this worker is not currently hosting an executing pthread. - var selfThreadId = 0; // The ID of this thread. 0 if not hosting a pthread. var parentThreadId = 0; // The ID of the parent pthread that launched this thread. - var tempDoublePtr = 0; // A temporary memory area for global float and double marshalling operations. -// Each thread has its own allocated stack space. +// Thread-local: Each thread has its own allocated stack space. var STACK_BASE = 0; var STACKTOP = 0; var STACK_MAX = 0; +// These are system-wide memory area parameters that are set at main runtime startup in main thread, and stay constant throughout the application. +var buffer; // All pthreads share the same Emscripten HEAP as SharedArrayBuffer with the main execution thread. +var DYNAMICTOP_PTR = 0; +var TOTAL_MEMORY = 0; +var STATICTOP = 0; +var staticSealed = true; // When threads are being initialized, the static memory area has been already sealed a long time ago. +var DYNAMIC_BASE = 0; + var ENVIRONMENT_IS_PTHREAD = true; // Cannot use console.log or console.error in a web worker, since that would risk a browser deadlock! https://bugzilla.mozilla.org/show_bug.cgi?id=1049091 @@ -42,8 +45,16 @@ this.alert = threadAlert; this.onmessage = function(e) { if (e.data.cmd === 'load') { // Preload command that is called once per worker to parse and load the Emscripten code. - buffer = e.data.buffer; + // Initialize the thread-local field(s): tempDoublePtr = e.data.tempDoublePtr; + + // Initialize the global "process"-wide fields: + buffer = e.data.buffer; + TOTAL_MEMORY = e.data.TOTAL_MEMORY; + STATICTOP = e.data.STATICTOP; + DYNAMIC_BASE = e.data.DYNAMIC_BASE; + DYNAMICTOP_PTR = e.data.DYNAMICTOP_PTR; + PthreadWorkerInit = e.data.PthreadWorkerInit; importScripts(e.data.url); FS.createStandardStreams(); diff --git a/tests/gauge_available_memory.cpp b/tests/gauge_available_memory.cpp new file mode 100644 index 0000000000000..2edc126321310 --- /dev/null +++ b/tests/gauge_available_memory.cpp @@ -0,0 +1,27 @@ +#include +#include +#include + +// Without noinline, Clang optimizes this whole application to a while(1) {} infinite loop. +void * __attribute__((noinline)) leak_alloc(uint64_t i) +{ + return malloc(i); +} + +int main() +{ + // Keep allocating in ~powers of two until we fail, then print out how much we got. + uint64_t availableMemory = 0; + uint64_t i = 0x80000000ULL; + while(i > 0) + { + void *ptr = leak_alloc(i); + if (ptr) availableMemory += i; + else i >>= 1; + } + printf("Total memory available: %llu\n", availableMemory); +#ifdef REPORT_RESULT + int result = (availableMemory > 10*1024*1024); + REPORT_RESULT(); +#endif +} diff --git a/tests/pthread/test_pthread_sbrk.cpp b/tests/pthread/test_pthread_sbrk.cpp new file mode 100644 index 0000000000000..922ad451cd4a2 --- /dev/null +++ b/tests/pthread/test_pthread_sbrk.cpp @@ -0,0 +1,133 @@ +#include +#include +#include +#include +#include +#include +#include + +#define NUM_THREADS 8 +#define NUM_ALLOCATIONS 10240 +#if ABORTING_MALLOC +#define ALLOCATION_SIZE 1280 // Malloc aborts, so allocate a bit less of memory so all fits +#else +#define ALLOCATION_SIZE 2560 // Malloc doesn't abort, allocate a bit more memory to test graceful allocation failures +#endif + +// Use barriers to make each thread synchronize their execution points, to maximize the possibility of seeing race conditions +// if those might occur. +static pthread_barrier_t barrierWaitToAlloc; +static pthread_barrier_t barrierWaitToVerify; +static pthread_barrier_t barrierWaitToFree; + +static void *thread_start(void *arg) +{ + int id = (int)(arg)+1; + int return_code = 0; + + uint8_t *allocated_buffers[NUM_ALLOCATIONS] = {}; + + int some_allocations_failed = 0; + + pthread_barrier_wait(&barrierWaitToAlloc); // Halt until all threads reach here, then proceed synchronously. + for(int i = 0; i < NUM_ALLOCATIONS; ++i) + { + allocated_buffers[i] = (uint8_t*)malloc(ALLOCATION_SIZE); + if (allocated_buffers[i]) + memset(allocated_buffers[i], id, ALLOCATION_SIZE); + else + some_allocations_failed = 1; + } + + pthread_barrier_wait(&barrierWaitToVerify); // Halt until all threads reach here, then proceed synchronously. + int reported_once = 0; + for(int i = 0; i < NUM_ALLOCATIONS; ++i) + { + if (!allocated_buffers[i]) continue; + for(int j = 0; j < ALLOCATION_SIZE; ++j) + if (allocated_buffers[i][j] != id) + { + ++return_code; // Failed! (but run to completion so that the barriers will all properly proceed without hanging) + if (!reported_once) { + EM_ASM_INT( { console.error('Memory corrupted! mem[i]: ' + $0 + ' != ' + $1 + ', i: ' + $2 + ', j: ' + $3); }, allocated_buffers[i][j], id, i, j); + reported_once = 1; // Avoid print flood that makes debugging hard. + } + } + } + + pthread_barrier_wait(&barrierWaitToFree); // Halt until all threads reach here, then proceed synchronously. + for(int i = 0; i < NUM_ALLOCATIONS; ++i) + free(allocated_buffers[i]); + +#if ABORTING_MALLOC + if (some_allocations_failed) + return_code = 12345678; // We expect allocations not to fail (if they did, shouldn't reach here, but we should have aborted) +#else + if (!some_allocations_failed) + return_code = 12345678; // We expect to be allocating so much memory that some of the allocations fail. +#endif + pthread_exit((void*)return_code); +/* + for(int i = 0; i < N; ++i) + { + mem[i] = (int*)malloc(4); + *mem[i] = n+i; + } + for(int i = 0; i < N; ++i) + { + int k = *mem[i]; + if (k != n+i) + { + EM_ASM_INT( { console.error('Memory corrupted! mem[i]: ' + $0 + ', i: ' + $1 + ', n: ' + $2); }, k, i, n); + pthread_exit((void*)1); + } + + assert(*mem[i] == n+i); + free(mem[i]); + } + EM_ASM_INT( { console.log('Worker with task number ' + $0 + ' finished.'); }, n); + pthread_exit(0); +*/ +} + +int main() +{ + int result = 0; + if (!emscripten_has_threading_support()) { +#ifdef REPORT_RESULT + REPORT_RESULT(); +#endif + printf("Skipped: threading support is not available!\n"); + return 0; + } + + int ret = pthread_barrier_init(&barrierWaitToAlloc, NULL, NUM_THREADS); + assert(ret == 0); + ret = pthread_barrier_init(&barrierWaitToVerify, NULL, NUM_THREADS); + assert(ret == 0); + ret = pthread_barrier_init(&barrierWaitToFree, NULL, NUM_THREADS); + assert(ret == 0); + + pthread_t thr[8/*NUM_THREADS*/]; + for(int i = 0; i < NUM_THREADS; ++i) + { + pthread_attr_t attr; + pthread_attr_init(&attr); + pthread_attr_setstacksize(&attr, NUM_ALLOCATIONS*80); + ret = pthread_create(&thr[i], &attr, thread_start, (void*)(i)); + assert(ret == 0); + } + + for(int i = 0; i < NUM_THREADS; ++i) { + int res = 0; + ret = pthread_join(thr[i], (void**)&res); + assert(ret == 0); + result += res; + if (res) printf("Thread %d failed with return code %d.\n", i, res); + } + printf("Test finished with result %d\n", result); + +#ifdef REPORT_RESULT + REPORT_RESULT(); +#endif +} diff --git a/tests/test_browser.py b/tests/test_browser.py index c372341d69c33..76d84bc9a4637 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -3018,6 +3018,20 @@ def test_pthread_custom_pthread_main_url(self): def test_pthread_proxying_in_futex_wait(self): self.btest(path_from_root('tests', 'pthread', 'test_pthread_proxying_in_futex_wait.cpp'), expected='0', args=['-O3', '-s', 'USE_PTHREADS=2', '-s', 'PTHREAD_POOL_SIZE=1', '--separate-asm'], timeout=30) + # Test that sbrk() operates properly in multithreaded conditions + def test_pthread_sbrk(self): + for aborting_malloc in [0, 1]: + print 'aborting malloc=' + str(aborting_malloc) + # With aborting malloc = 1, test allocating memory in threads + # With aborting malloc = 0, allocate so much memory in threads that some of the allocations fail. + self.btest(path_from_root('tests', 'pthread', 'test_pthread_sbrk.cpp'), expected='0', args=['-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '--separate-asm', '-s', 'ABORTING_MALLOC=' + str(aborting_malloc), '-DABORTING_MALLOC=' + str(aborting_malloc), '-s', 'TOTAL_MEMORY=134217728'], timeout=30) + + # Test that -s ABORTING_MALLOC=0 works in both pthreads and non-pthreads builds. (sbrk fails gracefully) + def test_pthread_gauge_available_memory(self): + for opts in [[], ['-O2']]: + for args in [[], ['-s', 'USE_PTHREADS=1']]: + self.btest(path_from_root('tests', 'gauge_available_memory.cpp'), expected='1', args=['-s', 'ABORTING_MALLOC=0'] + args + opts, timeout=30) + # test atomicrmw i64 def test_atomicrmw_i64(self): Popen([PYTHON, EMCC, path_from_root('tests', 'atomicrmw_i64.ll'), '-s', 'USE_PTHREADS=1', '-s', 'IN_TEST_HARNESS=1', '-o', 'test.html']).communicate() From 5094e6c59d775dd5ca03062b8957a9b0cd4de9ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Mon, 22 Aug 2016 21:51:38 +0300 Subject: [PATCH 14/26] abortOnCannotGrowMemory() is an asm.js basic built-in from preamble.js. --- emscripten.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/emscripten.py b/emscripten.py index 98f54ffc5571e..91fa30a81e9ff 100755 --- a/emscripten.py +++ b/emscripten.py @@ -651,7 +651,7 @@ def keyfunc(other): '(it is worth building your source files with -Werror (warnings are errors), as warnings can indicate undefined behavior which can cause this)' + \ '"); ' + extra - basic_funcs = ['abort', 'assert', 'enlargeMemory', 'getTotalMemory'] + [m.replace('.', '_') for m in math_envs] + basic_funcs = ['abort', 'assert', 'enlargeMemory', 'abortOnCannotGrowMemory', 'getTotalMemory'] + [m.replace('.', '_') for m in math_envs] if settings['STACK_OVERFLOW_CHECK']: basic_funcs += ['abortStackOverflow'] asm_safe_heap = settings['SAFE_HEAP'] and not settings['SAFE_HEAP_LOG'] and not settings['RELOCATABLE'] # optimized safe heap in asm, when we can @@ -1528,7 +1528,7 @@ def save_settings(): outfile.write(pre) pre = None - basic_funcs = ['abort', 'assert', 'enlargeMemory', 'getTotalMemory'] + basic_funcs = ['abort', 'assert', 'enlargeMemory', 'abortOnCannotGrowMemory', 'getTotalMemory'] access_quote = access_quoter(settings) From 59816c65eb411f730352e47b947e409a4f7c2202 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Mon, 22 Aug 2016 22:12:48 +0300 Subject: [PATCH 15/26] DYNAMICTOP is a signed 32-bit integer value, so read it as HEAP32. (This fixes -s USE_PTHREADS=2 builds when browser does not support pthreads, i.e. browser.test_pthread_create run on a non-supporting browser) --- emscripten.py | 10 +++++----- src/deterministic.js | 2 +- src/jsifier.js | 2 +- src/library.js | 14 +++++++------- src/library_bootstrap_structInfo.js | 4 ++-- src/library_trace.js | 2 +- src/memoryprofiler.js | 2 +- src/preamble.js | 14 +++++++------- src/runtime.js | 6 +++--- system/lib/split_malloc.cpp | 2 +- tests/test_other.py | 2 +- 11 files changed, 30 insertions(+), 30 deletions(-) diff --git a/emscripten.py b/emscripten.py index 91fa30a81e9ff..94101c80e90e4 100755 --- a/emscripten.py +++ b/emscripten.py @@ -1061,7 +1061,7 @@ def finalize_output(metadata, post, funcs_js, need_asyncify, provide_fround, asm '''] + ['' if not settings['SAFE_HEAP'] else ''' function setDynamicTop(value) { value = value | 0; - HEAPU32[DYNAMICTOP_PTR>>2] = value; + HEAP32[DYNAMICTOP_PTR>>2] = value; } '''] + ['' if not asm_safe_heap else ''' function SAFE_HEAP_STORE(dest, value, bytes) { @@ -1069,7 +1069,7 @@ def finalize_output(metadata, post, funcs_js, need_asyncify, provide_fround, asm value = value | 0; bytes = bytes | 0; if ((dest|0) <= 0) segfault(); - if (((dest + bytes)|0) > (HEAPU32[DYNAMICTOP_PTR>>2]|0)) segfault(); + if (((dest + bytes)|0) > (HEAP32[DYNAMICTOP_PTR>>2]|0)) segfault(); if ((bytes|0) == 4) { if ((dest&3)) alignfault(); HEAP32[dest>>2] = value; @@ -1085,7 +1085,7 @@ def finalize_output(metadata, post, funcs_js, need_asyncify, provide_fround, asm value = +value; bytes = bytes | 0; if ((dest|0) <= 0) segfault(); - if (((dest + bytes)|0) > (HEAPU32[DYNAMICTOP_PTR>>2]|0)) segfault(); + if (((dest + bytes)|0) > (HEAP32[DYNAMICTOP_PTR>>2]|0)) segfault(); if ((bytes|0) == 8) { if ((dest&7)) alignfault(); HEAPF64[dest>>3] = value; @@ -1099,7 +1099,7 @@ def finalize_output(metadata, post, funcs_js, need_asyncify, provide_fround, asm bytes = bytes | 0; unsigned = unsigned | 0; if ((dest|0) <= 0) segfault(); - if ((dest + bytes|0) > (HEAPU32[DYNAMICTOP_PTR>>2]|0)) segfault(); + if ((dest + bytes|0) > (HEAP32[DYNAMICTOP_PTR>>2]|0)) segfault(); if ((bytes|0) == 4) { if ((dest&3)) alignfault(); return HEAP32[dest>>2] | 0; @@ -1118,7 +1118,7 @@ def finalize_output(metadata, post, funcs_js, need_asyncify, provide_fround, asm dest = dest | 0; bytes = bytes | 0; if ((dest|0) <= 0) segfault(); - if ((dest + bytes|0) > (HEAPU32[DYNAMICTOP_PTR>>2]|0)) segfault(); + if ((dest + bytes|0) > (HEAP32[DYNAMICTOP_PTR>>2]|0)) segfault(); if ((bytes|0) == 8) { if ((dest&7)) alignfault(); return +HEAPF64[dest>>3]; diff --git a/src/deterministic.js b/src/deterministic.js index 5364f6f0769a4..e908dc5d94977 100644 --- a/src/deterministic.js +++ b/src/deterministic.js @@ -15,7 +15,7 @@ Module['thisProgram'] = 'thisProgram'; // for consistency between different buil function hashMemory(id) { var ret = 0; - var len = Math.max(HEAPU32[DYNAMICTOP_PTR>>2], STATICTOP); + var len = Math.max(HEAP32[DYNAMICTOP_PTR>>2], STATICTOP); for (var i = 0; i < len; i++) { ret = (ret*17 + HEAPU8[i])|0; } diff --git a/src/jsifier.js b/src/jsifier.js index 06ee30bf8121d..b009a433b47aa 100644 --- a/src/jsifier.js +++ b/src/jsifier.js @@ -386,7 +386,7 @@ function JSify(data, functionsOnly) { print('STACK_BASE = STACKTOP = Runtime.alignMemory(STATICTOP);\n'); print('STACK_MAX = STACK_BASE + TOTAL_STACK;\n'); print('DYNAMIC_BASE = Runtime.alignMemory(STACK_MAX);\n'); - print('HEAPU32[DYNAMICTOP_PTR>>2] = DYNAMIC_BASE;\n'); + print('HEAP32[DYNAMICTOP_PTR>>2] = DYNAMIC_BASE;\n'); print('staticSealed = true; // seal the static portion of memory\n'); if (ASSERTIONS) print('assert(DYNAMIC_BASE < TOTAL_MEMORY, "TOTAL_MEMORY not big enough for stack");\n'); if (USE_PTHREADS) print('}\n'); diff --git a/src/library.js b/src/library.js index 104ca46a5b06c..9c13063e1bea2 100644 --- a/src/library.js +++ b/src/library.js @@ -449,7 +449,7 @@ LibraryManager.library = { // Perform a compare-and-swap loop to update the new dynamic top value. This is because // this function can becalled simultaneously in multiple threads. do { - oldDynamicTop = Atomics_load(HEAPU32, DYNAMICTOP_PTR>>2); + oldDynamicTop = Atomics_load(HEAP32, DYNAMICTOP_PTR>>2); newDynamicTop = oldDynamicTop + increment | 0; // Asking to increase dynamic top to a too high value? In pthreads builds we cannot // enlarge memory, so this needs to fail. @@ -465,10 +465,10 @@ LibraryManager.library = { } // Attempt to update the dynamic top to new value. Another thread may have beat this thread to the update, // in which case we will need to start over by iterating the loop body again. - oldDynamicTopOnChange = Atomics_compareExchange(HEAPU32, DYNAMICTOP_PTR>>2, oldDynamicTop|0, newDynamicTop|0); + oldDynamicTopOnChange = Atomics_compareExchange(HEAP32, DYNAMICTOP_PTR>>2, oldDynamicTop|0, newDynamicTop|0); } while((oldDynamicTopOnChange|0) != (oldDynamicTop|0)); #else // singlethreaded build: (-s USE_PTHREADS=0) - oldDynamicTop = HEAPU32[DYNAMICTOP_PTR>>2]|0; + oldDynamicTop = HEAP32[DYNAMICTOP_PTR>>2]|0; newDynamicTop = oldDynamicTop + increment | 0; if (((increment|0) > 0 & (newDynamicTop|0) < (oldDynamicTop|0)) // Detect and fail if we would wrap around signed 32-bit int. @@ -480,7 +480,7 @@ LibraryManager.library = { return -1; } - HEAPU32[DYNAMICTOP_PTR>>2] = newDynamicTop; + HEAP32[DYNAMICTOP_PTR>>2] = newDynamicTop; totalMemory = getTotalMemory()|0; if ((newDynamicTop|0) > (totalMemory|0)) { // Warning: when -s USE_PTHREADS=1, this code block is called in a multithreaded context, @@ -489,7 +489,7 @@ LibraryManager.library = { // operation is performed as a single transaction. if ((enlargeMemory()|0) == 0) { ___setErrNo({{{ cDefine('ENOMEM') }}}); - HEAPU32[DYNAMICTOP_PTR>>2] = oldDynamicTop; + HEAP32[DYNAMICTOP_PTR>>2] = oldDynamicTop; return -1; } } @@ -502,9 +502,9 @@ LibraryManager.library = { brk: function(addr) { addr = addr|0; #if USE_PTHREADS - Atomics_set(HEAPU32, DYNAMICTOP_PTR>>2, addr); + Atomics_set(HEAP32, DYNAMICTOP_PTR>>2, addr); #else - HEAPU32[DYNAMICTOP_PTR>>2] = addr; + HEAP32[DYNAMICTOP_PTR>>2] = addr; #endif return 0; // TODO: error checking, return -1 on error and set errno. }, diff --git a/src/library_bootstrap_structInfo.js b/src/library_bootstrap_structInfo.js index b0069c35e7874..535265576f6bc 100644 --- a/src/library_bootstrap_structInfo.js +++ b/src/library_bootstrap_structInfo.js @@ -20,13 +20,13 @@ LibraryManager.library = { // We control the "dynamic" memory - DYNAMIC_BASE to DYNAMICTOP var self = _sbrk; if (!self.called) { - HEAPU32[DYNAMICTOP_PTR>>2] = alignMemoryPage(HEAPU32[DYNAMICTOP_PTR>>2]); // make sure we start out aligned + HEAP32[DYNAMICTOP_PTR>>2] = alignMemoryPage(HEAP32[DYNAMICTOP_PTR>>2]); // make sure we start out aligned self.called = true; assert(Runtime.dynamicAlloc); self.alloc = Runtime.dynamicAlloc; Runtime.dynamicAlloc = function() { abort('cannot dynamically allocate, sbrk now has control') }; } - var ret = HEAPU32[DYNAMICTOP_PTR>>2]; + var ret = HEAP32[DYNAMICTOP_PTR>>2]; if (bytes != 0) self.alloc(bytes); return ret; // Previous break location. }, diff --git a/src/library_trace.js b/src/library_trace.js index 957935f30e616..fdf458f1c7441 100644 --- a/src/library_trace.js +++ b/src/library_trace.js @@ -262,7 +262,7 @@ var LibraryTracing = { 'stack_top': STACKTOP, 'stack_max': STACK_MAX, 'dynamic_base': DYNAMIC_BASE, - 'dynamic_top': HEAPU32[DYNAMICTOP_PTR>>2], + 'dynamic_top': HEAP32[DYNAMICTOP_PTR>>2], 'total_memory': TOTAL_MEMORY }; var now = EmscriptenTrace.now(); diff --git a/src/memoryprofiler.js b/src/memoryprofiler.js index d0547e6ffd829..dbd47cba85859 100644 --- a/src/memoryprofiler.js +++ b/src/memoryprofiler.js @@ -302,7 +302,7 @@ var emscriptenMemoryProfiler = { html += '. STACK_MAX: ' + toHex(STACK_MAX, width) + '.'; html += '
STACK memory area used now (should be zero): ' + this.formatBytes(STACKTOP - STACK_BASE) + '.' + colorBar('#FFFF00') + ' STACK watermark highest seen usage (approximate lower-bound!): ' + this.formatBytes(this.stackTopWatermark - STACK_BASE); - var DYNAMICTOP = HEAPU32[DYNAMICTOP_PTR>>2]; + var DYNAMICTOP = HEAP32[DYNAMICTOP_PTR>>2]; html += '
' + colorBar('#70FF70') + 'DYNAMIC memory area size: ' + this.formatBytes(DYNAMICTOP-DYNAMIC_BASE); html += '. DYNAMIC_BASE: ' + toHex(DYNAMIC_BASE, width); html += '. DYNAMICTOP: ' + toHex(DYNAMICTOP, width) + '.'; diff --git a/src/preamble.js b/src/preamble.js index 45e3cb87b75c1..e02978574d1a6 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -53,9 +53,9 @@ function SAFE_HEAP_STORE(dest, value, bytes, isFloat) { if (dest <= 0) abort('segmentation fault storing ' + bytes + ' bytes to address ' + dest); if (dest % bytes !== 0) abort('alignment error storing to address ' + dest + ', which was expected to be aligned to a multiple of ' + bytes); if (staticSealed) { - if (dest + bytes > HEAPU32[DYNAMICTOP_PTR>>2]) abort('segmentation fault, exceeded the top of the available dynamic heap when storing ' + bytes + ' bytes to address ' + dest + '. STATICTOP=' + STATICTOP + ', DYNAMICTOP=' + HEAPU32[DYNAMICTOP_PTR>>2]); + if (dest + bytes > HEAP32[DYNAMICTOP_PTR>>2]) abort('segmentation fault, exceeded the top of the available dynamic heap when storing ' + bytes + ' bytes to address ' + dest + '. STATICTOP=' + STATICTOP + ', DYNAMICTOP=' + HEAP32[DYNAMICTOP_PTR>>2]); assert(DYNAMICTOP_PTR); - assert(HEAPU32[DYNAMICTOP_PTR>>2] <= TOTAL_MEMORY); + assert(HEAP32[DYNAMICTOP_PTR>>2] <= TOTAL_MEMORY); } else { if (dest + bytes > STATICTOP) abort('segmentation fault, exceeded the top of the available static heap when storing ' + bytes + ' bytes to address ' + dest + '. STATICTOP=' + STATICTOP); } @@ -69,9 +69,9 @@ function SAFE_HEAP_LOAD(dest, bytes, unsigned, isFloat) { if (dest <= 0) abort('segmentation fault loading ' + bytes + ' bytes from address ' + dest); if (dest % bytes !== 0) abort('alignment error loading from address ' + dest + ', which was expected to be aligned to a multiple of ' + bytes); if (staticSealed) { - if (dest + bytes > HEAPU32[DYNAMICTOP_PTR>>2]) abort('segmentation fault, exceeded the top of the available dynamic heap when loading ' + bytes + ' bytes from address ' + dest + '. STATICTOP=' + STATICTOP + ', DYNAMICTOP=' + HEAPU32[DYNAMICTOP_PTR>>2]); + if (dest + bytes > HEAP32[DYNAMICTOP_PTR>>2]) abort('segmentation fault, exceeded the top of the available dynamic heap when loading ' + bytes + ' bytes from address ' + dest + '. STATICTOP=' + STATICTOP + ', DYNAMICTOP=' + HEAP32[DYNAMICTOP_PTR>>2]); assert(DYNAMICTOP_PTR); - assert(HEAPU32[DYNAMICTOP_PTR>>2] <= TOTAL_MEMORY); + assert(HEAP32[DYNAMICTOP_PTR>>2] <= TOTAL_MEMORY); } else { if (dest + bytes > STATICTOP) abort('segmentation fault, exceeded the top of the available static heap when loading ' + bytes + ' bytes from address ' + dest + '. STATICTOP=' + STATICTOP); } @@ -1017,7 +1017,7 @@ function enlargeMemory() { #else // TOTAL_MEMORY is the current size of the actual array, and DYNAMICTOP is the new top. #if ASSERTIONS - assert(HEAPU32[DYNAMICTOP_PTR>>2] > TOTAL_MEMORY); // This function should only ever be called after the ceiling of the dynamic heap has already been bumped to exceed the current total size of the asm.js heap. + assert(HEAP32[DYNAMICTOP_PTR>>2] > TOTAL_MEMORY); // This function should only ever be called after the ceiling of the dynamic heap has already been bumped to exceed the current total size of the asm.js heap. assert(TOTAL_MEMORY > 4); // So the loop below will not be infinite #endif @@ -1030,9 +1030,9 @@ function enlargeMemory() { var LIMIT = Math.pow(2, 31); // 2GB is a practical maximum, as we use signed ints as pointers // and JS engines seem unhappy to give us 2GB arrays currently - if (HEAPU32[DYNAMICTOP_PTR>>2] >= LIMIT) return false; + if (HEAP32[DYNAMICTOP_PTR>>2] >= LIMIT) return false; - while (TOTAL_MEMORY < HEAPU32[DYNAMICTOP_PTR>>2]) { // Keep incrementing the heap size as long as it's less than what is requested. + while (TOTAL_MEMORY < HEAP32[DYNAMICTOP_PTR>>2]) { // Keep incrementing the heap size as long as it's less than what is requested. if (TOTAL_MEMORY < LIMIT/2) { TOTAL_MEMORY = alignMemoryPage(2*TOTAL_MEMORY); // // Simple heuristic: double until 1GB... } else { diff --git a/src/runtime.js b/src/runtime.js index db2d729985933..d5c7cf6e011a4 100644 --- a/src/runtime.js +++ b/src/runtime.js @@ -61,13 +61,13 @@ var RuntimeGenerator = { dynamicAlloc: function(size) { var ret = ''; if (ASSERTIONS) ret += 'assert(DYNAMICTOP_PTR);'; // dynamic area must be ready - ret += 'var ret = HEAPU32[DYNAMICTOP_PTR>>2];' + ret += 'var ret = HEAP32[DYNAMICTOP_PTR>>2];' + 'var end = (((ret + size + 15)|0) & -16);' - + 'HEAPU32[DYNAMICTOP_PTR>>2] = end;' + + 'HEAP32[DYNAMICTOP_PTR>>2] = end;' + 'if (end >= TOTAL_MEMORY) {' + 'var success = enlargeMemory();' + 'if (!success) {' - + 'HEAPU32[DYNAMICTOP_PTR>>2] = ret;' + + 'HEAP32[DYNAMICTOP_PTR>>2] = ret;' + 'return 0;' + '}' + '}' diff --git a/system/lib/split_malloc.cpp b/system/lib/split_malloc.cpp index da8f2f6865552..f4d60f69a5cba 100644 --- a/system/lib/split_malloc.cpp +++ b/system/lib/split_malloc.cpp @@ -71,7 +71,7 @@ struct Space { start = split_memory*index; } else { // small area in existing chunk 0 - start = EM_ASM_INT_V({ return (HEAPU32[DYNAMICTOP_PTR>>2]+3)&-4; }); + start = EM_ASM_INT_V({ return (HEAP32[DYNAMICTOP_PTR>>2]+3)&-4; }); assert(start < split_memory); } int size = (split_memory*(index+1)) - start; diff --git a/tests/test_other.py b/tests/test_other.py index a2be83a99c8ad..f9ffaddaa35f0 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -5135,7 +5135,7 @@ def test_massive_alloc(self): def test_failing_alloc(self): for pre_fail, post_fail, opts in [ ('', '', []), - ('EM_ASM( Module.temp = HEAPU32[DYNAMICTOP_PTR>>2] );', 'EM_ASM( assert(Module.temp === HEAPU32[DYNAMICTOP_PTR>>2], "must not adjust DYNAMICTOP when an alloc fails!") );', []), + ('EM_ASM( Module.temp = HEAP32[DYNAMICTOP_PTR>>2] );', 'EM_ASM( assert(Module.temp === HEAP32[DYNAMICTOP_PTR>>2], "must not adjust DYNAMICTOP when an alloc fails!") );', []), ('', '', ['-s', 'SPLIT_MEMORY=' + str(16*1024*1024)]), ]: for growth in [0, 1]: From 2beacd579549b5289301113a3ab47da46d47355d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Tue, 23 Aug 2016 00:43:59 +0300 Subject: [PATCH 16/26] Only import abortOnCannotGrowMemory() if not running with no memory growth enabled. --- emscripten.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/emscripten.py b/emscripten.py index 94101c80e90e4..f82d4755cddf7 100755 --- a/emscripten.py +++ b/emscripten.py @@ -651,7 +651,8 @@ def keyfunc(other): '(it is worth building your source files with -Werror (warnings are errors), as warnings can indicate undefined behavior which can cause this)' + \ '"); ' + extra - basic_funcs = ['abort', 'assert', 'enlargeMemory', 'abortOnCannotGrowMemory', 'getTotalMemory'] + [m.replace('.', '_') for m in math_envs] + basic_funcs = ['abort', 'assert', 'enlargeMemory', 'getTotalMemory'] + [m.replace('.', '_') for m in math_envs] + if not settings['ALLOW_MEMORY_GROWTH']: basic_funcs += ['abortOnCannotGrowMemory'] if settings['STACK_OVERFLOW_CHECK']: basic_funcs += ['abortStackOverflow'] asm_safe_heap = settings['SAFE_HEAP'] and not settings['SAFE_HEAP_LOG'] and not settings['RELOCATABLE'] # optimized safe heap in asm, when we can @@ -1528,7 +1529,8 @@ def save_settings(): outfile.write(pre) pre = None - basic_funcs = ['abort', 'assert', 'enlargeMemory', 'abortOnCannotGrowMemory', 'getTotalMemory'] + basic_funcs = ['abort', 'assert', 'enlargeMemory', 'getTotalMemory'] + if not settings['ALLOW_MEMORY_GROWTH']: basic_funcs += ['abortOnCannotGrowMemory'] access_quote = access_quoter(settings) From bc84947518825c3ee776d63da2212ff69d272e70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Tue, 23 Aug 2016 02:07:33 +0300 Subject: [PATCH 17/26] Even though ctor evaller doesn't use dynamic memory, it needs a valid address to dynamic memory base for safe heap assertion checking purposes. --- tools/ctor_evaller.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tools/ctor_evaller.py b/tools/ctor_evaller.py index 6d116828a3b7b..d2fb1ba72f73a 100644 --- a/tools/ctor_evaller.py +++ b/tools/ctor_evaller.py @@ -110,6 +110,7 @@ def add_func(asm, func): var buffer = new ArrayBuffer(totalMemory); var heap = new Uint8Array(buffer); +var heapi32 = new Int32Array(buffer); var memInit = %s; @@ -127,7 +128,8 @@ def add_func(asm, func): var stackMax = stackTop + totalStack; if (stackMax >= totalMemory) throw 'not enough room for stack'; -var dynamicTopPtr = 0; +var dynamicTopPtr = stackMax; +heapi32[dynamicTopPtr >> 2] = stackMax; if (!Math.imul) { Math.imul = Math.imul || function(a, b) { @@ -200,6 +202,10 @@ def add_func(asm, func): console.warn('globals modified'); break; } + if (heapi32[dynamicTopPtr >> 2] !== stackMax) { + console.warn('dynamic allocation was performend'); + break; + } // this one was ok. numSuccessful = i + 1; From 8490fd5713cb74556d20921c78e2c7a0db1b13ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Tue, 23 Aug 2016 11:15:47 +0300 Subject: [PATCH 18/26] Implement brk() and add test. --- src/library.js | 44 ++++++++++++++++----- tests/sbrk_brk.cpp | 96 ++++++++++++++++++++++++++++++++++++++++++++++ tests/test_core.py | 7 ++++ 3 files changed, 138 insertions(+), 9 deletions(-) create mode 100644 tests/sbrk_brk.cpp diff --git a/src/library.js b/src/library.js index 9c13063e1bea2..c7c0a7fa7d190 100644 --- a/src/library.js +++ b/src/library.js @@ -483,10 +483,6 @@ LibraryManager.library = { HEAP32[DYNAMICTOP_PTR>>2] = newDynamicTop; totalMemory = getTotalMemory()|0; if ((newDynamicTop|0) > (totalMemory|0)) { - // Warning: when -s USE_PTHREADS=1, this code block is called in a multithreaded context, - // so enlargeMemory() will be set to abort. If enlarging with pthreads is enabled at some point, - // sbrk will need to get mutexes to ensure that incrementing the heap size above and this enlarge - // operation is performed as a single transaction. if ((enlargeMemory()|0) == 0) { ___setErrNo({{{ cDefine('ENOMEM') }}}); HEAP32[DYNAMICTOP_PTR>>2] = oldDynamicTop; @@ -499,14 +495,44 @@ LibraryManager.library = { brk__asm: true, brk__sig: ['ii'], - brk: function(addr) { - addr = addr|0; + brk: function(newDynamicTop) { + newDynamicTop = newDynamicTop|0; + var oldDynamicTop = 0; + var totalMemory = 0; #if USE_PTHREADS - Atomics_set(HEAP32, DYNAMICTOP_PTR>>2, addr); + totalMemory = getTotalMemory()|0; + // Asking to increase dynamic top to a too high value? In pthreads builds we cannot + // enlarge memory, so this needs to fail. + if ((newDynamicTop|0) < 0 | (newDynamicTop|0) > (totalMemory|0)) { +#if ABORTING_MALLOC + abortOnCannotGrowMemory()|0; #else - HEAP32[DYNAMICTOP_PTR>>2] = addr; + ___setErrNo({{{ cDefine('ENOMEM') }}}); + return -1; +#endif + } + Atomics_store(HEAP32, DYNAMICTOP_PTR>>2, newDynamicTop|0); +#else // singlethreaded build: (-s USE_PTHREADS=0) + if ((newDynamicTop|0) < 0) { +#if ABORTING_MALLOC + abortOnCannotGrowMemory()|0; +#endif + ___setErrNo({{{ cDefine('ENOMEM') }}}); + return -1; + } + + oldDynamicTop = HEAP32[DYNAMICTOP_PTR>>2]|0; + HEAP32[DYNAMICTOP_PTR>>2] = newDynamicTop; + totalMemory = getTotalMemory()|0; + if ((newDynamicTop|0) > (totalMemory|0)) { + if ((enlargeMemory()|0) == 0) { + ___setErrNo({{{ cDefine('ENOMEM') }}}); + HEAP32[DYNAMICTOP_PTR>>2] = oldDynamicTop; + return -1; + } + } #endif - return 0; // TODO: error checking, return -1 on error and set errno. + return 0; }, system__deps: ['__setErrNo', '$ERRNO_CODES'], diff --git a/tests/sbrk_brk.cpp b/tests/sbrk_brk.cpp new file mode 100644 index 0000000000000..bca7690b616d6 --- /dev/null +++ b/tests/sbrk_brk.cpp @@ -0,0 +1,96 @@ +#include +#include +#include +#include +#include +#include + +// A custom tiny malloc & free impl +uintptr_t arena_end = 0; + +struct mem_node +{ + uint32_t size; + mem_node *next; +}; + +// List of reclaimed free memory nodes +mem_node free_memory_root = {}; + +void *mymalloc(size_t size) +{ + // Static init at program startup: + if (!arena_end) + { + arena_end = (uintptr_t)sbrk(0); + assert(arena_end != (uintptr_t)-1); + } + + // Find if there is an existing node we can reuse. + mem_node *prev = &free_memory_root; + mem_node *n = prev->next; + while(n) + { + if (n->size >= size) + { + prev->next = n->next; // Splice this node off from the free list. + return (void*)((uintptr_t)n + sizeof(mem_node)); + } + prev = n; + n = n->next; + } + + // If not, allocate new node from empty area + size_t allocated_size = sizeof(mem_node) + size; + +#if TEST_BRK // test brk() + uintptr_t new_brk = arena_end + allocated_size; + int failed = brk((void*)new_brk); + if (failed) return 0; + + mem_node *node = (mem_node*)arena_end; + arena_end = (uintptr_t)sbrk(0); + assert(arena_end == new_brk); +#else // test sbrk() + mem_node *node = (mem_node*)sbrk(allocated_size); + if ((uintptr_t)node == (uintptr_t)-1) + return 0; +#endif + node->size = size; + return (void*)((uintptr_t)node + sizeof(mem_node)); +} + +void myfree(void *ptr) +{ + mem_node *freed_node = (mem_node*)((uintptr_t)ptr - sizeof(mem_node)); + freed_node->next = free_memory_root.next; + free_memory_root.next = freed_node; +} + +int main() +{ +#define N 3000 // Arbitrary amount that fits within the default 16MB heap + + uint8_t *data[N]; + for(int i = 0; i < N; ++i) + { + int count = i&~15U; + uint8_t *memory = (uint8_t*)mymalloc(count); + assert(memory); + uint8_t *memory2 = (uint8_t*)mymalloc(count); + assert(memory2); + + myfree(memory); + data[i] = memory2; + memset(memory2, (uint8_t)i, count); + } + uintptr_t dynamicTop = (uintptr_t)sbrk(0); + for(int i = 0; i < N; ++i) + { + int count = i&~15U; + assert((uintptr_t)data[i] + count <= dynamicTop); + for(int j = 0; j < count; ++j) + assert(data[i][j] == (uint8_t)i); + } + printf("OK. brk at end: %u. \n", (uintptr_t)sbrk(0)); +} diff --git a/tests/test_core.py b/tests/test_core.py index 6261ca601561e..234cc461b2507 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -8204,6 +8204,13 @@ def test_binaryen(self): self.emcc_args += ['-s', 'BINARYEN=1', '-s', 'BINARYEN_METHOD="interpret-binary"'] self.do_run(open(path_from_root('tests', 'hello_world.c')).read(), 'hello, world!') + def test_sbrk(self): + self.do_run(open(path_from_root('tests', 'sbrk_brk.cpp')).read(), 'OK.') + + def test_brk(self): + self.emcc_args += ['-DTEST_BRK=1'] + self.do_run(open(path_from_root('tests', 'sbrk_brk.cpp')).read(), 'OK.') + # Generate tests for everything def make_run(fullname, name=-1, compiler=-1, embetter=0, quantum_size=0, typed_arrays=0, emcc_args=None, env=None): From 40b6e90f5e03f4e0730fd21459f2da2e6d91d3ee Mon Sep 17 00:00:00 2001 From: Jukka Jylanki Date: Tue, 23 Aug 2016 12:50:40 +0300 Subject: [PATCH 19/26] Fix abortOnCannotGrowMemory import to occur when aborting malloc is enabled. --- emscripten.py | 4 ++-- src/preamble.js | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/emscripten.py b/emscripten.py index f82d4755cddf7..25d046dd03a1b 100755 --- a/emscripten.py +++ b/emscripten.py @@ -652,7 +652,7 @@ def keyfunc(other): '"); ' + extra basic_funcs = ['abort', 'assert', 'enlargeMemory', 'getTotalMemory'] + [m.replace('.', '_') for m in math_envs] - if not settings['ALLOW_MEMORY_GROWTH']: basic_funcs += ['abortOnCannotGrowMemory'] + if settings['ABORTING_MALLOC']: basic_funcs += ['abortOnCannotGrowMemory'] if settings['STACK_OVERFLOW_CHECK']: basic_funcs += ['abortStackOverflow'] asm_safe_heap = settings['SAFE_HEAP'] and not settings['SAFE_HEAP_LOG'] and not settings['RELOCATABLE'] # optimized safe heap in asm, when we can @@ -1530,7 +1530,7 @@ def save_settings(): pre = None basic_funcs = ['abort', 'assert', 'enlargeMemory', 'getTotalMemory'] - if not settings['ALLOW_MEMORY_GROWTH']: basic_funcs += ['abortOnCannotGrowMemory'] + if settings['ABORTING_MALLOC']: basic_funcs += ['abortOnCannotGrowMemory'] access_quote = access_quoter(settings) diff --git a/src/preamble.js b/src/preamble.js index e02978574d1a6..e843b3cb2ce7c 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -979,11 +979,13 @@ function abortStackOverflow(allocSize) { } #endif -#if ALLOW_MEMORY_GROWTH == 0 +#if ABORTING_MALLOC function abortOnCannotGrowMemory() { abort('Cannot enlarge memory arrays. Either (1) compile with -s TOTAL_MEMORY=X with X higher than the current value ' + TOTAL_MEMORY + ', (2) compile with -s ALLOW_MEMORY_GROWTH=1 which adjusts the size at runtime but prevents some optimizations, (3) set Module.TOTAL_MEMORY to a higher value before the program runs, or if you want malloc to return NULL (0) instead of this abort, compile with -s ABORTING_MALLOC=0 '); } -#else // ALLOW_MEMORY_GROWTH +#endif + +#if ALLOW_MEMORY_GROWTH if (!Module['reallocBuffer']) Module['reallocBuffer'] = function(size) { var ret; try { From 7ac6a844365a44df48a9d3347d78034fc02e26c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Tue, 23 Aug 2016 17:28:56 +0300 Subject: [PATCH 20/26] Clean up leftover code in test_pthread_sbrk.cpp --- tests/pthread/test_pthread_sbrk.cpp | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/tests/pthread/test_pthread_sbrk.cpp b/tests/pthread/test_pthread_sbrk.cpp index 922ad451cd4a2..b39948a163930 100644 --- a/tests/pthread/test_pthread_sbrk.cpp +++ b/tests/pthread/test_pthread_sbrk.cpp @@ -67,27 +67,6 @@ static void *thread_start(void *arg) return_code = 12345678; // We expect to be allocating so much memory that some of the allocations fail. #endif pthread_exit((void*)return_code); -/* - for(int i = 0; i < N; ++i) - { - mem[i] = (int*)malloc(4); - *mem[i] = n+i; - } - for(int i = 0; i < N; ++i) - { - int k = *mem[i]; - if (k != n+i) - { - EM_ASM_INT( { console.error('Memory corrupted! mem[i]: ' + $0 + ', i: ' + $1 + ', n: ' + $2); }, k, i, n); - pthread_exit((void*)1); - } - - assert(*mem[i] == n+i); - free(mem[i]); - } - EM_ASM_INT( { console.log('Worker with task number ' + $0 + ' finished.'); }, n); - pthread_exit(0); -*/ } int main() From 210298736490fcde4a0a2c5db76f6004bb4f07e6 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 23 Aug 2016 10:25:55 -0700 Subject: [PATCH 21/26] fix test_exceptions_white_list* in binaryen/asm2wasm, which regressed in afa68d357e0186d0e40400e61de6f50d783d60d3 --- tests/test_core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index 77c5da3005164..c380387254b0d 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1434,7 +1434,7 @@ def test_exceptions_white_list(self): Settings.DISABLE_EXCEPTION_CATCHING = 2 # Wasm does not add an underscore to function names. For wasm, the # mismatches are fixed in fixImports() function in JS glue code. - if not Settings.BINARYEN: + if not self.is_wasm_backend(): Settings.EXCEPTION_CATCHING_WHITELIST = ["__Z12somefunctionv"] else: Settings.EXCEPTION_CATCHING_WHITELIST = ["_Z12somefunctionv"] @@ -1477,7 +1477,7 @@ def test_exceptions_white_list_2(self): Settings.DISABLE_EXCEPTION_CATCHING = 2 # Wasm does not add an underscore to function names. For wasm, the # mismatches are fixed in fixImports() function in JS glue code. - if not Settings.BINARYEN: + if not self.is_wasm_backend(): Settings.EXCEPTION_CATCHING_WHITELIST = ["_main"] else: Settings.EXCEPTION_CATCHING_WHITELIST = ["main"] From a8f5ddaf3a8c2a46ff9a4a07421edf1ba1d2fc77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Tue, 23 Aug 2016 23:26:50 +0300 Subject: [PATCH 22/26] Fix Windows handling when --output_eol when --proxy-to-worker mode is specified and outputting a .js file --- emcc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/emcc.py b/emcc.py index 0f8b5df41e8ae..5047b41f77aa5 100755 --- a/emcc.py +++ b/emcc.py @@ -2111,7 +2111,7 @@ def un_src(): # use this if you want to modify the script and need it to be inli shutil.move(js_target, js_target[:-3] + '.worker.js') # compiler output goes in .worker.js file worker_target_basename = target_basename + '.worker' target_contents = open(shared.path_from_root('src', 'webGLClient.js')).read() + '\n' + open(shared.path_from_root('src', 'proxyClient.js')).read().replace('{{{ filename }}}', shared.Settings.PROXY_TO_WORKER_FILENAME or worker_target_basename).replace('{{{ IDBStore.js }}}', open(shared.path_from_root('src', 'IDBStore.js')).read()) - open(target, 'wb').write(target_contents) + open(target, 'w').write(target_contents) for f in generated_text_files_with_native_eols: tools.line_endings.convert_line_endings_in_file(f, os.linesep, output_eol) From f20b4e5cc1cd2eaa687a493ce9291cbea5098187 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Tue, 23 Aug 2016 23:52:08 +0300 Subject: [PATCH 23/26] Fix test other.test_bad_triple on Windows if only Visual Studio 2015 is installed and no VS2013 or older. --- tools/shared.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/tools/shared.py b/tools/shared.py index 89592974b450b..25ba9c182762f 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -652,17 +652,21 @@ def get_clang_native_env(): return env if 'VSINSTALLDIR' in env: - visual_studio_2013_path = env['VSINSTALLDIR'] + visual_studio_path = env['VSINSTALLDIR'] elif 'VS120COMNTOOLS' in env: - visual_studio_2013_path = os.path.normpath(os.path.join(env['VS120COMNTOOLS'], '../..')) + visual_studio_path = os.path.normpath(os.path.join(env['VS120COMNTOOLS'], '../..')) + elif 'VS140COMNTOOLS' in env: + visual_studio_path = os.path.normpath(os.path.join(env['VS140COMNTOOLS'], '../..')) elif 'ProgramFiles(x86)' in env: - visual_studio_2013_path = os.path.normpath(os.path.join(env['ProgramFiles(x86)'], 'Microsoft Visual Studio 12.0')) + visual_studio_path = os.path.normpath(os.path.join(env['ProgramFiles(x86)'], 'Microsoft Visual Studio 12.0')) elif 'ProgramFiles' in env: - visual_studio_2013_path = os.path.normpath(os.path.join(env['ProgramFiles'], 'Microsoft Visual Studio 12.0')) + visual_studio_path = os.path.normpath(os.path.join(env['ProgramFiles'], 'Microsoft Visual Studio 12.0')) else: - visual_studio_2013_path = 'C:\\Program Files (x86)\\Microsoft Visual Studio 12.0' - if not os.path.isdir(visual_studio_2013_path): - raise Exception('Visual Studio 2013 was not found in "' + visual_studio_2013_path + '"! Run in Visual Studio command prompt to avoid the need to autoguess this location (or set VSINSTALLDIR env var).') + visual_studio_path = 'C:\\Program Files (x86)\\Microsoft Visual Studio 12.0' + if not os.path.isdir(visual_studio_path): + raise Exception('Visual Studio 2013 was not found in "' + visual_studio_path + '"! Run in Visual Studio command prompt to avoid the need to autoguess this location (or set VSINSTALLDIR env var).') + + env['INCLUDE'] = os.path.join(visual_studio_path, 'VC\\INCLUDE') if 'WindowsSdkDir' in env: windows_sdk_dir = env['WindowsSdkDir'] @@ -670,14 +674,18 @@ def get_clang_native_env(): windows_sdk_dir = os.path.normpath(os.path.join(env['ProgramFiles(x86)'], 'Windows Kits\\8.1')) elif 'ProgramFiles' in env: windows_sdk_dir = os.path.normpath(os.path.join(env['ProgramFiles'], 'Windows Kits\\8.1')) + elif os.path.isdir('C:\\Program Files (x86)\\Windows Kits\\10'): + windows_sdk_dir = 'C:\\Program Files (x86)\\Windows Kits\\10' + include_dir = 'C:\\Program Files (x86)\\Windows Kits\\10\\Include' + include_dir = [os.path.join(include_dir,x) for x in os.listdir(include_dir) if os.path.isdir(os.path.join(include_dir, x))][0] + '\\ucrt' + env['INCLUDE'] = env['INCLUDE'] + ';' + include_dir else: windows_sdk_dir = 'C:\\Program Files (x86)\\Windows Kits\\8.1' if not os.path.isdir(windows_sdk_dir): raise Exception('Windows SDK was not found in "' + windows_sdk_dir + '"! Run in Visual Studio command prompt to avoid the need to autoguess this location (or set WindowsSdkDir env var).') - env['INCLUDE'] = os.path.join(visual_studio_2013_path, 'VC\\INCLUDE') - env['LIB'] = os.path.join(visual_studio_2013_path, 'VC\\LIB\\amd64') + ';' + os.path.join(windows_sdk_dir, 'lib\\winv6.3\\um\\x64') - env['PATH'] = env['PATH'] + ';' + os.path.join(visual_studio_2013_path, 'VC\\BIN') + env['LIB'] = os.path.join(visual_studio_path, 'VC\\LIB\\amd64') + ';' + os.path.join(windows_sdk_dir, 'lib\\winv6.3\\um\\x64') + env['PATH'] = env['PATH'] + ';' + os.path.join(visual_studio_path, 'VC\\BIN') # Current configuration above is all Visual Studio -specific, so on non-Windowses, no action needed. From 78fd7d926cb203351699372ed4c9aac22b1da8f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Wed, 24 Aug 2016 00:00:31 +0300 Subject: [PATCH 24/26] Clean up previous commit to guess location of Program Files a bit smarter --- tools/shared.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/tools/shared.py b/tools/shared.py index 25ba9c182762f..33f7e92b3d91b 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -668,17 +668,22 @@ def get_clang_native_env(): env['INCLUDE'] = os.path.join(visual_studio_path, 'VC\\INCLUDE') + if 'ProgramFiles(x86)' in env: prog_files_x86 = env['ProgramFiles(x86)'] + elif 'ProgramFiles' in env: prog_files_x86 = env['ProgramFiles'] + elif os.path.isdir('C:\\Program Files (x86)'): prog_files_x86 = 'C:\\Program Files (x86)' + elif os.path.isdir('C:\\Program Files'): prog_files_x86 = 'C:\\Program Files' + else: + raise Exception('Unable to detect Program files directory for native Visual Studio build!') + if 'WindowsSdkDir' in env: windows_sdk_dir = env['WindowsSdkDir'] - elif 'ProgramFiles(x86)' in env: - windows_sdk_dir = os.path.normpath(os.path.join(env['ProgramFiles(x86)'], 'Windows Kits\\8.1')) - elif 'ProgramFiles' in env: - windows_sdk_dir = os.path.normpath(os.path.join(env['ProgramFiles'], 'Windows Kits\\8.1')) - elif os.path.isdir('C:\\Program Files (x86)\\Windows Kits\\10'): - windows_sdk_dir = 'C:\\Program Files (x86)\\Windows Kits\\10' - include_dir = 'C:\\Program Files (x86)\\Windows Kits\\10\\Include' + elif os.path.isdir(os.path.join(prog_files_x86, 'Windows Kits', '10')): + windows_sdk_dir = os.path.join(prog_files_x86, 'Windows Kits', '10') + include_dir = os.path.join(windows_sdk_dir, 'Include') include_dir = [os.path.join(include_dir,x) for x in os.listdir(include_dir) if os.path.isdir(os.path.join(include_dir, x))][0] + '\\ucrt' env['INCLUDE'] = env['INCLUDE'] + ';' + include_dir + elif os.path.isdir(os.path.join(prog_files_x86, 'Windows Kits', '8.1')): + windows_sdk_dir = os.path.join(prog_files_x86, 'Windows Kits', '8.1') else: windows_sdk_dir = 'C:\\Program Files (x86)\\Windows Kits\\8.1' if not os.path.isdir(windows_sdk_dir): From 48e5cca7132342e762ea1f395a060855f026db55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Wed, 24 Aug 2016 00:05:23 +0300 Subject: [PATCH 25/26] Skip other.test_outline on Windows if mingw32-make is not installed, since it is needed to drive the test build. --- tests/test_other.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_other.py b/tests/test_other.py index ddfcff9d2863c..340e200838263 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -754,6 +754,9 @@ def build(path, args): assert not os.path.exists('a.out') and not os.path.exists('a.exe'), 'Must not leave unneeded linker stubs' def test_outline(self): + if WINDOWS and not Building.which('mingw32-make'): + return self.skip('Skipping other.test_outline: This test requires "mingw32-make" tool in PATH on Windows to drive a Makefile build of zlib') + def test(name, src, libs, expected, expected_ranges, args=[], suffix='cpp'): print name From e3d94a2442aa712dfa005fa626c6406039d304b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Wed, 24 Aug 2016 01:19:09 +0300 Subject: [PATCH 26/26] Explicitly initialize global runtime vars to zero in main thread at startup. --- src/preamble.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/preamble.js b/src/preamble.js index e843b3cb2ce7c..60ceb77623e0e 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -947,6 +947,15 @@ var STATIC_BASE, STATICTOP, staticSealed; // static area var STACK_BASE, STACKTOP, STACK_MAX; // stack area var DYNAMIC_BASE, DYNAMICTOP_PTR; // dynamic area handled by sbrk +#if USE_PTHREADS +if (!ENVIRONMENT_IS_PTHREAD) { // Pthreads have already initialized these variables in src/pthread-main.js, where they were passed to the thread worker at startup time +#endif + STATIC_BASE = STATICTOP = STACK_BASE = STACKTOP = STACK_MAX = DYNAMIC_BASE = DYNAMICTOP_PTR = 0; + staticSealed = false; +#if USE_PTHREADS +} +#endif + #if USE_PTHREADS if (ENVIRONMENT_IS_PTHREAD) { staticSealed = true; // The static memory area has been initialized already in the main thread, pthreads skip this.