diff --git a/AUTHORS b/AUTHORS index 4f4d34de29bb3..c1e91a8342f9f 100644 --- a/AUTHORS +++ b/AUTHORS @@ -239,3 +239,9 @@ a license to everyone to use it as detailed in LICENSE.) * Jakob Stoklund Olesen * Jérémy Anger * Derek Schuff (copyright owned by Google, Inc.) +* Ashley Sommer +* Dave Fletcher +* Lars-Magnus Skog +* Pieter Vantorre +* Maher Sallam +* Andrey Burov diff --git a/ChangeLog.markdown b/ChangeLog.markdown index 7abc761b34273..46c8184cd6034 100644 --- a/ChangeLog.markdown +++ b/ChangeLog.markdown @@ -10,9 +10,312 @@ Not all changes are documented here. In particular, new features, user-oriented Current trunk code ------------------ - To see a list of commits in the active development branch 'incoming', which have not yet been packaged in a release, see - - Emscripten: https://github.com/kripken/emscripten/compare/1.34.12...incoming - - Emscripten-LLVM: https://github.com/kripken/emscripten-fastcomp/compare/1.34.12...incoming - - Emscripten-Clang: https://github.com/kripken/emscripten-fastcomp-clang/compare/1.34.12...incoming + - Emscripten: https://github.com/kripken/emscripten/compare/1.36.1...incoming + - Emscripten-LLVM: https://github.com/kripken/emscripten-fastcomp/compare/1.36.1...incoming + - Emscripten-Clang: https://github.com/kripken/emscripten-fastcomp-clang/compare/1.36.1...incoming + +v1.36.1: 3/8/2016 +----------------- + - Fixed glfwSetWindowSizeCallback to conform to GLFW2 API. + - Update OpenAL sources only when the browser window is visible to avoid occasional stuttering static glitches when the page tab is hidden. (#4107) + - Implemented LLVM math intrinsics powi, trunc and floor. + - Added support for SDL_GL_ALPHA_SIZE in GL context initialization. (#4125) + - Added no-op stubs for several pthread functions when building without pthreads enabled (#4130) + - Optimize glUniform*fv and glVertexAttrib*fv functions to generate less garbage and perform much faster (#4128) + - Added new EVAL_CTORS optimization pass which evaluates global data initializer constructors at link time, which would improve startup time and reduce code size of these ctors. + - Implemented support for OpenAL AL_PITCH option. + - Implemented new build options -s STACK_OVERFLOW_CHECK=0/1/2 which adds runtime stack overrun checks. 0: disabled, 1: minimal, between each frame, 2: at each explicit JS side stack allocation call to allocate(). + - Fixed an issue with -s SPLIT_MEMORY mode where an unsigned 32-bit memory access would come out as signed. (#4150) + - Fixed asm.js validation in call handlers to llvm_powi_f*. + - Full list of changes: + - Emscripten: https://github.com/kripken/emscripten/compare/1.36.0...1.36.1 + - Emscripten-LLVM: https://github.com/kripken/emscripten-fastcomp/compare/1.36.0...1.36.1 + - Emscripten-Clang: no changes. + +v1.36.0: 2/23/2016 +------------------ + - Fixed an OpenAL bug where OpenAL sources would not respect global volume setting. + - Fixed an issue where alGetListenerf() with AL_GAIN would not return the correct value. (#4091) + - Fixed an issue where setting alListenerf() with AL_GAIN would not set the correct value. (#4092) + - Implemented new JS optimizer "Duplicate Function Elimination" pass which collapses identical functions to save code size. + - Implemented the _Exit() function. + - Added support for SSE3 and SSSE3 intrinsics (#4099) and partially for SSE 4.1 intrinsics (#4030, #4101) + - Added support for -include-pch flag (#4086) + - Fixed a regex syntax in ccall on Chrome Canary (#4111) + - Full list of changes: + - Emscripten: https://github.com/kripken/emscripten/compare/1.35.23...1.36.0 + - Emscripten-LLVM: https://github.com/kripken/emscripten-fastcomp/compare/1.35.23...1.36.0 + - Emscripten-Clang: no changes. + +v1.35.23: 2/9/2016 +------------------ + - Provide $NM environment variable to point to llvm-nm when running emconfigure, which helps e.g. libjansson to build (#4036) + - Fixed glGetString(GL_SHADING_LANGUAGE_VERSION) to return appropriate result depending on if running on WebGL1 vs WebGL2, instead of hardcoding the result (#4040) + - Fixed a regression with CMake try_run() possibly failing, caused by the addition of CMAKE_CROSSCOMPILING_EMULATOR in v1.32.3. + - Fixed CMake to work in the case when NODE_JS is an array containing parameters to be passed to Node.js. (#4045) + - Fixed a memory issue that caused the Emscripten memory initializer file (.mem.js) to be unnecessarily retained in memory during runtime (#4044) + - Added support for complex valued mul and div ops. + - Added new option "Module.environment" which allows overriding the runtime ENVIRONMENT_IS_WEB/ENVIRONMENT_IS_WORKER/ENVIRONMENT_IS_NODE/ENVIRONMENT_IS_SHELL fields. + - Fixed an issue with SAFE_HEAP methods in async mode (#4046) + - Fixed WebSocket constructor to work in web worker environment (#3849) + - Fixed a potential issue with some browsers reporting gamepad axis values outside \[-1, 1\] (#3602) + - Changed libcxxabi to be linked in last, so that it does not override weakly linked methods in libcxx (#4053) + - Implemented new JSDCE code optimization pass which removes at JS link stage dead code that is not referenced anywhere (in addition to LLVM doing this for C++ link stage). + - Fixed a Windows issue where embedding memory initializer as a string in JS code might cause corrupted output. (#3854) + - Fixed an issue when spaces are present in directory names in response files (#4062) + - Fixed a build issue when using --tracing and -s ALLOW_MEMORY_GROWTH=1 simultaneously (#4064) + - Greatly updated Emscripten support for SIMD.js intrinsics (non-SSE or NEON) + - Fixed an issue where compiler would not generate a link error when JS library function depended on a nonexisting symbol. (#4077) + - Removed UTF16 and UTF32 marshalling code from being exported by default. + - Removed the -s NO_BROWSER linker option and automated the detection of when that option is needed. + - Removed the JS implemented C++ symbol name demangler, now always depend on the libcxxabi compiled one. + - Fixed an issue where Emscripten linker would redundantly generate missing function stubs for some functions that do exist. + - Full list of changes: + - Emscripten: https://github.com/kripken/emscripten/compare/1.35.22...1.35.23 + - Emscripten-LLVM: https://github.com/kripken/emscripten-fastcomp/compare/1.35.22...1.35.23 + - Emscripten-Clang: no changes. + +v1.35.22: 1/13/2016 +------------------- + - Updated to latest upstream LLVM trunk as of January 13th. + - Bumped up the required LLVM version from LLVM 3.8 to LLVM 3.9. + - Full list of changes: + - Emscripten: https://github.com/kripken/emscripten/compare/1.35.21...1.35.22 + - Emscripten-LLVM: https://github.com/kripken/emscripten-fastcomp/compare/1.35.21...1.35.22 + - Emscripten-Clang: https://github.com/kripken/emscripten-fastcomp-clang/compare/1.35.21...1.35.22 + +v1.35.21: 1/13/2016 +------------------- + - Improved support for handling GLFW2 keycodes. + - Improved emranlib, system/bin/sdl-config and system/bin/sdl2-config to be executable in both python2 and python3. + - Fixed build flags -s AGGRESSIVE_VARIABLE_ELIMINATION=1 and -s USE_PTHREADS=2 to correctly work when run on a browser that does not support pthreads. + - Fixed a build issue that caused sequences of \r\r\n to be emitted on Windows. + - Fixed an issue that prevented building LLVM on Visual Studio 2015 (emscripten-fastcomp-clang #7) + - Full list of changes: + - Emscripten: https://github.com/kripken/emscripten/compare/1.35.20...1.35.21 + - Emscripten-LLVM: https://github.com/kripken/emscripten-fastcomp/compare/1.35.20...1.35.21 + - Emscripten-Clang: https://github.com/kripken/emscripten-fastcomp-clang/compare/1.35.20...1.35.21 + +v1.35.20: 1/10/2016 +------------------- + - Fixed -s USE_PTHREADS compilation mode to account that SharedArrayBuffer specification no longer allows futex waiting on the main thread. (#4024) + - Added new python2 vs python3 compatibility wrappers for emcmake, emconfigure, emmake and emar. + - Fixed atomicrmw i64 codegen (#4025) + - Optimized codegen to simplify "x != 0" to just "x" when output is a boolean. + - Fixed a compiler crash when generating atomics code in debug builds of LLVM. + - Fixed a compiler crash when generating SIMD.js code that utilizes non-canonical length vectors (e.g. ) + - Full list of changes: + - Emscripten: https://github.com/kripken/emscripten/compare/1.35.19...1.35.20 + - Emscripten-LLVM: https://github.com/kripken/emscripten-fastcomp/compare/1.35.19...1.35.20 + - Emscripten-Clang: no changes. + +v1.35.19: 1/7/2016 +------------------ + - Updated to latest upstream LLVM trunk as of January 7th. + - Full list of changes: + - Emscripten: no changes. + - Emscripten-LLVM: https://github.com/kripken/emscripten-fastcomp/compare/1.35.18...1.35.19 + - Emscripten-Clang: https://github.com/kripken/emscripten-fastcomp-clang/compare/1.35.18...1.35.19 + +v1.35.18: 1/7/2016 +------------------ + - Implemented getpeername() and fixed issues with handling getsockname() (#3997) + - Fixed an issue with daylight saving time in mktime() (#4001) + - Optimized pthreads code to avoid unnecessary FFI transitions (#3504) + - Fixed issues with strftime() (#3993) + - Deprecated memory growth support in asm.js. + - Implemented llvm_bitreverse_i32() (#3976) + - Fixed missing include header that affected building relooper on some compilers. + - Full list of changes: + - Emscripten: https://github.com/kripken/emscripten/compare/1.35.17...1.35.18 + - Emscripten-LLVM: https://github.com/kripken/emscripten-fastcomp/compare/1.35.17...1.35.18 + - Emscripten-Clang: no changes. + +v1.35.17: 1/4/2016 +------------------ + - Updated to latest upstream LLVM trunk as of January 4th. + - Full list of changes: + - Emscripten: no changes. + - Emscripten-LLVM: https://github.com/kripken/emscripten-fastcomp/compare/1.35.16...1.35.17 + - Emscripten-Clang: https://github.com/kripken/emscripten-fastcomp/compare/1.35.16...1.35.17 + +v1.35.16: 1/4/2016 +------------------ + - Improved support for -s USE_PTHREADS=2 build mode and added support for Atomics.exchange(). + - Full list of changes: + - Emscripten: https://github.com/kripken/emscripten/compare/1.35.15...1.35.16 + - Emscripten-LLVM: https://github.com/kripken/emscripten-fastcomp/compare/1.35.15...1.35.16 + - Emscripten-Clang: no changes. + +v1.35.15: 1/4/2016 +------------------ + - Fixed an error with glClearbufferfv not working. (#3961) + - Improved file packager code so that file:// URLs work in Chrome too (#3965) + - Fixed issues with the --memoryprofiler UI. + - Fixed a Windows issue when generating system libraries in cache (#3939) + - Fixed a regression from v1.35.13 where GLES2 compilation would not work when -s USE_PTHREADS=1 was passed. + - Added support for WebIDL arrays as input parameters to WebIDL binder. + - Updated build support when using the LLVM wasm backend. + - Added new linker option --threadprofiler which generates a threads dashboard on the generated page for threads status overview. (#3971) + - Improved backwards compatibility of building on GCC 4.3 - 4.6. + - Fixed an asm.js validation issue when building against updated SIMD.js specification. (#3986) + - Improved Rust support. + - Full list of changes: + - Emscripten: https://github.com/kripken/emscripten/compare/1.35.14...1.35.15 + - Emscripten-LLVM: https://github.com/kripken/emscripten-fastcomp/compare/1.35.14...1.35.15 + - Emscripten-Clang: no changes. + +v1.35.14: 12/15/2015 +-------------------- + - Updated to latest upstream LLVM trunk as of December 15th. + - Full list of changes: + - Emscripten: https://github.com/kripken/emscripten/compare/1.35.13...1.35.14 + - Emscripten-LLVM: https://github.com/kripken/emscripten-fastcomp/compare/1.35.13...1.35.14 + - Emscripten-Clang: https://github.com/kripken/emscripten-fastcomp-clang/compare/1.35.13...1.35.14 + +v1.35.13: 12/15/2015 +-------------------- + - Updated -s USE_PTHREADS code generation to reflect that the SharedInt*Array hierarchy no longer exists in the SharedArrayBuffer spec. + - Removed references to Atomic.fence() which no longer is part of the SharedArrayBuffer specification. + - Fixed an issue where JS code minifiers might generate bad code for cwrap (#3945) + - Updated compiler to issue a warning when --separate-asm is being used and output suffix is .js. + - Added new build option -s ONLY_MY_CODE which aims to eliminate most of the Emscripten runtime and generate a very minimal compiler output. + - Added new build option -s WASM_BACKEND=0/1 which controls whether to utilize the upstream LLVM wasm emitting codegen backend. + - Full list of changes: + - Emscripten: https://github.com/kripken/emscripten/compare/1.35.12...1.35.13 + - Emscripten-LLVM: https://github.com/kripken/emscripten-fastcomp/compare/1.35.12...1.35.13 + - Emscripten-Clang: no changes. + +v1.35.12: 11/28/2015 +-------------------- + - Update to latest upstream LLVM trunk as of November 28th. + - Fix Emscripten to handle new style format outputted by llvm-nm. + - Added new build option BINARYEN_METHOD to allow choosing which wasm generation method to use. + - Updates to Binaryen support. + - Full list of changes: + - Emscripten: https://github.com/kripken/emscripten/compare/1.35.11...1.35.12 + - Emscripten-LLVM: https://github.com/kripken/emscripten-fastcomp/compare/1.35.11...1.35.12 + - Emscripten-Clang: https://github.com/kripken/emscripten-fastcomp-clang/compare/1.35.11...1.35.12 + +v1.35.11: 11/27/2015 +-------------------- + - Updated atomics test to stress 64-bit atomics better (#3892) + - Full list of changes: + - Emscripten: https://github.com/kripken/emscripten/compare/1.35.10...1.35.11 + - Emscripten-LLVM: https://github.com/kripken/emscripten-fastcomp/compare/1.35.10...1.35.11 + - Emscripten-Clang: no changes. + +v1.35.10: 11/25/2015 +-------------------- + - Integration with Binaryen. + - Add a performance warning when multiple FS.syncfs() calls are in flight simultaneously. + - Correctly pass GLFW_REPEAT when sending key press repeats. + - Improved filesystem performance when building in multithreaded mode (#3923) + - Improve error detection when data file fails to load. + - Clarified that -s NO_DYNAMIC_EXECUTION=1 and -s RELOCATABLE=1 build modes are mutually exclusive. + - Added new build option -s NO_DYNAMIC_EXECUTION=2 which demotes eval() errors to warnings at runtime, useful for iterating fixes in a codebase for multiple eval()s (#3930) + - Added support to Module.locateFile(filename) to locate the pthread-main.js file (#3500) + - Changed -s USE_PTHREADS=2 and -s PRECISE_F32=2 to imply --separate-asm instead of requiring it, to be backwards compatible (#3829, #3933) + - Fixed bad codegen for some 64-bit atomics (#3892, #3936) + - When emitting NaN canonicalization warning, also print the location in code where it occurs. + - Full list of changes: + - Emscripten: https://github.com/kripken/emscripten/compare/1.35.9...1.35.10 + - Emscripten-LLVM: https://github.com/kripken/emscripten-fastcomp/compare/1.35.9...1.35.10 + - Emscripten-Clang: no changes. + +v1.35.9: 11/12/2015 +------------------- + - Implement glfwSetInputMode when mode is GLFW_CURSOR and value is GLFW_CURSOR_NORMAL|GLFW_CURSOR_DISABLED + - Add explicit abort() when dlopen() is called without linking support + - Make emcc explicitly reinvoke itself from python2 if called from python3. + - Optimize memory initializer to omit zero-initialized values (#3907) + - Full list of changes: + - Emscripten: https://github.com/kripken/emscripten/compare/1.35.8...1.35.9 + - Emscripten-LLVM: https://github.com/kripken/emscripten-fastcomp/compare/1.35.8...1.35.9 + - Emscripten-Clang: no changes. + +v1.35.8: 11/10/2015 +------------------- + - Removed obsoleted EXPORTED_GLOBALS build option. + - Export filesystem as global object 'FS' in Emscripten runtime. + - Fixed realpath() function on directories. + - Fixed round() and roundf() to work when building without -s PRECISE_F32=1 and optimize these to be faster (#3876) + - Full list of changes: + - Emscripten: https://github.com/kripken/emscripten/compare/1.35.7...1.35.8 + - Emscripten-LLVM: no changes. + - Emscripten-Clang: no changes. + +v1.35.7: 11/4/2015 +------------------ + - Updated to latest upstream LLVM trunk version as of November 4th. + - Full list of changes: + - Emscripten: https://github.com/kripken/emscripten/compare/1.35.6...1.35.7 + - Emscripten-LLVM: https://github.com/kripken/emscripten-fastcomp/compare/1.35.6...1.35.7 + - Emscripten-Clang: https://github.com/kripken/emscripten-fastcomp-clang/compare/1.35.6...1.35.7 + +v1.35.6: 11/4/2015 +------------------ + - This tag was created for technical purposes, and has no changes compared to v1.35.6. + +v1.35.5: 11/4/2015 +------------------ + - Removed Content-Length and Connection: close headers in POST requests. + - Migrate to using the native C++11-implemented optimizer by default. + - Fixed call to glDrawBuffers(0, *); (#3890) + - Fixed lazy file system to work with closure (#3842) + - Fixed gzip compression with lazy file system (#3837) + - Added no-op gracefully failing stubs for process spawn functions (#3819) + - Clarified error message that memory growth is not supported with shared modules (#3893) + - Initial work on wasm support in optimizer + - Full list of changes: + - Emscripten: https://github.com/kripken/emscripten/compare/1.35.4...1.35.5 + - Emscripten-LLVM: no changes. + - Emscripten-Clang: no changes. + +v1.35.4: 10/26/2015 +------------------- + - Move to legalization in the JS backend. + - Full list of changes: + - Emscripten: https://github.com/kripken/emscripten/compare/1.35.3...1.35.4 + - Emscripten-LLVM: https://github.com/kripken/emscripten-fastcomp/compare/1.35.3...1.35.4 + - Emscripten-Clang: https://github.com/kripken/emscripten-fastcomp-clang/compare/1.35.3...1.35.4 + +v1.35.3: 10/26/2015 +------------------- + - Ignore O_CLOEXEC on NODEFS (#3862) + - Improved --js-library support in CMake by treating these as libraries (#3840) + - Still support -Wno-warn-absolute-paths (#3833) + - Add support to zext <4 x i1> to <4x i32> + - Emit emscripten versions of llvm and clang in clang --version + - Full list of changes: + - Emscripten: https://github.com/kripken/emscripten/compare/1.35.2...1.35.3 + - Emscripten-LLVM: https://github.com/kripken/emscripten-fastcomp/compare/1.35.2...1.35.3 + - Emscripten-Clang: https://github.com/kripken/emscripten-fastcomp-clang/compare/1.35.2...1.35.3 + +v1.35.2: 10/20/2015 +------------------- + - Rebase against upstream LLVM "google/stable" branch, bringing us to LLVM 3.8. + - Full list of changes: + - Emscripten: https://github.com/kripken/emscripten/compare/1.35.1...1.35.2 + - Emscripten-LLVM: https://github.com/kripken/emscripten-fastcomp/compare/1.35.1...1.35.2 + - Emscripten-Clang: https://github.com/kripken/emscripten-fastcomp-clang/compare/1.35.1...1.35.2 + +v1.35.1: 10/20/2015 +------------------- + - Fixed a bug where passing -s option to LLVM would not work. + - Work around a WebAudio bug on WebKit "pauseWebAudio failed: TypeError: Not enough arguments" (#3861) + - Full list of changes: + - Emscripten: https://github.com/kripken/emscripten/compare/1.35.0...1.35.1 + - Emscripten-LLVM: no changes. + - Emscripten-Clang: no changes. + +v1.35.0: 10/19/2015 +------------------- + - Fixed out of memory abort message. + - Full list of changes: + - Emscripten: https://github.com/kripken/emscripten/compare/1.34.12...1.35.0 + - Emscripten-LLVM: no changes. + - Emscripten-Clang: no changes. v1.34.12: 10/13/2015 -------------------- diff --git a/LICENSE b/LICENSE index 447ddba1a0482..70cabe388d27d 100644 --- a/LICENSE +++ b/LICENSE @@ -92,3 +92,11 @@ in accordance with the terms of the MIT license. Node's license follows: FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ + +The musl libc project is bundled in this repo, and it has the MIT license, see +system/lib/libc/musl/COPYRIGHT + +The third_party/ subdirectory contains code with other licenses. None of it is +used by default, but certain options use it (e.g., the optional closure compiler +flag will run closure compiler from third_party/). + diff --git a/embuilder.py b/embuilder.py index 7a0ff71b90e1a..e5227d0b8d761 100755 --- a/embuilder.py +++ b/embuilder.py @@ -33,7 +33,6 @@ libcxx_noexcept libcxxabi gl - struct_info native_optimizer bullet freetype @@ -42,6 +41,7 @@ sdl2 sdl2-image sdl2-ttf + sdl2-net vorbis zlib @@ -67,20 +67,21 @@ def build(src, result_libs, args=[]): temp_js = temp_files.get('.js').name shared.Building.emcc(temp, args, output_filename=temp_js) assert os.path.exists(temp_js), 'failed to build file' - for lib in result_libs: - assert os.path.exists(shared.Cache.get_path(lib)), 'not seeing that requested library %s has been built because file %s does not exist' % (lib, shared.Cache.get_path(lib)) + if result_libs: + for lib in result_libs: + assert os.path.exists(shared.Cache.get_path(lib)), 'not seeing that requested library %s has been built because file %s does not exist' % (lib, shared.Cache.get_path(lib)) def build_port(port_name, lib_name, params): build(''' int main() {} - ''', [os.path.join('ports-builds', port_name, lib_name)], params) + ''', [os.path.join('ports-builds', port_name, lib_name)] if lib_name else None, params) operation = sys.argv[1] if operation == 'build': tasks = sys.argv[2:] if 'ALL' in tasks: - tasks = ['libc', 'libc-mt', 'dlmalloc', 'dlmalloc_threadsafe', 'pthreads', 'libcxx', 'libcxx_noexcept', 'libcxxabi', 'gl', 'struct_info', 'bullet', 'freetype', 'libpng', 'ogg', 'sdl2', 'sdl2-image', 'vorbis', 'zlib'] + tasks = ['libc', 'libc-mt', 'dlmalloc', 'dlmalloc_threadsafe', 'pthreads', 'libcxx', 'libcxx_noexcept', 'libcxxabi', 'gl', 'bullet', 'freetype', 'libpng', 'ogg', 'sdl2', 'sdl2-image', 'sdl2-ttf', 'sdl2-net', 'vorbis', 'zlib'] if os.environ.get('EMSCRIPTEN_NATIVE_OPTIMIZER'): print 'Skipping building of native-optimizer since environment variable EMSCRIPTEN_NATIVE_OPTIMIZER is present and set to point to a prebuilt native optimizer path.' elif hasattr(shared, 'EMSCRIPTEN_NATIVE_OPTIMIZER'): @@ -138,10 +139,6 @@ def build_port(port_name, lib_name, params): return int(emscripten_GetProcAddress("waka waka")); } ''', ['gl.bc']) - elif what == 'struct_info': - build(''' - int main() {} - ''', ['struct_info.compiled.json']) elif what == 'native_optimizer': build(''' int main() {} @@ -160,10 +157,14 @@ def build_port(port_name, lib_name, params): build_port('sdl2', 'libsdl2.bc', ['-s', 'USE_SDL=2']) elif what == 'sdl2-image': build_port('sdl2-image', 'libsdl2_image.bc', ['-s', 'USE_SDL=2', '-s', 'USE_SDL_IMAGE=2']) + elif what == 'sdl2-net': + build_port('sdl2-net', 'libsdl2_net.bc', ['-s', 'USE_SDL=2', '-s', 'USE_SDL_NET=2']) elif what == 'freetype': build_port('freetype', 'libfreetype.a', ['-s', 'USE_FREETYPE=1']) elif what == 'sdl2-ttf': build_port('sdl2-ttf', 'libsdl2_ttf.bc', ['-s', 'USE_SDL=2', '-s', 'USE_SDL_TTF=2', '-s', 'USE_FREETYPE=1']) + elif what == 'binaryen': + build_port('binaryen', None, ['-s', 'BINARYEN=1']) else: shared.logging.error('unfamiliar build target: ' + what) sys.exit(1) diff --git a/emcc.py b/emcc.py index d98e95ae96823..11aa60efe46ab 100755 --- a/emcc.py +++ b/emcc.py @@ -95,7 +95,7 @@ def run(): global final, target, script_src, script_inline if DEBUG: logging.warning('invocation: ' + ' '.join(sys.argv) + (' + ' + EMCC_CFLAGS if EMCC_CFLAGS else '') + ' (in ' + os.getcwd() + ')') - if EMCC_CFLAGS: sys.argv.append(EMCC_CFLAGS) + if EMCC_CFLAGS: sys.argv.extend(shlex.split(EMCC_CFLAGS)) if DEBUG and LEAVE_INPUTS_RAW: logging.warning('leaving inputs raw') @@ -128,9 +128,9 @@ def run(): break if len(sys.argv) == 1 or '--help' in sys.argv: - # Documentation for emcc and its options must be updated in: + # Documentation for emcc and its options must be updated in: # site/source/docs/tools_reference/emcc.rst - # A prebuilt local version of the documentation is available at: + # A prebuilt local version of the documentation is available at: # site/build/text/docs/tools_reference/emcc.txt # (it is read from there and printed out when --help is invoked) # You can also build docs locally as HTML or other formats in site/ @@ -241,7 +241,7 @@ def filter_emscripten_options(argv): idx += 1 if compiler == shared.EMCC: compiler = [shared.PYTHON, shared.EMCC] - else: compiler = [compiler] + else: compiler = [compiler] cmd = compiler + list(filter_emscripten_options(sys.argv[1:])) if not use_js: cmd += shared.EMSDK_OPTS + ['-D__EMSCRIPTEN__', '-DEMSCRIPTEN'] if use_js: cmd += ['-s', 'ERROR_ON_UNDEFINED_SYMBOLS=1'] # configure tests should fail when an undefined symbol exists @@ -381,10 +381,16 @@ def filename_type_ending(filename): # Log out times for emcc stages class TimeLogger: last = time.time() + + @staticmethod + def update(): + TimeLogger.last = time.time() + def log_time(name): - now = time.time() - logging.debug('emcc step "%s" took %.2f seconds', name, now - TimeLogger.last) - TimeLogger.last = now + if DEBUG: + now = time.time() + logging.debug('emcc step "%s" took %.2f seconds', name, now - TimeLogger.last) + TimeLogger.update() misc_temp_files = shared.configuration.get_temp_files() @@ -631,8 +637,10 @@ def validate_arg_level(level_string, max_level, err_msg): shared.check_sanity(force=True) # this is a good time for a sanity check should_exit = True elif newargs[i] == '--clear-ports': - logging.warning('clearing ports') + logging.warning('clearing ports and cache') system_libs.Ports.erase() + shared.Cache.erase() + shared.check_sanity(force=True) # this is a good time for a sanity check should_exit = True elif newargs[i] == '--show-ports': system_libs.show_ports() @@ -741,8 +749,10 @@ def validate_arg_level(level_string, max_level, err_msg): for i in range(len(newargs)): if newargs[i] == '-s': if is_minus_s_for_emcc(newargs, i): - settings_changes.append(newargs[i+1]) + key = newargs[i+1] + settings_changes.append(key) newargs[i] = newargs[i+1] = '' + assert key != 'WASM_BACKEND', 'do not set -s WASM_BACKEND, instead set EMCC_WASM_BACKEND=1 in the environment' newargs = [arg for arg in newargs if arg is not ''] # Find input files @@ -854,12 +864,14 @@ def validate_arg_level(level_string, max_level, err_msg): js_target = unsuffixed(target) + '.js' asm_target = js_target[:-3] + '.asm.js' # might not be used, but if it is, this is the name - wasm_target = asm_target.replace('.asm.js', '.wast') # ditto, might not be used - if final_suffix == 'html' and not separate_asm and 'PRECISE_F32=2' in settings_changes or 'USE_PTHREADS=2' in settings_changes: + wasm_text_target = asm_target.replace('.asm.js', '.wast') # ditto, might not be used + wasm_binary_target = asm_target.replace('.asm.js', '.wasm') # ditto, might not be used + + if final_suffix == 'html' and not separate_asm and ('PRECISE_F32=2' in settings_changes or 'USE_PTHREADS=2' in settings_changes): separate_asm = True logging.warning('forcing separate asm output (--separate-asm), because -s PRECISE_F32=2 or -s USE_PTHREADS=2 was passed.') if separate_asm: - shared.Settings.SEPARATE_ASM = asm_target + shared.Settings.SEPARATE_ASM = os.path.basename(asm_target) # Find library files for i, lib in libs: @@ -939,6 +951,11 @@ def check(input_file): if shared.Settings.ASSERTIONS: shared.Settings.STACK_OVERFLOW_CHECK = 2 + if shared.get_llvm_target() == shared.WASM_TARGET: + shared.Settings.WASM_BACKEND = 1 + + # Use settings + try: assert shared.Settings.ASM_JS > 0, 'ASM_JS must be enabled in fastcomp' assert shared.Settings.SAFE_HEAP in [0, 1], 'safe heap must be 0 or 1 in fastcomp' @@ -1102,10 +1119,29 @@ def check(input_file): logging.error('-s MAIN_MODULE=1 is not supported with -s USE_PTHREADS=1!') exit(1) + if shared.Settings.WASM_BACKEND: + shared.Settings.BINARYEN = 1 + # Static linking is tricky with LLVM, since e.g. memset might not be used from libc, + # but be used as an intrinsic, and codegen will generate a libc call from that intrinsic + # *after* static linking would have thought it is all in there. In asm.js this is not an + # issue as we do JS linking anyhow, and have asm.js-optimized versions of all the LLVM + # intrinsics. But for wasm, we need a better solution. For now, just pin stuff. + shared.Settings.EXPORTED_FUNCTIONS += ['_memcpy', '_memmove', '_memset'] + # to bootstrap struct_info, we need binaryen + os.environ['EMCC_WASM_BACKEND_BINARYEN'] = '1' + if shared.Settings.BINARYEN: debug_level = max(1, debug_level) # keep whitespace readable, for asm.js parser simplicity shared.Settings.GLOBAL_BASE = 1024 # leave some room for mapping global vars assert not shared.Settings.SPLIT_MEMORY, 'WebAssembly does not support split memory' + if not shared.Settings.BINARYEN_METHOD: + shared.Settings.BINARYEN_METHOD = 'native-wasm,interpret-binary' + + if shared.Settings.CYBERDWARF: + newargs.append('-g') + shared.Settings.BUNDLED_CD_DEBUG_FILE = target + ".cd" + js_libraries.append(shared.path_from_root('src', 'library_cyberdwarf.js')) + js_libraries.append(shared.path_from_root('src', 'library_debugger_toolkit.js')) if tracing: if shared.Settings.ALLOW_MEMORY_GROWTH: @@ -1289,7 +1325,7 @@ def get_bitcode_args(input_files): log_time('calculate system libraries') - # final will be an array if linking is deferred, otherwise a normal string. + # final will be an array if linking is deferred, otherwise a normal string. DEFAULT_FINAL = in_temp(target_basename + '.bc') def get_final(): global final @@ -1304,7 +1340,11 @@ def get_final(): logging.debug('linking: ' + str(linker_inputs)) # force archive contents to all be included, if just archives, or if linking shared modules force_archive_contents = len([temp for i, temp in temp_files if not temp.endswith(STATICLIB_ENDINGS)]) == 0 or not shared.Building.can_build_standalone() - just_calculate = DEBUG != '2' # if EMCC_DEBUG=2 , link now, otherwise defer to be more efficient + + # if EMCC_DEBUG=2 then we must link now, so the temp files are complete. + # if using the wasm backend, we might be using vanilla LLVM, which does not allow our fastcomp deferred linking opts. + # TODO: we could check if this is a fastcomp build, and still speed things up here + just_calculate = DEBUG != '2' and not shared.Settings.WASM_BACKEND final = shared.Building.link(linker_inputs, DEFAULT_FINAL, force_archive_contents=force_archive_contents, temp_files=misc_temp_files, just_calculate=just_calculate) else: if not LEAVE_INPUTS_RAW: @@ -1340,7 +1380,7 @@ def save_intermediate(name=None, suffix='js'): # Optimize, if asked to if not LEAVE_INPUTS_RAW: - link_opts = [] if debug_level >= 4 else ['-strip-debug'] # remove LLVM debug if we are not asked for it + link_opts = [] if debug_level >= 4 or shared.Settings.CYBERDWARF else ['-strip-debug'] # remove LLVM debug if we are not asked for it if not shared.Settings.ASSERTIONS: link_opts += ['-disable-verify'] @@ -1403,10 +1443,14 @@ def save_intermediate(name=None, suffix='js'): if shared.Settings.WASM_BACKEND: # we also received wasm at this stage wasm_temp = final[:-3] + '.wast' - shutil.move(wasm_temp, wasm_target) - open(wasm_target + '.mappedGlobals', 'w').write('{}') # no need for mapped globals for now, but perhaps some day + shutil.move(wasm_temp, wasm_text_target) + open(wasm_text_target + '.mappedGlobals', 'w').write('{}') # no need for mapped globals for now, but perhaps some day - log_time('emscript (llvm=>%s)' % ('wasm' if shared.Settings.BINARYEN else 'js')) + if shared.Settings.CYBERDWARF: + cd_target = final + '.cd' + shutil.move(cd_target, target + '.cd') + + log_time('emscript (llvm => executable code)') # Embed and preload files if shared.Settings.SPLIT_MEMORY: @@ -1437,6 +1481,19 @@ def save_intermediate(name=None, suffix='js'): file_code = execute([shared.PYTHON, shared.FILE_PACKAGER, unsuffixed(target) + '.data'] + file_args, stdout=PIPE)[0] pre_js = file_code + pre_js + if shared.Settings.BINARYEN: + # add in the glue integration code as a pre-js, so it is optimized together with everything else + wasm_js_glue = open(os.path.join(shared.Settings.BINARYEN_ROOT, 'src', 'js', 'wasm.js-post.js')).read() + wasm_js_glue = wasm_js_glue.replace('{{{ asmjsCodeFile }}}', '"' + os.path.basename(asm_target) + '"') + wasm_js_glue = wasm_js_glue.replace('{{{ wasmTextFile }}}', '"' + os.path.basename(wasm_text_target) + '"') + wasm_js_glue = wasm_js_glue.replace('{{{ wasmBinaryFile }}}', '"' + os.path.basename(wasm_binary_target) + '"') + if shared.Settings.BINARYEN_METHOD: + wasm_js_glue = wasm_js_glue.replace('{{{ wasmJSMethod }}}', '(Module[\'wasmJSMethod\'] || "' + shared.Settings.BINARYEN_METHOD + '")') + else: + wasm_js_glue = wasm_js_glue.replace('{{{ wasmJSMethod }}}', 'null') + wasm_js_glue = wasm_js_glue.replace('{{{ WASM_BACKEND }}}', str(shared.Settings.WASM_BACKEND)) # if wasm backend, wasm contains memory segments + pre_js = wasm_js_glue + '\n' + pre_js + # Apply pre and postjs files if pre_js or post_js: logging.debug('applying pre/postjses') @@ -1445,7 +1502,12 @@ def save_intermediate(name=None, suffix='js'): if WINDOWS: # Avoid duplicating \r\n to \r\r\n when writing out. if pre_js: pre_js = pre_js.replace('\r\n', '\n') if post_js: post_js = post_js.replace('\r\n', '\n') - open(final, 'w').write(pre_js + src + post_js) + outfile = open(final, 'w') + outfile.write(pre_js) + outfile.write(src) # this may be large, don't join it to others + outfile.write(post_js) + outfile.close() + pre_js = src = post_js = None if DEBUG: save_intermediate('pre-post') # Apply a source code transformation, if requested @@ -1538,7 +1600,7 @@ def run_passes(passes, title, just_split, just_concat): passes = ['asm'] + passes if shared.Settings.PRECISE_F32: passes = ['asmPreciseF32'] + passes - if emit_symbol_map and 'minifyNames' in passes: + if (emit_symbol_map or shared.Settings.CYBERDWARF) and 'minifyNames' in passes: passes += ['symbolMap='+target+'.symbols'] if profiling_funcs and 'minifyNames' in passes: passes += ['profilingFuncs'] @@ -1761,6 +1823,10 @@ def do_minify(): # minifies the code. this is also when we do certain optimizati #src = re.sub(r'\n+[ \n]*\n+', '\n', src) #open(final, 'w').write(src) + # Bundle symbol data in with the cyberdwarf file + if shared.Settings.CYBERDWARF: + execute([shared.PYTHON, shared.path_from_root('tools', 'emdebug_cd_merger.py'), target + '.cd', target+'.symbols']) + # Emit source maps, if needed if debug_level >= 4: logging.debug('generating source maps') @@ -1775,8 +1841,9 @@ def do_minify(): # minifies the code. this is also when we do certain optimizati # Separate out the asm.js code, if asked. Or, if necessary for another option if (separate_asm or shared.Settings.BINARYEN) and not shared.Settings.WASM_BACKEND: + logging.debug('separating asm') temp_target = misc_temp_files.get(suffix='.js').name - execute([shared.PYTHON, shared.path_from_root('tools', 'separate_asm.py'), js_target, asm_target, temp_target]) + subprocess.check_call([shared.PYTHON, shared.path_from_root('tools', 'separate_asm.py'), js_target, asm_target, temp_target]) shutil.move(temp_target, js_target) # extra only-my-code logic @@ -1786,29 +1853,50 @@ def do_minify(): # minifies the code. this is also when we do certain optimizati shutil.move(temp, asm_target) if shared.Settings.BINARYEN: - # Emit wasm.js at the top of the js. TODO: for html, it could be a separate script tag - binaryen_bin = os.path.join(shared.BINARYEN_ROOT, 'bin') - wasm_js = open(os.path.join(binaryen_bin, 'wasm.js')).read() - wasm_js = wasm_js.replace("Module['asmjsCodeFile']", '"' + os.path.basename(asm_target) + '"') # " or '? who knows :) - wasm_js = wasm_js.replace('Module["asmjsCodeFile"]', '"' + os.path.basename(asm_target) + '"') - wasm_js = wasm_js.replace("Module['wasmCodeFile']", '"' + os.path.basename(wasm_target) + '"') # " or '? who knows :) - wasm_js = wasm_js.replace('Module["wasmCodeFile"]', '"' + os.path.basename(wasm_target) + '"') - wasm_js = wasm_js.replace("Module['providedTotalMemory']", str(shared.Settings.TOTAL_MEMORY)) - wasm_js = wasm_js.replace('Module["providedTotalMemory"]', str(shared.Settings.TOTAL_MEMORY)) - wasm_js = wasm_js.replace('EMSCRIPTEN_', 'emscripten_') # do not confuse the markers - if shared.Settings.BINARYEN_METHOD: - method = '(Module[\'wasmJSMethod\'] || "' + shared.Settings.BINARYEN_METHOD + '")' - wasm_js = wasm_js.replace("Module['wasmJSMethod']", method) # " or '? who knows :) - wasm_js = wasm_js.replace('Module["wasmJSMethod"]', method) - js = open(js_target).read() - combined = open(js_target, 'w') - combined.write(wasm_js) - combined.write('\n//^wasm.js\n') - combined.write(js) - combined.close() + logging.debug('using binaryen, with method: ' + shared.Settings.BINARYEN_METHOD) + binaryen_bin = os.path.join(shared.Settings.BINARYEN_ROOT, 'bin') + # Emit wasm.js at the top of the js. This is *not* optimized with the rest of the code, since + # (1) it contains asm.js, whose validation would be broken, and (2) it's very large so it would + # be slow in cleanup/JSDCE etc. + # TODO: for html, it could be a separate script tag + # We need wasm.js if there is a chance the polyfill will be used. If the user sets + # BINARYEN_METHOD with something that doesn't use the polyfill, then we don't need it. + if not shared.Settings.BINARYEN_METHOD or 'interpret' in shared.Settings.BINARYEN_METHOD: + logging.debug('integrating wasm.js polyfill interpreter') + wasm_js = open(os.path.join(binaryen_bin, 'wasm.js')).read() + wasm_js = wasm_js.replace('EMSCRIPTEN_', 'emscripten_') # do not confuse the markers + js = open(js_target).read() + combined = open(js_target, 'w') + combined.write(wasm_js) + combined.write('\n//^wasm.js\n') + combined.write(js) + combined.close() + # finish compiling to WebAssembly, using asm2wasm, if we didn't already emit WebAssembly directly using the wasm backend. if not shared.Settings.WASM_BACKEND: - # generate .wast file - subprocess.check_call([os.path.join(binaryen_bin, 'asm2wasm'), asm_target, '--mapped-globals=' + wasm_target + '.mappedGlobals'], stdout=open(wasm_target, 'w')) + cmd = [os.path.join(binaryen_bin, 'asm2wasm'), asm_target, '--mapped-globals=' + wasm_text_target + '.mappedGlobals', '--total-memory=' + str(shared.Settings.TOTAL_MEMORY)] + if shared.Settings.BINARYEN_IMPRECISE: + cmd += ['--imprecise'] + if opt_level == 0: + cmd += ['--no-opts'] + logging.debug('asm2wasm (asm.js => WebAssembly): ' + ' '.join(cmd)) + TimeLogger.update() + subprocess.check_call(cmd, stdout=open(wasm_text_target, 'w')) + log_time('asm2wasm') + if shared.Settings.BINARYEN_SCRIPTS: + binaryen_scripts = os.path.join(shared.Settings.BINARYEN_ROOT, 'scripts') + script_env = os.environ.copy() + root_dir = os.path.abspath(os.path.dirname(__file__)) + if script_env.get('PYTHONPATH'): + script_env['PYTHONPATH'] += ':' + root_dir + else: + script_env['PYTHONPATH'] = root_dir + for script in shared.Settings.BINARYEN_SCRIPTS.split(','): + logging.debug('running binaryen script: ' + script) + subprocess.check_call([shared.PYTHON, os.path.join(binaryen_scripts, script), js_target, wasm_text_target], env=script_env) + if 'native-wasm' in shared.Settings.BINARYEN_METHOD or 'interpret-binary' in shared.Settings.BINARYEN_METHOD: + logging.debug('wasm-as (wasm => binary)') + subprocess.check_call([os.path.join(binaryen_bin, 'wasm-as'), wasm_text_target, '-o', wasm_binary_target]) + shutil.copyfile(wasm_text_target + '.mappedGlobals', wasm_binary_target + '.mappedGlobals') # If we were asked to also generate HTML, do that if final_suffix == 'html': @@ -1908,7 +1996,7 @@ def un_src(): # use this if you want to modify the script and need it to be inli codeXHR = null; var src = URL.createObjectURL(blob); var script = document.createElement('script'); - script.src = URL.createObjectURL(blob); + script.src = src; script.onload = function() { setTimeout(function() { %s @@ -1922,6 +2010,20 @@ def un_src(): # use this if you want to modify the script and need it to be inli else: assert len(asm_mods) == 0, 'no --separate-asm means no client code mods are possible' + if shared.Settings.BINARYEN: + # We need to load the wasm file before anything else, it has to be synchronously ready TODO: optimize + un_src() + script_inline = ''' + var xhr = new XMLHttpRequest(); + xhr.open('GET', '%s', true); + xhr.responseType = 'arraybuffer'; + xhr.onload = function() { + Module.wasmBinary = xhr.response; +%s + }; + xhr.send(null); +''' % (os.path.basename(wasm_binary_target), script_inline) + html = open(target, 'w') assert (script_src or script_inline) and not (script_src and script_inline) if script_src: diff --git a/emscripten-version.txt b/emscripten-version.txt index 0a99febee11ed..66566adfeb390 100644 --- a/emscripten-version.txt +++ b/emscripten-version.txt @@ -1,2 +1,2 @@ -"1.36.1" +"1.36.2" diff --git a/emscripten.py b/emscripten.py index 3190f82c890e0..0b927e02cc569 100755 --- a/emscripten.py +++ b/emscripten.py @@ -9,7 +9,7 @@ headers, for the libc implementation in JS). ''' -import os, sys, json, optparse, subprocess, re, time, string, logging +import os, sys, json, optparse, subprocess, re, time, logging from tools import shared from tools import jsrun, cache as cache_module, tempfiles @@ -37,8 +37,25 @@ def get_configuration(): logging.info('logging stderr in js compiler phase into %s' % STDERR_FILE) STDERR_FILE = open(STDERR_FILE, 'w') -def emscript(infile, settings, outfile, outfile_name, libraries=[], compiler_engine=None, - temp_files=None, DEBUG=None, DEBUG_CACHE=None): +def quoter(settings): + def quote(prop): + if settings['USE_CLOSURE_COMPILER'] == 2: + return ''.join(map(lambda p: "'" + p + "'", prop.split('.'))) + else: + return prop + return quote + +def access_quoter(settings): + def access_quote(prop): + if settings['USE_CLOSURE_COMPILER'] == 2: + return ''.join(map(lambda p: "['" + p + "']", prop.split('.'))) + else: + return '.' + prop + return access_quote + + +def emscript(infile, settings, outfile, libraries=None, compiler_engine=None, + temp_files=None, DEBUG=None): """Runs the emscripten LLVM-to-JS compiler. Args: @@ -48,6 +65,8 @@ def emscript(infile, settings, outfile, outfile_name, libraries=[], compiler_eng outfile: The file where the output is written. """ + if libraries is None: libraries = [] + assert settings['ASM_JS'], 'fastcomp is asm.js-only (mode 1 or 2)' success = False @@ -60,6 +79,24 @@ def emscript(infile, settings, outfile, outfile_name, libraries=[], compiler_eng # * Run compiler.js on the metadata to emit the shell js code, pre/post-ambles, # JS library dependencies, etc. + # metadata and settings are modified by reference in some of the below + # these functions are split up to force variables to go out of scope and allow + # memory to be reclaimed + + funcs, metadata, mem_init = get_and_parse_backend(infile, settings, temp_files, DEBUG) + glue, forwarded_data = compiler_glue(metadata, settings, libraries, compiler_engine, temp_files, DEBUG) + + post, funcs_js, need_asyncify, provide_fround, asm_safe_heap, sending, receiving, asm_setup, the_global, asm_global_vars, asm_global_funcs, pre_tables, final_function_tables, exports, last_forwarded_json = function_tables_and_exports(funcs, metadata, mem_init, glue, forwarded_data, settings, outfile, DEBUG) + finalize_output(metadata, post, funcs_js, need_asyncify, provide_fround, asm_safe_heap, sending, receiving, asm_setup, the_global, asm_global_vars, asm_global_funcs, pre_tables, final_function_tables, exports, last_forwarded_json, settings, outfile, DEBUG) + + success = True + + finally: + outfile.close() + if not success: + shared.try_delete(outfile.name) # remove partial output + +def get_and_parse_backend(infile, settings, temp_files, DEBUG): temp_js = temp_files.get('.4.js').name backend_compiler = os.path.join(shared.LLVM_ROOT, 'llc') backend_args = [backend_compiler, infile, '-march=js', '-filetype=asm', '-o', temp_js] @@ -93,6 +130,10 @@ def emscript(infile, settings, outfile, outfile_name, libraries=[], compiler_eng backend_args += ['-emscripten-asyncify-whitelist=' + ','.join(settings['ASYNCIFY_WHITELIST'])] if settings['NO_EXIT_RUNTIME']: backend_args += ['-emscripten-no-exit-runtime'] + if settings['BINARYEN']: + backend_args += ['-emscripten-wasm'] + if settings['CYBERDWARF']: + backend_args += ['-enable-cyberdwarf'] if DEBUG: logging.debug('emscript: llvm backend: ' + ' '.join(backend_args)) @@ -100,7 +141,6 @@ def emscript(infile, settings, outfile, outfile_name, libraries=[], compiler_eng shared.jsrun.timeout_run(subprocess.Popen(backend_args, stdout=subprocess.PIPE)) if DEBUG: logging.debug(' emscript: llvm backend took %s seconds' % (time.time() - t)) - t = time.time() # Split up output backend_output = open(temp_js).read() @@ -156,17 +196,22 @@ def fix_dot_zero(m): num = m.group(3) # TODO: handle 0x floats? if num.find('.') < 0: - e = num.find('e'); + e = num.find('e') if e < 0: num += '.0' else: num = num[:e] + '.0' + num[e:] return m.group(1) + m.group(2) + num - funcs = re.sub(r'([(=,+\-*/%<>:?] *)\+(-?)((0x)?[0-9a-f]*\.?[0-9]+([eE][-+]?[0-9]+)?)', lambda m: fix_dot_zero(m), funcs) + funcs = re.sub(r'([(=,+\-*/%<>:?] *)\+(-?)((0x)?[0-9a-f]*\.?[0-9]+([eE][-+]?[0-9]+)?)', fix_dot_zero, funcs) + + return funcs, metadata, mem_init +def compiler_glue(metadata, settings, libraries, compiler_engine, temp_files, DEBUG): # js compiler - if DEBUG: logging.debug('emscript: js compiler glue') + if DEBUG: + logging.debug('emscript: js compiler glue') + t = time.time() # Settings changes i64_funcs = ['i64Add', 'i64Subtract', '__muldi3', '__divdi3', '__udivdi3', '__remdi3', '__uremdi3'] @@ -175,7 +220,7 @@ def fix_dot_zero(m): settings['PRECISE_I64_MATH'] = 2 break - metadata['declares'] = filter(lambda i64_func: i64_func not in ['getHigh32', 'setHigh32', '__muldi3', '__divdi3', '__remdi3', '__udivdi3', '__uremdi3'], metadata['declares']) # FIXME: do these one by one as normal js lib funcs + metadata['declares'] = filter(lambda i64_func: i64_func not in ['getHigh32', 'setHigh32'], metadata['declares']) # FIXME: do these one by one as normal js lib funcs # Syscalls optimization. Our syscalls are static, and so if we see a very limited set of them - in particular, # no open() syscall and just simple writing - then we don't need full filesystem support. @@ -197,6 +242,10 @@ def is_int(x): if DEBUG: logging.debug('very limited syscalls (%s) so disabling full filesystem support' % ', '.join(map(str, syscalls))) settings['NO_FILESYSTEM'] = 1 + if settings['CYBERDWARF']: + settings['DEFAULT_LIBRARY_FUNCS_TO_INCLUDE'].append("cyberdwarf_Debugger") + settings['EXPORTED_FUNCTIONS'].append("cyberdwarf_Debugger") + # Integrate info from backend if settings['SIDE_MODULE']: settings['DEFAULT_LIBRARY_FUNCS_TO_INCLUDE'] = [] # we don't need any JS library contents in side modules @@ -228,7 +277,6 @@ def save_settings(): save_settings() # Call js compiler - if DEBUG: t = time.time() out = jsrun.run_js(path_from_root('src', 'compiler.js'), compiler_engine, [settings_file] + libraries, stdout=subprocess.PIPE, stderr=STDERR_FILE, cwd=path_from_root('src'), error_limit=300) @@ -237,8 +285,18 @@ def save_settings(): if DEBUG: logging.debug(' emscript: glue took %s seconds' % (time.time() - t)) + + return glue, forwarded_data + +def function_tables_and_exports(funcs, metadata, mem_init, glue, forwarded_data, settings, outfile, DEBUG): + + if DEBUG: + logging.debug('emscript: python processing: function tables and exports') t = time.time() + access_quote = access_quoter(settings) + quote = quoter(settings) + last_forwarded_json = forwarded_json = json.loads(forwarded_data) # merge in information from llvm backend @@ -360,7 +418,7 @@ def move_preasm(m): outfile.write(contents + '\n') return '' if not settings['BOOTSTRAPPING_STRUCT_INFO'] and len(funcs_js) > 1: - funcs_js[1] = re.sub(r'/\* PRE_ASM \*/(.*)\n', lambda m: move_preasm(m), funcs_js[1]) + funcs_js[1] = re.sub(r'/\* PRE_ASM \*/(.*)\n', move_preasm, funcs_js[1]) class Counter: i = 0 @@ -461,8 +519,10 @@ def make_emulated_param(i): Counter.pre.append(specific_bad_func) return specific_bad if not newline else (specific_bad + '\n') clean_item = item.replace("asm['", '').replace("']", '') - if clean_item not in implemented_functions and not (settings['EMULATED_FUNCTION_POINTERS'] and not settings['RELOCATABLE']): # when emulating function pointers, we don't need wrappers - # but if relocating, then we also have the copies in-module, and do + # when emulating function pointers, we don't need wrappers + # but if relocating, then we also have the copies in-module, and do + # also if binaryen, as wasm requires wrappers for the function table + if clean_item not in implemented_functions and not (settings['EMULATED_FUNCTION_POINTERS'] and not settings['RELOCATABLE'] and not settings['BINARYEN']): # this is imported into asm, we must wrap it call_ident = clean_item if call_ident in metadata['redirects']: call_ident = metadata['redirects'][call_ident] @@ -713,18 +773,6 @@ def keyfunc(other): body = 'if (((ptr|0) >= (fb|0)) & ((ptr|0) < (fb + {{{ FTM_' + sig + ' }}} | 0))) { ' + maybe_return + ' ' + shared.JS.make_coercion('FUNCTION_TABLE_' + sig + '[(ptr-fb)&{{{ FTM_' + sig + ' }}}](' + mini_coerced_params + ')', sig[0], settings, ffi_arg=True) + '; ' + ('return;' if sig[0] == 'v' else '') + ' }' + final_return funcs_js.append(make_func('mftCall_' + sig, body, params, coercions) + '\n') - def quote(prop): - if settings['USE_CLOSURE_COMPILER'] == 2: - return ''.join(map(lambda p: "'" + p + "'", prop.split('.'))) - else: - return prop - - def access_quote(prop): - if settings['USE_CLOSURE_COMPILER'] == 2: - return ''.join(map(lambda p: "['" + p + "']", prop.split('.'))) - else: - return '.' + prop - # calculate exports exported_implemented_functions = list(exported_implemented_functions) + metadata['initializers'] if not settings['ONLY_MY_CODE']: @@ -759,7 +807,7 @@ def check(extern): asm_setup += 'var g$' + extern + ' = function() { ' + check(extern) + ' return ' + side + 'Module["' + extern + '"] };\n' def math_fix(g): return g if not g.startswith('Math_') else g.split('_')[1] - asm_global_funcs = ''.join([' var ' + g.replace('.', '_') + '=global' + access_quote(g) + ';\n' for g in maths]); + asm_global_funcs = ''.join([' var ' + g.replace('.', '_') + '=global' + access_quote(g) + ';\n' for g in maths]) asm_global_funcs += ''.join([' var ' + g + '=env' + access_quote(math_fix(g)) + ';\n' for g in basic_funcs + global_funcs]) if metadata['simd']: def string_contains_any(s, str_list): @@ -788,7 +836,7 @@ def string_contains_any(s, str_list): # Unofficial, Bool64x2 does not yet exist, but needed for Float64x2 comparisons. if metadata['simdFloat64x2']: - asm_global_funcs += ' var SIMD_Int32x4_fromBool64x2Bits = global.SIMD.Int32x4.fromBool64x2Bits;\n'; + asm_global_funcs += ' var SIMD_Int32x4_fromBool64x2Bits = global.SIMD.Int32x4.fromBool64x2Bits;\n' if settings['USE_PTHREADS']: asm_global_funcs += ''.join([' var Atomics_' + ty + '=global' + access_quote('Atomics') + access_quote(ty) + ';\n' for ty in ['load', 'store', 'exchange', 'compareExchange', 'add', 'sub', 'and', 'or', 'xor']]) asm_global_vars = ''.join([' var ' + g + '=env' + access_quote(g) + '|0;\n' for g in basic_vars + global_vars]) @@ -820,9 +868,7 @@ def string_contains_any(s, str_list): table = table.replace('var ' + tableName, 'var ' + tableName + ' = Module["' + tableName + '"]') receiving += table + '\n' - # finalize - - if DEBUG: logging.debug('asm text sizes' + str([map(len, funcs_js), len(asm_setup), len(asm_global_vars), len(asm_global_funcs), len(pre_tables), len('\n'.join(function_tables_impls)), len(function_tables_defs.replace('\n', '\n ')), len(exports), len(the_global), len(sending), len(receiving)])) + if DEBUG: logging.debug('asm text sizes' + str([map(len, funcs_js), len(asm_setup), len(asm_global_vars), len(asm_global_funcs), len(pre_tables), len('\n'.join(function_tables_impls)), len(function_tables_defs) + (function_tables_defs.count('\n') * len(' ')), len(exports), len(the_global), len(sending), len(receiving)])) final_function_tables = '\n'.join(function_tables_impls) + '\n' + function_tables_defs if settings.get('EMULATED_FUNCTION_POINTERS'): @@ -835,6 +881,19 @@ def string_contains_any(s, str_list): final_function_tables = final_function_tables.replace("asm['", '').replace("']", '').replace('var SIDE_FUNCTION_TABLE_', 'var FUNCTION_TABLE_').replace('var dynCall_', '//') + if DEBUG: + logging.debug(' emscript: python processing: function tables and exports took %s seconds' % (time.time() - t)) + + return post, funcs_js, need_asyncify, provide_fround, asm_safe_heap, sending, receiving, asm_setup, the_global, asm_global_vars, asm_global_funcs, pre_tables, final_function_tables, exports, last_forwarded_json + +def finalize_output(metadata, post, funcs_js, need_asyncify, provide_fround, asm_safe_heap, sending, receiving, asm_setup, the_global, asm_global_vars, asm_global_funcs, pre_tables, final_function_tables, exports, last_forwarded_json, settings, outfile, DEBUG): + + if DEBUG: + logging.debug('emscript: python processing: finalize') + t = time.time() + + access_quote = access_quoter(settings) + if settings['RELOCATABLE']: receiving += ''' var NAMED_GLOBALS = { %s }; @@ -984,24 +1043,6 @@ def string_contains_any(s, str_list): threwValue = value; } } -function copyTempFloat(ptr) { - ptr = ptr|0; - HEAP8[tempDoublePtr>>0] = HEAP8[ptr>>0]; - HEAP8[tempDoublePtr+1>>0] = HEAP8[ptr+1>>0]; - HEAP8[tempDoublePtr+2>>0] = HEAP8[ptr+2>>0]; - HEAP8[tempDoublePtr+3>>0] = HEAP8[ptr+3>>0]; -} -function copyTempDouble(ptr) { - ptr = ptr|0; - HEAP8[tempDoublePtr>>0] = HEAP8[ptr>>0]; - HEAP8[tempDoublePtr+1>>0] = HEAP8[ptr+1>>0]; - HEAP8[tempDoublePtr+2>>0] = HEAP8[ptr+2>>0]; - HEAP8[tempDoublePtr+3>>0] = HEAP8[ptr+3>>0]; - HEAP8[tempDoublePtr+4>>0] = HEAP8[ptr+4>>0]; - HEAP8[tempDoublePtr+5>>0] = HEAP8[ptr+5>>0]; - HEAP8[tempDoublePtr+6>>0] = HEAP8[ptr+6>>0]; - HEAP8[tempDoublePtr+7>>0] = HEAP8[ptr+7>>0]; -} '''] + ['' if not settings['SAFE_HEAP'] else ''' function setDynamicTop(value) { value = value | 0; @@ -1102,7 +1143,7 @@ def string_contains_any(s, str_list): access_quote('asmGlobalArg'), the_global, shared_array_buffer, access_quote('asmLibraryArg'), sending, - "'use asm';" if not metadata.get('hasInlineJS') and settings['ASM_JS'] == 1 else "'almost asm';", + "'use asm';" if not metadata.get('hasInlineJS') and settings['ASM_JS'] == 1 else "'almost asm';", first_in_asm, ''' var HEAP8 = new global%s(buffer); @@ -1153,8 +1194,8 @@ def string_contains_any(s, str_list): var undef = 0; var nan = global%s, inf = global%s; var tempInt = 0, tempBigInt = 0, tempBigIntP = 0, tempBigIntS = 0, tempBigIntR = 0.0, tempBigIntI = 0, tempBigIntD = 0, tempValue = 0, tempDouble = 0.0; -''' % (access_quote('NaN'), access_quote('Infinity'))) + ''.join([''' - var tempRet%d = 0;''' % i for i in range(10)]) + '\n' + asm_global_funcs] + \ + var tempRet0 = 0; +''' % (access_quote('NaN'), access_quote('Infinity'))) + '\n' + asm_global_funcs] + \ [' var tempFloat = %s;\n' % ('Math_fround(0)' if provide_fround else '0.0')] + \ [' var asyncState = 0;\n' if settings.get('EMTERPRETIFY_ASYNC') else ''] + \ ([' const f0 = Math_fround(0);\n'] if provide_fround else []) + \ @@ -1175,17 +1216,17 @@ def string_contains_any(s, str_list): '''] + [''' // EMSCRIPTEN_START_FUNCS '''] + runtime_funcs + funcs_js + [''' - %s + ''', pre_tables, final_function_tables, ''' return %s; }) // EMSCRIPTEN_END_ASM (%s, %s, buffer); -%s; -''' % (pre_tables + final_function_tables, exports, +''' % (exports, 'Module' + access_quote('asmGlobalArg'), - 'Module' + access_quote('asmLibraryArg'), - receiving)] + 'Module' + access_quote('asmLibraryArg')), ''' +''', receiving, '''; +'''] if not settings.get('SIDE_MODULE'): funcs_js.append(''' @@ -1216,13 +1257,14 @@ def function_table_maskize(js, masks): def fix(m): sig = m.groups(0)[0] return masks[sig] - return re.sub(r'{{{ FTM_([\w\d_$]+) }}}', lambda m: fix(m), js) # masks[m.groups(0)[0]] - funcs_js = map(lambda js: function_table_maskize(js, masks), funcs_js) + return re.sub(r'{{{ FTM_([\w\d_$]+) }}}', fix, js) # masks[m.groups(0)[0]] + for i in range(len(funcs_js)): # in-place as this can be large + funcs_js[i] = function_table_maskize(funcs_js[i], masks) if settings['SIDE_MODULE']: funcs_js.append(''' Runtime.registerFunctions(%(sigs)s, Module); -''' % { 'sigs': str(map(str, last_forwarded_json['Functions']['tables'].keys())) }) +''' % { 'sigs': str(map(str, last_forwarded_json['Functions']['tables'].keys())) }) for i in range(len(funcs_js)): # do this loop carefully to save memory if WINDOWS: funcs_js[i] = funcs_js[i].replace('\r\n', '\n') # Normalize to UNIX line endings, otherwise writing to text file will duplicate \r\n to \r\r\n! @@ -1232,27 +1274,28 @@ def fix(m): if WINDOWS: post = post.replace('\r\n', '\n') # Normalize to UNIX line endings, otherwise writing to text file will duplicate \r\n to \r\r\n! outfile.write(post) - outfile.close() - - if DEBUG: logging.debug(' emscript: final python processing took %s seconds' % (time.time() - t)) - - success = True + if DEBUG: + logging.debug(' emscript: python processing: finalize took %s seconds' % (time.time() - t)) - finally: - if not success: - outfile.close() - shared.try_delete(outfile.name) # remove partial output + if settings['CYBERDWARF']: + assert('cyberdwarf_data' in metadata) + cd_file_name = outfile.name + ".cd" + with open(cd_file_name, "w") as cd_file: + json.dump({ 'cyberdwarf': metadata['cyberdwarf_data'] }, cd_file) -def emscript_wasm_backend(infile, settings, outfile, outfile_name, libraries=[], compiler_engine=None, - temp_files=None, DEBUG=None, DEBUG_CACHE=None): +def emscript_wasm_backend(infile, settings, outfile, libraries=None, compiler_engine=None, + temp_files=None, DEBUG=None): # Overview: # * Run LLVM backend to emit .s # * Run Binaryen's s2wasm to generate WebAssembly. # * We may also run some Binaryen passes here. + if libraries is None: libraries = [] + temp_s = temp_files.get('.wb.s').name backend_compiler = os.path.join(shared.LLVM_ROOT, 'llc') backend_args = [backend_compiler, infile, '-march=wasm32', '-filetype=asm', '-o', temp_s] + backend_args += ['-thread-model=single'] # no threads support in backend, tell llc to not emit atomics if DEBUG: logging.debug('emscript: llvm wasm backend: ' + ' '.join(backend_args)) t = time.time() @@ -1263,14 +1306,16 @@ def emscript_wasm_backend(infile, settings, outfile, outfile_name, libraries=[], import shutil shutil.copyfile(temp_s, os.path.join(shared.CANONICAL_TEMP_DIR, 'emcc-llvm-backend-output.s')) - assert shared.BINARYEN_ROOT, 'need BINARYEN_ROOT config set so we can use Binaryen s2wasm on the backend output' - wasm = outfile_name[:-3] + '.wast' - s2wasm_args = [os.path.join(shared.BINARYEN_ROOT, 's2wasm'), temp_s] + assert shared.Settings.BINARYEN_ROOT, 'need BINARYEN_ROOT config set so we can use Binaryen s2wasm on the backend output' + wasm = outfile.name[:-3] + '.wast' + s2wasm_args = [os.path.join(shared.Settings.BINARYEN_ROOT, 'bin', 's2wasm'), temp_s] + s2wasm_args += ['--emscripten-glue'] s2wasm_args += ['--global-base=%d' % shared.Settings.GLOBAL_BASE] + s2wasm_args += ['--initial-memory=%d' % shared.Settings.TOTAL_MEMORY] if DEBUG: logging.debug('emscript: binaryen s2wasm: ' + ' '.join(s2wasm_args)) t = time.time() - s2wasm_args += ['--debug'] + #s2wasm_args += ['--debug'] shared.check_call(s2wasm_args, stdout=open(wasm, 'w')) if DEBUG: logging.debug(' emscript: binaryen s2wasm took %s seconds' % (time.time() - t)) @@ -1310,6 +1355,15 @@ def emscript_wasm_backend(infile, settings, outfile, outfile_name, libraries=[], for k, v in metadata_json.iteritems(): metadata[k] = v + def asmjs_mangle(name): + # Mangle a name the way asm.js/JSBackend globals are mangled (i.e. prepend + # '_' and replace non-alphanumerics with '_') + if name.startswith('dynCall_'): return name + return '_' + ''.join(['_' if not c.isalnum() else c for c in name]) + + # Initializers call the global var version of the export, so they get the mangled name. + metadata['initializers'] = [asmjs_mangle(i) for i in metadata['initializers']] + # TODO: emit it from s2wasm; for now, we parse it right here for line in open(wasm).readlines(): if line.startswith(' (import '): @@ -1321,7 +1375,11 @@ def emscript_wasm_backend(infile, settings, outfile, outfile_name, libraries=[], metadata['implementedFunctions'].append(func) elif line.startswith(' (export '): parts = line.split(' ') - metadata['exports'].append('_' + parts[3][1:-1]) + exportname = parts[3][1:-1] + assert asmjs_mangle(exportname) not in metadata['exports'] + metadata['exports'].append(exportname) + + metadata['declares'] = filter(lambda x: not x.startswith('emscripten_asm_const'), metadata['declares']) # we emit those ourselves if DEBUG: logging.debug(repr(metadata)) @@ -1335,6 +1393,8 @@ def emscript_wasm_backend(infile, settings, outfile, outfile_name, libraries=[], settings['MAX_GLOBAL_ALIGN'] = metadata['maxGlobalAlign'] + settings['IMPLEMENTED_FUNCTIONS'] = metadata['implementedFunctions'] + # Save settings to a file to work around v8 issue 1579 settings_file = temp_files.get('.txt').name def save_settings(): @@ -1357,7 +1417,7 @@ def save_settings(): logging.debug(' emscript: glue took %s seconds' % (time.time() - t)) t = time.time() - last_forwarded_json = forwarded_json = json.loads(forwarded_data) + forwarded_json = json.loads(forwarded_data) pre, post = glue.split('// EMSCRIPTEN_END_FUNCS') @@ -1434,23 +1494,7 @@ def save_settings(): basic_funcs = ['abort', 'assert'] - asm_runtime_funcs = ['stackAlloc', 'stackSave', 'stackRestore', 'establishStackSpace', 'setThrew'] - asm_runtime_funcs += ['setTempRet0', 'getTempRet0'] - - if settings['ONLY_MY_CODE']: - asm_runtime_funcs = [] - - def quote(prop): - if settings['USE_CLOSURE_COMPILER'] == 2: - return "'" + prop + "'" - else: - return prop - - def access_quote(prop): - if settings['USE_CLOSURE_COMPILER'] == 2: - return "['" + prop + "']" - else: - return '.' + prop + access_quote = access_quoter(settings) # calculate globals try: @@ -1476,17 +1520,17 @@ def math_fix(g): if settings['ASSERTIONS']: # assert on the runtime being in a valid state when calling into compiled code. The only exceptions are # some support code - receiving = '\n'.join(['var real_' + s + ' = asm["' + s + '"]; asm["' + s + '''"] = function() { + receiving = '\n'.join(['var real_' + asmjs_mangle(s) + ' = asm["' + s + '"]; asm["' + s + '''"] = function() { assert(runtimeInitialized, 'you need to wait for the runtime to be ready (e.g. wait for main() to be called)'); assert(!runtimeExited, 'the runtime was exited (use NO_EXIT_RUNTIME to keep it alive after main() exits)'); -return real_''' + s + '''.apply(null, arguments); +return real_''' + asmjs_mangle(s) + '''.apply(null, arguments); }; ''' for s in exported_implemented_functions if s not in ['_memcpy', '_memset', 'runPostSets', '_emscripten_replace_memory']]) if not settings['SWAPPABLE_ASM_MODULE']: - receiving += ';\n'.join(['var ' + s + ' = Module["' + s + '"] = asm["' + s[1:] + '"]' for s in exported_implemented_functions]) + receiving += ';\n'.join(['var ' + asmjs_mangle(s) + ' = Module["' + asmjs_mangle(s) + '"] = asm["' + s + '"]' for s in exported_implemented_functions]) else: - receiving += 'Module["asm"] = asm;\n' + ';\n'.join(['var ' + s + ' = Module["' + s + '"] = function() { return Module["asm"]["' + s[1:] + '"].apply(null, arguments) }' for s in exported_implemented_functions]) + receiving += 'Module["asm"] = asm;\n' + ';\n'.join(['var ' + asmjs_mangle(s) + ' = Module["' + asmjs_mangle(s) + '"] = function() { return Module["asm"]["' + s + '"].apply(null, arguments) }' for s in exported_implemented_functions]) receiving += ';\n' # finalize @@ -1496,8 +1540,6 @@ def math_fix(g): else: shared_array_buffer = '' - runtime_funcs = [] - funcs_js = [''' Module%s = %s; %s @@ -1511,12 +1553,16 @@ def math_fix(g): 'Module' + access_quote('asmLibraryArg'), receiving)] + # wasm backend stack goes down, and is stored in the first global var location funcs_js.append(''' +STACKTOP = STACK_BASE + TOTAL_STACK; +STACK_MAX = STACK_BASE; +HEAP32[%d >> 2] = STACKTOP; Runtime.stackAlloc = asm['stackAlloc']; Runtime.stackSave = asm['stackSave']; Runtime.stackRestore = asm['stackRestore']; Runtime.establishStackSpace = asm['establishStackSpace']; -''') +''' % shared.Settings.GLOBAL_BASE) funcs_js.append(''' Runtime.setTempRet0 = asm['setTempRet0']; @@ -1537,7 +1583,7 @@ def math_fix(g): logging.critical('Non-fastcomp compiler is no longer available, please use fastcomp or an older version of emscripten') sys.exit(1) -def main(args, compiler_engine, cache, temp_files, DEBUG, DEBUG_CACHE): +def main(args, compiler_engine, cache, temp_files, DEBUG): # Prepare settings for serialization to JSON. settings = {} for setting in args.settings: @@ -1547,18 +1593,18 @@ def main(args, compiler_engine, cache, temp_files, DEBUG, DEBUG_CACHE): # libraries libraries = args.libraries[0].split(',') if len(args.libraries) > 0 else [] - settings.setdefault('STRUCT_INFO', cache.get_path('struct_info.compiled.json')) + settings.setdefault('STRUCT_INFO', shared.path_from_root('src', 'struct_info.compiled.json')) struct_info = settings.get('STRUCT_INFO') - if not os.path.exists(struct_info) and not settings.get('BOOTSTRAPPING_STRUCT_INFO'): + if not os.path.exists(struct_info) and not settings.get('BOOTSTRAPPING_STRUCT_INFO') and not settings.get('ONLY_MY_CODE'): if DEBUG: logging.debug(' emscript: bootstrapping struct info...') shared.Building.ensure_struct_info(struct_info) if DEBUG: logging.debug(' emscript: bootstrapping struct info complete') emscripter = emscript_wasm_backend if settings['WASM_BACKEND'] else emscript - emscripter(args.infile, settings, args.outfile, args.outfile_name, libraries, compiler_engine=compiler_engine, - temp_files=temp_files, DEBUG=DEBUG, DEBUG_CACHE=DEBUG_CACHE) + emscripter(args.infile, settings, args.outfile, libraries, compiler_engine=compiler_engine, + temp_files=temp_files, DEBUG=DEBUG) def _main(args=None): if args is None: @@ -1631,7 +1677,6 @@ def _main(args=None): if len(positional) != 1: raise RuntimeError('Must provide exactly one positional argument. Got ' + str(len(positional)) + ': "' + '", "'.join(positional) + '"') keywords.infile = os.path.abspath(positional[0]) - keywords.outfile_name = keywords.outfile if isinstance(keywords.outfile, basestring): keywords.outfile = open(keywords.outfile, 'w') @@ -1649,10 +1694,8 @@ def _main(args=None): if keywords.verbose is None: DEBUG = get_configuration().DEBUG - DEBUG_CACHE = get_configuration().DEBUG_CACHE else: DEBUG = keywords.verbose - DEBUG_CACHE = keywords.verbose cache = cache_module.Cache() temp_files.run_and_clean(lambda: main( @@ -1661,7 +1704,6 @@ def _main(args=None): cache=cache, temp_files=temp_files, DEBUG=DEBUG, - DEBUG_CACHE=DEBUG_CACHE, )) if __name__ == '__main__': diff --git a/site/source/_themes/emscripten_sphinx_rtd_theme/static/css/theme.css b/site/source/_themes/emscripten_sphinx_rtd_theme/static/css/theme.css index 90d729cde171f..07f99a3d7a82a 100644 --- a/site/source/_themes/emscripten_sphinx_rtd_theme/static/css/theme.css +++ b/site/source/_themes/emscripten_sphinx_rtd_theme/static/css/theme.css @@ -264,7 +264,7 @@ code.code-large,.rst-content tt.code-large{font-size:90%}.wy-plain-list-disc,.rs @media print{.wy-nav-side{display:none}.wy-nav-content-wrap{margin-left:0}}nav.stickynav{position:absolute/* previously fixed hamishw */ ;top:0}.rst-versions{position:fixed;bottom:0;left:0;width:300px;color:#fcfcfc;background:#1f1d1d;border-top:solid 10px #343131;font-family:"Lato","proxima-nova","Helvetica Neue",Arial,sans-serif;z-index:400}.rst-versions a{color:#2980b9;text-decoration:none}.rst-versions .rst-badge-small{display:none}.rst-versions .rst-current-version{padding:12px;background-color:#272525;display:block;text-align:right;font-size:90%;cursor:pointer;color:#27ae60;*zoom:1}.rst-versions .rst-current-version:before,.rst-versions .rst-current-version:after{display:table;content:""}.rst-versions .rst-current-version:after{clear:both}.rst-versions .rst-current-version .fa,.rst-versions .rst-current-version .rst-content .admonition-title,.rst-content .rst-versions .rst-current-version .admonition-title,.rst-versions .rst-current-version .rst-content h1 .headerlink,.rst-content h1 .rst-versions .rst-current-version .headerlink,.rst-versions .rst-current-version .rst-content h2 .headerlink,.rst-content h2 .rst-versions .rst-current-version .headerlink,.rst-versions .rst-current-version .rst-content h3 .headerlink,.rst-content h3 .rst-versions .rst-current-version .headerlink,.rst-versions .rst-current-version .rst-content h4 .headerlink,.rst-content h4 .rst-versions .rst-current-version .headerlink,.rst-versions .rst-current-version .rst-content h5 .headerlink,.rst-content h5 .rst-versions .rst-current-version .headerlink,.rst-versions .rst-current-version .rst-content h6 .headerlink,.rst-content h6 .rst-versions .rst-current-version .headerlink,.rst-versions .rst-current-version .rst-content dl dt .headerlink,.rst-content dl dt .rst-versions .rst-current-version .headerlink,.rst-versions .rst-current-version .icon{color:#fcfcfc}.rst-versions .rst-current-version .fa-book,.rst-versions .rst-current-version .icon-book{float:left}.rst-versions .rst-current-version .icon-book{float:left}.rst-versions .rst-current-version.rst-out-of-date{background-color:#e74c3c;color:#fff}.rst-versions .rst-current-version.rst-active-old-version{background-color:#f1c40f;color:#000}.rst-versions.shift-up .rst-other-versions{display:block}.rst-versions .rst-other-versions{font-size:90%;padding:12px;color:gray;display:none}.rst-versions .rst-other-versions hr{display:block;height:1px;border:0;margin:20px 0;padding:0;border-top:solid 1px #413d3d}.rst-versions .rst-other-versions dd{display:inline-block;margin:0}.rst-versions .rst-other-versions dd a{display:inline-block;padding:6px;color:#fcfcfc}.rst-versions.rst-badge{width:auto;bottom:20px;right:20px;left:auto;border:none;max-width:300px}.rst-versions.rst-badge .icon-book{float:none}.rst-versions.rst-badge .fa-book,.rst-versions.rst-badge .icon-book{float:none}.rst-versions.rst-badge.shift-up .rst-current-version{text-align:right}.rst-versions.rst-badge.shift-up .rst-current-version .fa-book,.rst-versions.rst-badge.shift-up .rst-current-version .icon-book{float:left}.rst-versions.rst-badge.shift-up .rst-current-version .icon-book{float:left}.rst-versions.rst-badge .rst-current-version{width:auto;height:30px;line-height:30px;padding:0 6px;display:block;text-align:center} @media screen and (max-width: 768px){ - .rst-versions{width:85%;display:none}.rst-versions.shift{display:block}img{width:100%;height:auto}}.rst-content img{max-width:100%;height:auto !important}.rst-content div.figure{margin-bottom:24px}.rst-content div.figure.align-center{text-align:center}.rst-content .section>img{margin-bottom:24px}.rst-content blockquote{margin-left:24px;line-height:24px;margin-bottom:24px}.rst-content .note .last,.rst-content .attention .last,.rst-content .caution .last,.rst-content .danger .last,.rst-content .error .last,.rst-content .hint .last,.rst-content .important .last,.rst-content .tip .last,.rst-content .warning .last,.rst-content .seealso .last{margin-bottom:0}.rst-content .admonition-title:before{margin-right:4px}.rst-content .admonition table{border-color:rgba(0,0,0,0.1)}.rst-content .admonition table td,.rst-content .admonition table th{background:transparent !important;border-color:rgba(0,0,0,0.1) !important}.rst-content .section ol.loweralpha,.rst-content .section ol.loweralpha li{list-style:lower-alpha}.rst-content .section ol.upperalpha,.rst-content .section ol.upperalpha li{list-style:upper-alpha}.rst-content .section ol p,.rst-content .section ul p{margin-bottom:12px}.rst-content .line-block{margin-left:24px}.rst-content .topic-title{font-weight:bold;margin-bottom:12px}.rst-content .toc-backref{color:#404040}.rst-content .align-right{float:right;margin:0px 0px 24px 24px}.rst-content .align-left{float:left;margin:0px 24px 24px 0px}.rst-content .align-center{margin:auto;display:block}.rst-content h1 .headerlink,.rst-content h2 .headerlink,.rst-content h3 .headerlink,.rst-content h4 .headerlink,.rst-content h5 .headerlink,.rst-content h6 .headerlink,.rst-content dl dt .headerlink{display:none;visibility:hidden;font-size:14px}.rst-content h1 .headerlink:after,.rst-content h2 .headerlink:after,.rst-content h3 .headerlink:after,.rst-content h4 .headerlink:after,.rst-content h5 .headerlink:after,.rst-content h6 .headerlink:after,.rst-content dl dt .headerlink:after{visibility:visible;content:"\f0c1";font-family:FontAwesome;display:inline-block}.rst-content h1:hover .headerlink,.rst-content h2:hover .headerlink,.rst-content h3:hover .headerlink,.rst-content h4:hover .headerlink,.rst-content h5:hover .headerlink,.rst-content h6:hover .headerlink,.rst-content dl dt:hover .headerlink{display:inline-block}.rst-content .sidebar{float:right;width:40%;display:block;margin:0 0 24px 24px;padding:24px;background:#f3f6f6;border:solid 1px #e1e4e5}.rst-content .sidebar p,.rst-content .sidebar ul,.rst-content .sidebar dl{font-size:90%}.rst-content .sidebar .last{margin-bottom:0}.rst-content .sidebar .sidebar-title{display:block;font-family:"Roboto Slab","ff-tisa-web-pro","Georgia",Arial,sans-serif;font-weight:bold;background:#e1e4e5;padding:6px 12px;margin:-24px;margin-bottom:24px;font-size:100%}.rst-content .highlighted{background:#f1c40f;display:inline-block;font-weight:bold;padding:0 6px}.rst-content .footnote-reference,.rst-content .citation-reference{vertical-align:super;font-size:90%}.rst-content table.docutils.citation,.rst-content table.docutils.footnote{background:none;border:none;color:#999}.rst-content table.docutils.citation td,.rst-content table.docutils.citation tr,.rst-content table.docutils.footnote td,.rst-content table.docutils.footnote tr{border:none;background-color:transparent !important;white-space:normal}.rst-content table.docutils.citation td.label,.rst-content table.docutils.footnote td.label{padding-left:0;padding-right:0;vertical-align:top}.rst-content table.field-list{border:none}.rst-content table.field-list td{border:none;padding-top:5px}.rst-content table.field-list td>strong{display:inline-block;margin-top:3px}.rst-content table.field-list .field-name{padding-right:10px;text-align:left;white-space:nowrap}.rst-content table.field-list .field-body{text-align:left;padding-left:0}.rst-content tt{color:#000}.rst-content tt big,.rst-content tt em{font-size:100% !important;line-height:normal}.rst-content tt .xref,a .rst-content tt{font-weight:bold}.rst-content a tt{color:#2980b9}.rst-content dl{margin-bottom:24px}.rst-content dl dt{font-weight:bold}.rst-content dl p,.rst-content dl table,.rst-content dl ul,.rst-content dl ol{margin-bottom:12px !important}.rst-content dl dd{margin:0 0 12px 24px}.rst-content dl:not(.docutils){margin-bottom:24px}.rst-content dl:not(.docutils) dt{display:inline-block;margin:6px 0;font-size:90%;line-height:normal;background:#e7f2fa;color:#2980b9;border-top:solid 3px #6ab0de;padding:6px;position:relative}.rst-content dl:not(.docutils) dt:before{color:#6ab0de}.rst-content dl:not(.docutils) dt .headerlink{color:#404040;font-size:100% !important}.rst-content dl:not(.docutils) dl dt{margin-bottom:6px;border:none;border-left:solid 3px #ccc;background:#f0f0f0;color:gray}.rst-content dl:not(.docutils) dl dt .headerlink{color:#404040;font-size:100% !important}.rst-content dl:not(.docutils) dt:first-child{margin-top:0}.rst-content dl:not(.docutils) tt{font-weight:bold}.rst-content dl:not(.docutils) tt.descname,.rst-content dl:not(.docutils) tt.descclassname{background-color:transparent;border:none;padding:0;font-size:100% !important}.rst-content dl:not(.docutils) tt.descname{font-weight:bold}.rst-content dl:not(.docutils) .optional{display:inline-block;padding:0 4px;color:#000;font-weight:bold}.rst-content dl:not(.docutils) .property{display:inline-block;padding-right:8px}.rst-content .viewcode-link,.rst-content .viewcode-back{display:inline-block;color:#27ae60;font-size:80%;padding-left:24px}.rst-content .viewcode-back{display:block;float:right} + .rst-versions{width:85%;display:none}.rst-versions.shift{display:block}img{height:auto}}.rst-content img{max-width:100%;height:auto !important}.rst-content div.figure{margin-bottom:24px}.rst-content div.figure.align-center{text-align:center}.rst-content .section>img{margin-bottom:24px}.rst-content blockquote{margin-left:24px;line-height:24px;margin-bottom:24px}.rst-content .note .last,.rst-content .attention .last,.rst-content .caution .last,.rst-content .danger .last,.rst-content .error .last,.rst-content .hint .last,.rst-content .important .last,.rst-content .tip .last,.rst-content .warning .last,.rst-content .seealso .last{margin-bottom:0}.rst-content .admonition-title:before{margin-right:4px}.rst-content .admonition table{border-color:rgba(0,0,0,0.1)}.rst-content .admonition table td,.rst-content .admonition table th{background:transparent !important;border-color:rgba(0,0,0,0.1) !important}.rst-content .section ol.loweralpha,.rst-content .section ol.loweralpha li{list-style:lower-alpha}.rst-content .section ol.upperalpha,.rst-content .section ol.upperalpha li{list-style:upper-alpha}.rst-content .section ol p,.rst-content .section ul p{margin-bottom:12px}.rst-content .line-block{margin-left:24px}.rst-content .topic-title{font-weight:bold;margin-bottom:12px}.rst-content .toc-backref{color:#404040}.rst-content .align-right{float:right;margin:0px 0px 24px 24px}.rst-content .align-left{float:left;margin:0px 24px 24px 0px}.rst-content .align-center{margin:auto;display:block}.rst-content h1 .headerlink,.rst-content h2 .headerlink,.rst-content h3 .headerlink,.rst-content h4 .headerlink,.rst-content h5 .headerlink,.rst-content h6 .headerlink,.rst-content dl dt .headerlink{display:none;visibility:hidden;font-size:14px}.rst-content h1 .headerlink:after,.rst-content h2 .headerlink:after,.rst-content h3 .headerlink:after,.rst-content h4 .headerlink:after,.rst-content h5 .headerlink:after,.rst-content h6 .headerlink:after,.rst-content dl dt .headerlink:after{visibility:visible;content:"\f0c1";font-family:FontAwesome;display:inline-block}.rst-content h1:hover .headerlink,.rst-content h2:hover .headerlink,.rst-content h3:hover .headerlink,.rst-content h4:hover .headerlink,.rst-content h5:hover .headerlink,.rst-content h6:hover .headerlink,.rst-content dl dt:hover .headerlink{display:inline-block}.rst-content .sidebar{float:right;width:40%;display:block;margin:0 0 24px 24px;padding:24px;background:#f3f6f6;border:solid 1px #e1e4e5}.rst-content .sidebar p,.rst-content .sidebar ul,.rst-content .sidebar dl{font-size:90%}.rst-content .sidebar .last{margin-bottom:0}.rst-content .sidebar .sidebar-title{display:block;font-family:"Roboto Slab","ff-tisa-web-pro","Georgia",Arial,sans-serif;font-weight:bold;background:#e1e4e5;padding:6px 12px;margin:-24px;margin-bottom:24px;font-size:100%}.rst-content .highlighted{background:#f1c40f;display:inline-block;font-weight:bold;padding:0 6px}.rst-content .footnote-reference,.rst-content .citation-reference{vertical-align:super;font-size:90%}.rst-content table.docutils.citation,.rst-content table.docutils.footnote{background:none;border:none;color:#999}.rst-content table.docutils.citation td,.rst-content table.docutils.citation tr,.rst-content table.docutils.footnote td,.rst-content table.docutils.footnote tr{border:none;background-color:transparent !important;white-space:normal}.rst-content table.docutils.citation td.label,.rst-content table.docutils.footnote td.label{padding-left:0;padding-right:0;vertical-align:top}.rst-content table.field-list{border:none}.rst-content table.field-list td{border:none;padding-top:5px}.rst-content table.field-list td>strong{display:inline-block;margin-top:3px}.rst-content table.field-list .field-name{padding-right:10px;text-align:left;white-space:nowrap}.rst-content table.field-list .field-body{text-align:left;padding-left:0}.rst-content tt{color:#000}.rst-content tt big,.rst-content tt em{font-size:100% !important;line-height:normal}.rst-content tt .xref,a .rst-content tt{font-weight:bold}.rst-content a tt{color:#2980b9}.rst-content dl{margin-bottom:24px}.rst-content dl dt{font-weight:bold}.rst-content dl p,.rst-content dl table,.rst-content dl ul,.rst-content dl ol{margin-bottom:12px !important}.rst-content dl dd{margin:0 0 12px 24px}.rst-content dl:not(.docutils){margin-bottom:24px}.rst-content dl:not(.docutils) dt{display:inline-block;margin:6px 0;font-size:90%;line-height:normal;background:#e7f2fa;color:#2980b9;border-top:solid 3px #6ab0de;padding:6px;position:relative}.rst-content dl:not(.docutils) dt:before{color:#6ab0de}.rst-content dl:not(.docutils) dt .headerlink{color:#404040;font-size:100% !important}.rst-content dl:not(.docutils) dl dt{margin-bottom:6px;border:none;border-left:solid 3px #ccc;background:#f0f0f0;color:gray}.rst-content dl:not(.docutils) dl dt .headerlink{color:#404040;font-size:100% !important}.rst-content dl:not(.docutils) dt:first-child{margin-top:0}.rst-content dl:not(.docutils) tt{font-weight:bold}.rst-content dl:not(.docutils) tt.descname,.rst-content dl:not(.docutils) tt.descclassname{background-color:transparent;border:none;padding:0;font-size:100% !important}.rst-content dl:not(.docutils) tt.descname{font-weight:bold}.rst-content dl:not(.docutils) .optional{display:inline-block;padding:0 4px;color:#000;font-weight:bold}.rst-content dl:not(.docutils) .property{display:inline-block;padding-right:8px}.rst-content .viewcode-link,.rst-content .viewcode-back{display:inline-block;color:#27ae60;font-size:80%;padding-left:24px}.rst-content .viewcode-back{display:block;float:right} @media screen and (max-width: 480px){ .rst-content .sidebar{width:100%}}span[id*='MathJax-Span']{color:#404040} diff --git a/site/source/docs/building_from_source/toolchain_what_is_needed.rst b/site/source/docs/building_from_source/toolchain_what_is_needed.rst index 5c7b34faf1924..33d111fd49143 100644 --- a/site/source/docs/building_from_source/toolchain_what_is_needed.rst +++ b/site/source/docs/building_from_source/toolchain_what_is_needed.rst @@ -69,6 +69,7 @@ When building Emscripten from source code, whether "manually" or using the SDK, - Install XCode from the `Mac OS X App Store `_. - In **XCode | Preferences | Downloads**, install *Command Line Tools*. +.. note:: Building LLVM and Clang from source can require a lot of memory and hard drive space. The specific requirements change from LLVM version to another, but you probably need at least 2GB of RAM, preferably 4GB or more. Debug builds or builds with assertions can require even more memory. .. _toolchain-test-which-dependencies-are-installed: diff --git a/site/source/docs/compiling/Building-Projects.rst b/site/source/docs/compiling/Building-Projects.rst index 437856e2ae0ee..95fdada0613f3 100644 --- a/site/source/docs/compiling/Building-Projects.rst +++ b/site/source/docs/compiling/Building-Projects.rst @@ -176,6 +176,8 @@ You should see some notifications about SDL2 being used, and built if it wasn't .. note:: *SDL_image* has also been added to ports, use it with ``-s USE_SDL_IMAGE=2``. To see a list of all available ports, run ``emcc --show-ports``. For SDL2_image to be useful, you generally need to specify the image formats you are planning on using with -s SDL2_IMAGE_FORMATS='["png"]'. This will also ensure that ``IMG_Init`` works properly. Alternatively, you can use specify ``emcc --use-preload-plugins`` (and ``--preload-file`` your images, so the browser codecs decode them), but then your calls to ``IMG_Init`` will fail. +.. note:: *SDL_net* has also been added to ports, use it with ``-s USE_SDL_NET=2``. To see a list of all available ports, run ``emcc --show-ports``. + .. note:: Emscripten also has support for older SDL1, which is built-in. If you do not specify SDL2 as in the command above, then SDL1 is linked in and the SDL1 include paths are used. SDL1 has support for *sdl-config*, which is present in `system/bin `_. Using the native *sdl-config* may result in compilation or missing-symbol errors. You will need to modify the build system to look for files in **emscripten/system** or **emscripten/system/bin** in order to use the Emscripten *sdl-config*. Adding more ports diff --git a/site/source/docs/debugging/CyberDWARF.rst b/site/source/docs/debugging/CyberDWARF.rst new file mode 100644 index 0000000000000..33831b8a39655 --- /dev/null +++ b/site/source/docs/debugging/CyberDWARF.rst @@ -0,0 +1,94 @@ +.. _CyberDWARF: + +==================== +CyberDWARF Debugging +==================== + +Building +======== + +To add CyberDWARF support to a build, pass ``-s CYBERDWARF=1`` to ``emcc``. This generates a ``.cd`` file containing type information for debugging and adds a debugging toolkit to the output Javascript. + +Using +===== + +The CyberDWARF debugger is designed to be used from the Javascript devtool console available in most modern browsers. + +Heap Pretty Printer +------------------- + +This small example will show how to use CyberDWARF to visualize a simple struct + +.. code-block:: cpp + + #include + #include + #include + + struct TinyStruct { + short len; + char * chars; + }; + + extern "C" { + + int example() { + TinyStruct example; + example.chars = "Hello World"; + example.len = strlen(example.chars); + + printf("%p\n", &example); + EM_ASM({ debugger }); + + return 0; + } + + } + + int main(int argc, char *argv[]) { + return 0; + } + +**Compile the code** + +.. code-block:: bash + + em++ -O1 -s CYBERDWARF=1 example.cpp -o example.html -s EXPORTED_FUNCTIONS="['_example']" + +**Visualizing** + +After the page loads, open a Javascript console. + +.. code-block:: bash + + > Module['cyberdwarf'].initialize_debugger() + Debugger ready + > Module['_example']() + 0x1078 + > Module['cyberdwarf'].set_current_function("_example") + > JSON.stringify(Module['cyberdwarf'].decode_var_by_var_name(0x1078, "example", 10), null, "\t") + '{ + "struct TinyStruct": { + "short : len": 5, + "char * : chars": 87 + } + }' + +API +--- + +.. js:function:: Module['cyberdwarf'].initialize_debugger + + Called to load the CyberDWARF file for the script. + +.. js:function:: Module['cyberdwarf'].set_current_function(name) + + Sets the function to lookup variable type by ``name``. Supports either C/mangled name or minified name from symbol file. + +.. js:function:: Module['cyberdwarf'].decode_var_by_var_name(address, name, depth) + + Looks up the type given by variable ``name`` in the current function, then dumps a JSON formatted representation of that type at ``address`` up to ``depth`` *(defaults to 1)* + +.. js:function:: Module['cyberdwarf'].decode_var_by_type_name(address, type, depth) + + Using the type given in ``type``, then dumps a JSON formatted representation of that type at ``address`` up to depth *(defaults to 1)* diff --git a/site/source/docs/getting_started/FAQ.rst b/site/source/docs/getting_started/FAQ.rst index 3274aa521323f..7d5a8e60e369a 100644 --- a/site/source/docs/getting_started/FAQ.rst +++ b/site/source/docs/getting_started/FAQ.rst @@ -184,7 +184,7 @@ Why can't my code access a file in the same directory? Emscripten-generated code running *in the browser* cannot access files in the local file system. Instead you can use :ref:`preloading ` and :ref:`embedding ` to work around the lack of synchronous file IO. See :ref:`file-system-overview` for more information. -It is possible to allow access to local file system for code running in *node.js*. +It is possible to allow access to local file system for code running in *node.js*, use the :ref:`NODEFS ` filesystem option. .. _faq-when-safe-to-call-compiled-functions: @@ -192,7 +192,7 @@ It is possible to allow access to local file system for code running in *node.js How can I tell when the page is fully loaded and it is safe to call compiled functions? ======================================================================================= -(You may need this answer if you see an error saying something like ``you need to wait for the runtime to be ready (e.g. wait for main() to be called)``.) +(You may need this answer if you see an error saying something like ``you need to wait for the runtime to be ready (e.g. wait for main() to be called)``, which is a check enabled in ``ASSERTIONS`` builds.) Calling a compiled function before a page has fully loaded can result in an error, if the function relies on files that may not be present (for example the :ref:`.mem ` file and :ref:`preloaded ` files are loaded asynchronously). diff --git a/site/source/docs/index.rst b/site/source/docs/index.rst index ea2165e5e52e1..17c346f858394 100644 --- a/site/source/docs/index.rst +++ b/site/source/docs/index.rst @@ -27,6 +27,7 @@ This comprehensive documentation set contains everything you need to know to use - :ref:`api-reference-index` is a reference for the Emscripten toolchain. - :ref:`tools-reference` is a reference for the Emscripten integration APIs. +- :ref:`CyberDWARF` shows how to use the CyberDWARF debugging system The full hierarchy of articles, opened to the second level, is shown below. @@ -42,6 +43,7 @@ The full hierarchy of articles, opened to the second level, is shown below. contributing/index api_reference/index tools_reference/index + debugging/CyberDWARF site/index diff --git a/site/source/docs/porting/asyncify.rst b/site/source/docs/porting/asyncify.rst new file mode 100644 index 0000000000000..977f1e7c63bb4 --- /dev/null +++ b/site/source/docs/porting/asyncify.rst @@ -0,0 +1,126 @@ +.. Asyncify: + +======================== +Asyncify +======================== + +In general, you want to make your code as asynchronous-friendly as possible, e.g. by never calling sleep, having a single main loop and so on. + +Sometimes refactoring a codebase with this in mind isn't feasible, so there are a couple of alternatives (each with downsides). Asyncify is covered here. + +**Asyncify is experimental, and not recommended. See https://kripken.github.io/emscripten-site/docs/porting/emterpreter.html for a more recent option with similar functionality, that is currently supported.** + +``ASYNCIFY`` allows you to use some asynchronous function in C, through several transformation of LLVM IR. + +Intro +===== + +If you call ``sleep()`` in C/C++, what kind of JavaScript code would you expect emscripten to produce? + +:: + + // test.c + #include + #include + int main() { + int i = 100; + printf("Hello\n"); + sleep(1); + printf("World %d!\n"); + return 0; + } + +Note that we cannot implement ``sleep`` like this:: + + function sleep(ms) { + var t = Date.now() + ms; + while(Date.now() < t) ; + } + +because this would block the JavaScript engine, such that pending events cannot be processed. + + +A hand written counterpart in JavaScript would be + +:: + + function main() { + var i = 100; + console.log('Hello'); + setTimeout(function() { + console.log('World ' + i + '!'); + async_return_value = 0; + }, 1000); + } + +Specifically, a number of aspects should be taken into consideration: + - Split the function when an async function is called, and the second function should be registered as the callback for the async function + - Any function that calls an async function also becomes an async function. + - Keep all local variables available to the callback + - Closure cannot be used in order to make asm.js validated code. + - Take care of loops and branches + - Make the return value available to the callee + - Some functions could be both sync or async, depending on the input. + +And the ``ASYNCIFY`` option does all above automatically, through a number of transformations on LLVM IR. + + +Usage +===== + +Call ``emscripten_sleep()`` whenever you need to pause the program, and add ``-s ASYNCIFY=1`` to emscripten. + +Sometimes it's a good replacement of ``emscripten_set_main_loop``, you may replace all ``sleep``-alike functions with ``emscripten_sleep``, instead of refactoring the whole main loop. + +Also ``emscripten_sleep(1)`` can be used to 'interrupt' your code, such that the JavaScript engine can do the rendering and process events. + +Extensions +========== + +It is possible to implement more new async functions that appears to be sync in C. + +- Implement the function normally in a JavaScript library, suppose the function name is ``func``. +- Add ``func`` into ``ASYNCIFY_FUNCTIONS`` +- When ``func`` is called and finished, the program will NOT continue, instead it just save the context and exit. +- Call ``_emscripten_async_resume`` when you want to resume the program, usually in the callback functions of some async calls. + +Please read ``src/library_async.js`` for details. + +Limitations +=========== + +Code size increase should be expected, depending on the specific input. +``-Os`` (or ``-Oz`` for linking) is recommended when ``ASYNCIFY`` is turned on. +E.g. usually the following loop is expanded to speed up:: + + for(int i = 0; i < 3; ++i) { + // do something + emscripten_sleep(1000); + // do something else + } + +However by expanding the loop, two more async calls are introduced, such that more callback functions will be produced during the asyncify transformation. + +**Asyncify can make performance much slower, if it ends up splitting a function which you need to be fast.** + +``setjmp/longjmp`` and C++ exception are not working well when there are async function calls in the scope, but they still work when there's no async calls. E.g. + +:: + + try { + // do something + if(error) throw 0; // works + emscripten_sleep(1000); + // do something else + if(error) throw 0; // does not work + } + +Currently all function pointer calls are considered as aync, and some functions might be recognized as async incorrectly. This can be corrected by manually setting the ``ASYNCIFY_WHITELIST`` option. + + +Other possible implementations +============================== + + - Closures (breaking asm.js) + - Generators (too slow currently) + - Blocking message (in workers) diff --git a/site/source/docs/porting/connecting_cpp_and_javascript/Interacting-with-code.rst b/site/source/docs/porting/connecting_cpp_and_javascript/Interacting-with-code.rst index 258b92f8d2c1d..e0fc28c8f9abb 100644 --- a/site/source/docs/porting/connecting_cpp_and_javascript/Interacting-with-code.rst +++ b/site/source/docs/porting/connecting_cpp_and_javascript/Interacting-with-code.rst @@ -38,6 +38,12 @@ to more detailed information. environment, see :ref:`emscripten-runtime-environment`. For file system related manners, see the :ref:`file-system-overview`. +.. note:: Before you can call your code, the runtime environment may need + to load a memory initialization file, preload files, or + do other asynchronous operations depending on optimization + and build settings. + See :ref:`faq-when-safe-to-call-compiled-functions` in the FAQ. + .. _interacting-with-code-ccall-cwrap: diff --git a/site/source/docs/porting/connecting_cpp_and_javascript/WebIDL-Binder.rst b/site/source/docs/porting/connecting_cpp_and_javascript/WebIDL-Binder.rst index a5fe999c36761..75c4ec8fb0b6a 100644 --- a/site/source/docs/porting/connecting_cpp_and_javascript/WebIDL-Binder.rst +++ b/site/source/docs/porting/connecting_cpp_and_javascript/WebIDL-Binder.rst @@ -264,7 +264,7 @@ You can bind to C++ operators using ``[Operator=]``: .. note:: - The operator name can be anything (``add`` is just an example). - - Support is currently limited to operators that contain ``=``: ``+=``, ``*=``, ``-=`` etc. + - Support is currently limited to operators that contain ``=``: ``+=``, ``*=``, ``-=`` etc., and to the array indexing operator ``[]``. enums @@ -433,4 +433,4 @@ Test and example code For a complete working example, see `test_webidl `_ in the `test suite `_. The test suite code is guaranteed to work and covers more cases than this article alone. -Another good example is `ammo.js `_, which uses the *WebIDL Binder* to port the `Bullet Physics engine `_ to the Web. \ No newline at end of file +Another good example is `ammo.js `_, which uses the *WebIDL Binder* to port the `Bullet Physics engine `_ to the Web. diff --git a/site/source/docs/porting/emterpreter.rst b/site/source/docs/porting/emterpreter.rst new file mode 100644 index 0000000000000..a3d84c6345efd --- /dev/null +++ b/site/source/docs/porting/emterpreter.rst @@ -0,0 +1,155 @@ +.. Emterpreter: + +============================== +Emterpreter +============================== + +The **Emterpreter** is an option that compiles asm.js output from Emscripten into a binary bytecode. It also generates an interpreter ("Emscripten interpreter", hence *Emterpreter*) capable of executing that bytecode. This lets you compile your project, or parts of your project, into bytecode that will be interpreted, as opposed to asm.js that will be executed directly by the JavaScript engine. + +Why does this option exist? To provide an alternative in situations where normal direct execution by the JavaScript engine has issues. The two main motivations are + + * JavaScript must be parsed and compiled before it is executed, which can take a long time in large codebases, whereas a binary bytecode is just data, so you can get to the point of *something* executing earlier. Executing in an interpreter might be slower, but it can can start earlier. Or in other words, it can already be running (albeit slowly) before the code would normally be running at all. + * JavaScript has high-level control flow (no gotos) and must be written as short-running events, not long-running synchronous code. However, sometimes you have code that is written in the latter form that you can't easily refactor. The Emterpreter can handle that, because running the code in an interpreter allows us to manually control the flow of execution, as well as pause and resume the entire call stack, letting us turn synchronous code into asynchronous code. + +For more background on the Emterpreter, see + * [Emterpreter startup time blogpost](https://blog.mozilla.org/research/2015/02/23/the-emterpreter-run-code-before-it-can-be-parsed/) + * [Emterpreter synchronous execution blogpost](https://hacks.mozilla.org/2015/02/synchronous-execution-and-filesystem-access-in-emscripten/) + +General Usage +============= + +To use the Emterpreter, build with ``-s EMTERPRETIFY=1``. This runs all code in the interpreter by default. You can also use the ``EMTERPRETIFY_BLACKLIST`` option to specify the only methods **not** to be interpreted, or ``EMTERPRETIFY_WHITELIST`` to specify the only methods that **are** to be interpreted. + +You can optionally use ``-s 'EMTERPRETIFY_FILE="data.binary"'`` to store the emterpreter bytecode in a file (called ``data.binary`` in this example). Otherwise, the default is to store the bytecode in the JS file itself, which is very inefficient for binary data. For small amounts of bytecode it is fine, but for large applications you should really use a file, otherwise code size and startup can be bad (you should receive a warning when building a large application without this option). + +As usual, you can grep the ``tests/`` folder for examples of emterpreter usage in the test suite (search for ``EMTERPRETIFY``). + +Specific Use Cases +================== + +A couple of examples are given below of how the Emterpreter can be useful in general. + +Improving Startup 1: Swapping +----------------------------- + +As mentioned earlier, the Emterpreter and its binary bytecode load faster than JavaScript and asm.js can. Just building with the Emterpreter option gives you that, but it also makes the code run more slowly. A hybrid solution is to start up quickly in the Emterpreter, then switch to faster execution in full asm.js speed later. This is possible by **swapping** the asm.js module - first load the Emterpreted one, then load the fast one in the background and switch to it when it's ready. + +To do this, build the project twice: + + * Once with the Emterpreter option enabled, and ``SWAPPABLE_ASM_MODULE``. This is the module you will start up with, and swap out when the fast one is ready. + * Again to normal asm.js, then run ``tools/distill_asm.py infile.js outfile.js swap-in``. The output, ``outfile.js``, will be just the asm module itself. You can then load this in a script tag on the same page, and it will swap itself in when it is ready. + +Improving Startup 2: Set Aside Cold Code +---------------------------------------- + +If you have a method that you know will only ever run exactly once, and doesn't need to be fast, you can run that specific method in the Emterpreter: As mentioned above, the JavaScript engine won't need to compile it, and the bytecode is smaller than asm.js, so both download and startup will be faster. Another example is exception-handling or assertion reporting code, something that should never run, and if it does, is ok to run at a slower speed. + +To do this, simply use the whitelist option mentioned before, with a list of the methods you want to be run as bytecode. + +Emterpreter-Async: Run Synchronous Code +======================================= + +The emterpreter runs the code in an interpreter, which makes it feasible to manually control the call stack and so forth. To enable this support, build with ``-s EMTERPRETIFY_ASYNC=1``. You can then write synchronous-looking code, and it will "just work", e.g. + +:: + + while (1) { + do_frame(); + emscripten_sleep(10); + } + +is a simple way to do a main loop, which typically you would refactor your code for and use ``emscripten_set_main_loop``. Instead, in the emterpreter the call to ``emscripten_sleep`` will save the execution state, including call stack, do a ``setTimeout`` for the specified amount of milliseconds, and after that delay, reconstruct the execution state exactly as it was before. From the perspective of the source code, it looks like synchronous sleep, but under the hood it is converted to a form that can work in a web browser asynchronously. + +For a list of the APIs that can be used in this synchronous manner, see [the docs](http://kripken.github.io/emscripten-site/docs/api_reference/emscripten.h.html#emterpreter-async-functions). + +When using sleep in this manner, you can likely use the emterpreter whitelist very efficiently: Only things that can lead to a call to sleep (or another synchronous method) need to be emterpreted. In the example above, ``do_frame`` and everything that could call it should be in the whitelist, so that everything else runs at full asm.js speed. More specifically, for the interpreter to be able to save and later restore the state of execution, the current call stack must only contain emterpreted functions, not normal asm.js functions, and not functions from outside that are not compiled code. To save the state of execution, the interpreter records its current location and all variables on the stack, both of which we cannot do for code that is not run in the interpreter. Note that this makes using ``ccall`` or ``cwrap`` to call code which does an asynchronous operation a little tricky. There is some support for this (see the [async option](https://kripken.github.io/emscripten-site/docs/api_reference/preamble.js.html#ccall) on ccall), but if assumes there isn't anything else on the JS stack you want paused and resumed, and also, it works like a promise in that it returns to your JS code, but continues to work later on (this should probably be turned into a proper JS Promise). + +Semantics +--------- + +``emscripten_sleep`` and other synchronous methods are meant to actually **be** synchronous. When they execute, control returns to the browser's main event loop (which allows rendering to show up), however, we attempt to block other asynchronous events. That is, if you do an ``emscripten_async_wget`` and then an ``emscripten_sleep``, the asynchronous wget will **not** execute during the sleep. The sleep must complete first. This keeps things in alignment with how synchronous code would work in C. + +If you **do** want asynchronous events during sleep, use ``emscripten_sleep_with_yield``. This is not fully tested yet, however, and may need rethinking. + +Deciding on which methods to Emterpret for async +------------------------------------------------ + +There are both static and dynamic tools that can help here. + +Static Analysis +~~~~~~~~~~~~~~~ + +Building with ``EMTERPRETIFY_ADVISE`` will process the project and perform a static analysis to determine which methods should probably be run in the interpreter. This checks which methods *could* be on the stack underneath a call to a synchronous method, in which case they must be interpreted so that we can save and restore the stack later in an asynchronous way. + +The analysis is pessimistic, in that it checks what *could* possibly be called, but might not in practice. For exmaple, function pointers are hard to figure out: Even though the analysis takes into account the **type** of function pointer, if you call a ``void (int)`` method by a function pointer, then the analysis must assume that any ``void (int)`` method (that ever has its address taken, i.e., *could* be called via a function pointer) could be called there. For example, on Doom it suggests that 31% (!) of all methods should be interpreted, while in practice only 1% need to be (as is easy to verify by reading the code). + +Dynamic Tools +~~~~~~~~~~~~~ + +By building with ``-s ASSERTIONS=1``, you can get runtime errors on not having methods interpreted that should be. This will catch only problems that occur **in practice**, so it is an optimistic approach (the opposite of the static analysis route). But by running your codecase on a representative workload, this approach should give you very useful results. Here is how you can do this: + +First, make sure all synchronous execution works correctly when interpreting **everything**, by building with ``-s EMTERPRETIFY=1 -s EMTERPRETIFY_ASYNC=1``. Might be slow, but it should work! + +Then, run only ``main()`` in the interpreter, by adding ``-s EMTERPRETIFY_WHITELIST='["_main"]'``, and building with assertions, ``-s ASSERTIONS=1,`` as well as preserving (not minifying) function names, ``--profiling-funcs``. Any synchronous execution not happening in main itself is in non-interpreted code, and therefore bad; in a a build with assertions, this will trigger a runtime error you can view in the web console in your browser, and thanks to the profiling option, the error will have clearly readable function names, for example, you could see this on Doom:: + + This error happened during an emterpreter-async save or load of the stack. Was there non-emterpreted code on the stack during save (which is unallowed)? This is what the stack looked like when we tried to save it: + + jsStackTrace@file:///home/alon/Dev/boon/boon.js:1:26546 + stackTrace@file:///home/alon/Dev/boon/boon.js:1:26729 + EmterpreterAsync.handle@file:///home/alon/Dev/boon/boon.js:1:196637 + _emscripten_sleep@file:///home/alon/Dev/boon/boon.js:1:196851 + _D_DoomLoop@file:///home/alon/Dev/boon/boon.js:12:188349 + _D_DoomMain@file:///home/alon/Dev/boon/boon.js:12:186664 + emterpret@file:///home/alon/Dev/boon/boon.js:11:10149 + _main@file:///home/alon/Dev/boon/boon.js:12:76663 + asm._main@file:///home/alon/Dev/boon/boon.js:19:985 + callMain@file:///home/alon/Dev/boon/boon.js:19:31106 + doRun@file:///home/alon/Dev/boon/boon.js:19:32001 + run/<@file:///home/alon/Dev/boon/boon.js:19:32169 + +You can see ``main()`` at the bottom (below it is how main is invoked, which you can ignore), then a call into the ``emterpret()`` function, which is how main invokes the interpreter for itself (``main()`` itself is just a little "trampoline" that jumps into the interpreter and tells it which bytecode to run). Above that, we can see ``_D_DoomMain`` and ``_D_DoomLoop``. Those two methods must be interpreted, since higher up on the stack trace you can see a call to emscripten_sleep(), which is synchronous. + +Adding those methods to the whitelist of interpreted functions, you can then build and run the application again, and repeat this process until everything works properly. You should still carefully review your codebase and see what should be interpreted, but the semi-automatic process described here is easy to use and can be very effective in practice, if you test all relevant code paths. + +**Warning**: The runtime checks that ASSERTIONS adds guard against compiled code that is not interpreted. But it does not protect you from non-compiled code. For example, if a compiled method calls a non-compiled method, which then calls back into compiled code, we cannot save and restore the stack: Even if the compiled methods are interpreted, the non-compiled one has no way for us to save it's current execution state. If you try to run synchronous code in this incorrect manner, things will fail in potentially confusing ways: what happens is the emterpreted code returns immediately (in order to wait for the asynchronous callback), and your handwritten code underneath it will then continue to execute, not knowing that the code just returning has not yet completed. + +Inlining +~~~~~~~~ + +A potentially confusing issue can arise through function inlining: If a parent method calls a method that will sleep, and another that won't, only the former of the two children needs to be interpreted (as well as the parent function). But, if both are inlined into the parent, then they are all now one function, which must be interpreted. + +To obtain optimal performance, you may want to mark some non-interpreted methods called from interpreted methods as no-inline, using ``__attribute__((noinline))``. That avoids their code ending up running in the interpreter. + +Note that this will only be a performance issue, not correctness - by inlining, a child's code ends up in the parent function, and since we need everything on the stack during a sync call to be interpreted, it is ok to inline among those, or even inline from non-interpreted functions as well. In other words, it just adds methods to be interpreted, which is always safe, at the cost of performance. + +Comparison to ASYNCIFY +---------------------- + +ASYNCIFY is an earlier experiment on running synchronous code. It does a whole-program analysis in LLVM and modifies all relevant methods to they can be saved and resumed, by breaking them up and so forth. Comparing the two, + + * ASYNCIFY has a bad worst-case of large code size: If it needs to modify many methods, it can grow code size very significantly (even 10x more was seen). The emterpreter on the other hand has a guarantee of having smaller code size than normal emscripten output, simply because emterpreter bytecode is smaller than JS source. (Note: you should use ``EMTERPRETIFY_FILE`` to reduce code size, as without it the bytecode is stored in JS which is inefficient for binary data.) + * ASYNCIFY is slower than normal emscripten output, but probably not hugely so, while the emterpreter can be much slower, because it interprets code. Using a whitelist or blacklist with the emterpreter, this can be mitigated. + * There are some known bugs with ASYNCIFY on things like exceptions and setjmp. The emterpreter has not been tested on those feature yet, so it's unclear if it would work. Update: there are known issues with doing and async operation when there is a try-catch (llvm invoke) on the stack. + * ASYNCIFY focused on a static analysis, while the Emterpreter-Async option has both a static analysis and dynamic tools to help figure out which methods should be treated in a special way to enable synchronous code. + * As the emterpreter is useful for other things than synchronous code, it will likely continue to be worked on, while the ASYNCIFY option currently does not have activity. + +Further reading +~~~~~~~~~~~~~~~ + + * [DOSBox usage](http://dreamlayers.blogspot.com/2015/02/fixing-hard-problem-in-em-dosbox-using.html) + +Debugging +========= + +Stack traces when running the emterpreter can be a little confusing. Keep these things in mind: + + * When non-emterpreted code calls into emterpreted code, it has to go through a "trampoline", a little function that just calls ``emterpret()`` with the location of the code to execute. That's why you'll see ``main() -> emterpret()`` in your stack traces, ``main()`` is just a trampoline. + * When calling between emterpreted code, there is an ``INTCALL`` opcode which does a direct call from ``emterpret()`` to another invocation of ``emterpret()``. That means that you do see a stack trace of the right size, but the names are all the same. Invoke emcc with ``--profiling-funcs`` or ``--profiling`` to have the emterpreter take a slower path of calling through trampolines all the time. This is useful for profiling. + +Bytecode Design +=============== + +The bytecode is a simple register-based bytecode invented for this purpose, just enough to support the asm.js code that Emscripten emits. It is designed more for speed of execution and quick startup (no preprocessing necessary at all), than size. + +It also has a bunch of "combo" opcodes for things like test+branch, etc. See ``tools/emterpretify.py`` for the list of opcodes. + diff --git a/site/source/docs/porting/index.rst b/site/source/docs/porting/index.rst index 4352ad73c8346..c1d3e4355cda7 100644 --- a/site/source/docs/porting/index.rst +++ b/site/source/docs/porting/index.rst @@ -17,5 +17,7 @@ The topics in this section cover the main integration points that you need to co Debugging pthreads simd + asyncify + emterpreter diff --git a/site/source/docs/tools_reference/emcc.rst b/site/source/docs/tools_reference/emcc.rst index 6f8700edd6ecb..288315d7bee53 100644 --- a/site/source/docs/tools_reference/emcc.rst +++ b/site/source/docs/tools_reference/emcc.rst @@ -362,7 +362,7 @@ Options that are modified or new in *emcc* are listed below: .. _emcc-clear-ports: ``--clear-ports`` - Manually clears the local copies and builds of projects from the Emscripten Ports repos (sdl2, etc.) + Manually clears the local copies of ports from the Emscripten Ports repos (sdl2, etc.). This also clears the cache, to remove their builds. You should only need to do this if a problem happens and you want all ports that you use to be downloaded and built from scratch. After this operation is complete, this process will exit. diff --git a/site/source/index.rst b/site/source/index.rst index 56323ddb37af8..6e108771f8355 100644 --- a/site/source/index.rst +++ b/site/source/index.rst @@ -28,6 +28,6 @@ docs/building_from_source/index docs/contributing/index docs/api_reference/index + docs/debugging/CyberDWARF docs/tools_reference/index docs/site/about - diff --git a/src/compiler.js b/src/compiler.js index fc70683980990..5b2c1022a8c17 100644 --- a/src/compiler.js +++ b/src/compiler.js @@ -179,14 +179,9 @@ RUNTIME_DEBUG = LIBRARY_DEBUG || GL_DEBUG; if (VERBOSE) printErr('VERBOSE is on, this generates a lot of output and can slow down compilation'); -if (!BOOTSTRAPPING_STRUCT_INFO) { +if (!BOOTSTRAPPING_STRUCT_INFO && !ONLY_MY_CODE) { // Load struct and define information. - //try { - var temp = JSON.parse(read(STRUCT_INFO)); - //} catch(e) { - // printErr('cannot load struct info at ' + STRUCT_INFO + ' : ' + e + ', trying in current dir'); - // temp = JSON.parse(read('struct_info.compiled.json')); - //} + var temp = JSON.parse(read(STRUCT_INFO)); C_STRUCTS = temp.structs; C_DEFINES = temp.defines; } else { diff --git a/src/ecmascript_simd.js b/src/ecmascript_simd.js index 81efacadaec72..7bd5dea7f7632 100644 --- a/src/ecmascript_simd.js +++ b/src/ecmascript_simd.js @@ -158,9 +158,9 @@ function simdReplaceLane(type, a, i, s) { function simdFrom(toType, fromType, a) { a = fromType.fn.check(a); for (var i = 0; i < fromType.lanes; i++) { - var v = fromType.fn.extractLane(a, i); + var v = Math.trunc(fromType.fn.extractLane(a, i)); if (toType.minVal !== undefined && - (v < toType.minVal || v > toType.maxVal)) { + !(toType.minVal <= v && v <= toType.maxVal)) { throw new RangeError("Can't convert value"); } lanes[i] = v; @@ -306,13 +306,10 @@ function simdLoad(type, tarray, index, count) { for (var i = 0; i < bytes; i++) { dst[i] = src[i]; } - - // XXX Emscripten: Work around to fix bug https://github.com/tc39/ecmascript_simd/issues/313 - for (var i = bytes; i < 16; ++i) { + var typeBytes = type.lanes * type.laneSize; + for (var i = bytes; i < typeBytes; i++) { dst[i] = 0; } - // XXX Emscripten - return newValue; } @@ -1207,8 +1204,7 @@ var simdFns = { shiftLeftByScalar: function(type) { return function(a, bits) { - if (bits>>>0 >= type.laneSize * 8) - return type.fn.splat(0); + bits &= type.laneSize * 8 - 1; return simdShiftOp(type, binaryShiftLeft, a, bits); } }, @@ -1217,14 +1213,12 @@ var simdFns = { function(type) { if (type.unsigned) { return function(a, bits) { - if (bits>>>0 >= type.laneSize * 8) - return type.fn.splat(0); + bits &= type.laneSize * 8 - 1; return simdShiftOp(type, binaryShiftRightLogical, a, bits); } } else { return function(a, bits) { - if (bits>>>0 >= type.laneSize * 8) - bits = type.laneSize * 8 - 1; + bits &= type.laneSize * 8 - 1; return simdShiftOp(type, binaryShiftRightArithmetic, a, bits); } } diff --git a/src/embind/embind.js b/src/embind/embind.js index dc7ad2fe823b7..b1a1f1deed693 100644 --- a/src/embind/embind.js +++ b/src/embind/embind.js @@ -1487,7 +1487,7 @@ var LibraryEmbind = { } }, - $makeClassHandle__deps: ['throwInternalError'], + $makeClassHandle__deps: ['$throwInternalError'], $makeClassHandle: function(prototype, record) { if (!record.ptrType || !record.ptr) { throwInternalError('makeClassHandle requires ptr and ptrType'); diff --git a/src/fastLong.js b/src/fastLong.js deleted file mode 100644 index 31289fc6510d9..0000000000000 --- a/src/fastLong.js +++ /dev/null @@ -1,299 +0,0 @@ -// ======== compiled code from system/lib/compiler-rt , see readme therein -function ___muldsi3($a, $b) { - $a = $a | 0; - $b = $b | 0; - var $1 = 0, $2 = 0, $3 = 0, $6 = 0, $8 = 0, $11 = 0, $12 = 0; - $1 = $a & 65535; - $2 = $b & 65535; - $3 = Math_imul($2, $1) | 0; - $6 = $a >>> 16; - $8 = ($3 >>> 16) + (Math_imul($2, $6) | 0) | 0; - $11 = $b >>> 16; - $12 = Math_imul($11, $1) | 0; - return ({{{ makeSetTempRet0('(($8 >>> 16) + (Math_imul($11, $6) | 0) | 0) + ((($8 & 65535) + $12 | 0) >>> 16) | 0') }}}, 0 | ($8 + $12 << 16 | $3 & 65535)) | 0; -} -function ___divdi3($a$0, $a$1, $b$0, $b$1) { - $a$0 = $a$0 | 0; - $a$1 = $a$1 | 0; - $b$0 = $b$0 | 0; - $b$1 = $b$1 | 0; - var $1$0 = 0, $1$1 = 0, $2$0 = 0, $2$1 = 0, $4$0 = 0, $4$1 = 0, $6$0 = 0, $7$0 = 0, $7$1 = 0, $8$0 = 0, $10$0 = 0; - $1$0 = $a$1 >> 31 | (($a$1 | 0) < 0 ? -1 : 0) << 1; - $1$1 = (($a$1 | 0) < 0 ? -1 : 0) >> 31 | (($a$1 | 0) < 0 ? -1 : 0) << 1; - $2$0 = $b$1 >> 31 | (($b$1 | 0) < 0 ? -1 : 0) << 1; - $2$1 = (($b$1 | 0) < 0 ? -1 : 0) >> 31 | (($b$1 | 0) < 0 ? -1 : 0) << 1; - $4$0 = _i64Subtract($1$0 ^ $a$0 | 0, $1$1 ^ $a$1 | 0, $1$0 | 0, $1$1 | 0) | 0; - $4$1 = {{{ makeGetTempRet0() }}}; - $6$0 = _i64Subtract($2$0 ^ $b$0 | 0, $2$1 ^ $b$1 | 0, $2$0 | 0, $2$1 | 0) | 0; - $7$0 = $2$0 ^ $1$0; - $7$1 = $2$1 ^ $1$1; - $8$0 = ___udivmoddi4($4$0, $4$1, $6$0, {{{ makeGetTempRet0() }}}, 0) | 0; - $10$0 = _i64Subtract($8$0 ^ $7$0 | 0, {{{ makeGetTempRet0() }}} ^ $7$1 | 0, $7$0 | 0, $7$1 | 0) | 0; - return $10$0 | 0; -} -function ___remdi3($a$0, $a$1, $b$0, $b$1) { - $a$0 = $a$0 | 0; - $a$1 = $a$1 | 0; - $b$0 = $b$0 | 0; - $b$1 = $b$1 | 0; - var $rem = 0, $1$0 = 0, $1$1 = 0, $2$0 = 0, $2$1 = 0, $4$0 = 0, $4$1 = 0, $6$0 = 0, $10$0 = 0, $10$1 = 0, __stackBase__ = 0; - __stackBase__ = STACKTOP; - STACKTOP = STACKTOP + 16 | 0; - $rem = __stackBase__ | 0; - $1$0 = $a$1 >> 31 | (($a$1 | 0) < 0 ? -1 : 0) << 1; - $1$1 = (($a$1 | 0) < 0 ? -1 : 0) >> 31 | (($a$1 | 0) < 0 ? -1 : 0) << 1; - $2$0 = $b$1 >> 31 | (($b$1 | 0) < 0 ? -1 : 0) << 1; - $2$1 = (($b$1 | 0) < 0 ? -1 : 0) >> 31 | (($b$1 | 0) < 0 ? -1 : 0) << 1; - $4$0 = _i64Subtract($1$0 ^ $a$0 | 0, $1$1 ^ $a$1 | 0, $1$0 | 0, $1$1 | 0) | 0; - $4$1 = {{{ makeGetTempRet0() }}}; - $6$0 = _i64Subtract($2$0 ^ $b$0 | 0, $2$1 ^ $b$1 | 0, $2$0 | 0, $2$1 | 0) | 0; - ___udivmoddi4($4$0, $4$1, $6$0, {{{ makeGetTempRet0() }}}, $rem) | 0; - $10$0 = _i64Subtract(HEAP32[$rem >> 2] ^ $1$0 | 0, HEAP32[$rem + 4 >> 2] ^ $1$1 | 0, $1$0 | 0, $1$1 | 0) | 0; - $10$1 = {{{ makeGetTempRet0() }}}; - STACKTOP = __stackBase__; - return ({{{ makeSetTempRet0('$10$1') }}}, $10$0) | 0; -} -function ___muldi3($a$0, $a$1, $b$0, $b$1) { - $a$0 = $a$0 | 0; - $a$1 = $a$1 | 0; - $b$0 = $b$0 | 0; - $b$1 = $b$1 | 0; - var $x_sroa_0_0_extract_trunc = 0, $y_sroa_0_0_extract_trunc = 0, $1$0 = 0, $1$1 = 0, $2 = 0; - $x_sroa_0_0_extract_trunc = $a$0; - $y_sroa_0_0_extract_trunc = $b$0; - $1$0 = ___muldsi3($x_sroa_0_0_extract_trunc, $y_sroa_0_0_extract_trunc) | 0; - $1$1 = {{{ makeGetTempRet0() }}}; - $2 = Math_imul($a$1, $y_sroa_0_0_extract_trunc) | 0; - return ({{{ makeSetTempRet0('((Math_imul($b$1, $x_sroa_0_0_extract_trunc) | 0) + $2 | 0) + $1$1 | $1$1 & 0') }}}, 0 | $1$0 & -1) | 0; -} -function ___udivdi3($a$0, $a$1, $b$0, $b$1) { - $a$0 = $a$0 | 0; - $a$1 = $a$1 | 0; - $b$0 = $b$0 | 0; - $b$1 = $b$1 | 0; - var $1$0 = 0; - $1$0 = ___udivmoddi4($a$0, $a$1, $b$0, $b$1, 0) | 0; - return $1$0 | 0; -} -function ___uremdi3($a$0, $a$1, $b$0, $b$1) { - $a$0 = $a$0 | 0; - $a$1 = $a$1 | 0; - $b$0 = $b$0 | 0; - $b$1 = $b$1 | 0; - var $rem = 0, __stackBase__ = 0; - __stackBase__ = STACKTOP; - STACKTOP = STACKTOP + 16 | 0; - $rem = __stackBase__ | 0; - ___udivmoddi4($a$0, $a$1, $b$0, $b$1, $rem) | 0; - STACKTOP = __stackBase__; - return ({{{ makeSetTempRet0('HEAP32[$rem + 4 >> 2] | 0') }}}, HEAP32[$rem >> 2] | 0) | 0; -} -function ___udivmoddi4($a$0, $a$1, $b$0, $b$1, $rem) { - $a$0 = $a$0 | 0; - $a$1 = $a$1 | 0; - $b$0 = $b$0 | 0; - $b$1 = $b$1 | 0; - $rem = $rem | 0; - var $n_sroa_0_0_extract_trunc = 0, $n_sroa_1_4_extract_shift$0 = 0, $n_sroa_1_4_extract_trunc = 0, $d_sroa_0_0_extract_trunc = 0, $d_sroa_1_4_extract_shift$0 = 0, $d_sroa_1_4_extract_trunc = 0, $4 = 0, $17 = 0, $37 = 0, $49 = 0, $51 = 0, $57 = 0, $58 = 0, $66 = 0, $78 = 0, $86 = 0, $88 = 0, $89 = 0, $91 = 0, $92 = 0, $95 = 0, $105 = 0, $117 = 0, $119 = 0, $125 = 0, $126 = 0, $130 = 0, $q_sroa_1_1_ph = 0, $q_sroa_0_1_ph = 0, $r_sroa_1_1_ph = 0, $r_sroa_0_1_ph = 0, $sr_1_ph = 0, $d_sroa_0_0_insert_insert99$0 = 0, $d_sroa_0_0_insert_insert99$1 = 0, $137$0 = 0, $137$1 = 0, $carry_0203 = 0, $sr_1202 = 0, $r_sroa_0_1201 = 0, $r_sroa_1_1200 = 0, $q_sroa_0_1199 = 0, $q_sroa_1_1198 = 0, $147 = 0, $149 = 0, $r_sroa_0_0_insert_insert42$0 = 0, $r_sroa_0_0_insert_insert42$1 = 0, $150$1 = 0, $151$0 = 0, $152 = 0, $154$0 = 0, $r_sroa_0_0_extract_trunc = 0, $r_sroa_1_4_extract_trunc = 0, $155 = 0, $carry_0_lcssa$0 = 0, $carry_0_lcssa$1 = 0, $r_sroa_0_1_lcssa = 0, $r_sroa_1_1_lcssa = 0, $q_sroa_0_1_lcssa = 0, $q_sroa_1_1_lcssa = 0, $q_sroa_0_0_insert_ext75$0 = 0, $q_sroa_0_0_insert_ext75$1 = 0, $q_sroa_0_0_insert_insert77$1 = 0, $_0$0 = 0, $_0$1 = 0; - $n_sroa_0_0_extract_trunc = $a$0; - $n_sroa_1_4_extract_shift$0 = $a$1; - $n_sroa_1_4_extract_trunc = $n_sroa_1_4_extract_shift$0; - $d_sroa_0_0_extract_trunc = $b$0; - $d_sroa_1_4_extract_shift$0 = $b$1; - $d_sroa_1_4_extract_trunc = $d_sroa_1_4_extract_shift$0; - if (($n_sroa_1_4_extract_trunc | 0) == 0) { - $4 = ($rem | 0) != 0; - if (($d_sroa_1_4_extract_trunc | 0) == 0) { - if ($4) { - HEAP32[$rem >> 2] = ($n_sroa_0_0_extract_trunc >>> 0) % ($d_sroa_0_0_extract_trunc >>> 0); - HEAP32[$rem + 4 >> 2] = 0; - } - $_0$1 = 0; - $_0$0 = ($n_sroa_0_0_extract_trunc >>> 0) / ($d_sroa_0_0_extract_trunc >>> 0) >>> 0; - return ({{{ makeSetTempRet0('$_0$1') }}}, $_0$0) | 0; - } else { - if (!$4) { - $_0$1 = 0; - $_0$0 = 0; - return ({{{ makeSetTempRet0('$_0$1') }}}, $_0$0) | 0; - } - HEAP32[$rem >> 2] = $a$0 & -1; - HEAP32[$rem + 4 >> 2] = $a$1 & 0; - $_0$1 = 0; - $_0$0 = 0; - return ({{{ makeSetTempRet0('$_0$1') }}}, $_0$0) | 0; - } - } - $17 = ($d_sroa_1_4_extract_trunc | 0) == 0; - do { - if (($d_sroa_0_0_extract_trunc | 0) == 0) { - if ($17) { - if (($rem | 0) != 0) { - HEAP32[$rem >> 2] = ($n_sroa_1_4_extract_trunc >>> 0) % ($d_sroa_0_0_extract_trunc >>> 0); - HEAP32[$rem + 4 >> 2] = 0; - } - $_0$1 = 0; - $_0$0 = ($n_sroa_1_4_extract_trunc >>> 0) / ($d_sroa_0_0_extract_trunc >>> 0) >>> 0; - return ({{{ makeSetTempRet0('$_0$1') }}}, $_0$0) | 0; - } - if (($n_sroa_0_0_extract_trunc | 0) == 0) { - if (($rem | 0) != 0) { - HEAP32[$rem >> 2] = 0; - HEAP32[$rem + 4 >> 2] = ($n_sroa_1_4_extract_trunc >>> 0) % ($d_sroa_1_4_extract_trunc >>> 0); - } - $_0$1 = 0; - $_0$0 = ($n_sroa_1_4_extract_trunc >>> 0) / ($d_sroa_1_4_extract_trunc >>> 0) >>> 0; - return ({{{ makeSetTempRet0('$_0$1') }}}, $_0$0) | 0; - } - $37 = $d_sroa_1_4_extract_trunc - 1 | 0; - if (($37 & $d_sroa_1_4_extract_trunc | 0) == 0) { - if (($rem | 0) != 0) { - HEAP32[$rem >> 2] = 0 | $a$0 & -1; - HEAP32[$rem + 4 >> 2] = $37 & $n_sroa_1_4_extract_trunc | $a$1 & 0; - } - $_0$1 = 0; - $_0$0 = $n_sroa_1_4_extract_trunc >>> ((_llvm_cttz_i32($d_sroa_1_4_extract_trunc | 0) | 0) >>> 0); - return ({{{ makeSetTempRet0('$_0$1') }}}, $_0$0) | 0; - } - $49 = Math_clz32($d_sroa_1_4_extract_trunc | 0) | 0; - $51 = $49 - (Math_clz32($n_sroa_1_4_extract_trunc | 0) | 0) | 0; - if ($51 >>> 0 <= 30) { - $57 = $51 + 1 | 0; - $58 = 31 - $51 | 0; - $sr_1_ph = $57; - $r_sroa_0_1_ph = $n_sroa_1_4_extract_trunc << $58 | $n_sroa_0_0_extract_trunc >>> ($57 >>> 0); - $r_sroa_1_1_ph = $n_sroa_1_4_extract_trunc >>> ($57 >>> 0); - $q_sroa_0_1_ph = 0; - $q_sroa_1_1_ph = $n_sroa_0_0_extract_trunc << $58; - break; - } - if (($rem | 0) == 0) { - $_0$1 = 0; - $_0$0 = 0; - return ({{{ makeSetTempRet0('$_0$1') }}}, $_0$0) | 0; - } - HEAP32[$rem >> 2] = 0 | $a$0 & -1; - HEAP32[$rem + 4 >> 2] = $n_sroa_1_4_extract_shift$0 | $a$1 & 0; - $_0$1 = 0; - $_0$0 = 0; - return ({{{ makeSetTempRet0('$_0$1') }}}, $_0$0) | 0; - } else { - if (!$17) { - $117 = Math_clz32($d_sroa_1_4_extract_trunc | 0) | 0; - $119 = $117 - (Math_clz32($n_sroa_1_4_extract_trunc | 0) | 0) | 0; - if ($119 >>> 0 <= 31) { - $125 = $119 + 1 | 0; - $126 = 31 - $119 | 0; - $130 = $119 - 31 >> 31; - $sr_1_ph = $125; - $r_sroa_0_1_ph = $n_sroa_0_0_extract_trunc >>> ($125 >>> 0) & $130 | $n_sroa_1_4_extract_trunc << $126; - $r_sroa_1_1_ph = $n_sroa_1_4_extract_trunc >>> ($125 >>> 0) & $130; - $q_sroa_0_1_ph = 0; - $q_sroa_1_1_ph = $n_sroa_0_0_extract_trunc << $126; - break; - } - if (($rem | 0) == 0) { - $_0$1 = 0; - $_0$0 = 0; - return ({{{ makeSetTempRet0('$_0$1') }}}, $_0$0) | 0; - } - HEAP32[$rem >> 2] = 0 | $a$0 & -1; - HEAP32[$rem + 4 >> 2] = $n_sroa_1_4_extract_shift$0 | $a$1 & 0; - $_0$1 = 0; - $_0$0 = 0; - return ({{{ makeSetTempRet0('$_0$1') }}}, $_0$0) | 0; - } - $66 = $d_sroa_0_0_extract_trunc - 1 | 0; - if (($66 & $d_sroa_0_0_extract_trunc | 0) != 0) { - $86 = (Math_clz32($d_sroa_0_0_extract_trunc | 0) | 0) + 33 | 0; - $88 = $86 - (Math_clz32($n_sroa_1_4_extract_trunc | 0) | 0) | 0; - $89 = 64 - $88 | 0; - $91 = 32 - $88 | 0; - $92 = $91 >> 31; - $95 = $88 - 32 | 0; - $105 = $95 >> 31; - $sr_1_ph = $88; - $r_sroa_0_1_ph = $91 - 1 >> 31 & $n_sroa_1_4_extract_trunc >>> ($95 >>> 0) | ($n_sroa_1_4_extract_trunc << $91 | $n_sroa_0_0_extract_trunc >>> ($88 >>> 0)) & $105; - $r_sroa_1_1_ph = $105 & $n_sroa_1_4_extract_trunc >>> ($88 >>> 0); - $q_sroa_0_1_ph = $n_sroa_0_0_extract_trunc << $89 & $92; - $q_sroa_1_1_ph = ($n_sroa_1_4_extract_trunc << $89 | $n_sroa_0_0_extract_trunc >>> ($95 >>> 0)) & $92 | $n_sroa_0_0_extract_trunc << $91 & $88 - 33 >> 31; - break; - } - if (($rem | 0) != 0) { - HEAP32[$rem >> 2] = $66 & $n_sroa_0_0_extract_trunc; - HEAP32[$rem + 4 >> 2] = 0; - } - if (($d_sroa_0_0_extract_trunc | 0) == 1) { - $_0$1 = $n_sroa_1_4_extract_shift$0 | $a$1 & 0; - $_0$0 = 0 | $a$0 & -1; - return ({{{ makeSetTempRet0('$_0$1') }}}, $_0$0) | 0; - } else { - $78 = _llvm_cttz_i32($d_sroa_0_0_extract_trunc | 0) | 0; - $_0$1 = 0 | $n_sroa_1_4_extract_trunc >>> ($78 >>> 0); - $_0$0 = $n_sroa_1_4_extract_trunc << 32 - $78 | $n_sroa_0_0_extract_trunc >>> ($78 >>> 0) | 0; - return ({{{ makeSetTempRet0('$_0$1') }}}, $_0$0) | 0; - } - } - } while (0); - if (($sr_1_ph | 0) == 0) { - $q_sroa_1_1_lcssa = $q_sroa_1_1_ph; - $q_sroa_0_1_lcssa = $q_sroa_0_1_ph; - $r_sroa_1_1_lcssa = $r_sroa_1_1_ph; - $r_sroa_0_1_lcssa = $r_sroa_0_1_ph; - $carry_0_lcssa$1 = 0; - $carry_0_lcssa$0 = 0; - } else { - $d_sroa_0_0_insert_insert99$0 = 0 | $b$0 & -1; - $d_sroa_0_0_insert_insert99$1 = $d_sroa_1_4_extract_shift$0 | $b$1 & 0; - $137$0 = _i64Add($d_sroa_0_0_insert_insert99$0 | 0, $d_sroa_0_0_insert_insert99$1 | 0, -1, -1) | 0; - $137$1 = {{{ makeGetTempRet0() }}}; - $q_sroa_1_1198 = $q_sroa_1_1_ph; - $q_sroa_0_1199 = $q_sroa_0_1_ph; - $r_sroa_1_1200 = $r_sroa_1_1_ph; - $r_sroa_0_1201 = $r_sroa_0_1_ph; - $sr_1202 = $sr_1_ph; - $carry_0203 = 0; - while (1) { - $147 = $q_sroa_0_1199 >>> 31 | $q_sroa_1_1198 << 1; - $149 = $carry_0203 | $q_sroa_0_1199 << 1; - $r_sroa_0_0_insert_insert42$0 = 0 | ($r_sroa_0_1201 << 1 | $q_sroa_1_1198 >>> 31); - $r_sroa_0_0_insert_insert42$1 = $r_sroa_0_1201 >>> 31 | $r_sroa_1_1200 << 1 | 0; - _i64Subtract($137$0 | 0, $137$1 | 0, $r_sroa_0_0_insert_insert42$0 | 0, $r_sroa_0_0_insert_insert42$1 | 0) | 0; - $150$1 = {{{ makeGetTempRet0() }}}; - $151$0 = $150$1 >> 31 | (($150$1 | 0) < 0 ? -1 : 0) << 1; - $152 = $151$0 & 1; - $154$0 = _i64Subtract($r_sroa_0_0_insert_insert42$0 | 0, $r_sroa_0_0_insert_insert42$1 | 0, $151$0 & $d_sroa_0_0_insert_insert99$0 | 0, ((($150$1 | 0) < 0 ? -1 : 0) >> 31 | (($150$1 | 0) < 0 ? -1 : 0) << 1) & $d_sroa_0_0_insert_insert99$1 | 0) | 0; - $r_sroa_0_0_extract_trunc = $154$0; - $r_sroa_1_4_extract_trunc = {{{ makeGetTempRet0() }}}; - $155 = $sr_1202 - 1 | 0; - if (($155 | 0) == 0) { - break; - } else { - $q_sroa_1_1198 = $147; - $q_sroa_0_1199 = $149; - $r_sroa_1_1200 = $r_sroa_1_4_extract_trunc; - $r_sroa_0_1201 = $r_sroa_0_0_extract_trunc; - $sr_1202 = $155; - $carry_0203 = $152; - } - } - $q_sroa_1_1_lcssa = $147; - $q_sroa_0_1_lcssa = $149; - $r_sroa_1_1_lcssa = $r_sroa_1_4_extract_trunc; - $r_sroa_0_1_lcssa = $r_sroa_0_0_extract_trunc; - $carry_0_lcssa$1 = 0; - $carry_0_lcssa$0 = $152; - } - $q_sroa_0_0_insert_ext75$0 = $q_sroa_0_1_lcssa; - $q_sroa_0_0_insert_ext75$1 = 0; - $q_sroa_0_0_insert_insert77$1 = $q_sroa_1_1_lcssa | $q_sroa_0_0_insert_ext75$1; - if (($rem | 0) != 0) { - HEAP32[$rem >> 2] = 0 | $r_sroa_0_1_lcssa; - HEAP32[$rem + 4 >> 2] = $r_sroa_1_1_lcssa | 0; - } - $_0$1 = (0 | $q_sroa_0_0_insert_ext75$0) >>> 31 | $q_sroa_0_0_insert_insert77$1 << 1 | ($q_sroa_0_0_insert_ext75$1 << 1 | $q_sroa_0_0_insert_ext75$0 >>> 31) & 0 | $carry_0_lcssa$1; - $_0$0 = ($q_sroa_0_0_insert_ext75$0 << 1 | 0 >>> 31) & -2 | $carry_0_lcssa$0; - return ({{{ makeSetTempRet0('$_0$1') }}}, $_0$0) | 0; -} -// ======================================================================= - diff --git a/src/jsifier.js b/src/jsifier.js index e31c7ed9b5cbb..281ac5df3e76c 100644 --- a/src/jsifier.js +++ b/src/jsifier.js @@ -16,8 +16,6 @@ var INDENTATION = ' '; var functionStubSigs = {}; -var ALWAYS_EMITTED_I64_FUNCS = set('i64Add', 'i64Subtract', 'bitshift64Shl', 'bitshift64Lshr', 'bitshift64Ashr'); // even in side modules - // JSifier function JSify(data, functionsOnly) { //B.start('jsifier'); @@ -152,7 +150,6 @@ function JSify(data, functionsOnly) { LibraryManager.library[shortident] = new Function("Module['printErr']('missing function: " + shortident + "'); abort(-1);"); } else { var target = (MAIN_MODULE ? '' : 'parent') + "Module['_" + shortident + "']"; - if (SIDE_MODULE && (ident in ALWAYS_EMITTED_I64_FUNCS)) return ''; // we emit i64Add etc. even in side modules (small, and should be fast) var assertion = ''; if (ASSERTIONS) assertion = 'if (!' + target + ') abort("external function \'' + shortident + '\' is missing. perhaps a side module was not linked in? if this function was expected to arrive from a system library, try to build the MAIN_MODULE with EMCC_FORCE_STDLIBS=1 in the environment");'; LibraryManager.library[shortident] = new Function(assertion + "return " + target + ".apply(null, arguments);"); @@ -409,39 +406,7 @@ function JSify(data, functionsOnly) { // This is the main 'post' pass. Print out the generated code that we have here, together with the // rest of the output that we started to print out earlier (see comment on the // "Final shape that will be created"). - if (PRECISE_I64_MATH && (Types.preciseI64MathUsed || PRECISE_I64_MATH == 2)) { - if (SIDE_MODULE) { - print('// ASM_LIBRARY FUNCTIONS'); // fastLong.js etc. code is indeed asm library code - } - if (!INCLUDE_FULL_LIBRARY) { - // first row are utilities called from generated code, second are needed from fastLong - ['i64Add', 'i64Subtract', 'bitshift64Shl', 'bitshift64Lshr', 'bitshift64Ashr', - 'llvm_cttz_i32'].forEach(function(ident) { - var finalName = '_' + ident; - if (!Functions.libraryFunctions[finalName] || (ident[0] === 'l' && !addedLibraryItems[ident])) { // TODO: one-by-one in fastcomp glue mode - print(processLibraryFunction(LibraryManager.library[ident], ident, finalName)); // must be first to be close to generated code - Functions.implementedFunctions[finalName] = LibraryManager.library[ident + '__sig']; - Functions.libraryFunctions[finalName] = 2; // XXX - // limited dependency handling - var deps = LibraryManager.library[ident + '__deps']; - if (deps) { - deps.forEach(function(dep) { - assert(typeof dep == 'function'); - var text = dep(); - assert(text.indexOf('\n') < 0); - text = text.replace('ALLOC_STATIC', 'ALLOC_DYNAMIC'); - print('/* PRE_ASM */ ' + text + '\n'); - }); - } - } - }); - } - // these may be duplicated in side modules and the main module without issue - print(processMacros(read('fastLong.js'))); - print('// EMSCRIPTEN_END_FUNCS\n'); - } else { - print('// EMSCRIPTEN_END_FUNCS\n'); - } + print('// EMSCRIPTEN_END_FUNCS\n'); if (HEADLESS) { print('if (!ENVIRONMENT_IS_WEB) {'); diff --git a/src/library.js b/src/library.js index 18dc55197488a..9917fc79862f1 100644 --- a/src/library.js +++ b/src/library.js @@ -893,7 +893,9 @@ LibraryManager.library = { return 'var cttz_i8 = allocate([' + range(256).map(function(x) { return cttz(x) }).join(',') + '], "i8", ALLOC_STATIC);'; #endif }], +#if BINARYEN == 0 // binaryen will convert these calls to wasm anyhow llvm_cttz_i32__asm: true, +#endif llvm_cttz_i32__sig: 'ii', llvm_cttz_i32: function(x) { x = x|0; @@ -1211,51 +1213,6 @@ LibraryManager.library = { {{{ makeThrow('ptr') }}} }, - llvm_uadd_with_overflow_i8: function(x, y) { - x = x & 0xff; - y = y & 0xff; - {{{ makeStructuralReturn(['(x+y) & 0xff', 'x+y > 255']) }}}; - }, - - llvm_umul_with_overflow_i8: function(x, y) { - x = x & 0xff; - y = y & 0xff; - {{{ makeStructuralReturn(['(x*y) & 0xff', 'x*y > 255']) }}}; - }, - - llvm_uadd_with_overflow_i16: function(x, y) { - x = x & 0xffff; - y = y & 0xffff; - {{{ makeStructuralReturn(['(x+y) & 0xffff', 'x+y > 65535']) }}}; - }, - - llvm_umul_with_overflow_i16: function(x, y) { - x = x & 0xffff; - y = y & 0xffff; - {{{ makeStructuralReturn(['(x*y) & 0xffff', 'x*y > 65535']) }}}; - }, - - llvm_uadd_with_overflow_i32: function(x, y) { - x = x>>>0; - y = y>>>0; - {{{ makeStructuralReturn(['(x+y)>>>0', 'x+y > 4294967295']) }}}; - }, - - llvm_umul_with_overflow_i32: function(x, y) { - x = x>>>0; - y = y>>>0; - {{{ makeStructuralReturn(['(x*y)>>>0', 'x*y > 4294967295']) }}}; - }, - - llvm_umul_with_overflow_i64__deps: [function() { Types.preciseI64MathUsed = 1 }], - llvm_umul_with_overflow_i64: function(xl, xh, yl, yh) { -#if ASSERTIONS - Runtime.warnOnce('no overflow support in llvm_umul_with_overflow_i64'); -#endif - var low = ___muldi3(xl, xh, yl, yh); - {{{ makeStructuralReturn(['low', makeGetTempRet0(), '0']) }}}; - }, - llvm_stacksave: function() { var self = _llvm_stacksave; if (!self.LLVM_SAVEDSTACKS) { @@ -3679,25 +3636,21 @@ LibraryManager.library = { return Math.random(); }, - emscripten_get_now: function() { - if (!_emscripten_get_now.actual) { - if (ENVIRONMENT_IS_NODE) { - _emscripten_get_now.actual = function _emscripten_get_now_actual() { - var t = process['hrtime'](); - return t[0] * 1e3 + t[1] / 1e6; - } - } else if (typeof dateNow !== 'undefined') { - _emscripten_get_now.actual = dateNow; - } else if (typeof self === 'object' && self['performance'] && typeof self['performance']['now'] === 'function') { - _emscripten_get_now.actual = function _emscripten_get_now_actual() { return self['performance']['now'](); }; - } else if (typeof performance === 'object' && typeof performance['now'] === 'function') { - _emscripten_get_now.actual = function _emscripten_get_now_actual() { return performance['now'](); }; - } else { - _emscripten_get_now.actual = Date.now; - } - } - return _emscripten_get_now.actual(); - }, + emscripten_get_now: function() { abort() }, // replaced by the postset at startup time + emscripten_get_now__postset: "if (ENVIRONMENT_IS_NODE) {\n" + + " _emscripten_get_now = function _emscripten_get_now_actual() {\n" + + " var t = process['hrtime']();\n" + + " return t[0] * 1e3 + t[1] / 1e6;\n" + + " };\n" + + "} else if (typeof dateNow !== 'undefined') {\n" + + " _emscripten_get_now = dateNow;\n" + + "} else if (typeof self === 'object' && self['performance'] && typeof self['performance']['now'] === 'function') {\n" + + " _emscripten_get_now = function() { return self['performance']['now'](); };\n" + + "} else if (typeof performance === 'object' && typeof performance['now'] === 'function') {\n" + + " _emscripten_get_now = function() { return performance['now'](); };\n" + + "} else {\n" + + " _emscripten_get_now = Date.now;\n" + + "}", emscripten_get_now_res: function() { // return resolution of get_now, in nanoseconds if (ENVIRONMENT_IS_NODE) { @@ -3947,20 +3900,6 @@ LibraryManager.library = { h = (b + d + (((l>>>0) < (a>>>0))|0))>>>0; // Add carry from low word to high word on overflow. {{{ makeStructuralReturn(['l|0', 'h'], true) }}}; }, - llvm_uadd_with_overflow_i64__asm: true, - llvm_uadd_with_overflow_i64__sig: 'iiiii', - llvm_uadd_with_overflow_i64: function(a, b, c, d) { - a = a|0; b = b|0; c = c|0; d = d|0; - var l = 0, h = 0, overflow = 0; - l = (a + c)>>>0; - h = (b + d)>>>0; - overflow = ((h>>>0) < (b>>>0))|0; // Return whether addition overflowed even the high word. - if ((l>>>0) < (a>>>0)) { - h = (h + 1)>>>0; // Add carry from low word to high word on overflow. - overflow = overflow | (!h); // Check again for overflow. - } - {{{ makeStructuralReturn(['l|0', 'h', 'overflow'], true) }}}; - }, i64Subtract__asm: true, i64Subtract__sig: 'iiiii', @@ -4111,6 +4050,326 @@ LibraryManager.library = { emscripten_asm_const: true, emscripten_asm_const_int: true, emscripten_asm_const_double: true, + + // ======== compiled code from system/lib/compiler-rt , see readme therein + __muldsi3__asm: true, + __muldsi3__sig: 'iii', + __muldsi3: function($a, $b) { + $a = $a | 0; + $b = $b | 0; + var $1 = 0, $2 = 0, $3 = 0, $6 = 0, $8 = 0, $11 = 0, $12 = 0; + $1 = $a & 65535; + $2 = $b & 65535; + $3 = Math_imul($2, $1) | 0; + $6 = $a >>> 16; + $8 = ($3 >>> 16) + (Math_imul($2, $6) | 0) | 0; + $11 = $b >>> 16; + $12 = Math_imul($11, $1) | 0; + return ({{{ makeSetTempRet0('(($8 >>> 16) + (Math_imul($11, $6) | 0) | 0) + ((($8 & 65535) + $12 | 0) >>> 16) | 0') }}}, 0 | ($8 + $12 << 16 | $3 & 65535)) | 0; + }, + __divdi3__sig: 'iiiii', + __divdi3__asm: true, + __divdi3__deps: ['__udivmoddi4', 'i64Subtract'], + __divdi3: function($a$0, $a$1, $b$0, $b$1) { + $a$0 = $a$0 | 0; + $a$1 = $a$1 | 0; + $b$0 = $b$0 | 0; + $b$1 = $b$1 | 0; + var $1$0 = 0, $1$1 = 0, $2$0 = 0, $2$1 = 0, $4$0 = 0, $4$1 = 0, $6$0 = 0, $7$0 = 0, $7$1 = 0, $8$0 = 0, $10$0 = 0; + $1$0 = $a$1 >> 31 | (($a$1 | 0) < 0 ? -1 : 0) << 1; + $1$1 = (($a$1 | 0) < 0 ? -1 : 0) >> 31 | (($a$1 | 0) < 0 ? -1 : 0) << 1; + $2$0 = $b$1 >> 31 | (($b$1 | 0) < 0 ? -1 : 0) << 1; + $2$1 = (($b$1 | 0) < 0 ? -1 : 0) >> 31 | (($b$1 | 0) < 0 ? -1 : 0) << 1; + $4$0 = _i64Subtract($1$0 ^ $a$0 | 0, $1$1 ^ $a$1 | 0, $1$0 | 0, $1$1 | 0) | 0; + $4$1 = {{{ makeGetTempRet0() }}}; + $6$0 = _i64Subtract($2$0 ^ $b$0 | 0, $2$1 ^ $b$1 | 0, $2$0 | 0, $2$1 | 0) | 0; + $7$0 = $2$0 ^ $1$0; + $7$1 = $2$1 ^ $1$1; + $8$0 = ___udivmoddi4($4$0, $4$1, $6$0, {{{ makeGetTempRet0() }}}, 0) | 0; + $10$0 = _i64Subtract($8$0 ^ $7$0 | 0, {{{ makeGetTempRet0() }}} ^ $7$1 | 0, $7$0 | 0, $7$1 | 0) | 0; + return $10$0 | 0; + }, + __remdi3__sig: 'iiiii', + __remdi3__asm: true, + __remdi3__deps: ['__udivmoddi4', 'i64Subtract'], + __remdi3: function($a$0, $a$1, $b$0, $b$1) { + $a$0 = $a$0 | 0; + $a$1 = $a$1 | 0; + $b$0 = $b$0 | 0; + $b$1 = $b$1 | 0; + var $rem = 0, $1$0 = 0, $1$1 = 0, $2$0 = 0, $2$1 = 0, $4$0 = 0, $4$1 = 0, $6$0 = 0, $10$0 = 0, $10$1 = 0, __stackBase__ = 0; + __stackBase__ = STACKTOP; + STACKTOP = STACKTOP + 16 | 0; + $rem = __stackBase__ | 0; + $1$0 = $a$1 >> 31 | (($a$1 | 0) < 0 ? -1 : 0) << 1; + $1$1 = (($a$1 | 0) < 0 ? -1 : 0) >> 31 | (($a$1 | 0) < 0 ? -1 : 0) << 1; + $2$0 = $b$1 >> 31 | (($b$1 | 0) < 0 ? -1 : 0) << 1; + $2$1 = (($b$1 | 0) < 0 ? -1 : 0) >> 31 | (($b$1 | 0) < 0 ? -1 : 0) << 1; + $4$0 = _i64Subtract($1$0 ^ $a$0 | 0, $1$1 ^ $a$1 | 0, $1$0 | 0, $1$1 | 0) | 0; + $4$1 = {{{ makeGetTempRet0() }}}; + $6$0 = _i64Subtract($2$0 ^ $b$0 | 0, $2$1 ^ $b$1 | 0, $2$0 | 0, $2$1 | 0) | 0; + ___udivmoddi4($4$0, $4$1, $6$0, {{{ makeGetTempRet0() }}}, $rem) | 0; + $10$0 = _i64Subtract(HEAP32[$rem >> 2] ^ $1$0 | 0, HEAP32[$rem + 4 >> 2] ^ $1$1 | 0, $1$0 | 0, $1$1 | 0) | 0; + $10$1 = {{{ makeGetTempRet0() }}}; + STACKTOP = __stackBase__; + return ({{{ makeSetTempRet0('$10$1') }}}, $10$0) | 0; + }, + __muldi3__sig: 'iiiii', + __muldi3__asm: true, + __muldi3__deps: ['__muldsi3'], + __muldi3: function($a$0, $a$1, $b$0, $b$1) { + $a$0 = $a$0 | 0; + $a$1 = $a$1 | 0; + $b$0 = $b$0 | 0; + $b$1 = $b$1 | 0; + var $x_sroa_0_0_extract_trunc = 0, $y_sroa_0_0_extract_trunc = 0, $1$0 = 0, $1$1 = 0, $2 = 0; + $x_sroa_0_0_extract_trunc = $a$0; + $y_sroa_0_0_extract_trunc = $b$0; + $1$0 = ___muldsi3($x_sroa_0_0_extract_trunc, $y_sroa_0_0_extract_trunc) | 0; + $1$1 = {{{ makeGetTempRet0() }}}; + $2 = Math_imul($a$1, $y_sroa_0_0_extract_trunc) | 0; + return ({{{ makeSetTempRet0('((Math_imul($b$1, $x_sroa_0_0_extract_trunc) | 0) + $2 | 0) + $1$1 | $1$1 & 0') }}}, 0 | $1$0 & -1) | 0; + }, + __udivdi3__sig: 'iiiii', + __udivdi3__asm: true, + __udivdi3__deps: ['__udivmoddi4'], + __udivdi3: function($a$0, $a$1, $b$0, $b$1) { + $a$0 = $a$0 | 0; + $a$1 = $a$1 | 0; + $b$0 = $b$0 | 0; + $b$1 = $b$1 | 0; + var $1$0 = 0; + $1$0 = ___udivmoddi4($a$0, $a$1, $b$0, $b$1, 0) | 0; + return $1$0 | 0; + }, + __uremdi3__sig: 'iiiii', + __uremdi3__asm: true, + __uremdi3__deps: ['__udivmoddi4'], + __uremdi3: function($a$0, $a$1, $b$0, $b$1) { + $a$0 = $a$0 | 0; + $a$1 = $a$1 | 0; + $b$0 = $b$0 | 0; + $b$1 = $b$1 | 0; + var $rem = 0, __stackBase__ = 0; + __stackBase__ = STACKTOP; + STACKTOP = STACKTOP + 16 | 0; + $rem = __stackBase__ | 0; + ___udivmoddi4($a$0, $a$1, $b$0, $b$1, $rem) | 0; + STACKTOP = __stackBase__; + return ({{{ makeSetTempRet0('HEAP32[$rem + 4 >> 2] | 0') }}}, HEAP32[$rem >> 2] | 0) | 0; + }, + __udivmoddi4__sig: 'iiiiii', + __udivmoddi4__asm: true, + __udivmoddi4__deps: ['i64Add', 'i64Subtract', 'llvm_cttz_i32'], + __udivmoddi4: function($a$0, $a$1, $b$0, $b$1, $rem) { + $a$0 = $a$0 | 0; + $a$1 = $a$1 | 0; + $b$0 = $b$0 | 0; + $b$1 = $b$1 | 0; + $rem = $rem | 0; + var $n_sroa_0_0_extract_trunc = 0, $n_sroa_1_4_extract_shift$0 = 0, $n_sroa_1_4_extract_trunc = 0, $d_sroa_0_0_extract_trunc = 0, $d_sroa_1_4_extract_shift$0 = 0, $d_sroa_1_4_extract_trunc = 0, $4 = 0, $17 = 0, $37 = 0, $49 = 0, $51 = 0, $57 = 0, $58 = 0, $66 = 0, $78 = 0, $86 = 0, $88 = 0, $89 = 0, $91 = 0, $92 = 0, $95 = 0, $105 = 0, $117 = 0, $119 = 0, $125 = 0, $126 = 0, $130 = 0, $q_sroa_1_1_ph = 0, $q_sroa_0_1_ph = 0, $r_sroa_1_1_ph = 0, $r_sroa_0_1_ph = 0, $sr_1_ph = 0, $d_sroa_0_0_insert_insert99$0 = 0, $d_sroa_0_0_insert_insert99$1 = 0, $137$0 = 0, $137$1 = 0, $carry_0203 = 0, $sr_1202 = 0, $r_sroa_0_1201 = 0, $r_sroa_1_1200 = 0, $q_sroa_0_1199 = 0, $q_sroa_1_1198 = 0, $147 = 0, $149 = 0, $r_sroa_0_0_insert_insert42$0 = 0, $r_sroa_0_0_insert_insert42$1 = 0, $150$1 = 0, $151$0 = 0, $152 = 0, $154$0 = 0, $r_sroa_0_0_extract_trunc = 0, $r_sroa_1_4_extract_trunc = 0, $155 = 0, $carry_0_lcssa$0 = 0, $carry_0_lcssa$1 = 0, $r_sroa_0_1_lcssa = 0, $r_sroa_1_1_lcssa = 0, $q_sroa_0_1_lcssa = 0, $q_sroa_1_1_lcssa = 0, $q_sroa_0_0_insert_ext75$0 = 0, $q_sroa_0_0_insert_ext75$1 = 0, $q_sroa_0_0_insert_insert77$1 = 0, $_0$0 = 0, $_0$1 = 0; + $n_sroa_0_0_extract_trunc = $a$0; + $n_sroa_1_4_extract_shift$0 = $a$1; + $n_sroa_1_4_extract_trunc = $n_sroa_1_4_extract_shift$0; + $d_sroa_0_0_extract_trunc = $b$0; + $d_sroa_1_4_extract_shift$0 = $b$1; + $d_sroa_1_4_extract_trunc = $d_sroa_1_4_extract_shift$0; + if (($n_sroa_1_4_extract_trunc | 0) == 0) { + $4 = ($rem | 0) != 0; + if (($d_sroa_1_4_extract_trunc | 0) == 0) { + if ($4) { + HEAP32[$rem >> 2] = ($n_sroa_0_0_extract_trunc >>> 0) % ($d_sroa_0_0_extract_trunc >>> 0); + HEAP32[$rem + 4 >> 2] = 0; + } + $_0$1 = 0; + $_0$0 = ($n_sroa_0_0_extract_trunc >>> 0) / ($d_sroa_0_0_extract_trunc >>> 0) >>> 0; + return ({{{ makeSetTempRet0('$_0$1') }}}, $_0$0) | 0; + } else { + if (!$4) { + $_0$1 = 0; + $_0$0 = 0; + return ({{{ makeSetTempRet0('$_0$1') }}}, $_0$0) | 0; + } + HEAP32[$rem >> 2] = $a$0 & -1; + HEAP32[$rem + 4 >> 2] = $a$1 & 0; + $_0$1 = 0; + $_0$0 = 0; + return ({{{ makeSetTempRet0('$_0$1') }}}, $_0$0) | 0; + } + } + $17 = ($d_sroa_1_4_extract_trunc | 0) == 0; + do { + if (($d_sroa_0_0_extract_trunc | 0) == 0) { + if ($17) { + if (($rem | 0) != 0) { + HEAP32[$rem >> 2] = ($n_sroa_1_4_extract_trunc >>> 0) % ($d_sroa_0_0_extract_trunc >>> 0); + HEAP32[$rem + 4 >> 2] = 0; + } + $_0$1 = 0; + $_0$0 = ($n_sroa_1_4_extract_trunc >>> 0) / ($d_sroa_0_0_extract_trunc >>> 0) >>> 0; + return ({{{ makeSetTempRet0('$_0$1') }}}, $_0$0) | 0; + } + if (($n_sroa_0_0_extract_trunc | 0) == 0) { + if (($rem | 0) != 0) { + HEAP32[$rem >> 2] = 0; + HEAP32[$rem + 4 >> 2] = ($n_sroa_1_4_extract_trunc >>> 0) % ($d_sroa_1_4_extract_trunc >>> 0); + } + $_0$1 = 0; + $_0$0 = ($n_sroa_1_4_extract_trunc >>> 0) / ($d_sroa_1_4_extract_trunc >>> 0) >>> 0; + return ({{{ makeSetTempRet0('$_0$1') }}}, $_0$0) | 0; + } + $37 = $d_sroa_1_4_extract_trunc - 1 | 0; + if (($37 & $d_sroa_1_4_extract_trunc | 0) == 0) { + if (($rem | 0) != 0) { + HEAP32[$rem >> 2] = 0 | $a$0 & -1; + HEAP32[$rem + 4 >> 2] = $37 & $n_sroa_1_4_extract_trunc | $a$1 & 0; + } + $_0$1 = 0; + $_0$0 = $n_sroa_1_4_extract_trunc >>> ((_llvm_cttz_i32($d_sroa_1_4_extract_trunc | 0) | 0) >>> 0); + return ({{{ makeSetTempRet0('$_0$1') }}}, $_0$0) | 0; + } + $49 = Math_clz32($d_sroa_1_4_extract_trunc | 0) | 0; + $51 = $49 - (Math_clz32($n_sroa_1_4_extract_trunc | 0) | 0) | 0; + if ($51 >>> 0 <= 30) { + $57 = $51 + 1 | 0; + $58 = 31 - $51 | 0; + $sr_1_ph = $57; + $r_sroa_0_1_ph = $n_sroa_1_4_extract_trunc << $58 | $n_sroa_0_0_extract_trunc >>> ($57 >>> 0); + $r_sroa_1_1_ph = $n_sroa_1_4_extract_trunc >>> ($57 >>> 0); + $q_sroa_0_1_ph = 0; + $q_sroa_1_1_ph = $n_sroa_0_0_extract_trunc << $58; + break; + } + if (($rem | 0) == 0) { + $_0$1 = 0; + $_0$0 = 0; + return ({{{ makeSetTempRet0('$_0$1') }}}, $_0$0) | 0; + } + HEAP32[$rem >> 2] = 0 | $a$0 & -1; + HEAP32[$rem + 4 >> 2] = $n_sroa_1_4_extract_shift$0 | $a$1 & 0; + $_0$1 = 0; + $_0$0 = 0; + return ({{{ makeSetTempRet0('$_0$1') }}}, $_0$0) | 0; + } else { + if (!$17) { + $117 = Math_clz32($d_sroa_1_4_extract_trunc | 0) | 0; + $119 = $117 - (Math_clz32($n_sroa_1_4_extract_trunc | 0) | 0) | 0; + if ($119 >>> 0 <= 31) { + $125 = $119 + 1 | 0; + $126 = 31 - $119 | 0; + $130 = $119 - 31 >> 31; + $sr_1_ph = $125; + $r_sroa_0_1_ph = $n_sroa_0_0_extract_trunc >>> ($125 >>> 0) & $130 | $n_sroa_1_4_extract_trunc << $126; + $r_sroa_1_1_ph = $n_sroa_1_4_extract_trunc >>> ($125 >>> 0) & $130; + $q_sroa_0_1_ph = 0; + $q_sroa_1_1_ph = $n_sroa_0_0_extract_trunc << $126; + break; + } + if (($rem | 0) == 0) { + $_0$1 = 0; + $_0$0 = 0; + return ({{{ makeSetTempRet0('$_0$1') }}}, $_0$0) | 0; + } + HEAP32[$rem >> 2] = 0 | $a$0 & -1; + HEAP32[$rem + 4 >> 2] = $n_sroa_1_4_extract_shift$0 | $a$1 & 0; + $_0$1 = 0; + $_0$0 = 0; + return ({{{ makeSetTempRet0('$_0$1') }}}, $_0$0) | 0; + } + $66 = $d_sroa_0_0_extract_trunc - 1 | 0; + if (($66 & $d_sroa_0_0_extract_trunc | 0) != 0) { + $86 = (Math_clz32($d_sroa_0_0_extract_trunc | 0) | 0) + 33 | 0; + $88 = $86 - (Math_clz32($n_sroa_1_4_extract_trunc | 0) | 0) | 0; + $89 = 64 - $88 | 0; + $91 = 32 - $88 | 0; + $92 = $91 >> 31; + $95 = $88 - 32 | 0; + $105 = $95 >> 31; + $sr_1_ph = $88; + $r_sroa_0_1_ph = $91 - 1 >> 31 & $n_sroa_1_4_extract_trunc >>> ($95 >>> 0) | ($n_sroa_1_4_extract_trunc << $91 | $n_sroa_0_0_extract_trunc >>> ($88 >>> 0)) & $105; + $r_sroa_1_1_ph = $105 & $n_sroa_1_4_extract_trunc >>> ($88 >>> 0); + $q_sroa_0_1_ph = $n_sroa_0_0_extract_trunc << $89 & $92; + $q_sroa_1_1_ph = ($n_sroa_1_4_extract_trunc << $89 | $n_sroa_0_0_extract_trunc >>> ($95 >>> 0)) & $92 | $n_sroa_0_0_extract_trunc << $91 & $88 - 33 >> 31; + break; + } + if (($rem | 0) != 0) { + HEAP32[$rem >> 2] = $66 & $n_sroa_0_0_extract_trunc; + HEAP32[$rem + 4 >> 2] = 0; + } + if (($d_sroa_0_0_extract_trunc | 0) == 1) { + $_0$1 = $n_sroa_1_4_extract_shift$0 | $a$1 & 0; + $_0$0 = 0 | $a$0 & -1; + return ({{{ makeSetTempRet0('$_0$1') }}}, $_0$0) | 0; + } else { + $78 = _llvm_cttz_i32($d_sroa_0_0_extract_trunc | 0) | 0; + $_0$1 = 0 | $n_sroa_1_4_extract_trunc >>> ($78 >>> 0); + $_0$0 = $n_sroa_1_4_extract_trunc << 32 - $78 | $n_sroa_0_0_extract_trunc >>> ($78 >>> 0) | 0; + return ({{{ makeSetTempRet0('$_0$1') }}}, $_0$0) | 0; + } + } + } while (0); + if (($sr_1_ph | 0) == 0) { + $q_sroa_1_1_lcssa = $q_sroa_1_1_ph; + $q_sroa_0_1_lcssa = $q_sroa_0_1_ph; + $r_sroa_1_1_lcssa = $r_sroa_1_1_ph; + $r_sroa_0_1_lcssa = $r_sroa_0_1_ph; + $carry_0_lcssa$1 = 0; + $carry_0_lcssa$0 = 0; + } else { + $d_sroa_0_0_insert_insert99$0 = 0 | $b$0 & -1; + $d_sroa_0_0_insert_insert99$1 = $d_sroa_1_4_extract_shift$0 | $b$1 & 0; + $137$0 = _i64Add($d_sroa_0_0_insert_insert99$0 | 0, $d_sroa_0_0_insert_insert99$1 | 0, -1, -1) | 0; + $137$1 = {{{ makeGetTempRet0() }}}; + $q_sroa_1_1198 = $q_sroa_1_1_ph; + $q_sroa_0_1199 = $q_sroa_0_1_ph; + $r_sroa_1_1200 = $r_sroa_1_1_ph; + $r_sroa_0_1201 = $r_sroa_0_1_ph; + $sr_1202 = $sr_1_ph; + $carry_0203 = 0; + while (1) { + $147 = $q_sroa_0_1199 >>> 31 | $q_sroa_1_1198 << 1; + $149 = $carry_0203 | $q_sroa_0_1199 << 1; + $r_sroa_0_0_insert_insert42$0 = 0 | ($r_sroa_0_1201 << 1 | $q_sroa_1_1198 >>> 31); + $r_sroa_0_0_insert_insert42$1 = $r_sroa_0_1201 >>> 31 | $r_sroa_1_1200 << 1 | 0; + _i64Subtract($137$0 | 0, $137$1 | 0, $r_sroa_0_0_insert_insert42$0 | 0, $r_sroa_0_0_insert_insert42$1 | 0) | 0; + $150$1 = {{{ makeGetTempRet0() }}}; + $151$0 = $150$1 >> 31 | (($150$1 | 0) < 0 ? -1 : 0) << 1; + $152 = $151$0 & 1; + $154$0 = _i64Subtract($r_sroa_0_0_insert_insert42$0 | 0, $r_sroa_0_0_insert_insert42$1 | 0, $151$0 & $d_sroa_0_0_insert_insert99$0 | 0, ((($150$1 | 0) < 0 ? -1 : 0) >> 31 | (($150$1 | 0) < 0 ? -1 : 0) << 1) & $d_sroa_0_0_insert_insert99$1 | 0) | 0; + $r_sroa_0_0_extract_trunc = $154$0; + $r_sroa_1_4_extract_trunc = {{{ makeGetTempRet0() }}}; + $155 = $sr_1202 - 1 | 0; + if (($155 | 0) == 0) { + break; + } else { + $q_sroa_1_1198 = $147; + $q_sroa_0_1199 = $149; + $r_sroa_1_1200 = $r_sroa_1_4_extract_trunc; + $r_sroa_0_1201 = $r_sroa_0_0_extract_trunc; + $sr_1202 = $155; + $carry_0203 = $152; + } + } + $q_sroa_1_1_lcssa = $147; + $q_sroa_0_1_lcssa = $149; + $r_sroa_1_1_lcssa = $r_sroa_1_4_extract_trunc; + $r_sroa_0_1_lcssa = $r_sroa_0_0_extract_trunc; + $carry_0_lcssa$1 = 0; + $carry_0_lcssa$0 = $152; + } + $q_sroa_0_0_insert_ext75$0 = $q_sroa_0_1_lcssa; + $q_sroa_0_0_insert_ext75$1 = 0; + $q_sroa_0_0_insert_insert77$1 = $q_sroa_1_1_lcssa | $q_sroa_0_0_insert_ext75$1; + if (($rem | 0) != 0) { + HEAP32[$rem >> 2] = 0 | $r_sroa_0_1_lcssa; + HEAP32[$rem + 4 >> 2] = $r_sroa_1_1_lcssa | 0; + } + $_0$1 = (0 | $q_sroa_0_0_insert_ext75$0) >>> 31 | $q_sroa_0_0_insert_insert77$1 << 1 | ($q_sroa_0_0_insert_ext75$1 << 1 | $q_sroa_0_0_insert_ext75$0 >>> 31) & 0 | $carry_0_lcssa$1; + $_0$0 = ($q_sroa_0_0_insert_ext75$0 << 1 | 0 >>> 31) & -2 | $carry_0_lcssa$0; + return ({{{ makeSetTempRet0('$_0$1') }}}, $_0$0) | 0; + }, + // ======================================================================= + }; function autoAddDeps(object, name) { diff --git a/src/library_browser.js b/src/library_browser.js index fae44b42734bf..b610d133601d9 100644 --- a/src/library_browser.js +++ b/src/library_browser.js @@ -372,7 +372,8 @@ var LibraryBrowser = { canvasContainer.requestFullScreen = canvasContainer['requestFullScreen'] || canvasContainer['mozRequestFullScreen'] || canvasContainer['msRequestFullscreen'] || - (canvasContainer['webkitRequestFullScreen'] ? function() { canvasContainer['webkitRequestFullScreen'](Element['ALLOW_KEYBOARD_INPUT']) } : null); + (canvasContainer['webkitRequestFullScreen'] ? function() { canvasContainer['webkitRequestFullScreen'](Element['ALLOW_KEYBOARD_INPUT']) } : null) || + (canvasContainer['webkitRequestFullscreen'] ? function() { canvasContainer['webkitRequestFullscreen'](Element['ALLOW_KEYBOARD_INPUT']) } : null); if (vrDevice) { canvasContainer.requestFullScreen({ vrDisplay: vrDevice }); @@ -1056,7 +1057,8 @@ var LibraryBrowser = { if (mode == 0 /*EM_TIMING_SETTIMEOUT*/) { Browser.mainLoop.scheduler = function Browser_mainLoop_scheduler_setTimeout() { - setTimeout(Browser.mainLoop.runner, value); // doing this each time means that on exception, we stop + var timeUntilNextTick = Math.max(0, Browser.mainLoop.tickStartTime + value - _emscripten_get_now())|0; + setTimeout(Browser.mainLoop.runner, timeUntilNextTick); // doing this each time means that on exception, we stop }; Browser.mainLoop.method = 'timeout'; } else if (mode == 1 /*EM_TIMING_RAF*/) { @@ -1089,7 +1091,7 @@ var LibraryBrowser = { return 0; }, - emscripten_set_main_loop__deps: ['emscripten_set_main_loop_timing'], + emscripten_set_main_loop__deps: ['emscripten_set_main_loop_timing', 'emscripten_get_now'], emscripten_set_main_loop: function(func, fps, simulateInfiniteLoop, arg, noSetTiming) { Module['noExitRuntime'] = true; @@ -1098,6 +1100,18 @@ var LibraryBrowser = { Browser.mainLoop.func = func; Browser.mainLoop.arg = arg; + var browserIterationFunc; + if (typeof arg !== 'undefined') { + var argArray = [arg]; + browserIterationFunc = function() { + Runtime.dynCall('vi', func, argArray); + }; + } else { + browserIterationFunc = function() { + Runtime.dynCall('v', func); + }; + } + var thisMainLoopId = Browser.mainLoop.currentlyRunningMainloop; Browser.mainLoop.runner = function Browser_mainLoop_runner() { @@ -1136,6 +1150,8 @@ var LibraryBrowser = { // Not the scheduled time to render this frame - skip. Browser.mainLoop.scheduler(); return; + } else if (Browser.mainLoop.timingMode == 0/*EM_TIMING_SETTIMEOUT*/) { + Browser.mainLoop.tickStartTime = _emscripten_get_now(); } // Signal GL rendering layer that processing of a new frame is about to start. This helps it optimize @@ -1149,13 +1165,7 @@ var LibraryBrowser = { Browser.mainLoop.method = ''; // just warn once per call to set main loop } - Browser.mainLoop.runIter(function() { - if (typeof arg !== 'undefined') { - Runtime.dynCall('vi', func, [arg]); - } else { - Runtime.dynCall('v', func); - } - }); + Browser.mainLoop.runIter(browserIterationFunc); #if STACK_OVERFLOW_CHECK checkStackCookie(); diff --git a/src/library_cyberdwarf.js b/src/library_cyberdwarf.js new file mode 100644 index 0000000000000..30c06936c376d --- /dev/null +++ b/src/library_cyberdwarf.js @@ -0,0 +1,9 @@ +var LibraryCyberdwarf = { + // These are empty, designed to be replaced when a debugger is invoked + metadata_llvm_dbg_value_constant: function(a,b,c,d) { + }, + metadata_llvm_dbg_value_local: function(a,b,c,d) { + } +}; + +mergeInto(LibraryManager.library, LibraryCyberdwarf); diff --git a/src/library_debugger_toolkit.js b/src/library_debugger_toolkit.js new file mode 100644 index 0000000000000..29cb532aecaaf --- /dev/null +++ b/src/library_debugger_toolkit.js @@ -0,0 +1,485 @@ +var CyberDWARFHeapPrinter = function(cdFileLocation) { + var BASIC_TYPE = 0, + DERIVED_TYPE = 1, + COMPOSITE_TYPE = 2, + SUBROUTINE_TYPE = 3, + SUBRANGE_INFO = 4, + SUBPROGRAM_TYPE = 5, + ENUMERATOR_TYPE = 6, + STRING_REFERENCE = 10; + + var DW_TAG_array_type = 0x01, + DW_TAG_class_type = 0x02, + DW_TAG_entry_point = 0x03, + DW_TAG_enumeration_type = 0x04, + DW_TAG_formal_parameter = 0x05, + DW_TAG_imported_declaration = 0x08, + DW_TAG_label = 0x0a, + DW_TAG_lexical_block = 0x0b, + DW_TAG_member = 0x0d, + DW_TAG_pointer_type = 0x0f, + DW_TAG_reference_type = 0x10, + DW_TAG_compile_unit = 0x11, + DW_TAG_string_type = 0x12, + DW_TAG_structure_type = 0x13, + DW_TAG_subroutine_type = 0x15, + DW_TAG_typedef = 0x16, + DW_TAG_union_type = 0x17, + DW_TAG_unspecified_parameters = 0x18, + DW_TAG_variant = 0x19, + DW_TAG_common_block = 0x1a, + DW_TAG_common_inclusion = 0x1b, + DW_TAG_inheritance = 0x1c, + DW_TAG_inlined_subroutine = 0x1d, + DW_TAG_module = 0x1e, + DW_TAG_ptr_to_member_type = 0x1f, + DW_TAG_set_type = 0x20, + DW_TAG_subrange_type = 0x21, + DW_TAG_with_stmt = 0x22, + DW_TAG_access_declaration = 0x23, + DW_TAG_base_type = 0x24, + DW_TAG_catch_block = 0x25, + DW_TAG_const_type = 0x26, + DW_TAG_constant = 0x27, + DW_TAG_enumerator = 0x28, + DW_TAG_file_type = 0x29, + DW_TAG_friend = 0x2a, + DW_TAG_namelist = 0x2b, + DW_TAG_namelist_item = 0x2c, + DW_TAG_packed_type = 0x2d, + DW_TAG_subprogram = 0x2e, + DW_TAG_template_type_parameter = 0x2f, + DW_TAG_template_value_parameter = 0x30, + DW_TAG_thrown_type = 0x31, + DW_TAG_try_block = 0x32, + DW_TAG_variant_part = 0x33, + DW_TAG_variable = 0x34, + DW_TAG_volatile_type = 0x35, + DW_TAG_dwarf_procedure = 0x36, + DW_TAG_restrict_type = 0x37, + DW_TAG_interface_type = 0x38, + DW_TAG_namespace = 0x39, + DW_TAG_imported_module = 0x3a, + DW_TAG_unspecified_type = 0x3b, + DW_TAG_partial_unit = 0x3c, + DW_TAG_imported_unit = 0x3d, + DW_TAG_condition = 0x3f, + DW_TAG_shared_type = 0x40, + DW_TAG_lo_user = 0x4080, + DW_TAG_hi_user = 0xffff; + + var DW_ATE_address = 0x01, + DW_ATE_boolean = 0x02, + DW_ATE_complex_float = 0x03, + DW_ATE_float = 0x04, + DW_ATE_signed = 0x05, + DW_ATE_signed_char = 0x06, + DW_ATE_unsigned = 0x07, + DW_ATE_unsigned_char = 0x08, + DW_ATE_imaginary_float = 0x09, + DW_ATE_packed_decimal = 0x0a, + DW_ATE_numeric_string = 0x0b, + DW_ATE_edited = 0x0c, + DW_ATE_signed_fixed = 0x0d, + DW_ATE_unsigned_fixed = 0x0e, + DW_ATE_decimal_float = 0x0f, + DW_ATE_lo_user = 0x80, + DW_ATE_hi_user = 0xff; + + var TYPE_ID_IDX = 0, + NAME_IDX = 1, + TAG_IDX = 2, + BASE_TYPE_IDX = 3, + OFFSET_IDX = 4, + SIZE_IDX = 5, + C_ELEMS_IDX = 7; + + function TypedVariable(base_ptr, type_id) { + this.derives = []; + this.primary = null; + this.primary_type_id = ""; + this.base_ptr = base_ptr; + + this.type_id = type_id; + this.built = false; + this.resolved = false; + this.name = ""; + this.value = ""; + this.pointerCount = 0; + this.offset = 0; + this.size = 0; + this.standalone_id = 0; + this.dereference = true; + this.isMember = false; + this.isInherited = false + } + + TypedVariable.prototype.toString = function() { + return "[TypedVariable < " + JSON.stringify(this.derives) + " > " + JSON.stringify(this.primary) + " (&" + this._getMyAddress() + ")]"; + } + + TypedVariable.prototype._initialBuild = function() { + if (this.built) { + return; + } + + this.built = true; + this._buildDeriveChain(); + this._processTypes(); + } + + TypedVariable.prototype._buildDeriveChain = function() { + var cur_td = type_id_to_type_descriptor(this.type_id, this.base_ptr); + + var ptr = this.base_ptr; + + while (cur_td[TYPE_ID_IDX] == 1 || cur_td[TYPE_ID_IDX] == 10) { + if (cur_td[TYPE_ID_IDX] == 1) { + this.derives.push(cur_td); + this.primary_type_id = cur_td[BASE_TYPE_IDX]; + if (cur_td[TAG_IDX] == DW_TAG_member) { + ptr += cur_td[OFFSET_IDX]; + } + cur_td = type_id_to_type_descriptor(cur_td[BASE_TYPE_IDX], ptr); + } else { + this.primary_type_id = cur_td[1]; + cur_td = type_id_to_type_descriptor(cur_td[1], ptr); + } + } + + this.primary = cur_td; + } + + TypedVariable.prototype._processTypes = function() { + var name_vec = []; + + for (var i in this.derives) { + switch (this.derives[i][TAG_IDX]) { + case DW_TAG_reference_type: { + name_vec.unshift("&"); + this.pointerCount++; + } break; + case DW_TAG_pointer_type: { + name_vec.unshift("*"); + this.pointerCount++; + } break; + case DW_TAG_const_type: { + name_vec.unshift("const"); + } break; + case DW_TAG_inheritance: { + this.isInherited = true; + name_vec.unshift(">"); + this.offset = this.derives[i][OFFSET_IDX] / 8; + } break; + case DW_TAG_member: { + this.isMember = true; + name_vec.push(":"); + name_vec.push(this.derives[i][NAME_IDX]) + this.offset = this.derives[i][OFFSET_IDX] / 8; + } break; + case DW_TAG_typedef: { + // Ignoring typedefs for the time being + } break; + case DW_TAG_volatile_type: { + name_vec.unshift("volatile"); + } + default: + console.error("Unimplemented " + type_descriptor); + } + } + + if (this.primary[TYPE_ID_IDX] == BASIC_TYPE) { + this.size = this.primary[4]; + name_vec.unshift(this.primary[NAME_IDX]); + } else if (this.primary[TYPE_ID_IDX] == COMPOSITE_TYPE) { + this.size = this.primary[SIZE_IDX]; + name_vec.unshift(this.primary[NAME_IDX]); + + switch (this.primary[TAG_IDX]) { + case DW_TAG_structure_type: { + name_vec.unshift("struct"); + } break; + case DW_TAG_class_type: { + name_vec.unshift("class"); + } break; + case DW_TAG_enumeration_type: { + name_vec.unshift("class"); + } break; + case DW_TAG_union_type: { + name_vec.unshift("union"); + } break; + case DW_TAG_array_type: { + name_vec.unshift("array"); + } break; + default: + console.error("Unimplemented for composite " + this.primary); + } + } + + this.name = name_vec.join(" "); + } + + TypedVariable.prototype._getMyAddress = function() { + var base_addr = this.base_ptr + this.offset; + for (var i = 0; i < this.pointerCount; i++) { + if (base_addr == 0) { + return 0; + } + base_addr = heap["u32"][base_addr >> 2]; + } + return base_addr; + } + + TypedVariable.prototype._resolveBaseTypeValue = function() { + this.resolved = true; + if (this.primary[TYPE_ID_IDX] == BASIC_TYPE) { + if (!this.dereference) { + switch (this.primary[TAG_IDX]) { + case DW_ATE_unsigned: + case DW_ATE_float: + case DW_ATE_signed: { + this.value = this.base_ptr; + }; break; + } + } else { + if (this.base_ptr == 0) { + this.value = null; + } else { + var heap_id = type_descriptor_to_heap_id(this.primary); + var ptr = this._getMyAddress(); + this.value = heap[heap_id][ptr >> heap_shift[heap_id]]; + } + } + } else { + this.value = {}; + + for (var i in this.primary[C_ELEMS_IDX]) { + var cur_elem = new TypedVariable(this._getMyAddress(), this.primary[C_ELEMS_IDX][i]); + + cur_elem._initialBuild(); + + if (cur_elem.size > 0 && (cur_elem.isInherited || cur_elem.isMember)) { + var pointed_elem = new TypedVariable(cur_elem._getMyAddress(), cur_elem.primary_type_id); + pointed_elem._initialBuild(); + this.value[cur_elem.name] = pointed_elem; + } + } + } + } + + TypedVariable.prototype._buildForPrettyPrint = function(depth) { + if (typeof(depth) === "undefined") { + depth = 1; + } + + if (depth < 1) { + return; + } + + this._initialBuild(); + this._resolveBaseTypeValue(); + + if (this.primary[TYPE_ID_IDX] != 2 || this._getMyAddress() == 0) { + return; + } + + for (var i in this.value) { + this.value[i]._buildForPrettyPrint(depth - 1); + } + } + + TypedVariable.prototype._prettyPrintToObjectHelper = function() { + + if (typeof(this.value) !== "object") { + return this.value; + } + + if (this.value == null) { + return "null"; + } + + var childNames = Object.keys(this.value); + + var retval = {}; + + for (var i in childNames) { + if (this.value[childNames[i]].resolved) { + retval[childNames[i]] = this.value[childNames[i]]._prettyPrintToObjectHelper(); + } else { + retval[childNames[i]] = "cd_tvptr(0x" + this.value[childNames[i]]._getMyAddress().toString(16) + ",'" + this.value[childNames[i]].type_id + "')"; + } + } + + return retval; + } + + TypedVariable.prototype.prettyPrintToObject = function(depth) { + this._buildForPrettyPrint(depth); + var retval = {}; + retval[this.name] = this._prettyPrintToObjectHelper(); + return retval; + } + + var heap = {}; + var cyberdwarf = undefined; + var heap_shift = { + "u8" : 0, + "u16": 1, + "u32": 2, + "i8" : 0, + "i16": 1, + "i32": 2, + "f32": 2, + "f64": 3 + } + function install_heap($heap) { + heap = { + "u8" : new Uint8Array($heap), + "u16": new Uint16Array($heap), + "u32": new Uint32Array($heap), + "i8" : new Int8Array($heap), + "i16": new Int16Array($heap), + "i32": new Int32Array($heap), + "f32": new Float32Array($heap), + "f64": new Float64Array($heap) + }; + } + + var symbols = {}; + + function install_cyberdwarf(data_cd) { + cyberdwarf = JSON.parse(data_cd)["cyberdwarf"]; + invert_vtables(); + } + + function type_descriptor_to_heap_id(type_descriptor) { + var id = ""; + switch (type_descriptor[TAG_IDX]) { + case DW_ATE_unsigned: case DW_ATE_unsigned_char: id = "u"; break; + case DW_ATE_signed: case DW_ATE_signed_char: id = "i"; break; + case DW_ATE_float: id = "f"; break; + } + id += type_descriptor[4]; + return id; + } + + function type_id_to_type_descriptor(type_id, ptr, dit) { + if (!isNaN(+type_id)) { + return cyberdwarf["types"][type_id]; + } + if (typeof(type_id) === "string") { + if (dit) { + type_id = resolve_from_vtable(ptr, type_id); + } + type_id = cyberdwarf["type_name_map"][type_id]; + } + return cyberdwarf["types"][type_id]; + } + + function invert_vtables() { + cyberdwarf["has_vtable"] = {}; + for (var i in cyberdwarf["vtable_offsets"]) { + cyberdwarf["has_vtable"][cyberdwarf["vtable_offsets"][i]] = true; + } + } + + function resolve_from_vtable(val, type_name) { + var lookup_name = "_ZTV" + type_name.substr(4); + if (cyberdwarf["has_vtable"][lookup_name]) { + var potential_vtable = "" + (heap["u32"][val >> 2] - 8); + if (cyberdwarf["vtable_offsets"][potential_vtable]) { + var ans = type_name.substr(0,4) + cyberdwarf["vtable_offsets"][potential_vtable].substr(4); + return ans; + } + } + return type_name; + } + + var current_function = ""; + function set_current_function(func_name) { + if (typeof(cyberdwarf["function_name_map"][func_name]) !== "undefined") { + func_name = cyberdwarf["function_name_map"][func_name]; + } + if (func_name.substring(0,1) == "_") { + func_name = func_name.substring(1); + } + current_function = cyberdwarf["functions"][func_name]; + } + + function pretty_print_to_object(val, type_id, depth) { + install_heap(Module.HEAPU8.buffer); + var base_type = new TypedVariable(val, current_function[type_id]); + return base_type.prettyPrintToObject(depth); + } + + function pretty_print_from_typename(val, type_name, depth) { + install_heap(Module.HEAPU8.buffer); + var base_type = new TypedVariable(val, type_name); + return base_type.prettyPrintToObject(depth); + } + + function stack_decoder(val, name, depth, on_heap) { + var stack_frames = jsStackTrace().split("\n"); + + // To find the function that called us, look for the first reference to calling an EM_ASM block + var stack_offset = 0; + for (; stack_offset < stack_frames.length; stack_offset++) { + if (stack_frames[stack_offset].match("emscripten_asm_const")) break; + } + + // Subtract 1 since we shouldn't have found it on the last frame + if (stack_offset == stack_frames.length - 1) { + console.error("Couldn't find the function the decoder was called from"); + return {}; + } + + var func_finder = new RegExp("at (?:Object\\.|)([^\\s]+)", "g"); + var result = func_finder.exec(stack_frames[stack_offset + 1]); + + if (result.length > 1) { + set_current_function(result[1]); + } + + var decoded = pretty_print_to_object(val, name, depth); + return decoded; + } + + function initialize_debugger(cb) { + var cdFile; + if (typeof Module['locateFile'] === 'function') { + cdFileLocation = Module['locateFile'](cdFileLocation); + } else if (Module['cdInitializerPrefixURL']) { + cdFileLocation = Module['cdInitializerPrefixURL'] + cdFileLocation; + } + if (ENVIRONMENT_IS_NODE || ENVIRONMENT_IS_SHELL) { + var data = Module['read'](cdFileLocation); + install_cyberdwarf(data); + } else { + var applyCDFile = function(data) { + var decoder = new TextDecoder("utf8"); + install_cyberdwarf(decoder.decode(data)); + console.info("Debugger ready"); + if (typeof(cb) !== "undefined") { + cb(); + } + } + function doBrowserLoad() { + Module['readAsync'](cdFileLocation, applyCDFile, function() { + throw 'could not load debug data ' + cdFileLocation; + }); + } + // fetch it from the network ourselves + doBrowserLoad(); + } + } + + return { + "decode_from_stack": stack_decoder, + "initialize_debugger": initialize_debugger, + "set_current_function": set_current_function, + "decode_var_by_var_name": pretty_print_to_object, + "decode_var_by_type_name": pretty_print_from_typename + }; +}; + +mergeInto(LibraryManager.library, { "cyberdwarf_Debugger": CyberDWARFHeapPrinter }); diff --git a/src/library_gl.js b/src/library_gl.js index 9e6fc6d5d87a3..347987ff649af 100644 --- a/src/library_gl.js +++ b/src/library_gl.js @@ -622,7 +622,7 @@ var LibraryGL = { "OES_texture_float_linear", "OES_texture_half_float_linear", "WEBGL_compressed_texture_atc", "WEBGL_compressed_texture_pvrtc", "EXT_color_buffer_half_float", "WEBGL_color_buffer_float", "EXT_frag_depth", "EXT_sRGB", "WEBGL_draw_buffers", "WEBGL_shared_resources", - "EXT_shader_texture_lod" ]; + "EXT_shader_texture_lod", "EXT_color_buffer_float"]; function shouldEnableAutomatically(extension) { var ret = false; @@ -1051,18 +1051,30 @@ var LibraryGL = { case 0x1906 /* GL_ALPHA */: case 0x1909 /* GL_LUMINANCE */: case 0x1902 /* GL_DEPTH_COMPONENT */: +#if USE_WEBGL2 case 0x1903 /* GL_RED */: + case 0x8D94 /* GL_RED_INTEGER */: +#endif numChannels = 1; break; case 0x190A /* GL_LUMINANCE_ALPHA */: +#if USE_WEBGL2 case 0x8227 /* GL_RG */: + case 0x8228 /* GL_RG_INTEGER*/: +#endif numChannels = 2; break; case 0x1907 /* GL_RGB */: +#if USE_WEBGL2 + case 0x8D98 /* GL_RGB_INTEGER */: +#endif case 0x8C40 /* GL_SRGB_EXT */: numChannels = 3; break; case 0x1908 /* GL_RGBA */: +#if USE_WEBGL2 + case 0x8D99 /* GL_RGBA_INTEGER */: +#endif case 0x8C42 /* GL_SRGB_ALPHA_EXT */: numChannels = 4; break; diff --git a/src/library_glfw.js b/src/library_glfw.js index 00de71c25b293..6ed1ac8e9b0fb 100644 --- a/src/library_glfw.js +++ b/src/library_glfw.js @@ -1195,7 +1195,7 @@ var LibraryGLFW = { glfwJoystickPresent: function(joy) { throw "glfwJoystickPresent is not implemented."; }, - glfwGetJoystickAxes: function(joy, count) { throw "glfwGetJoystickParam is not implemented."; }, + glfwGetJoystickAxes: function(joy, count) { throw "glfwGetJoystickAxes is not implemented."; }, glfwGetJoystickButtons: function(joy, count) { throw "glfwGetJoystickButtons is not implemented."; }, diff --git a/src/library_pthread.js b/src/library_pthread.js index 63daebd39516a..e94f2abd7187e 100644 --- a/src/library_pthread.js +++ b/src/library_pthread.js @@ -767,7 +767,7 @@ var LibraryPThread = { return 0; }, - // Stores the memory address that the main thread is futexWaiting on, if any. + // Stores the memory address that the main thread is waiting on, if any. _main_thread_futex_wait_address: '; if (ENVIRONMENT_IS_PTHREAD) __main_thread_futex_wait_address = PthreadWorkerInit.__main_thread_futex_wait_address; else PthreadWorkerInit.__main_thread_futex_wait_address = __main_thread_futex_wait_address = allocate(1, "i32*", ALLOC_STATIC)', // Returns 0 on success, or one of the values -ETIMEDOUT, -EWOULDBLOCK or -EINVAL on error. @@ -779,17 +779,17 @@ var LibraryPThread = { #if PTHREADS_PROFILING PThread.setThreadStatusConditional(_pthread_self(), {{{ cDefine('EM_THREAD_STATUS_RUNNING') }}}, {{{ cDefine('EM_THREAD_STATUS_WAITFUTEX') }}}); #endif - var ret = Atomics.futexWait(HEAP32, addr >> 2, val, timeout); + var ret = Atomics.wait(HEAP32, addr >> 2, val, timeout); // dump('futex_wait done by thread: ' + _pthread_self() + (ENVIRONMENT_IS_PTHREAD?'(pthread)':'') + '\n'); #if PTHREADS_PROFILING PThread.setThreadStatusConditional(_pthread_self(), {{{ cDefine('EM_THREAD_STATUS_WAITFUTEX') }}}, {{{ cDefine('EM_THREAD_STATUS_RUNNING') }}}); #endif - if (ret == Atomics.TIMEDOUT) return -{{{ cDefine('ETIMEDOUT') }}}; - if (ret == Atomics.NOTEQUAL) return -{{{ cDefine('EWOULDBLOCK') }}}; - if (ret == 0) return 0; - throw 'Atomics.futexWait returned an unexpected value ' + ret; + if (ret === Atomics.TIMEDOUT) return -{{{ cDefine('ETIMEDOUT') }}}; + if (ret === Atomics.NOTEQUAL) return -{{{ cDefine('EWOULDBLOCK') }}}; + if (ret === Atomics.OK) return 0; + throw 'Atomics.wait returned an unexpected value ' + ret; } else { - // Atomics.futexWait is not available in the main browser thread, so simulate it via busy spinning. + // Atomics.wait is not available in the main browser thread, so simulate it via busy spinning. var loadedVal = Atomics.load(HEAP32, addr >> 2); if (val != loadedVal) return -{{{ cDefine('EWOULDBLOCK') }}}; @@ -800,7 +800,7 @@ var LibraryPThread = { PThread.setThreadStatusConditional(_pthread_self(), {{{ cDefine('EM_THREAD_STATUS_RUNNING') }}}, {{{ cDefine('EM_THREAD_STATUS_WAITFUTEX') }}}); #endif - // Register globally which address the main thread is simulating to be futexWaiting on. When zero, main thread is not waiting on anything, + // Register globally which address the main thread is simulating to be waiting on. When zero, main thread is not waiting on anything, // and on nonzero, the contents of address pointed by __main_thread_futex_wait_address tell which address the main thread is simulating its wait on. Atomics.store(HEAP32, __main_thread_futex_wait_address >> 2, addr); while (addr) { @@ -843,9 +843,9 @@ var LibraryPThread = { } // Wake any workers waiting on this address. - var ret = Atomics.futexWake(HEAP32, addr >> 2, count); + var ret = Atomics.wake(HEAP32, addr >> 2, count); if (ret >= 0) return ret + mainThreadWoken; - throw 'Atomics.futexWake returned an unexpected value ' + ret; + throw 'Atomics.wake returned an unexpected value ' + ret; }, // Returns the number of threads (>= 0) woken up, or one of the values -EINVAL or -EAGAIN on error. @@ -876,10 +876,10 @@ var LibraryPThread = { } // Wake any workers waiting on this address. - var ret = Atomics.futexWakeOrRequeue(HEAP32, addr >> 2, count, addr2 >> 2, cmpValue); + var ret = Atomics.wakeOrRequeue(HEAP32, addr >> 2, count, addr2 >> 2, cmpValue); if (ret == Atomics.NOTEQUAL) return -{{{ cDefine('EAGAIN') }}}; if (ret >= 0) return ret + mainThreadWoken; - throw 'Atomics.futexWakeOrRequeue returned an unexpected value ' + ret; + throw 'Atomics.wakeOrRequeue returned an unexpected value ' + ret; }, __atomic_is_lock_free: function(size, ptr) { diff --git a/src/library_pthread_stub.js b/src/library_pthread_stub.js index 31fb67d7ed282..31e77848ff5ac 100644 --- a/src/library_pthread_stub.js +++ b/src/library_pthread_stub.js @@ -171,6 +171,7 @@ var LibraryPThreadStub = { pthread_rwlock_wrlock: function() { return 0; }, pthread_rwlock_trywrlock: function() { return 0; }, pthread_rwlock_timedwrlock: function() { return 0; }, + pthread_rwlock_unlock: function() { return 0; }, pthread_rwlockattr_init: function() { return 0; }, pthread_rwlockattr_destroy: function() { return 0; }, @@ -261,11 +262,11 @@ var LibraryPThreadStub = { } }, - __atomic_fetch_add_8__deps: ['llvm_uadd_with_overflow_i64'], + __atomic_fetch_add_8__deps: ['i64Add'], __atomic_fetch_add_8: function(ptr, vall, valh, memmodel) { var l = {{{ makeGetValue('ptr', 0, 'i32') }}}; var h = {{{ makeGetValue('ptr', 4, 'i32') }}}; - {{{ makeSetValue('ptr', 0, '_llvm_uadd_with_overflow_i64(l, h, vall, valh)', 'i32') }}}; + {{{ makeSetValue('ptr', 0, '_i64Add(l, h, vall, valh)', 'i32') }}}; {{{ makeSetValue('ptr', 4, 'Runtime["getTempRet0"]()', 'i32') }}}; {{{ makeStructuralReturn(['l', 'h']) }}}; }, diff --git a/src/modules.js b/src/modules.js index 13efa7217f5bf..aaff60faefe9f 100644 --- a/src/modules.js +++ b/src/modules.js @@ -179,6 +179,15 @@ var LibraryManager = { } } + if (WASM_BACKEND) { + // all asm.js methods should just be run in JS. We should optimize them eventually into wasm. TODO + for (var x in lib) { + if (lib[x + '__asm']) { + lib[x + '__asm'] = undefined; + } + } + } + /* // export code for CallHandlers.h printErr('============================'); diff --git a/src/postamble.js b/src/postamble.js index 7fea7bde421bd..cd966c19bf74b 100644 --- a/src/postamble.js +++ b/src/postamble.js @@ -94,6 +94,10 @@ if (memoryInitializer) { #endif #endif +#if CYBERDWARF + Module['cyberdwarf'] = _cyberdwarf_Debugger(cyberDWARFFile); +#endif + function ExitStatus(status) { this.name = "ExitStatus"; this.message = "Program terminated with exit(" + status + ")"; @@ -204,7 +208,7 @@ function run(args) { if (Module['calledRun']) return; // run may have just been called while the async setStatus time below was happening Module['calledRun'] = true; - if (ABORT) return; + if (ABORT) return; ensureInitRuntime(); @@ -401,4 +405,3 @@ var workerResponded = false, workerCallbackId = -1; })(); #endif - diff --git a/src/preamble.js b/src/preamble.js index 99355cd3dd24c..a8525a6bf51a0 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -2,9 +2,9 @@ // === Preamble library stuff === -// Documentation for the public APIs defined in this file must be updated in: +// Documentation for the public APIs defined in this file must be updated in: // site/source/docs/api_reference/preamble.js.rst -// A prebuilt local version of the documentation is available at: +// A prebuilt local version of the documentation is available at: // site/build/text/docs/api_reference/preamble.js.txt // You can also build docs locally as HTML or other formats in site/ // An online HTML version (which may be of a different version of Emscripten) @@ -152,7 +152,7 @@ var cwrap, ccall; // For fast lookup of conversion functions var toC = {'string' : JSfuncs['stringToC'], 'array' : JSfuncs['arrayToC']}; - // C calling interface. + // C calling interface. ccall = function ccallFunc(ident, returnType, argTypes, args, opts) { var func = getCFunc(ident); var cArgs = []; @@ -213,7 +213,7 @@ var cwrap, ccall; } } } - + cwrap = function cwrap(ident, returnType, argTypes) { argTypes = argTypes || []; var cfunc = getCFunc(ident); @@ -568,7 +568,7 @@ function UTF8ToString(ptr) { // str: the Javascript string to copy. // outU8Array: the array to copy to. Each index in this array is assumed to be one 8-byte element. // outIdx: The starting offset in the array to begin the copying. -// maxBytesToWrite: The maximum number of bytes this function can write to the array. This count should include the null +// maxBytesToWrite: The maximum number of bytes this function can write to the array. This count should include the null // terminator, i.e. if maxBytesToWrite=1, only the null terminator will be written and nothing else. // maxBytesToWrite=0 does not write any bytes to the output, not even the null terminator. // Returns the number of bytes written, EXCLUDING the null terminator. @@ -690,7 +690,7 @@ function UTF16ToString(ptr) { // Parameters: // str: the Javascript string to copy. // outPtr: Byte address in Emscripten HEAP where to write the string to. -// maxBytesToWrite: The maximum number of bytes this function can write to the array. This count should include the null +// maxBytesToWrite: The maximum number of bytes this function can write to the array. This count should include the null // terminator, i.e. if maxBytesToWrite=2, only the null terminator will be written and nothing else. // maxBytesToWrite<2 does not write any bytes to the output, not even the null terminator. // Returns the number of bytes written, EXCLUDING the null terminator. @@ -753,7 +753,7 @@ function UTF32ToString(ptr) { // Parameters: // str: the Javascript string to copy. // outPtr: Byte address in Emscripten HEAP where to write the string to. -// maxBytesToWrite: The maximum number of bytes this function can write to the array. This count should include the null +// maxBytesToWrite: The maximum number of bytes this function can write to the array. This count should include the null // terminator, i.e. if maxBytesToWrite=4, only the null terminator will be written and nothing else. // maxBytesToWrite<4 does not write any bytes to the output, not even the null terminator. // Returns the number of bytes written, EXCLUDING the null terminator. @@ -814,15 +814,16 @@ function demangle(func) { if (getValue(status, 'i32') === 0 && ret) { return Pointer_stringify(ret); } - // otherwise, libcxxabi failed, we can try ours which may return a partial result + // otherwise, libcxxabi failed } catch(e) { - // failure when using libcxxabi, we can try ours which may return a partial result - return func; + // ignore problems here } finally { if (buf) _free(buf); if (status) _free(status); if (ret) _free(ret); } + // failure when using libcxxabi, don't demangle + return func; } Runtime.warnOnce('warning: build with -s DEMANGLE_SUPPORT=1 to link in libcxxabi demangling'); return func; @@ -850,7 +851,9 @@ function jsStackTrace() { } function stackTrace() { - return demangleAll(jsStackTrace()); + var js = jsStackTrace(); + if (Module['extraStackTrace']) js += '\n' + Module['extraStackTrace'](); + return demangleAll(js); } {{{ maybeExport('stackTrace') }}} @@ -1104,9 +1107,9 @@ if (typeof Atomics === 'undefined') { Atomics['add'] = function(t, i, v) { var w = t[i]; t[i] += v; return w; } Atomics['and'] = function(t, i, v) { var w = t[i]; t[i] &= v; return w; } Atomics['compareExchange'] = function(t, i, e, r) { var w = t[i]; if (w == e) t[i] = r; return w; } - Atomics['futexWait'] = function(t, i, v, o) { if (t[i] != v) abort('Multithreading is not supported, cannot sleep to wait for futex!'); } - Atomics['futexWake'] = function(t, i, c) {} - Atomics['futexWakeOrRequeue'] = function(t, i1, c, i2, v) {} + Atomics['wait'] = function(t, i, v, o) { if (t[i] != v) abort('Multithreading is not supported, cannot sleep to wait for futex!'); } + Atomics['wake'] = function(t, i, c) {} + Atomics['wakeOrRequeue'] = function(t, i1, c, i2, v) {} Atomics['isLockFree'] = function(s) { return true; } Atomics['load'] = function(t, i) { return t[i]; } Atomics['or'] = function(t, i, v) { var w = t[i]; t[i] |= v; return w; } @@ -1115,6 +1118,22 @@ if (typeof Atomics === 'undefined') { Atomics['xor'] = function(t, i, v) { var w = t[i]; t[i] ^= v; return w; } } +// In old Atomics spec, Atomics.OK/.TIMEDOUT/.NOTEQUAL were integers. In new spec, Atomics functions return strings, so if we are in the new spec, +// assign the strings to the place where the integers would have been to keep implementation of emscripten_futex_wait straightforward without dynamic checks for both. +// See https://github.com/tc39/ecmascript_sharedmem/issues/69 for details. +if (typeof Atomics['OK'] === 'undefined') { + Atomics['OK'] = 'ok'; + Atomics['TIMEDOUT'] = 'timed-out'; + Atomics['NOTEQUAL'] = 'not-equal'; +} + +// If running browser with old API names, account for function renames. See https://bugzilla.mozilla.org/show_bug.cgi?id=1260910. +if (typeof Atomics['wait'] === 'undefined') { + Atomics['wait'] = Atomics['futexWait']; + Atomics['wake'] = Atomics['futexWake']; + Atomics['wakeOrRequeue'] = Atomics['futexWakeOrRequeue']; +} + #else // USE_PTHREADS #if SPLIT_MEMORY == 0 @@ -1273,7 +1292,7 @@ function freeSplitChunk(i) { function checkPtr(ptr, shifts) { if (ptr <= 0) abort('segmentation fault storing to address ' + ptr); if (ptr !== ((ptr >> shifts) << shifts)) abort('alignment error storing to address ' + ptr + ', which was expected to be aligned to a shift of ' + shifts); - if ((ptr >> SPLIT_MEMORY_BITS) !== (ptr + Math.pow(2, shifts) - 1 >> SPLIT_MEMORY_BITS)) abort('segmentation fault, write spans split chunks ' + [ptr, shifts]); + if ((ptr >> SPLIT_MEMORY_BITS) !== (ptr + Math.pow(2, shifts) - 1 >> SPLIT_MEMORY_BITS)) abort('segmentation fault, write spans split chunks ' + [ptr, shifts]); } #endif @@ -1886,4 +1905,8 @@ Module['FS_createPreloadedFile'] = FS.createPreloadedFile; #endif #endif +#if CYBERDWARF +var cyberDWARFFile = '{{{ BUNDLED_CD_DEBUG_FILE }}}'; +#endif + // === Body === diff --git a/src/runtime.js b/src/runtime.js index 90eca027dacfd..cabeffa850745 100644 --- a/src/runtime.js +++ b/src/runtime.js @@ -196,12 +196,10 @@ var Runtime = { #if ASSERTIONS assert(args.length == sig.length-1); #endif - if (!args.splice) args = Array.prototype.slice.call(args); - args.splice(0, 0, ptr); #if ASSERTIONS assert(('dynCall_' + sig) in Module, 'bad function pointer type - no table for sig \'' + sig + '\''); #endif - return Module['dynCall_' + sig].apply(null, args); + return Module['dynCall_' + sig].apply(null, [ptr].concat(args)); } else { #if ASSERTIONS assert(sig.length == 1); @@ -346,9 +344,21 @@ var Runtime = { } var sigCache = Runtime.funcWrappers[sig]; if (!sigCache[func]) { - sigCache[func] = function dynCall_wrapper() { - return Runtime.dynCall(sig, func, arguments); - }; + // optimize away arguments usage in common cases + if (sig.length === 1) { + sigCache[func] = function dynCall_wrapper() { + return Runtime.dynCall(sig, func); + }; + } else if (sig.length === 2) { + sigCache[func] = function dynCall_wrapper(arg) { + return Runtime.dynCall(sig, func, [arg]); + }; + } else { + // general case + sigCache[func] = function dynCall_wrapper() { + return Runtime.dynCall(sig, func, Array.prototype.slice.call(arguments)); + }; + } } return sigCache[func]; }, diff --git a/src/settings.js b/src/settings.js index 96e4fa3711a0d..0bc325bf21985 100644 --- a/src/settings.js +++ b/src/settings.js @@ -253,7 +253,7 @@ var LEGACY_GL_EMULATION = 0; // Includes code to emulate various desktop GL feat // in some cases, see http://kripken.github.io/emscripten-site/docs/porting/multimedia_and_graphics/OpenGL-support.html var GL_FFP_ONLY = 0; // If you specified LEGACY_GL_EMULATION = 1 and only use fixed function pipeline in your code, // you can also set this to 1 to signal the GL emulation layer that it can perform extra - // optimizations by knowing that the user code does not use shaders at all. If + // optimizations by knowing that the user code does not use shaders at all. If // LEGACY_GL_EMULATION = 0, this setting has no effect. var STB_IMAGE = 0; // Enables building of stb-image, a tiny public-domain library for decoding images, allowing @@ -299,11 +299,11 @@ var ASYNCIFY_FUNCTIONS = ['emscripten_sleep', // Functions that call any functio 'emscripten_wget', // will be transformed 'emscripten_yield']; var ASYNCIFY_WHITELIST = ['qsort', // Functions in this list are never considered async, even if they appear in ASYNCIFY_FUNCTIONS - 'trinkle', // In the asyncify transformation, any function that calls a function pointer is considered async + 'trinkle', // In the asyncify transformation, any function that calls a function pointer is considered async '__toread', // This whitelist is useful when a function is known to be sync '__uflow', // currently this link contains some functions in libc - '__fwritex', - 'MUSL_vfprintf']; + '__fwritex', + 'MUSL_vfprintf']; var EXPORTED_RUNTIME_METHODS = [ // Methods that are exported on Module. By default we export quite a bit, you can reduce this list to lower your code size, // especially when closure is run (exporting prevents closure from eliminating code) @@ -361,7 +361,7 @@ var FS_LOG = 0; // Log all FS operations. This is especially helpful when you'r // so that you can create a virtual file system with all of the required files. var CASE_INSENSITIVE_FS = 0; // If set to nonzero, the provided virtual filesystem if treated case-insensitive, like // Windows and OSX do. If set to 0, the VFS is case-sensitive, like on Linux. -var MEMFS_APPEND_TO_TYPED_ARRAYS = 0; // If set to nonzero, MEMFS will always utilize typed arrays as the backing store +var MEMFS_APPEND_TO_TYPED_ARRAYS = 0; // If set to nonzero, MEMFS will always utilize typed arrays as the backing store // for appending data to files. The default behavior is to use typed arrays for files // when the file size doesn't change after initial creation, and for files that do // change size, use normal JS arrays instead. @@ -551,14 +551,13 @@ var PGO = 0; // Enables profile-guided optimization in the form of runtime check // which functions are actually called. Emits a list during shutdown that you // can pass to DEAD_FUNCTIONS (you can also emit the list manually by // calling PGOMonitor.dump()); -var DEAD_FUNCTIONS = []; // Functions on this list are not converted to JS, and calls to +var DEAD_FUNCTIONS = []; // JS library functions on this list are not converted to JS, and calls to // them are turned into abort()s. This is potentially useful for // reducing code size. // If a dead function is actually called, you will get a runtime // error. - // This can affect both functions in compiled code, and system - // library functions (e.g., you can use this to kill printf). - // TODO: options to lazily load such functions + // TODO: make this work on compiled methods as well, perhaps by + // adding a JS optimizer pass? var EXPLICIT_ZEXT = 0; // If 1, generate an explicit conversion of zext i1 to i32, using ?: @@ -621,18 +620,25 @@ var USE_GLFW = 2; // Specify the GLFW version that is being linked against. // Only relevant, if you are linking against the GLFW library. // Valid options are 2 for GLFW2 and 3 for GLFW3. -var BINARYEN = ""; // Path to [Binaryen](https://github.com/WebAssembly/binaryen), which we use to - // compile (at runtime) our asm.js output into WebAssembly. That then runs in - // a shipped Binaryen interpreter for WebAssembly, or, once browsers get native - // WebAssembly support, it will run directly. - // This path should be to the root Binaryen directory (not the /bin subfolder). - // You need to build Binaryen, so that /bin/wasm.js under the Binaryen - // directory exists. +var BINARYEN = 0; // Whether to use [Binaryen](https://github.com/WebAssembly/binaryen) to + // compile (at runtime) our asm.js output into WebAssembly. + // This will fetch the binaryen port and build it. (If, instead, you set + // BINARYEN_ROOT in your ~/.emscripten file, then we use that instead + // of the port, which can useful for local dev work on binaryen itself). var BINARYEN_METHOD = ""; // See binaryen's src/js/post.js for details. +var BINARYEN_SCRIPTS = ""; // An optional comma-separated list of script hooks to run after binaryen, + // in binaryen's /scripts dir. +var BINARYEN_IMPRECISE = 0; // Whether to apply imprecise/unsafe binaryen optimizations. If enabled, + // code will run faster, but some types of undefined behavior might + // trap in wasm. +var BINARYEN_ROOT = ""; // Directory where we can find Binaryen. Will be automatically set for you, + // but you can set it to override if you are a Binaryen developer. var WASM_BACKEND = 0; // Whether to use the WebAssembly backend that is in development in LLVM. // This requires that BINARYEN be set, as we use Binaryen's s2wasm to // translate the backend output. + // You should not set this yourself, instead set EMCC_WASM_BACKEND=1 in the + // environment. // Ports @@ -641,6 +647,7 @@ var USE_SDL = 1; // Specify the SDL version that is being linked against. // 2 is a port of the SDL C code on emscripten-ports var USE_SDL_IMAGE = 1; // Specify the SDL_image version that is being linked against. Must match USE_SDL var USE_SDL_TTF = 1; // Specify the SDL_ttf version that is being linked against. Must match USE_SDL +var USE_SDL_NET = 1; // Specify the SDL_net version that is being linked against. Must match USE_SDL var USE_ZLIB = 0; // 1 = use zlib from emscripten-ports var USE_LIBPNG = 0; // 1 = use libpng from emscripten-ports var USE_BULLET = 0; // 1 = use bullet from emscripten-ports @@ -735,4 +742,8 @@ var EVAL_CTORS = 0; // This tries to evaluate global ctors at compile-time, appl // further code, so it only tries to optimize ctors with lowest // priority. We do know that, and can optimize all the ctors. +var CYBERDWARF = 0; // see http://kripken.github.io/emscripten-site/docs/debugging/CyberDWARF.html + +var BUNDLED_CD_DEBUG_FILE = ""; // Path to the CyberDWARF debug file passed to the compiler + // Reserved: variables containing POINTER_MASKING. diff --git a/src/shell.js b/src/shell.js index 65fc9deea2d95..94f241eac6e6a 100644 --- a/src/shell.js +++ b/src/shell.js @@ -83,16 +83,9 @@ if (ENVIRONMENT_IS_NODE) { Module['read'] = function read(filename, binary) { if (!nodeFS) nodeFS = require('fs'); if (!nodePath) nodePath = require('path'); - filename = nodePath['normalize'](filename); var ret = nodeFS['readFileSync'](filename); - // The path is absolute if the normalized version is the same as the resolved. - if (!ret && filename != nodePath['resolve'](filename)) { - filename = path.join(__dirname, '..', 'src', filename); - ret = nodeFS['readFileSync'](filename); - } - if (ret && !binary) ret = ret.toString(); - return ret; + return binary ? ret : ret.toString(); }; Module['readBinary'] = function readBinary(filename) { diff --git a/src/struct_info.compiled.json b/src/struct_info.compiled.json new file mode 100644 index 0000000000000..09f47d8b882e0 --- /dev/null +++ b/src/struct_info.compiled.json @@ -0,0 +1 @@ +{"structs":{"utsname":{"sysname":0,"nodename":65,"domainname":325,"machine":260,"version":195,"release":130,"__size__":390},"sockaddr":{"sa_data":2,"sa_family":0,"__size__":16},"addrinfo":{"ai_flags":0,"ai_next":28,"ai_canonname":24,"ai_socktype":8,"ai_addr":20,"ai_protocol":12,"ai_family":4,"ai_addrlen":16,"__size__":32},"timespec":{"tv_sec":0,"tv_nsec":4,"__size__":8},"utimbuf":{"modtime":4,"actime":0,"__size__":8},"EmscriptenVisibilityChangeEvent":{"hidden":0,"visibilityState":4,"__size__":8},"SDL_MouseButtonEvent":{"timestamp":4,"button":16,"state":17,"windowID":8,"which":12,"y":24,"x":20,"padding2":19,"type":0,"padding1":18,"__size__":28},"sockaddr_in":{"sin_port":2,"sin_addr":{"s_addr":4,"__size__":4},"sin_family":0,"sin_zero":8,"__size__":16},"pthread":{"tsd":116,"attr":120,"canceldisable":72,"threadStatus":0,"tsd_used":56,"pid":52,"stack":92,"cancelasync":76,"tid":48,"threadExitCode":4,"detached":80,"profilerBlock":20,"self":24,"stack_size":96,"__size__":216},"WebVRFieldOfView":{"leftDegrees":24,"upDegrees":0,"downDegrees":16,"rightDegrees":8,"__size__":32},"SDL_KeyboardEvent":{"repeat":9,"keysym":12,"state":8,"windowID":4,"__size__":28,"type":0,"padding3":11,"padding2":10},"SDL_MouseMotionEvent":{"yrel":32,"timestamp":4,"state":16,"windowID":8,"which":12,"xrel":28,"y":24,"x":20,"type":0,"__size__":36},"SDL_Rect":{"y":4,"x":0,"h":12,"w":8,"__size__":16},"itimerspec":{"it_interval":{"tv_sec":0,"tv_nsec":4,"__size__":8},"it_value":{"tv_sec":8,"tv_nsec":12,"__size__":8},"__size__":16},"iovec":{"iov_len":4,"iov_base":0,"__size__":8},"timezone":{"tz_dsttime":4,"tz_minuteswest":0,"__size__":8},"flock":{"l_whence":2,"l_type":0,"l_start":4,"__size__":16,"l_len":8,"l_pid":12},"EmscriptenOrientationChangeEvent":{"orientationIndex":0,"orientationAngle":4,"__size__":8},"EmscriptenMouseEvent":{"clientX":16,"clientY":20,"targetX":52,"buttons":42,"timestamp":0,"button":40,"targetY":56,"altKey":32,"canvasY":64,"metaKey":36,"movementX":44,"movementY":48,"shiftKey":28,"ctrlKey":24,"screenY":12,"screenX":8,"canvasX":60,"__size__":72},"SDL_ResizeEvent":{"h":8,"type":0,"w":4,"__size__":12},"tms":{"tms_stime":4,"tms_utime":0,"tms_cstime":12,"tms_cutime":8,"__size__":16},"SDL_Color":{"unused":3,"r":0,"b":2,"g":1,"__size__":4},"EmscriptenKeyboardEvent":{"code":32,"charValue":120,"locale":88,"shiftKey":72,"altKey":76,"which":160,"metaKey":80,"location":64,"key":0,"ctrlKey":68,"charCode":152,"keyCode":156,"repeat":84,"__size__":164},"rusage":{"ru_msgrcv":56,"ru_utime":{"tv_sec":0,"tv_usec":4,"__size__":8},"ru_isrss":28,"ru_stime":{"tv_sec":8,"tv_usec":12,"__size__":8},"ru_nsignals":60,"ru_nivcsw":68,"ru_msgsnd":52,"ru_nswap":40,"ru_minflt":32,"ru_nvcsw":64,"ru_ixrss":20,"ru_inblock":44,"ru_idrss":24,"ru_maxrss":16,"ru_oublock":48,"ru_majflt":36,"__size__":136},"div_t":{"quot":0,"rem":4,"__size__":8},"timeval":{"tv_sec":0,"tv_usec":4,"__size__":8},"rlimit":{"rlim_cur":0,"rlim_max":8,"__size__":16},"in6_addr":{"__in6_union":{"__s6_addr16":0,"__s6_addr":0,"__s6_addr32":0,"__size__":16},"__size__":16},"tm":{"tm_sec":0,"tm_hour":8,"tm_mday":12,"tm_isdst":32,"tm_year":20,"tm_zone":40,"tm_mon":16,"tm_yday":28,"tm_gmtoff":36,"tm_wday":24,"tm_min":4,"__size__":44},"EmscriptenWebGLContextAttributes":{"majorVersion":32,"stencil":8,"preserveDrawingBuffer":20,"failIfMajorPerformanceCaveat":28,"antialias":12,"depth":4,"minorVersion":36,"premultipliedAlpha":16,"enableExtensionsByDefault":40,"alpha":0,"preferLowPowerToHighPerformance":24,"__size__":44},"EmscriptenBatteryEvent":{"dischargingTime":8,"level":16,"charging":24,"chargingTime":0,"__size__":32},"protoent":{"p_aliases":4,"p_proto":8,"p_name":0,"__size__":12},"SDL_Surface":{"userdata":24,"locked":28,"clip_rect":36,"format":4,"h":12,"refcount":56,"map":52,"flags":0,"w":8,"pitch":16,"lock_data":32,"pixels":20,"__size__":60},"EmscriptenTouchEvent":{"touches":20,"shiftKey":8,"altKey":12,"metaKey":16,"ctrlKey":4,"__size__":1684,"numTouches":0},"dirent":{"d_name":11,"d_off":4,"d_ino":0,"d_reclen":8,"d_type":10,"__size__":268},"sockaddr_in6":{"sin6_family":0,"sin6_flowinfo":4,"sin6_scope_id":24,"sin6_addr":{"__in6_union":{"__s6_addr16":8,"__s6_addr":8,"__s6_addr32":8,"__size__":16},"__size__":16},"__size__":28,"sin6_port":2},"SDL_JoyAxisEvent":{"__size__":12,"type":0,"value":8,"which":4,"padding2":7,"padding1":6,"axis":5},"netent":{"n_name":0,"n_net":12,"n_addrtype":8,"n_aliases":4,"__size__":16},"SDL_PixelFormat":{"palette":4,"Gloss":29,"Bmask":20,"Bloss":30,"Rloss":28,"format":0,"Gshift":33,"Aloss":31,"BitsPerPixel":8,"refcount":36,"next":40,"padding":10,"Rmask":12,"Bshift":34,"Gmask":16,"BytesPerPixel":9,"Amask":24,"Rshift":32,"Ashift":35,"__size__":44},"SDL_JoyButtonEvent":{"type":0,"button":5,"state":6,"which":4,"padding1":7,"__size__":8},"EmscriptenPointerlockChangeEvent":{"id":132,"nodeName":4,"isActive":0,"__size__":260},"in_addr":{"s_addr":0,"__size__":4},"EmscriptenDeviceOrientationEvent":{"timestamp":0,"beta":16,"alpha":8,"__size__":40,"gamma":24,"absolute":32},"SDL_WindowEvent":{"data2":16,"type":0,"data1":12,"windowID":4,"__size__":20,"padding1":9,"event":8,"padding3":11,"padding2":10},"SDL_Keysym":{"scancode":0,"mod":8,"unicode":12,"sym":4,"__size__":16},"cmsghdr":{"cmsg_type":8,"cmsg_level":4,"cmsg_len":0,"__size__":12},"EmscriptenUiEvent":{"windowInnerWidth":12,"detail":0,"scrollLeft":32,"documentBodyClientHeight":8,"windowInnerHeight":16,"scrollTop":28,"windowOuterHeight":24,"windowOuterWidth":20,"documentBodyClientWidth":4,"__size__":36},"thread_profiler_block":{"threadStatus":0,"timeSpentInStatus":16,"currentStatusStartTime":8,"name":72,"__size__":104},"stat":{"st_rdev":28,"st_mtim":{"tv_sec":56,"tv_nsec":60,"__size__":8},"st_blocks":44,"st_atim":{"tv_sec":48,"tv_nsec":52,"__size__":8},"st_nlink":16,"__st_ino_truncated":8,"st_ctim":{"tv_sec":64,"tv_nsec":68,"__size__":8},"st_mode":12,"st_blksize":40,"__st_dev_padding":4,"st_dev":0,"st_size":36,"st_gid":24,"__st_rdev_padding":32,"st_uid":20,"st_ino":72,"__size__":76},"pollfd":{"fd":0,"events":4,"revents":6,"__size__":8},"WebVRPositionState":{"linearVelocity":{"y":56,"x":48,"z":64,"w":72,"__size__":32},"orientation":{"y":128,"x":120,"z":136,"w":144,"__size__":32},"timeStamp":0,"angularVelocity":{"y":160,"x":152,"z":168,"w":176,"__size__":32},"hasPosition":8,"angularAcceleration":{"y":192,"x":184,"z":200,"w":208,"__size__":32},"linearAcceleration":{"y":88,"x":80,"z":96,"w":104,"__size__":32},"hasOrientation":112,"position":{"y":24,"x":16,"z":32,"w":40,"__size__":32},"__size__":216},"SDL_TextInputEvent":{"text":8,"windowID":4,"type":0,"__size__":40},"EmscriptenTouchPoint":{"clientX":12,"clientY":16,"identifier":0,"targetX":36,"targetY":40,"isChanged":28,"canvasY":48,"canvasX":44,"pageX":20,"pageY":24,"screenY":8,"screenX":4,"onTarget":32,"__size__":52},"EmscriptenDeviceMotionEvent":{"timestamp":0,"accelerationIncludingGravityZ":48,"accelerationIncludingGravityX":32,"accelerationIncludingGravityY":40,"accelerationY":16,"accelerationX":8,"rotationRateBeta":64,"accelerationZ":24,"rotationRateGamma":72,"rotationRateAlpha":56,"__size__":80},"SDL_AudioSpec":{"padding":10,"userdata":20,"format":4,"channels":6,"callback":16,"samples":8,"freq":0,"size":12,"silence":7,"__size__":24},"hostent":{"h_addrtype":8,"h_addr_list":16,"h_name":0,"__size__":20,"h_aliases":4,"h_length":12},"SDL_MouseWheelEvent":{"timestamp":4,"windowID":8,"which":12,"y":20,"x":16,"type":0,"__size__":24},"EmscriptenFocusEvent":{"id":128,"nodeName":0,"__size__":256},"SDL_version":{"major":0,"patch":2,"minor":1,"__size__":3},"statvfs":{"f_bsize":0,"f_bavail":16,"f_fsid":32,"f_favail":28,"f_files":20,"f_frsize":4,"f_blocks":8,"f_ffree":24,"f_bfree":12,"f_flag":40,"f_namemax":44,"__size__":72},"linger":{"l_onoff":0,"l_linger":4,"__size__":8},"EmscriptenFullscreenChangeEvent":{"elementWidth":264,"screenWidth":272,"nodeName":8,"elementHeight":268,"fullscreenEnabled":4,"screenHeight":276,"isFullscreen":0,"id":136,"__size__":280},"EmscriptenWheelEvent":{"deltaX":72,"deltaY":80,"deltaZ":88,"deltaMode":96,"mouse":0,"__size__":104},"WebVRIntRect":{"y":4,"x":0,"height":12,"width":8,"__size__":16},"SDL_TouchFingerEvent":{"timestamp":4,"dy":36,"touchId":8,"pressure":40,"dx":32,"type":0,"y":28,"x":24,"fingerId":16,"__size__":48},"SDL_AudioCVT":{"len_ratio":32,"len_cvt":24,"rate_incr":8,"filters":40,"len":20,"needed":0,"filter_index":80,"src_format":4,"len_mult":28,"__size__":88,"buf":16,"dst_format":6},"WebVRPoint":{"y":8,"x":0,"z":16,"w":24,"__size__":32},"statfs":{"f_bsize":4,"f_bavail":16,"f_fsid":28,"f_files":20,"f_frsize":40,"f_namelen":36,"f_blocks":8,"f_ffree":24,"f_bfree":12,"f_flags":44,"__size__":64},"msghdr":{"msg_iov":8,"msg_iovlen":12,"msg_namelen":4,"msg_controllen":20,"msg_flags":24,"msg_name":0,"msg_control":16,"__size__":28},"EmscriptenGamepadEvent":{"index":1300,"analogButton":528,"timestamp":0,"numButtons":12,"mapping":1368,"digitalButton":1040,"connected":1296,"numAxes":8,"__size__":1432,"id":1304,"axis":16},"SDL_Palette":{"ncolors":0,"colors":4,"version":8,"refcount":12,"__size__":16},"EmscriptenFullscreenStrategy":{"canvasResizedCallbackUserData":16,"canvasResolutionScaleMode":4,"scaleMode":0,"canvasResizedCallback":12,"filteringMode":8,"__size__":20},"timeb":{"dstflag":8,"timezone":6,"time":0,"millitm":4,"__size__":12},"WebVREyeParameters":{"currentFieldOfView":{"leftDegrees":152,"upDegrees":128,"downDegrees":144,"rightDegrees":136,"__size__":32},"recommendedFieldOfView":{"leftDegrees":88,"upDegrees":64,"downDegrees":80,"rightDegrees":72,"__size__":32},"eyeTranslation":{"y":104,"x":96,"z":112,"w":120,"__size__":32},"renderRect":{"y":164,"x":160,"height":172,"width":168,"__size__":16},"minimumFieldOfView":{"leftDegrees":24,"upDegrees":0,"downDegrees":16,"rightDegrees":8,"__size__":32},"maximumFieldOfView":{"leftDegrees":56,"upDegrees":32,"downDegrees":48,"rightDegrees":40,"__size__":32},"__size__":176}},"defines":{"ETXTBSY":26,"EOF":-1,"EMSCRIPTEN_EVENT_MOUSEOVER":35,"ETOOMANYREFS":109,"ENAMETOOLONG":36,"ENOPKG":65,"UUID_TYPE_DCE_TIME":1,"_SC_XOPEN_LEGACY":129,"_SC_XOPEN_VERSION":89,"F_UNLCK":2,"_SC_BC_DIM_MAX":37,"EL3HLT":46,"S_IFDIR":16384,"EMSCRIPTEN_EVENT_KEYPRESS":1,"EINPROGRESS":115,"_SC_BARRIERS":133,"EMSCRIPTEN_EVENT_TOUCHMOVE":24,"SDL_AUDIO_ALLOW_FREQUENCY_CHANGE":1,"AUDIO_U8":8,"EAI_AGAIN":-3,"_PC_MAX_CANON":1,"ENOTSUP":95,"EFBIG":27,"O_CREAT":64,"_SC_2_PBS_LOCATE":170,"EM_PROXIED_SETENV":113,"_CS_POSIX_V6_LP64_OFF64_LIBS":1126,"ENOLINK":67,"ABDAY_7":131078,"ABDAY_6":131077,"ABDAY_5":131076,"ABDAY_4":131075,"ABDAY_3":131074,"ABDAY_2":131073,"ABDAY_1":131072,"EL3RST":47,"YESEXPR":327680,"_SC_V6_ILP32_OFFBIG":177,"SDL_MINOR_VERSION":3,"EM_PROXIED_CLEARENV":112,"_SC_MEMLOCK":17,"ENOTUNIQ":76,"EMSCRIPTEN_RESULT_FAILED":-6,"ABMON_1":131086,"ELNRNG":48,"UUID_VARIANT_MICROSOFT":2,"EMSCRIPTEN_EVENT_TOUCHSTART":22,"ENOANO":55,"EMSCRIPTEN_EVENT_FOCUSIN":14,"EMSCRIPTEN_EVENT_MOUSEUP":6,"ENOPROTOOPT":92,"POLLIN":1,"S_IALLUGO":4095,"_SC_THREAD_KEYS_MAX":74,"EM_THREAD_STATUS_WAITPROXY":5,"O_RDWR":2,"EREMCHG":78,"EMSCRIPTEN_EVENT_GAMEPADDISCONNECTED":27,"_SC_2_PBS":168,"_SC_TRACE_INHERIT":183,"_SC_REGEXP":155,"_CS_POSIX_V6_LP64_OFF64_CFLAGS":1124,"_SC_DELAYTIMER_MAX":26,"S_IWUGO":146,"S_IFREG":32768,"F_GETLK64":12,"O_DIRECTORY":65536,"EM_PROXIED_UTIMES":13,"POLLHUP":16,"S_IFMT":61440,"F_SETLK64":13,"_SC_XOPEN_CRYPT":92,"_SC_CLOCK_SELECTION":137,"_PC_CHOWN_RESTRICTED":6,"E2BIG":7,"ABMON_3":131088,"AM_STR":131110,"SDL_AUDIO_MASK_ENDIAN":4096,"ALT_DIGITS":131119,"EHOSTDOWN":112,"EBFONT":59,"ENOTEMPTY":39,"AUDIO_S16":32784,"TIOCGPGRP":21519,"EBUSY":16,"_SC_MQ_PRIO_MAX":28,"_SC_PAGE_SIZE":30,"EADDRINUSE":98,"ENOTSOCK":88,"PM_STR":131111,"O_WRONLY":1,"_SC_STREAM_MAX":5,"ABMON_9":131094,"ELIBACC":79,"S_IFIFO":4096,"EDQUOT":122,"EAI_SYSTEM":-11,"ENOENT":2,"_SC_TIMERS":11,"O_SYNC":1052672,"SEEK_END":2,"EM_THREAD_STATUS_FINISHED":6,"_PC_REC_MIN_XFER_SIZE":16,"_PC_PATH_MAX":4,"_SC_SPORADIC_SERVER":160,"ECOMM":70,"_SC_NPROCESSORS_ONLN":84,"_CS_POSIX_V6_LPBIG_OFFBIG_LIBS":1130,"_PC_MAX_INPUT":2,"_SC_VERSION":29,"_SC_XBS5_LPBIG_OFFBIG":128,"_SC_CLK_TCK":2,"ABMON_2":131087,"EXFULL":54,"ABMON_7":131092,"ABMON_6":131091,"ABMON_5":131090,"ABMON_4":131089,"ENOTDIR":20,"ABMON_8":131093,"_SC_AIO_MAX":24,"ERA":131116,"EM_PROXIED_UNSETENV":114,"_SC_THREAD_PRIO_INHERIT":80,"_PC_2_SYMLINKS":20,"_SC_XBS5_LP64_OFF64":127,"EMSCRIPTEN_EVENT_BATTERYLEVELCHANGE":30,"ENETRESET":102,"EAFNOSUPPORT":97,"MON_2":131099,"MON_3":131100,"MON_1":131098,"EMSCRIPTEN_EVENT_DEVICEORIENTATION":16,"MON_7":131104,"MON_4":131101,"MON_5":131102,"_SC_SPAWN":159,"MON_8":131105,"MON_9":131106,"_CS_POSIX_V6_ILP32_OFF32_LDFLAGS":1117,"S_IFSOCK":49152,"S_IRUGO":292,"SOCK_DGRAM":2,"POLLERR":8,"EINVAL":22,"_CS_POSIX_V6_LPBIG_OFFBIG_CFLAGS":1128,"POLLRDNORM":64,"AUDIO_F32SYS":33056,"_SC_TRACE_SYS_MAX":244,"AI_V4MAPPED":8,"AI_NUMERICHOST":4,"_CS_POSIX_V6_WIDTH_RESTRICTED_ENVS":1,"EHOSTUNREACH":113,"ENOCSI":50,"EPROTONOSUPPORT":93,"_SC_AIO_PRIO_DELTA_MAX":25,"_SC_MONOTONIC_CLOCK":149,"ETIME":62,"ENOTTY":25,"_SC_XOPEN_ENH_I18N":93,"EAI_SERVICE":-8,"EAGAIN":11,"F_SETLKW64":14,"EMSGSIZE":90,"ELIBEXEC":83,"_SC_MEMORY_PROTECTION":19,"EMSCRIPTEN_FULLSCREEN_SCALE_CENTER":3,"SDL_AUDIO_ALLOW_FORMAT_CHANGE":2,"ECANCELED":125,"_SC_SPIN_LOCKS":154,"_SC_XOPEN_SHM":94,"_PC_LINK_MAX":0,"TIOCSPGRP":21520,"EOPNOTSUPP":95,"EMSCRIPTEN_EVENT_MOUSEENTER":33,"EAI_FAIL":-4,"NOEXPR":327681,"_SC_FSYNC":15,"_SC_GETGR_R_SIZE_MAX":69,"EDESTADDRREQ":89,"EADDRNOTAVAIL":99,"AUDIO_S32SYS":32800,"_SC_TRACE_NAME_MAX":243,"_SC_BC_BASE_MAX":36,"EMSCRIPTEN_EVENT_CANVASRESIZED":37,"EPERM":1,"EAI_FAMILY":-6,"O_NOFOLLOW":131072,"SOCK_STREAM":1,"O_APPEND":1024,"_SC_XOPEN_STREAMS":246,"_SC_GETPW_R_SIZE_MAX":70,"MON_6":131103,"EPROTOTYPE":91,"_SC_CPUTIME":138,"EISCONN":106,"_SC_XBS5_ILP32_OFFBIG":126,"S_IFBLK":24576,"T_FMT_AMPM":131115,"EM_PROXIED_FPATHCONF":46,"F_SETLKW":14,"SDL_TOUCH_MOUSEID":-1,"EMSCRIPTEN_EVENT_SCROLL":11,"ELOOP":40,"_SC_OPEN_MAX":4,"_SC_2_FORT_RUN":50,"EMSCRIPTEN_EVENT_VISIBILITYCHANGE":21,"EREMOTE":66,"_SC_RE_DUP_MAX":44,"_SC_THREAD_PRIO_PROTECT":81,"_SC_2_PBS_CHECKPOINT":175,"_SC_2_PBS_TRACK":172,"MON_10":131107,"MON_11":131108,"MON_12":131109,"TCGETS":21505,"_SC_THREAD_PROCESS_SHARED":82,"AF_INET":2,"_SC_SHARED_MEMORY_OBJECTS":22,"F_GETFD":1,"EMSCRIPTEN_EVENT_DEVICEMOTION":17,"SDL_MIX_MAXVOLUME":128,"_PC_ALLOC_SIZE_MIN":18,"TCSETS":21506,"ELIBMAX":82,"_SC_READER_WRITER_LOCKS":153,"EMULTIHOP":72,"_SC_PHYS_PAGES":85,"_SC_MEMLOCK_RANGE":18,"_SC_PRIORITY_SCHEDULING":10,"T_FMT":131114,"AI_ALL":16,"_PC_VDISABLE":8,"THOUSEP":65537,"_SC_TRACE_EVENT_FILTER":182,"ERA_T_FMT":131121,"_SC_THREAD_ATTR_STACKADDR":77,"_SC_THREAD_THREADS_MAX":76,"_SC_LOGIN_NAME_MAX":71,"_SC_2_C_BIND":47,"_PC_NO_TRUNC":7,"ECONNABORTED":103,"EMSCRIPTEN_RESULT_SUCCESS":0,"_SC_SHELL":157,"EFAULT":14,"_SC_V6_LP64_OFF64":178,"_CS_GNU_LIBC_VERSION":2,"ENODATA":61,"_SC_SEM_VALUE_MAX":33,"_SC_MQ_OPEN_MAX":27,"AI_ADDRCONFIG":32,"_SC_HOST_NAME_MAX":180,"_SC_THREAD_STACK_MIN":75,"_SC_TIMEOUTS":164,"POLLOUT":4,"_SC_IPV6":235,"_SC_CHILD_MAX":1,"EDOM":33,"_SC_2_PBS_MESSAGE":171,"EILSEQ":84,"UUID_VARIANT_DCE":1,"_SC_2_C_DEV":48,"_SC_TIMER_MAX":35,"FP_ZERO":2,"EPFNOSUPPORT":96,"ENONET":64,"ECHRNG":44,"_SC_THREADS":67,"_SC_REALTIME_SIGNALS":9,"CLOCKS_PER_SEC":1000000,"ERA_D_T_FMT":131120,"ESRCH":3,"D_FMT":131113,"POLLPRI":2,"_PC_ASYNC_IO":10,"DAY_2":131080,"DAY_3":131081,"DAY_1":131079,"DAY_6":131084,"DAY_7":131085,"DAY_4":131082,"DAY_5":131083,"_SC_SYNCHRONIZED_IO":14,"EL2HLT":51,"EMSCRIPTEN_FULLSCREEN_CANVAS_SCALE_STDDEF":1,"IPPROTO_UDP":17,"_SC_MAPPED_FILES":16,"EL2NSYNC":45,"_SC_NGROUPS_MAX":3,"ENOMSG":42,"EISDIR":21,"_SC_SEMAPHORES":21,"AI_NUMERICSERV":1024,"EDEADLOCK":35,"EMSCRIPTEN_EVENT_WEBGLCONTEXTLOST":31,"EMSCRIPTEN_EVENT_BATTERYCHARGINGCHANGE":29,"AUDIO_F32LSB":33056,"_SC_COLL_WEIGHTS_MAX":40,"SO_ERROR":4,"ECONNRESET":104,"AT_SYMLINK_NOFOLLOW":256,"_SC_TRACE_LOG":184,"AUDIO_U16LSB":16,"ESTRPIPE":86,"ESHUTDOWN":108,"_PC_SOCK_MAXBUF":12,"_CS_POSIX_V6_LPBIG_OFFBIG_LDFLAGS":1129,"EDEADLK":35,"_CS_POSIX_V6_ILP32_OFF32_CFLAGS":1116,"EBADRQC":56,"_SC_THREAD_DESTRUCTOR_ITERATIONS":73,"_SC_TYPED_MEMORY_OBJECTS":165,"_SC_TRACE_EVENT_NAME_MAX":242,"_SC_BC_STRING_MAX":39,"_SC_2_SW_DEV":51,"FP_NAN":0,"F_SETOWN":8,"EMSCRIPTEN_EVENT_RESIZE":10,"_SC_ARG_MAX":0,"_SC_THREAD_PRIORITY_SCHEDULING":79,"F_GETLK":12,"EMSCRIPTEN_FULLSCREEN_CANVAS_SCALE_HIDEF":2,"FIONREAD":21531,"_SC_THREAD_CPUTIME":139,"EMSCRIPTEN_EVENT_POINTERLOCKCHANGE":20,"EM_THREAD_STATUS_NOTSTARTED":0,"_CS_POSIX_V6_ILP32_OFF32_LIBS":1118,"EUNATCH":49,"AUDIO_S8":32776,"AUDIO_S32LSB":32800,"SDL_AUDIO_MASK_BITSIZE":255,"ERA_D_FMT":131118,"AUDIO_F32MSB":37152,"_CS_POSIX_V6_LP64_OFF64_LDFLAGS":1125,"FP_INFINITE":1,"ECHILD":10,"EAI_MEMORY":-10,"O_TRUNC":512,"ETIMEDOUT":110,"EALREADY":114,"ENXIO":6,"NI_NUMERICHOST":1,"EMFILE":24,"F_GETOWN":9,"EMLINK":31,"F_SETFD":2,"ENFILE":23,"EM_PROXIED_SYSCONF":72,"EM_PROXIED_GETENV":111,"SDL_MAJOR_VERSION":1,"ENOMEM":12,"ENOSR":63,"SDL_AUDIO_ALLOW_ANY_CHANGE":7,"EOWNERDEAD":130,"_PC_PRIO_IO":11,"ELIBSCN":81,"_SC_V6_LPBIG_OFFBIG":179,"EM_PROXIED_CHROOT":37,"EMSCRIPTEN_EVENT_CLICK":4,"EPIPE":32,"_SC_EXPR_NEST_MAX":42,"_CS_POSIX_V6_ILP32_OFFBIG_CFLAGS":1120,"EBADSLT":57,"AUDIO_S16MSB":36880,"S_ISVTX":512,"EMSCRIPTEN_RESULT_DEFERRED":1,"EMSCRIPTEN_RESULT_UNKNOWN_TARGET":-4,"S_IRWXUGO":511,"EM_PROXIED_TZSET":119,"_CS_GNU_LIBPTHREAD_VERSION":3,"_PC_REC_MAX_XFER_SIZE":15,"UUID_VARIANT_OTHER":3,"EMSCRIPTEN_EVENT_WEBGLCONTEXTRESTORED":32,"EM_PROXIED_PTHREAD_CREATE":137,"EMSCRIPTEN_FULLSCREEN_FILTERING_DEFAULT":0,"RADIXCHAR":65536,"AF_UNSPEC":0,"ENOSTR":60,"W_OK":2,"AUDIO_S32":32800,"EACCES":13,"R_OK":4,"S_IRWXO":7,"_SC_V6_ILP32_OFF32":176,"EMSCRIPTEN_EVENT_FULLSCREENCHANGE":19,"EIO":5,"EMSCRIPTEN_RESULT_NOT_SUPPORTED":-1,"EM_PROXIED_CONFSTR":68,"_SC_SIGQUEUE_MAX":34,"EWOULDBLOCK":11,"AUDIO_U16SYS":16,"EMSCRIPTEN_EVENT_FOCUSOUT":15,"EAI_OVERFLOW":-12,"SDL_AUDIO_MASK_DATATYPE":256,"MAP_PRIVATE":2,"_SC_TZNAME_MAX":6,"_CS_PATH":0,"SEEK_SET":0,"EBADE":52,"EMSCRIPTEN_RESULT_FAILED_NOT_DEFERRED":-2,"INT_MAX":2147483647,"EMSCRIPTEN_EVENT_KEYDOWN":2,"EMSCRIPTEN_FULLSCREEN_SCALE_STRETCH":1,"_SC_MESSAGE_PASSING":20,"_SC_THREAD_SAFE_FUNCTIONS":68,"_SC_SYMLOOP_MAX":173,"_PC_NAME_MAX":3,"O_EXCL":128,"_SC_TRACE_USER_EVENT_MAX":245,"_PC_REC_XFER_ALIGN":17,"EMSCRIPTEN_FULLSCREEN_SCALE_DEFAULT":0,"_SC_RAW_SOCKETS":236,"_SC_2_UPE":97,"EMSCRIPTEN_RESULT_NO_DATA":-7,"EMSCRIPTEN_EVENT_BLUR":12,"_SC_RTSIG_MAX":31,"ESOCKTNOSUPPORT":94,"_SC_PRIORITIZED_IO":13,"_SC_XOPEN_UNIX":91,"CODESET":14,"IPPROTO_TCP":6,"_PC_REC_INCR_XFER_SIZE":14,"F_SETLK":13,"_PC_FILESIZEBITS":13,"_SC_XBS5_ILP32_OFF32":125,"RAND_MAX":2147483647,"EM_PROXIED_SYSCALL":138,"ENOLCK":37,"EM_PROXIED_PUTENV":115,"AUDIO_U16":16,"EMSCRIPTEN_EVENT_MOUSELEAVE":34,"EMSCRIPTEN_EVENT_MOUSEOUT":36,"_SC_2_VERSION":46,"_PC_SYNC_IO":9,"EEXIST":17,"FP_NORMAL":4,"O_RDONLY":0,"_SC_SEM_NSEMS_MAX":32,"_SC_IOV_MAX":60,"EPROTO":71,"_SC_TRACE":181,"ESRMNT":69,"_CS_POSIX_V6_ILP32_OFFBIG_LDFLAGS":1121,"SDL_PIXELFORMAT_RGBA8888":-2042224636,"INADDR_LOOPBACK":2130706433,"EXDEV":18,"EM_THREAD_STATUS_RUNNING":1,"EMSCRIPTEN_EVENT_BEFOREUNLOAD":28,"EM_THREAD_STATUS_WAITFUTEX":3,"EMSCRIPTEN_RESULT_INVALID_TARGET":-3,"_SC_THREAD_SPORADIC_SERVER":161,"F_SETFL":4,"AI_PASSIVE":1,"ELIBBAD":80,"_SC_LINE_MAX":43,"D_T_FMT":131112,"ERANGE":34,"ESTALE":116,"F_DUPFD":0,"AUDIO_F32":33056,"CLOCK_MONOTONIC":1,"EMSCRIPTEN_EVENT_GAMEPADCONNECTED":26,"F_GETOWN_EX":16,"_SC_ASYNCHRONOUS_IO":12,"ENOTRECOVERABLE":131,"ENOBUFS":105,"EIDRM":43,"EMSCRIPTEN_EVENT_ORIENTATIONCHANGE":18,"CRNCYSTR":262159,"EINTR":4,"EADV":68,"ENOSYS":38,"_CS_POSIX_V6_ILP32_OFFBIG_LIBS":1122,"EM_PROXIED_UTIME":12,"F_GETFL":3,"S_IXUGO":73,"_SC_2_FORT_DEV":49,"SDL_COMPILEDVERSION":1300,"EBADMSG":74,"EUSERS":87,"CLOCK_REALTIME":0,"ENODEV":19,"AF_INET6":10,"_SC_ATEXIT_MAX":87,"_SC_SAVED_IDS":8,"SOL_SOCKET":1,"S_IFLNK":40960,"AUDIO_S16LSB":32784,"POLLNVAL":32,"EMSCRIPTEN_EVENT_TOUCHCANCEL":25,"EMSCRIPTEN_RESULT_INVALID_PARAM":-5,"EMSCRIPTEN_EVENT_MOUSEDOWN":5,"EM_THREAD_STATUS_SLEEPING":2,"_SC_JOB_CONTROL":7,"NI_NAMEREQD":8,"EMSCRIPTEN_FULLSCREEN_SCALE_ASPECT":2,"EMSCRIPTEN_EVENT_MOUSEMOVE":8,"UUID_TYPE_DCE_RANDOM":4,"ENOTCONN":107,"_SC_ADVISORY_INFO":132,"ENETUNREACH":101,"_SC_XOPEN_REALTIME_THREADS":131,"_SC_2_LOCALEDEF":52,"_PC_SYMLINK_MAX":19,"X_OK":1,"EMSCRIPTEN_EVENT_KEYUP":3,"AI_CANONNAME":2,"UUID_VARIANT_NCS":0,"ESPIPE":29,"AUDIO_S32MSB":36896,"EMSCRIPTEN_EVENT_WHEEL":9,"SDL_AUDIO_ALLOW_CHANNELS_CHANGE":4,"_SC_XOPEN_REALTIME":130,"EAI_NONAME":-2,"_PC_PIPE_BUF":5,"EROFS":30,"EM_PROXIED_ATEXIT":110,"ECONNREFUSED":111,"_SC_2_PBS_ACCOUNTING":169,"EMSCRIPTEN_EVENT_FOCUS":13,"AUDIO_S16SYS":32784,"ENETDOWN":100,"ENOEXEC":8,"ENOSPC":28,"EBADF":9,"EAI_SOCKTYPE":-7,"EDOTDOT":73,"_SC_THREAD_ATTR_STACKSIZE":78,"EBADFD":77,"O_ACCMODE":2097155,"EBADR":53,"EM_PROXIED_SBRK":73,"S_IFCHR":8192,"SDL_PATCHLEVEL":0,"ABMON_12":131097,"PTHREAD_KEYS_MAX":128,"ENOMEDIUM":123,"EMSCRIPTEN_FULLSCREEN_CANVAS_SCALE_NONE":0,"AUDIO_U16MSB":4112,"EMSCRIPTEN_FULLSCREEN_FILTERING_BILINEAR":2,"_SC_2_CHAR_TERM":95,"EMSCRIPTEN_EVENT_TOUCHEND":23,"_SC_AIO_LISTIO_MAX":23,"_SC_BC_SCALE_MAX":38,"ENOTBLK":15,"EAI_BADFLAGS":-1,"EOVERFLOW":75,"EMSCRIPTEN_EVENT_DBLCLICK":7,"SDL_AUDIO_MASK_SIGNED":32768,"EMSCRIPTEN_FULLSCREEN_FILTERING_NEAREST":1,"ABMON_11":131096,"ABMON_10":131095,"AT_FDCWD":-100,"_SC_TTY_NAME_MAX":72}} \ No newline at end of file diff --git a/system/include/emscripten/emmintrin.h b/system/include/emscripten/emmintrin.h index d216dd5377a32..81637b1b91ab2 100644 --- a/system/include/emscripten/emmintrin.h +++ b/system/include/emscripten/emmintrin.h @@ -1412,7 +1412,7 @@ static __inline__ __m128i __attribute__((__always_inline__, __nodebug__)) _mm_slli_epi16(__m128i __a, int __count) { #ifdef __EMSCRIPTEN__ - return emscripten_int16x8_shiftLeftByScalar(__a, __count); + return ((unsigned int)__count < 16) ? emscripten_int16x8_shiftLeftByScalar(__a, __count) : ((int16x8){ 0, 0, 0, 0, 0, 0, 0, 0 }); #else return (__m128i)__builtin_ia32_psllwi128((__v8hi)__a, __count); #endif @@ -1422,7 +1422,7 @@ static __inline__ __m128i __attribute__((__always_inline__, __nodebug__)) _mm_sll_epi16(__m128i __a, __m128i __count) { #ifdef __EMSCRIPTEN__ - return emscripten_int16x8_shiftLeftByScalar(__a, __count[1] == 0 ? __count[0] : 16); + return (__count[1] == 0 && (unsigned int)__count[0] < 16) ? emscripten_int16x8_shiftLeftByScalar(__a, __count[0]) : ((int16x8){ 0, 0, 0, 0, 0, 0, 0, 0 }); #else return (__m128i)__builtin_ia32_psllw128((__v8hi)__a, (__v8hi)__count); #endif @@ -1432,7 +1432,7 @@ static __inline__ __m128i __attribute__((__always_inline__, __nodebug__)) _mm_slli_epi32(__m128i __a, int __count) { #ifdef __EMSCRIPTEN__ - return emscripten_int32x4_shiftLeftByScalar(__a, __count); + return ((unsigned int)__count < 32) ? emscripten_int32x4_shiftLeftByScalar(__a, __count) : ((int32x4){ 0, 0, 0, 0 }); #else return (__m128i)__builtin_ia32_pslldi128((__v4si)__a, __count); #endif @@ -1442,7 +1442,7 @@ static __inline__ __m128i __attribute__((__always_inline__, __nodebug__)) _mm_sll_epi32(__m128i __a, __m128i __count) { #ifdef __EMSCRIPTEN__ - return emscripten_int32x4_shiftLeftByScalar(__a, __count[1] == 0 ? __count[0] : 32); + return (__count[1] == 0 && (unsigned int)__count[0] < 32) ? emscripten_int32x4_shiftLeftByScalar(__a, __count[0]) : ((int32x4){ 0, 0, 0, 0 }); #else return (__m128i)__builtin_ia32_pslld128((__v4si)__a, (__v4si)__count); #endif @@ -1498,7 +1498,7 @@ static __inline__ __m128i __attribute__((__always_inline__, __nodebug__)) _mm_srai_epi16(__m128i __a, int __count) { #ifdef __EMSCRIPTEN__ - return emscripten_int16x8_shiftRightByScalar(__a, __count); + return emscripten_int16x8_shiftRightByScalar(__a, (unsigned int)__count < 15 ? __count : 15); #else return (__m128i)__builtin_ia32_psrawi128((__v8hi)__a, __count); #endif @@ -1508,7 +1508,7 @@ static __inline__ __m128i __attribute__((__always_inline__, __nodebug__)) _mm_sra_epi16(__m128i __a, __m128i __count) { #ifdef __EMSCRIPTEN__ - return emscripten_int16x8_shiftRightByScalar(__a, __count[1] == 0 ? __count[0] : 16); + return emscripten_int16x8_shiftRightByScalar(__a, (__count[1] == 0 && (unsigned int)__count[0] < 15) ? __count[0] : 15); #else return (__m128i)__builtin_ia32_psraw128((__v8hi)__a, (__v8hi)__count); #endif @@ -1518,7 +1518,7 @@ static __inline__ __m128i __attribute__((__always_inline__, __nodebug__)) _mm_srai_epi32(__m128i __a, int __count) { #ifdef __EMSCRIPTEN__ - return emscripten_int32x4_shiftRightByScalar(__a, __count); + return emscripten_int32x4_shiftRightByScalar(__a, (unsigned int)__count < 31 ? __count : 31); #else return (__m128i)__builtin_ia32_psradi128((__v4si)__a, __count); #endif @@ -1528,7 +1528,7 @@ static __inline__ __m128i __attribute__((__always_inline__, __nodebug__)) _mm_sra_epi32(__m128i __a, __m128i __count) { #ifdef __EMSCRIPTEN__ - return emscripten_int32x4_shiftRightByScalar(__a, __count[1] == 0 ? __count[0] : 32); + return emscripten_int32x4_shiftRightByScalar(__a, (__count[1] == 0 && (unsigned int)__count[0] < 31) ? __count[0] : 31); #else return (__m128i)__builtin_ia32_psrad128((__v4si)__a, (__v4si)__count); #endif @@ -1561,7 +1561,7 @@ static __inline__ __m128i __attribute__((__always_inline__, __nodebug__)) _mm_srli_epi16(__m128i __a, int __count) { #ifdef __EMSCRIPTEN__ - return (int16x8)emscripten_uint16x8_shiftRightByScalar((uint16x8)__a, __count); + return ((unsigned int)__count < 16) ? ((int16x8)emscripten_uint16x8_shiftRightByScalar((uint16x8)__a, __count)) : ((int16x8){ 0, 0, 0, 0, 0, 0, 0, 0 }); #else return (__m128i)__builtin_ia32_psrlwi128((__v8hi)__a, __count); #endif @@ -1571,7 +1571,7 @@ static __inline__ __m128i __attribute__((__always_inline__, __nodebug__)) _mm_srl_epi16(__m128i __a, __m128i __count) { #ifdef __EMSCRIPTEN__ - return (int16x8)emscripten_uint16x8_shiftRightByScalar((uint16x8)__a, __count[1] == 0 ? __count[0] : 16); + return (__count[1] == 0 && (unsigned int)__count[0] < 32) ? ((int16x8)emscripten_uint16x8_shiftRightByScalar((uint16x8)__a, __count[0])) : ((int16x8){ 0, 0, 0, 0, 0, 0, 0, 0 }); #else return (__m128i)__builtin_ia32_psrlw128((__v8hi)__a, (__v8hi)__count); #endif @@ -1581,7 +1581,7 @@ static __inline__ __m128i __attribute__((__always_inline__, __nodebug__)) _mm_srli_epi32(__m128i __a, int __count) { #ifdef __EMSCRIPTEN__ - return (int32x4)emscripten_uint32x4_shiftRightByScalar((uint32x4)__a, __count); + return ((unsigned int)__count < 32) ? ((int32x4)emscripten_uint32x4_shiftRightByScalar((uint32x4)__a, __count)) : ((int32x4){ 0, 0, 0, 0 }); #else return (__m128i)__builtin_ia32_psrldi128((__v4si)__a, __count); #endif @@ -1591,7 +1591,7 @@ static __inline__ __m128i __attribute__((__always_inline__, __nodebug__)) _mm_srl_epi32(__m128i __a, __m128i __count) { #ifdef __EMSCRIPTEN__ - return (int32x4)emscripten_uint32x4_shiftRightByScalar((uint32x4)__a, __count[1] == 0 ? __count[0] : 32); + return (__count[1] == 0 && (unsigned int)__count[0] < 32) ? ((int32x4)emscripten_uint32x4_shiftRightByScalar((uint32x4)__a, __count[0])) : ((int32x4){ 0, 0, 0, 0 }); #else return (__m128i)__builtin_ia32_psrld128((__v4si)__a, (__v4si)__count); #endif diff --git a/system/include/emscripten/threading.h b/system/include/emscripten/threading.h index 68e7987b28126..6c686ed3b381e 100644 --- a/system/include/emscripten/threading.h +++ b/system/include/emscripten/threading.h @@ -81,7 +81,7 @@ uint64_t emscripten_atomic_xor_u64(void/*uint64_t*/ *addr, uint64_t val); // Emu int emscripten_futex_wait(void/*uint32_t*/ *addr, uint32_t val, double maxWaitMilliseconds); int emscripten_futex_wake(void/*uint32_t*/ *addr, int count); -int emscripten_futex_wake_or_requeue(void/*uint32_t*/ *addr, int count, int cmpValue, void/*uint32_t*/ *addr2); +int emscripten_futex_wake_or_requeue(void/*uint32_t*/ *addr, int count, void/*uint32_t*/ *addr2, int cmpValue); typedef union em_variant_val { diff --git a/system/lib/compiler-rt/LICENSE.TXT b/system/lib/compiler-rt/LICENSE.TXT index d6966ab7ac336..a17dc12b272fb 100644 --- a/system/lib/compiler-rt/LICENSE.TXT +++ b/system/lib/compiler-rt/LICENSE.TXT @@ -14,7 +14,7 @@ Full text of the relevant licenses is included below. University of Illinois/NCSA Open Source License -Copyright (c) 2009-2015 by the contributors listed in CREDITS.TXT +Copyright (c) 2009-2016 by the contributors listed in CREDITS.TXT All rights reserved. @@ -89,4 +89,3 @@ other licenses gives permission to use the names of the LLVM Team or the University of Illinois to endorse or promote products derived from this Software. - diff --git a/system/lib/compiler-rt/int_lib.h b/system/lib/compiler-rt/int_lib.h deleted file mode 100644 index a87426c513c66..0000000000000 --- a/system/lib/compiler-rt/int_lib.h +++ /dev/null @@ -1,46 +0,0 @@ -/* ===-- int_lib.h - configuration header for compiler-rt -----------------=== - * - * The LLVM Compiler Infrastructure - * - * This file is dual licensed under the MIT and the University of Illinois Open - * Source Licenses. See LICENSE.TXT for details. - * - * ===----------------------------------------------------------------------=== - * - * This file is a configuration header for compiler-rt. - * This file is not part of the interface of this library. - * - * ===----------------------------------------------------------------------=== - */ - -#ifndef INT_LIB_H -#define INT_LIB_H - -/* Assumption: Signed integral is 2's complement. */ -/* Assumption: Right shift of signed negative is arithmetic shift. */ -/* Assumption: Endianness is little or big (not mixed). */ - -/* ABI macro definitions */ - -#if __ARM_EABI__ -# define ARM_EABI_FNALIAS(aeabi_name, name) \ - void __aeabi_##aeabi_name() __attribute__((alias("__" #name))); -# define COMPILER_RT_ABI __attribute__((pcs("aapcs"))) -#else -# define ARM_EABI_FNALIAS(aeabi_name, name) -# define COMPILER_RT_ABI -#endif - -/* Include the standard compiler builtin headers we use functionality from. */ -#include -#include -#include -#include - -/* Include the commonly used internal type definitions. */ -#include "int_types.h" - -/* Include internal utility function declarations. */ -#include "int_util.h" - -#endif /* INT_LIB_H */ diff --git a/system/lib/compiler-rt/divdc3.c b/system/lib/compiler-rt/lib/builtins/divdc3.c similarity index 99% rename from system/lib/compiler-rt/divdc3.c rename to system/lib/compiler-rt/lib/builtins/divdc3.c index a869afa0f785a..3c88390b5e77f 100644 --- a/system/lib/compiler-rt/divdc3.c +++ b/system/lib/compiler-rt/lib/builtins/divdc3.c @@ -58,4 +58,3 @@ __divdc3(double __a, double __b, double __c, double __d) } return z; } - diff --git a/system/lib/compiler-rt/divdi3.c b/system/lib/compiler-rt/lib/builtins/divdi3.c similarity index 64% rename from system/lib/compiler-rt/divdi3.c rename to system/lib/compiler-rt/lib/builtins/divdi3.c index 09f4a04edd65c..b8eebcb204656 100644 --- a/system/lib/compiler-rt/divdi3.c +++ b/system/lib/compiler-rt/lib/builtins/divdi3.c @@ -14,8 +14,6 @@ #include "int_lib.h" -du_int COMPILER_RT_ABI __udivmoddi4(du_int a, du_int b, du_int* rem); - /* Returns: a / b */ COMPILER_RT_ABI di_int @@ -29,19 +27,3 @@ __divdi3(di_int a, di_int b) s_a ^= s_b; /*sign of quotient */ return (__udivmoddi4(a, b, (du_int*)0) ^ s_a) - s_a; /* negate if s_a == -1 */ } - -/* XXX EMSCRIPTEN */ - -COMPILER_RT_ABI di_int -__remdi3(di_int a, di_int b) -{ - const int bits_in_dword_m1 = (int)(sizeof(di_int) * CHAR_BIT) - 1; - di_int s_a = a >> bits_in_dword_m1; /* s_a = a < 0 ? -1 : 0 */ - di_int s_b = b >> bits_in_dword_m1; /* s_b = b < 0 ? -1 : 0 */ - a = (a ^ s_a) - s_a; /* negate if s_a == -1 */ - b = (b ^ s_b) - s_b; /* negate if s_b == -1 */ - du_int rem; - __udivmoddi4(a, b, &rem); - return (rem ^ s_a) - s_a; /* negate if s_a == -1 */ -} - diff --git a/system/lib/compiler-rt/divsc3.c b/system/lib/compiler-rt/lib/builtins/divsc3.c similarity index 99% rename from system/lib/compiler-rt/divsc3.c rename to system/lib/compiler-rt/lib/builtins/divsc3.c index 063cc048c0f4f..42a48315e66d1 100644 --- a/system/lib/compiler-rt/divsc3.c +++ b/system/lib/compiler-rt/lib/builtins/divsc3.c @@ -58,4 +58,3 @@ __divsc3(float __a, float __b, float __c, float __d) } return z; } - diff --git a/system/lib/compiler-rt/int_endianness.h b/system/lib/compiler-rt/lib/builtins/int_endianness.h similarity index 78% rename from system/lib/compiler-rt/int_endianness.h rename to system/lib/compiler-rt/lib/builtins/int_endianness.h index fa294c49eb2c2..7995ddbb953cd 100644 --- a/system/lib/compiler-rt/int_endianness.h +++ b/system/lib/compiler-rt/lib/builtins/int_endianness.h @@ -16,22 +16,39 @@ #ifndef INT_ENDIANNESS_H #define INT_ENDIANNESS_H +#if defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && \ + defined(__ORDER_LITTLE_ENDIAN__) + +/* Clang and GCC provide built-in endianness definitions. */ +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#define _YUGA_LITTLE_ENDIAN 0 +#define _YUGA_BIG_ENDIAN 1 +#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define _YUGA_LITTLE_ENDIAN 1 +#define _YUGA_BIG_ENDIAN 0 +#endif /* __BYTE_ORDER__ */ + +#else /* Compilers other than Clang or GCC. */ + #if defined(__SVR4) && defined(__sun) #include -#if _BYTE_ORDER == _BIG_ENDIAN +#if defined(_BIG_ENDIAN) #define _YUGA_LITTLE_ENDIAN 0 #define _YUGA_BIG_ENDIAN 1 -#elif _BYTE_ORDER == _LITTLE_ENDIAN +#elif defined(_LITTLE_ENDIAN) #define _YUGA_LITTLE_ENDIAN 1 #define _YUGA_BIG_ENDIAN 0 -#endif /* _BYTE_ORDER */ +#else /* !_LITTLE_ENDIAN */ +#error "unknown endianness" +#endif /* !_LITTLE_ENDIAN */ #endif /* Solaris and AuroraUX. */ /* .. */ -#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__DragonFly__) || defined(__minix) +#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__DragonFly__) || \ + defined(__minix) #include #if _BYTE_ORDER == _BIG_ENDIAN @@ -59,8 +76,9 @@ /* .. */ -/* Mac OSX has __BIG_ENDIAN__ or __LITTLE_ENDIAN__ automatically set by the compiler (at least with GCC) */ -#if defined(__APPLE__) && defined(__MACH__) || defined(__ellcc__ ) +/* Mac OSX has __BIG_ENDIAN__ or __LITTLE_ENDIAN__ automatically set by the + * compiler (at least with GCC) */ +#if defined(__APPLE__) || defined(__ellcc__ ) #ifdef __BIG_ENDIAN__ #if __BIG_ENDIAN__ @@ -80,19 +98,6 @@ /* .. */ -#if defined(__linux__) -#include - -#if __BYTE_ORDER == __BIG_ENDIAN -#define _YUGA_LITTLE_ENDIAN 0 -#define _YUGA_BIG_ENDIAN 1 -#elif __BYTE_ORDER == __LITTLE_ENDIAN -#define _YUGA_LITTLE_ENDIAN 1 -#define _YUGA_BIG_ENDIAN 0 -#endif /* __BYTE_ORDER */ - -#endif /* GNU/Linux */ - #if defined(_WIN32) #define _YUGA_LITTLE_ENDIAN 1 @@ -100,12 +105,7 @@ #endif /* Windows */ -#if defined(__EMSCRIPTEN__) - -#define _YUGA_LITTLE_ENDIAN 1 -#define _YUGA_BIG_ENDIAN 0 - -#endif /* emscripten */ +#endif /* Clang or GCC. */ /* . */ diff --git a/system/lib/compiler-rt/lib/builtins/int_lib.h b/system/lib/compiler-rt/lib/builtins/int_lib.h new file mode 100644 index 0000000000000..e66cda3fffb4b --- /dev/null +++ b/system/lib/compiler-rt/lib/builtins/int_lib.h @@ -0,0 +1,133 @@ +/* ===-- int_lib.h - configuration header for compiler-rt -----------------=== + * + * The LLVM Compiler Infrastructure + * + * This file is dual licensed under the MIT and the University of Illinois Open + * Source Licenses. See LICENSE.TXT for details. + * + * ===----------------------------------------------------------------------=== + * + * This file is a configuration header for compiler-rt. + * This file is not part of the interface of this library. + * + * ===----------------------------------------------------------------------=== + */ + +#ifndef INT_LIB_H +#define INT_LIB_H + +/* Assumption: Signed integral is 2's complement. */ +/* Assumption: Right shift of signed negative is arithmetic shift. */ +/* Assumption: Endianness is little or big (not mixed). */ + +#if defined(__ELF__) +#define FNALIAS(alias_name, original_name) \ + void alias_name() __attribute__((alias(#original_name))) +#else +#define FNALIAS(alias, name) _Pragma("GCC error(\"alias unsupported on this file format\")") +#endif + +/* ABI macro definitions */ + +#if __ARM_EABI__ +# define ARM_EABI_FNALIAS(aeabi_name, name) \ + void __aeabi_##aeabi_name() __attribute__((alias("__" #name))); +# define COMPILER_RT_ABI __attribute__((pcs("aapcs"))) +#else +# define ARM_EABI_FNALIAS(aeabi_name, name) +# if defined(__arm__) && defined(_WIN32) && (!defined(_MSC_VER) || defined(__clang__)) +# define COMPILER_RT_ABI __attribute__((pcs("aapcs"))) +# else +# define COMPILER_RT_ABI +# endif +#endif + +#ifdef _MSC_VER +#define ALWAYS_INLINE __forceinline +#define NOINLINE __declspec(noinline) +#define NORETURN __declspec(noreturn) +#define UNUSED +#else +#define ALWAYS_INLINE __attribute__((always_inline)) +#define NOINLINE __attribute__((noinline)) +#define NORETURN __attribute__((noreturn)) +#define UNUSED __attribute__((unused)) +#endif + +#if defined(__NetBSD__) && (defined(_KERNEL) || defined(_STANDALONE)) +/* + * Kernel and boot environment can't use normal headers, + * so use the equivalent system headers. + */ +# include +# include +# include +#else +/* Include the standard compiler builtin headers we use functionality from. */ +# include +# include +# include +# include +#endif + +/* Include the commonly used internal type definitions. */ +#include "int_types.h" + +/* Include internal utility function declarations. */ +#include "int_util.h" + +COMPILER_RT_ABI si_int __paritysi2(si_int a); +COMPILER_RT_ABI si_int __paritydi2(di_int a); + +COMPILER_RT_ABI di_int __divdi3(di_int a, di_int b); +COMPILER_RT_ABI si_int __divsi3(si_int a, si_int b); +COMPILER_RT_ABI su_int __udivsi3(su_int n, su_int d); + +COMPILER_RT_ABI su_int __udivmodsi4(su_int a, su_int b, su_int* rem); +COMPILER_RT_ABI du_int __udivmoddi4(du_int a, du_int b, du_int* rem); +#ifdef CRT_HAS_128BIT +COMPILER_RT_ABI si_int __clzti2(ti_int a); +COMPILER_RT_ABI tu_int __udivmodti4(tu_int a, tu_int b, tu_int* rem); +#endif + +/* Definitions for builtins unavailable on MSVC */ +#if defined(_MSC_VER) && !defined(__clang__) +#include + +uint32_t __inline __builtin_ctz(uint32_t value) { + uint32_t trailing_zero = 0; + if (_BitScanForward(&trailing_zero, value)) + return trailing_zero; + return 32; +} + +uint32_t __inline __builtin_clz(uint32_t value) { + uint32_t leading_zero = 0; + if (_BitScanReverse(&leading_zero, value)) + return 31 - leading_zero; + return 32; +} + +#if defined(_M_ARM) || defined(_M_X64) +uint32_t __inline __builtin_clzll(uint64_t value) { + uint32_t leading_zero = 0; + if (_BitScanReverse64(&leading_zero, value)) + return 63 - leading_zero; + return 64; +} +#else +uint32_t __inline __builtin_clzll(uint64_t value) { + if (value == 0) + return 64; + uint32_t msh = (uint32_t)(value >> 32); + uint32_t lsh = (uint32_t)(value & 0xFFFFFFFF); + if (msh != 0) + return __builtin_clz(msh); + return 32 + __builtin_clz(lsh); +} +#endif + +#define __builtin_clzl __builtin_clzll +#endif /* defined(_MSC_VER) && !defined(__clang__) */ + +#endif /* INT_LIB_H */ diff --git a/system/lib/compiler-rt/int_math.h b/system/lib/compiler-rt/lib/builtins/int_math.h similarity index 62% rename from system/lib/compiler-rt/int_math.h rename to system/lib/compiler-rt/lib/builtins/int_math.h index d6b4bdae162b9..fc81fb7f0220d 100644 --- a/system/lib/compiler-rt/int_math.h +++ b/system/lib/compiler-rt/lib/builtins/int_math.h @@ -25,43 +25,90 @@ # define __has_builtin(x) 0 #endif -#define CRT_INFINITY __builtin_huge_valf() +#if defined(_MSC_VER) && !defined(__clang__) +#include +#include +#include +#endif -#define crt_isinf(x) __builtin_isinf((x)) -#define crt_isnan(x) __builtin_isnan((x)) +#if defined(_MSC_VER) && !defined(__clang__) +#define CRT_INFINITY INFINITY +#else +#define CRT_INFINITY __builtin_huge_valf() +#endif +#if defined(_MSC_VER) && !defined(__clang__) +#define crt_isfinite(x) _finite((x)) +#define crt_isinf(x) !_finite((x)) +#define crt_isnan(x) _isnan((x)) +#else /* Define crt_isfinite in terms of the builtin if available, otherwise provide * an alternate version in terms of our other functions. This supports some * versions of GCC which didn't have __builtin_isfinite. */ #if __has_builtin(__builtin_isfinite) # define crt_isfinite(x) __builtin_isfinite((x)) -#else +#elif defined(__GNUC__) # define crt_isfinite(x) \ __extension__(({ \ __typeof((x)) x_ = (x); \ !crt_isinf(x_) && !crt_isnan(x_); \ })) -#endif +#else +# error "Do not know how to check for infinity" +#endif /* __has_builtin(__builtin_isfinite) */ +#define crt_isinf(x) __builtin_isinf((x)) +#define crt_isnan(x) __builtin_isnan((x)) +#endif /* _MSC_VER */ +#if defined(_MSC_VER) && !defined(__clang__) +#define crt_copysign(x, y) copysign((x), (y)) +#define crt_copysignf(x, y) copysignf((x), (y)) +#define crt_copysignl(x, y) copysignl((x), (y)) +#else #define crt_copysign(x, y) __builtin_copysign((x), (y)) #define crt_copysignf(x, y) __builtin_copysignf((x), (y)) #define crt_copysignl(x, y) __builtin_copysignl((x), (y)) +#endif +#if defined(_MSC_VER) && !defined(__clang__) +#define crt_fabs(x) fabs((x)) +#define crt_fabsf(x) fabsf((x)) +#define crt_fabsl(x) fabs((x)) +#else #define crt_fabs(x) __builtin_fabs((x)) #define crt_fabsf(x) __builtin_fabsf((x)) #define crt_fabsl(x) __builtin_fabsl((x)) +#endif +#if defined(_MSC_VER) && !defined(__clang__) +#define crt_fmax(x, y) __max((x), (y)) +#define crt_fmaxf(x, y) __max((x), (y)) +#define crt_fmaxl(x, y) __max((x), (y)) +#else #define crt_fmax(x, y) __builtin_fmax((x), (y)) #define crt_fmaxf(x, y) __builtin_fmaxf((x), (y)) #define crt_fmaxl(x, y) __builtin_fmaxl((x), (y)) +#endif +#if defined(_MSC_VER) && !defined(__clang__) +#define crt_logb(x) logb((x)) +#define crt_logbf(x) logbf((x)) +#define crt_logbl(x) logbl((x)) +#else #define crt_logb(x) __builtin_logb((x)) #define crt_logbf(x) __builtin_logbf((x)) #define crt_logbl(x) __builtin_logbl((x)) +#endif +#if defined(_MSC_VER) && !defined(__clang__) +#define crt_scalbn(x, y) scalbn((x), (y)) +#define crt_scalbnf(x, y) scalbnf((x), (y)) +#define crt_scalbnl(x, y) scalbnl((x), (y)) +#else #define crt_scalbn(x, y) __builtin_scalbn((x), (y)) #define crt_scalbnf(x, y) __builtin_scalbnf((x), (y)) #define crt_scalbnl(x, y) __builtin_scalbnl((x), (y)) +#endif #endif /* INT_MATH_H */ diff --git a/system/lib/compiler-rt/int_types.h b/system/lib/compiler-rt/lib/builtins/int_types.h similarity index 78% rename from system/lib/compiler-rt/int_types.h rename to system/lib/compiler-rt/lib/builtins/int_types.h index 001f0d7d0252a..660385ecd6aed 100644 --- a/system/lib/compiler-rt/int_types.h +++ b/system/lib/compiler-rt/lib/builtins/int_types.h @@ -20,6 +20,10 @@ #include "int_endianness.h" +/* si_int is defined in Linux sysroot's asm-generic/siginfo.h */ +#ifdef si_int +#undef si_int +#endif typedef int si_int; typedef unsigned su_int; @@ -56,8 +60,13 @@ typedef union }s; } udwords; -#if __x86_64 +/* MIPS64 issue: PR 20098 */ +#if (defined(__LP64__) || defined(__wasm__)) && \ + !(defined(__mips__) && defined(__clang__)) +#define CRT_HAS_128BIT +#endif +#ifdef CRT_HAS_128BIT typedef int ti_int __attribute__ ((mode (TI))); typedef unsigned tu_int __attribute__ ((mode (TI))); @@ -91,21 +100,21 @@ typedef union }s; } utwords; -static inline ti_int make_ti(di_int h, di_int l) { +static __inline ti_int make_ti(di_int h, di_int l) { twords r; r.s.high = h; r.s.low = l; return r.all; } -static inline tu_int make_tu(du_int h, du_int l) { +static __inline tu_int make_tu(du_int h, du_int l) { utwords r; r.s.high = h; r.s.low = l; return r.all; } -#endif /* __x86_64 */ +#endif /* CRT_HAS_128BIT */ typedef union { @@ -136,12 +145,22 @@ typedef union long double f; } long_double_bits; +#if __STDC_VERSION__ >= 199901L typedef float _Complex Fcomplex; typedef double _Complex Dcomplex; typedef long double _Complex Lcomplex; #define COMPLEX_REAL(x) __real__(x) #define COMPLEX_IMAGINARY(x) __imag__(x) +#else +typedef struct { float real, imaginary; } Fcomplex; + +typedef struct { double real, imaginary; } Dcomplex; + +typedef struct { long double real, imaginary; } Lcomplex; +#define COMPLEX_REAL(x) (x).real +#define COMPLEX_IMAGINARY(x) (x).imaginary +#endif #endif /* INT_TYPES_H */ diff --git a/system/lib/compiler-rt/int_util.h b/system/lib/compiler-rt/lib/builtins/int_util.h similarity index 68% rename from system/lib/compiler-rt/int_util.h rename to system/lib/compiler-rt/lib/builtins/int_util.h index 1348b85eb923e..a7b20ed66244f 100644 --- a/system/lib/compiler-rt/int_util.h +++ b/system/lib/compiler-rt/lib/builtins/int_util.h @@ -20,10 +20,14 @@ #define INT_UTIL_H /** \brief Trigger a program abort (or panic for kernel code). */ -#define compilerrt_abort() compilerrt_abort_impl(__FILE__, __LINE__, \ - __FUNCTION__) +#define compilerrt_abort() compilerrt_abort_impl(__FILE__, __LINE__, __func__) -void compilerrt_abort_impl(const char *file, int line, - const char *function) __attribute__((noreturn)); +NORETURN void compilerrt_abort_impl(const char *file, int line, + const char *function); + +#define COMPILE_TIME_ASSERT(expr) COMPILE_TIME_ASSERT1(expr, __COUNTER__) +#define COMPILE_TIME_ASSERT1(expr, cnt) COMPILE_TIME_ASSERT2(expr, cnt) +#define COMPILE_TIME_ASSERT2(expr, cnt) \ + typedef char ct_assert_##cnt[(expr) ? 1 : -1] UNUSED #endif /* INT_UTIL_H */ diff --git a/system/lib/compiler-rt/muldc3.c b/system/lib/compiler-rt/lib/builtins/muldc3.c similarity index 99% rename from system/lib/compiler-rt/muldc3.c rename to system/lib/compiler-rt/lib/builtins/muldc3.c index 5c14ff1301a83..16d8e98390a34 100644 --- a/system/lib/compiler-rt/muldc3.c +++ b/system/lib/compiler-rt/lib/builtins/muldc3.c @@ -71,5 +71,3 @@ __muldc3(double __a, double __b, double __c, double __d) } return z; } - - diff --git a/system/lib/compiler-rt/muldi3.c b/system/lib/compiler-rt/lib/builtins/muldi3.c similarity index 100% rename from system/lib/compiler-rt/muldi3.c rename to system/lib/compiler-rt/lib/builtins/muldi3.c diff --git a/system/lib/compiler-rt/mulsc3.c b/system/lib/compiler-rt/lib/builtins/mulsc3.c similarity index 99% rename from system/lib/compiler-rt/mulsc3.c rename to system/lib/compiler-rt/lib/builtins/mulsc3.c index d6d4a296bf022..c89cfd247a153 100644 --- a/system/lib/compiler-rt/mulsc3.c +++ b/system/lib/compiler-rt/lib/builtins/mulsc3.c @@ -71,5 +71,3 @@ __mulsc3(float __a, float __b, float __c, float __d) } return z; } - - diff --git a/system/lib/compiler-rt/udivdi3.c b/system/lib/compiler-rt/lib/builtins/udivdi3.c similarity index 100% rename from system/lib/compiler-rt/udivdi3.c rename to system/lib/compiler-rt/lib/builtins/udivdi3.c diff --git a/system/lib/compiler-rt/udivmoddi4.c b/system/lib/compiler-rt/lib/builtins/udivmoddi4.c similarity index 100% rename from system/lib/compiler-rt/udivmoddi4.c rename to system/lib/compiler-rt/lib/builtins/udivmoddi4.c diff --git a/system/lib/compiler-rt/readme.txt b/system/lib/compiler-rt/readme.txt index d10f53e4e6443..0daaf416dc647 100644 --- a/system/lib/compiler-rt/readme.txt +++ b/system/lib/compiler-rt/readme.txt @@ -1,20 +1,17 @@ These files are from compiler-rt, -Last Changed Rev: 179380 -Last Changed Date: 2013-04-12 07:57:03 -0700 (Fri, 12 Apr 2013) +Last Changed Rev: 266813 +Last Changed Date: Tue Apr 19 13:29:59 2016 =========================================================================== -Changes: - - * add emscripten endianness to int_endianness.h - * add rem functions + No changes from upstream (but not all files are included). =========================================================================== Compile with something like -./emcc system/lib/compiler-rt/*.c -Isystem/lib/compiler-rt/ -o rt.bc +./emcc system/lib/compiler-rt/lib/builtins/*.c -Isystem/lib/compiler-rt/lib/builtins -o rt.bc ./emcc -O2 -s ASM_JS=1 -g rt.bc -s LINKABLE=1 manually replace Math_imul with Math.imul diff --git a/system/lib/libc/musl/arch/emscripten/bits/alltypes.h b/system/lib/libc/musl/arch/emscripten/bits/alltypes.h index 0b9f239b14b18..e927d2ecf9ce1 100644 --- a/system/lib/libc/musl/arch/emscripten/bits/alltypes.h +++ b/system/lib/libc/musl/arch/emscripten/bits/alltypes.h @@ -1,4 +1,11 @@ +// wasm breaks musl convention and defines size_t as long, to benefit wasm32/64 similarity +// on wasm32, in practice it should be the same as asmjs's int. +#ifdef __wasm__ +#define _Addr long +#else #define _Addr int +#endif + #define _Int64 long long #define _Reg int diff --git a/system/lib/libc/musl/src/thread/pthread_rwlock_timedwrlock.c b/system/lib/libc/musl/src/thread/pthread_rwlock_timedwrlock.c index fc45a8b63a856..5434f575d9c6f 100644 --- a/system/lib/libc/musl/src/thread/pthread_rwlock_timedwrlock.c +++ b/system/lib/libc/musl/src/thread/pthread_rwlock_timedwrlock.c @@ -5,7 +5,7 @@ int pthread_rwlock_timedwrlock(pthread_rwlock_t *restrict rw, const struct times #ifdef __EMSCRIPTEN__ /// XXX Emscripten: The spec allows detecting when multiple write locks would deadlock, which we do here to avoid hangs. /// If attempting to lock the write lock that we already own, error out. - if (rw->_rw_wr_owner == pthread_self()) return EDEADLK; + if (rw->_rw_wr_owner == (int)pthread_self()) return EDEADLK; #endif int r, t; while ((r=pthread_rwlock_trywrlock(rw))==EBUSY) { @@ -20,7 +20,7 @@ int pthread_rwlock_timedwrlock(pthread_rwlock_t *restrict rw, const struct times #ifdef __EMSCRIPTEN__ /// XXX Emscripten: The spec allows detecting when multiple write locks would deadlock, which we do here to avoid hangs. /// Mark this thread as the owner of this write lock. - rw->_rw_wr_owner = pthread_self(); + rw->_rw_wr_owner = (int)pthread_self(); #endif return r; } diff --git a/system/lib/libc/musl/src/thread/pthread_rwlock_trywrlock.c b/system/lib/libc/musl/src/thread/pthread_rwlock_trywrlock.c index d6779e58bf5e1..fdd08d39dbcc4 100644 --- a/system/lib/libc/musl/src/thread/pthread_rwlock_trywrlock.c +++ b/system/lib/libc/musl/src/thread/pthread_rwlock_trywrlock.c @@ -6,7 +6,7 @@ int pthread_rwlock_trywrlock(pthread_rwlock_t *rw) #ifdef __EMSCRIPTEN__ /// XXX Emscripten: The spec allows detecting when multiple write locks would deadlock, which we do here to avoid hangs. /// Mark this thread to own the write lock, to ignore multiple attempts to lock. - rw->_rw_wr_owner = pthread_self(); + rw->_rw_wr_owner = (int)pthread_self(); #endif return 0; } diff --git a/system/lib/libc/musl/src/thread/pthread_rwlock_unlock.c b/system/lib/libc/musl/src/thread/pthread_rwlock_unlock.c index 418b6b07a44f2..97e6a79bbc61a 100644 --- a/system/lib/libc/musl/src/thread/pthread_rwlock_unlock.c +++ b/system/lib/libc/musl/src/thread/pthread_rwlock_unlock.c @@ -7,7 +7,7 @@ int pthread_rwlock_unlock(pthread_rwlock_t *rw) #ifdef __EMSCRIPTEN__ /// XXX Emscripten: The spec allows detecting when multiple write locks would deadlock, which we do here to avoid hangs. /// Mark this thread to not own the write lock anymore. - if (rw->_rw_wr_owner == pthread_self()) rw->_rw_wr_owner = 0; + if (rw->_rw_wr_owner == (int)pthread_self()) rw->_rw_wr_owner = 0; #endif do { diff --git a/system/lib/pthread/library_pthread.c b/system/lib/pthread/library_pthread.c index caf16e9f7ea3f..5c7bef81c1671 100644 --- a/system/lib/pthread/library_pthread.c +++ b/system/lib/pthread/library_pthread.c @@ -348,7 +348,7 @@ void * EMSCRIPTEN_KEEPALIVE emscripten_sync_run_in_main_thread_xprintf_varargs(i len = vsnprintf(s, len+1, format, args); } em_queued_call q = { function, 0 }; - q.args[0].vp = param0; + q.args[0].vp = (void*)param0; q.args[1].vp = s; q.returnValue.vp = 0; emscripten_sync_run_in_main_thread(&q); diff --git a/system/lib/wasm-libc.symbols b/system/lib/wasm-libc.symbols new file mode 100644 index 0000000000000..f19c1be17162b --- /dev/null +++ b/system/lib/wasm-libc.symbols @@ -0,0 +1,59 @@ + U __cos + U __cosdf + U __rem_pio2 + U __rem_pio2f + U __sin + U __sindf + U __tan + U __tandf +-------- T acos +-------- T acosf +-------- T acosl +-------- T asin +-------- T asinf +-------- T asinl +-------- T atan +-------- T atan2 +-------- T atan2f +-------- T atan2l +-------- T atanf +-------- d atanhi +-------- d atanhi.1 +-------- T atanl +-------- d atanlo +-------- d atanlo.2 +-------- d bp +-------- d bp.4 +-------- T cos +-------- T cosf +-------- T cosl +-------- d dp_h +-------- d dp_h.6 +-------- d dp_l +-------- d dp_l.5 +-------- T exp +-------- T expf +-------- T expl + U fabs + U fabsf +-------- d half +-------- d half.3 +-------- T log +-------- T logf +-------- T logl +-------- T memcpy +-------- T memmove +-------- T memset +-------- T pow +-------- T powf +-------- T powl + U scalbn + U scalbnf +-------- T sin +-------- T sinf +-------- T sinl + U sqrt + U sqrtf +-------- T tan +-------- T tanf +-------- T tanl diff --git a/tests/browser_harness.html b/tests/browser_harness.html index 86f3749f79c52..0eed6d625f9b2 100644 --- a/tests/browser_harness.html +++ b/tests/browser_harness.html @@ -19,6 +19,7 @@

Running tests...

setTimeout(check, 333); } catch(e) { document.write('Tests complete. View log in console.'); + window.close(); return; } } diff --git a/tests/cases/insertvalue_recursive.ll b/tests/cases/insertvalue_recursive.ll new file mode 100644 index 0000000000000..80376a4d18711 --- /dev/null +++ b/tests/cases/insertvalue_recursive.ll @@ -0,0 +1,26 @@ +; ModuleID = 'insertvalue_recursive.ll' +target datalayout = "e-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-p:32:32:32-v128:32:128-n32-S128" +target triple = "asmjs-unknown-emscripten" + +%nested_struct = type { [1 x i64] } +%struct = type { [1 x i64], %nested_struct } + +@.str = private unnamed_addr constant [15 x i8] c"hello, world!\0A\00", align 1 + +declare i32 @printf(i8*, ...) + +define i32 @main() { +entry: + %.1 = insertvalue [1 x i64] undef, i64 1, 0 + %inserted.first = insertvalue %struct undef, [1 x i64] %.1, 0 + + %.2 = insertvalue [1 x i64] undef, i64 2, 0 + %.nested_struct = insertvalue %nested_struct undef, [1 x i64] %.2, 0 + %inserted.second = insertvalue %struct %inserted.first, %nested_struct %.nested_struct, 1 + + %.3 = alloca %struct, align 8 + store %struct %inserted.second, %struct* %.3, align 8 + + %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @.str, i32 0, i32 0)) ; [#uses=0 type=i32] + ret i32 0 +} \ No newline at end of file diff --git a/tests/core/test_simd_int16x8.out b/tests/core/test_simd_int16x8.out index 2493a0a122ca9..0784dcd7894c3 100644 --- a/tests/core/test_simd_int16x8.out +++ b/tests/core/test_simd_int16x8.out @@ -17,15 +17,15 @@ emscripten_int16x8_notEqual(v, w): -1 -1 -1 -1 -1 -1 -1 -1 emscripten_int16x8_shiftLeftByScalar(v, 0): 1 0 1 32767 -2 9 13 -32768 emscripten_int16x8_shiftLeftByScalar(v, 1): 2 0 2 -2 -4 18 26 0 emscripten_int16x8_shiftLeftByScalar(v, 2): 4 0 4 -4 -8 36 52 0 -emscripten_int16x8_shiftLeftByScalar(v, 16): 0 0 0 0 0 0 0 0 -emscripten_int16x8_shiftLeftByScalar(v, 32): 0 0 0 0 0 0 0 0 -emscripten_int16x8_shiftLeftByScalar(v, 48): 0 0 0 0 0 0 0 0 +emscripten_int16x8_shiftLeftByScalar(v, 16): 1 0 1 32767 -2 9 13 -32768 +emscripten_int16x8_shiftLeftByScalar(v, 32): 1 0 1 32767 -2 9 13 -32768 +emscripten_int16x8_shiftLeftByScalar(v, 48): 1 0 1 32767 -2 9 13 -32768 emscripten_int16x8_shiftRightByScalar(v, 0): 1 0 1 32767 -2 9 13 -32768 emscripten_int16x8_shiftRightByScalar(v, 1): 0 0 0 16383 -1 4 6 -16384 emscripten_int16x8_shiftRightByScalar(v, 2): 0 0 0 8191 -1 2 3 -8192 -emscripten_int16x8_shiftRightByScalar(v, 16): 0 0 0 0 -1 0 0 -1 -emscripten_int16x8_shiftRightByScalar(v, 32): 0 0 0 0 -1 0 0 -1 -emscripten_int16x8_shiftRightByScalar(v, 48): 0 0 0 0 -1 0 0 -1 +emscripten_int16x8_shiftRightByScalar(v, 16): 1 0 1 32767 -2 9 13 -32768 +emscripten_int16x8_shiftRightByScalar(v, 32): 1 0 1 32767 -2 9 13 -32768 +emscripten_int16x8_shiftRightByScalar(v, 48): 1 0 1 32767 -2 9 13 -32768 emscripten_int16x8_select(b, v, w): 2 0 2 32767 2 9 2 -32768 emscripten_int16x8_addSaturate(v, w): 3 2 3 32767 0 11 15 -32766 emscripten_int16x8_subSaturate(v, w): -1 -2 -1 32765 -4 7 11 -32768 @@ -50,4 +50,4 @@ emscripten_int16x8_shuffle(v, z, 4, 4, 4, 4, 4, 4, 4, 4): -2 -2 -2 -2 -2 -2 -2 - emscripten_int16x8_shuffle(v, z, 15, 15, 15, 15, 7, 7, 7, 7): -32750 -32750 -32750 -32750 -32768 -32768 -32768 -32768 emscripten_int16x8_shuffle(v, z, 0, 2, 4, 6, 8, 10, 12, 14): 1 1 -2 13 -5 14 0 32764 emscripten_int16x8_shuffle(v, z, 7, 0, 3, 5, 9, 11, 1, 4): -32768 1 32767 9 20 9 0 -2 -Done! \ No newline at end of file +Done! diff --git a/tests/core/test_simd_int32x4.out b/tests/core/test_simd_int32x4.out index 471de66e1189a..1455415647e19 100644 --- a/tests/core/test_simd_int32x4.out +++ b/tests/core/test_simd_int32x4.out @@ -18,13 +18,13 @@ emscripten_int32x4_shiftLeftByScalar(v, 0): 1 0 1 4 emscripten_int32x4_shiftLeftByScalar(v, 1): 2 0 2 8 emscripten_int32x4_shiftLeftByScalar(v, 2): 4 0 4 16 emscripten_int32x4_shiftLeftByScalar(v, 16): 65536 0 65536 262144 -emscripten_int32x4_shiftLeftByScalar(v, 32): 0 0 0 0 -emscripten_int32x4_shiftLeftByScalar(v, 48): 0 0 0 0 +emscripten_int32x4_shiftLeftByScalar(v, 32): 1 0 1 4 +emscripten_int32x4_shiftLeftByScalar(v, 48): 65536 0 65536 262144 emscripten_int32x4_shiftRightByScalar(v, 0): 1 0 1 4 emscripten_int32x4_shiftRightByScalar(v, 1): 0 0 0 2 emscripten_int32x4_shiftRightByScalar(v, 2): 0 0 0 1 emscripten_int32x4_shiftRightByScalar(v, 16): 0 0 0 0 -emscripten_int32x4_shiftRightByScalar(v, 32): 0 0 0 0 +emscripten_int32x4_shiftRightByScalar(v, 32): 1 0 1 4 emscripten_int32x4_shiftRightByScalar(v, 48): 0 0 0 0 emscripten_int32x4_select(b, v, w): 2 0 2 4 emscripten_int32x4_replaceLane(v, 0, 9): 9 0 1 4 @@ -50,4 +50,4 @@ emscripten_int32x4_shuffle(v, z, 4, 4, 4, 4): -5 -5 -5 -5 emscripten_int32x4_shuffle(v, z, 7, 7, 7, 7): 9 9 9 9 emscripten_int32x4_shuffle(v, z, 0, 2, 4, 6): 1 1 -5 14 emscripten_int32x4_shuffle(v, z, 7, 0, 3, 5): 9 1 4 20 -Done! \ No newline at end of file +Done! diff --git a/tests/core/test_simd_int8x16.out b/tests/core/test_simd_int8x16.out index f56a2e7163e98..7d0f3749745a4 100644 --- a/tests/core/test_simd_int8x16.out +++ b/tests/core/test_simd_int8x16.out @@ -17,15 +17,15 @@ emscripten_int8x16_notEqual(v, w): -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 emscripten_int8x16_shiftLeftByScalar(v, 0): 1 0 1 127 -2 9 13 -128 3 19 73 -53 42 66 10 100 emscripten_int8x16_shiftLeftByScalar(v, 1): 2 0 2 -2 -4 18 26 0 6 38 -110 -106 84 -124 20 -56 emscripten_int8x16_shiftLeftByScalar(v, 2): 4 0 4 -4 -8 36 52 0 12 76 36 44 -88 8 40 -112 -emscripten_int8x16_shiftLeftByScalar(v, 16): 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 -emscripten_int8x16_shiftLeftByScalar(v, 32): 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 -emscripten_int8x16_shiftLeftByScalar(v, 48): 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +emscripten_int8x16_shiftLeftByScalar(v, 16): 1 0 1 127 -2 9 13 -128 3 19 73 -53 42 66 10 100 +emscripten_int8x16_shiftLeftByScalar(v, 32): 1 0 1 127 -2 9 13 -128 3 19 73 -53 42 66 10 100 +emscripten_int8x16_shiftLeftByScalar(v, 48): 1 0 1 127 -2 9 13 -128 3 19 73 -53 42 66 10 100 emscripten_int8x16_shiftRightByScalar(v, 0): 1 0 1 127 -2 9 13 -128 3 19 73 -53 42 66 10 100 emscripten_int8x16_shiftRightByScalar(v, 1): 0 0 0 63 -1 4 6 -64 1 9 36 -27 21 33 5 50 emscripten_int8x16_shiftRightByScalar(v, 2): 0 0 0 31 -1 2 3 -32 0 4 18 -14 10 16 2 25 -emscripten_int8x16_shiftRightByScalar(v, 16): 0 0 0 0 -1 0 0 -1 0 0 0 -1 0 0 0 0 -emscripten_int8x16_shiftRightByScalar(v, 32): 0 0 0 0 -1 0 0 -1 0 0 0 -1 0 0 0 0 -emscripten_int8x16_shiftRightByScalar(v, 48): 0 0 0 0 -1 0 0 -1 0 0 0 -1 0 0 0 0 +emscripten_int8x16_shiftRightByScalar(v, 16): 1 0 1 127 -2 9 13 -128 3 19 73 -53 42 66 10 100 +emscripten_int8x16_shiftRightByScalar(v, 32): 1 0 1 127 -2 9 13 -128 3 19 73 -53 42 66 10 100 +emscripten_int8x16_shiftRightByScalar(v, 48): 1 0 1 127 -2 9 13 -128 3 19 73 -53 42 66 10 100 emscripten_int8x16_select(b, v, w): 2 0 2 127 2 9 2 -128 2 19 2 -53 2 66 2 100 emscripten_int8x16_addSaturate(v, w): 3 2 3 127 0 11 15 -126 5 21 75 -51 44 68 12 102 emscripten_int8x16_subSaturate(v, w): -1 -2 -1 125 -4 7 11 -128 1 17 71 -55 40 64 8 98 @@ -58,4 +58,4 @@ emscripten_int8x16_shuffle(v, z, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4) emscripten_int8x16_shuffle(v, z, 15, 15, 15, 15, 7, 7, 7, 7, 31, 31, 31, 31, 15, 14, 13, 12): 100 100 100 100 -128 -128 -128 -128 110 110 110 110 100 10 66 42 emscripten_int8x16_shuffle(v, z, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30): 1 1 -2 13 3 73 42 10 -5 14 0 120 40 60 80 100 emscripten_int8x16_shuffle(v, z, 7, 0, 3, 5, 9, 11, 1, 4, 6, 12, 19, 4, 9, 0, 2, 3): -128 1 127 9 19 -53 0 -2 13 42 9 -2 19 1 1 127 -Done! \ No newline at end of file +Done! diff --git a/tests/debugger/test_pointers.cpp b/tests/debugger/test_pointers.cpp new file mode 100644 index 0000000000000..1b2d61548280a --- /dev/null +++ b/tests/debugger/test_pointers.cpp @@ -0,0 +1,97 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct TestBase { + float a; + char b; + char *c; + char *d; + char **e; + char ***f; + short **g; + + // Test a reference + int &h; + TestBase(int& rt) : h(rt) {} +}; + +struct TinyStruct { + short len; + char * chars; +}; + +struct StructOfStructs { + TestBase *tb; + TinyStruct *ts; +}; + +void test_1() { + int z = 42; + TestBase tb(z); + tb.a = 42.1f; + tb.b = 'b'; + tb.c = nullptr; + + tb.d = static_cast(malloc(10)); + tb.d[0] = 'd'; + + tb.e = static_cast(malloc(sizeof(char*))); + tb.e[0] = static_cast(malloc(sizeof(char))); + tb.e[0][0] = 'e'; + + tb.f = static_cast(malloc(sizeof(char**))); + tb.f[0] = static_cast(malloc(sizeof(char*))); + tb.f[0][0] = static_cast(malloc(sizeof(char))); + tb.f[0][0][0] = 'f'; + + tb.g = static_cast(malloc(sizeof(short*))); + tb.g[0] = static_cast(malloc(sizeof(short))); + tb.g[0][0] = 12345; + + EM_ASM_INT({ + var decoded = Module['cyberdwarf'].decode_from_stack($0, "tb", 100)["struct TestBase"]; + test_assert("Found the answer", decoded["float : a"] - 42 < 1.0); + test_assert("Found 'b'", decoded["char : b"] == 98); + test_assert("Found null", decoded["char * : c"] == "null"); + test_assert("Found 'd'", decoded["char * : d"] == 100); + test_assert("Found 'e'", decoded["char * * : e"] == 101); + test_assert("Found 'f'", decoded["char * * * : f"] == 102); + test_assert("Found simple short", decoded["short * * : g"] == 12345); + test_assert("Found the answer by reference", decoded["int & : h"] == 42); + + }, &tb); + +} + +void test_2() { + StructOfStructs sos; + int z = 50; + sos.tb = new TestBase(z); + sos.tb->a = 1337; + + sos.ts = static_cast(malloc(sizeof(TinyStruct))); + sos.ts->chars = "Hello in there"; + sos.ts->len = strlen(sos.ts->chars); + + EM_ASM_INT({ + var decoded = Module['cyberdwarf'].decode_from_stack($0, "sos", 100)["struct StructOfStructs"]; + test_assert("Found the answer", decoded["struct TestBase * : tb"]["float : a"] == 1337); + }, &sos); + +} + + +int main(int argc, char *argv[]) { + EM_ASM(init_cd_test("test_pointers")); + + test_1(); + test_2(); +} diff --git a/tests/debugger/test_preamble.js b/tests/debugger/test_preamble.js new file mode 100644 index 0000000000000..816d3e34c945f --- /dev/null +++ b/tests/debugger/test_preamble.js @@ -0,0 +1,15 @@ +function init_cd_test(name) { + console.log("-------------- Starting test " + name + " --------------"); + Module['cyberdwarf'].initialize_debugger(); +} + +function TestException(message) { + this.message = message; + this.name = "TestException"; +} + +function test_assert(desc, value) { + if (!value) { + throw new TestException("Test case " + desc + " failed!"); + } +} diff --git a/tests/debugger/test_union.cpp b/tests/debugger/test_union.cpp new file mode 100644 index 0000000000000..82891353c9237 --- /dev/null +++ b/tests/debugger/test_union.cpp @@ -0,0 +1,51 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +union TestBase { + char c; + int i; + float f; +}; + + +// Keep in mind that it is undefined behavior to read from the non active member of a union in C++ +// Don't add tests that assign to 1 member but check another +void test_1() { + TestBase tb; + tb.c = 11; + + EM_ASM_INT({ + var decoded = Module['cyberdwarf'].decode_from_stack($0, "tb", 100)["union TestBase"]; + test_assert("char == 11", decoded["char : c"] == 11); + }, &tb); + + tb.i = 1337; + + EM_ASM_INT({ + var decoded = Module['cyberdwarf'].decode_from_stack($0, "tb", 100)["union TestBase"]; + test_assert("int == 1337", decoded["int : i"] == 1337); + }, &tb); + + tb.f = 10000; + + EM_ASM_INT({ + var decoded = Module['cyberdwarf'].decode_from_stack($0, "tb", 100)["union TestBase"]; + test_assert("float == 10000", decoded["float : f"] == 10000); + }, &tb); + + +} + +int main(int argc, char *argv[]) { + EM_ASM(init_cd_test("test_union")); + + test_1(); +} diff --git a/tests/emscripten_main_loop_and_blocker.cpp b/tests/emscripten_main_loop_and_blocker.cpp index 70829869a2d08..7ee5fd8e8a042 100644 --- a/tests/emscripten_main_loop_and_blocker.cpp +++ b/tests/emscripten_main_loop_and_blocker.cpp @@ -26,10 +26,11 @@ void looper() { frame++; double curTime = emscripten_get_now(); double timeSincePrevious = curTime - prevTime; - printf("frame: %d. dt: %g\n", frame, timeSincePrevious); - if (timeSincePrevious <= 16.0) + prevTime = curTime; + printf("frame: %d. dt: %g. absolute: %g\n", frame, timeSincePrevious, curTime); + if (frame > 1 && timeSincePrevious <= 14.5) // should be 16, but browser jitter { - printf("Abort: main loop tick was called too quickly after the previous frame!\n"); + printf("Abort: main loop tick was called too quickly (%f ms > 16) after the previous frame!\n", timeSincePrevious); int result = 1; #ifdef REPORT_RESULT REPORT_RESULT(); @@ -38,8 +39,6 @@ void looper() { exit(0); } - prevTime = curTime; - if (frame == 20) { emscripten_cancel_main_loop(); emscripten_async_call(final, (void*)0, 1000); diff --git a/tests/emscripten_main_loop_settimeout.cpp b/tests/emscripten_main_loop_settimeout.cpp new file mode 100644 index 0000000000000..acb73d604c497 --- /dev/null +++ b/tests/emscripten_main_loop_settimeout.cpp @@ -0,0 +1,36 @@ +#include +#include +#include +#include + +int numFrames = 0; + +void looper() { + static double frame0 = emscripten_get_now(); + + double start = emscripten_get_now(); + + ++numFrames; + printf("Frame %d\n", numFrames); + if (numFrames == 10) { + double now = emscripten_get_now(); + double msecsPerFrame = (now - frame0) / (numFrames-1); // Sub one to account for intervals vs endpoints + printf("Avg. msecs/frame: %f\n", msecsPerFrame); +#ifdef REPORT_RESULT + int result = (msecsPerFrame > 350 && msecsPerFrame < 650); // Expecting 500msecs/frame, but allow a lot of leeway. Bad value would be 900msecs/frame (400msecs of processing below and 500msecs of delay) + REPORT_RESULT(); +#endif + emscripten_cancel_main_loop(); + } + + // Busy wait 400 msecs. + double now = start; + while (now - start < 400) { + now = emscripten_get_now(); + } +} + +int main() { + // Want to run at 2 fps, or 500msecs/frame. + emscripten_set_main_loop(looper, 2, 0); +} diff --git a/tests/fuzz/csmith_driver.py b/tests/fuzz/csmith_driver.py index 514fa29f58b97..c5e1a5464b466 100755 --- a/tests/fuzz/csmith_driver.py +++ b/tests/fuzz/csmith_driver.py @@ -122,6 +122,13 @@ def try_js(args=[]): js_args += ["--memory-init-file", "0", "-s", "MEM_INIT_METHOD=2"] if random.random() < 0.5: js_args += ['-s', 'ASSERTIONS=1'] + #js_args += ['-s', 'BINARYEN=1'] + #if random.random() < 0.333: + # js_args += ['-s', 'BINARYEN_METHOD="interpret-s-expr"'] + #elif random.random() < 0.5: + # js_args += ['-s', 'BINARYEN_METHOD="interpret-binary"'] + #else: + # js_args += ['-s', 'BINARYEN_METHOD="interpret-asm2wasm"'] print '(compile)', ' '.join(js_args) open(fullname, 'a').write('\n// ' + ' '.join(js_args) + '\n\n') try: @@ -133,7 +140,7 @@ def try_js(args=[]): def execute_js(engine): print '(run in %s)' % engine - js = shared.run_js(filename + '.js', engine=engine1, check_timeout=True, assert_returncode=None) + js = shared.run_js(filename + '.js', engine=engine, check_timeout=True, assert_returncode=None) js = js.split('\n')[0] + '\n' # remove any extra printed stuff (node workarounds) return correct1 == js or correct2 == js diff --git a/tests/pthread/test_pthread_atomics.cpp b/tests/pthread/test_pthread_atomics.cpp index b0212290b478d..a1200f7602b91 100644 --- a/tests/pthread/test_pthread_atomics.cpp +++ b/tests/pthread/test_pthread_atomics.cpp @@ -150,6 +150,9 @@ int main() int result = 0; + emscripten_atomic_fence(); + __sync_synchronize(); + if (!emscripten_has_threading_support()) { #ifdef REPORT_RESULT diff --git a/tests/runner.py b/tests/runner.py index 57094158d36b0..ae47453ab6686 100755 --- a/tests/runner.py +++ b/tests/runner.py @@ -9,7 +9,7 @@ from subprocess import Popen, PIPE, STDOUT -import os, unittest, tempfile, shutil, time, inspect, sys, math, glob, re, difflib, webbrowser, hashlib, threading, platform, BaseHTTPServer, SimpleHTTPServer, multiprocessing, functools, stat, string, random, operator, fnmatch +import os, unittest, tempfile, shutil, time, inspect, sys, math, glob, re, difflib, webbrowser, hashlib, threading, platform, BaseHTTPServer, SimpleHTTPServer, multiprocessing, functools, stat, string, random, operator, fnmatch, httplib from urllib import unquote # Setup @@ -660,7 +660,7 @@ def log_request(code=0, size=0): httpd = BaseHTTPServer.HTTPServer(('localhost', 9999), TestServerHandler) httpd.serve_forever() # test runner will kill us -def server_func(dir, q, back_queue): +def server_func(dir, q): class TestServerHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): def do_GET(self): if 'report_' in self.path: @@ -675,7 +675,6 @@ def log_request(code=0, size=0): pass os.chdir(dir) httpd = BaseHTTPServer.HTTPServer(('localhost', 8888), TestServerHandler) - back_queue.put(True) # Send a message back to parent to signal that the server hosting the test page is now running httpd.serve_forever() # test runner will kill us class BrowserCore(RunnerCore): @@ -685,6 +684,7 @@ def __init__(self, *args, **kwargs): @classmethod def setUpClass(self): super(BrowserCore, self).setUpClass() + self.browser_timeout = 30 self.harness_queue = multiprocessing.Queue() self.harness_server = multiprocessing.Process(target=harness_server_func, args=(self.harness_queue,)) self.harness_server.start() @@ -705,15 +705,20 @@ def run_browser(self, html_file, message, expectedResult=None, timeout=None): if expectedResult is not None: try: queue = multiprocessing.Queue() - back_queue = multiprocessing.Queue() - server = multiprocessing.Process(target=functools.partial(server_func, self.get_dir()), args=(queue, back_queue)) + server = multiprocessing.Process(target=functools.partial(server_func, self.get_dir()), args=(queue,)) server.start() # Starting the web page server above is an asynchronous procedure, so before we tell the browser below to navigate to # the test page, we need to know that the server has started up and is ready to process the site navigation. - # Therefore block until we get a message from the server telling that the server has started up. - try: - back_queue.get(True, 10) - except: + # Therefore block until we can make a connection to the server. + for i in range(10): + httpconn = httplib.HTTPConnection('localhost:8888', timeout=1) + try: + httpconn.connect() + httpconn.close() + break + except: + time.sleep(1) + else: raise Exception('[Test harness server failed to start up in a timely manner]') self.harness_queue.put('http://localhost:8888/' + html_file) output = '[no http server activity]' diff --git a/tests/sdl2_net_client.c b/tests/sdl2_net_client.c new file mode 100644 index 0000000000000..004fe3b477881 --- /dev/null +++ b/tests/sdl2_net_client.c @@ -0,0 +1,133 @@ +/* + * Compile with: + * + * gcc -Wall `sdl-config --cflags` sdl2_net_client.c -o sdl2_net_client `sdl-config --libs` -lSDL_net + * + * or + * + * emcc -Wall sdl2_net_client.c -s USE_SDL_NET=2 -s USE_SDL=2 -o sdl2_net_client.js + */ + +#include +#include +#include +#include +#include + +#include "SDL_net.h" + +typedef enum { + MSG_READ, + MSG_WRITE +} msg_state_t; + +typedef struct { + TCPsocket sd; /* Socket descriptor */ + msg_state_t msg_state; + int msg_i; +} state_t; + +state_t state; + +void finish(int result) { + if (state.sd) { + SDLNet_TCP_Close(state.sd); + SDLNet_Quit(); + } +#ifdef __EMSCRIPTEN__ + REPORT_RESULT(); + emscripten_force_exit(result); +#else + exit(result); +#endif +} + +char *msgs[] = { + "testmsg1", + "anothertestmsg", + "exit", +}; + +void main_loop() +{ + char *sendbuf = msgs[state.msg_i]; + char recvbuf[256] = {0}; + int actual = 0, len = strlen(sendbuf) + 1; + printf("main loop with string %s and len %d\n", sendbuf, len); + + if (state.msg_state == MSG_WRITE) { + printf("trying to send %s\n", sendbuf); + if ((actual = SDLNet_TCP_Send(state.sd, (void *)sendbuf, len)) != len) + { + fprintf(stderr, "SDLNet_TCP_Send: count:%d/%d errno:%d msg:%s\n", + actual, len, errno, SDLNet_GetError()); + if (errno == EAGAIN) { + if (actual > 0) { + assert(0); + } + return; + } + finish(EXIT_FAILURE); + } + printf("send success\n"); + state.msg_state = MSG_READ; + } + if (state.msg_state == MSG_READ) { + printf("trying to receive %s\n", sendbuf); + if ((actual = SDLNet_TCP_Recv(state.sd, (void *)recvbuf, len)) != len) + { + fprintf(stderr, "SDLNet_TCP_Recv: count:%d/%d errno:%d msg:%s\n", + actual, len, errno, SDLNet_GetError()); + if (errno == EAGAIN) { + if (actual > 0) { + assert(0); + } + return; + } + finish(EXIT_FAILURE); + } + printf("receive success\n"); + assert(strcmp(sendbuf, recvbuf) == 0); + if (!strcmp(recvbuf, "exit")) { + finish(EXIT_SUCCESS); + } + state.msg_i++; + state.msg_state = MSG_WRITE; + } +} + +int main(int argc, char **argv) +{ + IPaddress ip; /* Server address */ + memset(&state, 0, sizeof(state_t)); + state.msg_state = MSG_WRITE; + + if (SDLNet_Init() < 0) + { + fprintf(stderr, "SDLNet_Init: %s\n", SDLNet_GetError()); + finish(EXIT_FAILURE); + } + + /* Resolve the host we are connecting to */ + if (SDLNet_ResolveHost(&ip, "localhost", SOCKK) < 0) + { + fprintf(stderr, "SDLNet_ResolveHost: %s\n", SDLNet_GetError()); + finish(EXIT_FAILURE); + } + + /* Open a connection with the IP provided */ + if (!(state.sd = SDLNet_TCP_Open(&ip))) + { + fprintf(stderr, "SDLNet_TCP_Open: %s\n", SDLNet_GetError()); + finish(EXIT_FAILURE); + } + + /* Send messages */ +#ifdef __EMSCRIPTEN__ + emscripten_set_main_loop(main_loop, 60, 0); +#else + while (1) main_loop(); +#endif + + return EXIT_SUCCESS; +} diff --git a/tests/sdl2_net_server.c b/tests/sdl2_net_server.c new file mode 100644 index 0000000000000..821f544f1b08c --- /dev/null +++ b/tests/sdl2_net_server.c @@ -0,0 +1,165 @@ +/* + * Compile with: + * + * gcc -Wall `sdl-config --cflags` sdl2_net_server.c -o sdl2_net_server `sdl-config --libs` -lSDL_net + * + * or + * + * emcc -Wall sdl2_net_server.c -s USE_SDL_NET=2 -s USE_SDL=2 -o sdl2_net_server.js + */ + +#include +#include +#include +#include +#ifdef __EMSCRIPTEN__ +#include +#endif + +#include "SDL_net.h" + +#define MAX_SOCKETS 128 +#define MAX_CLIENTS MAX_SOCKETS - 1 + +typedef struct { + TCPsocket clientSocket[MAX_CLIENTS]; + SDLNet_SocketSet socketSet; + TCPsocket sd; /* Socket descriptor, Client socket descriptor */ +} state_t; + +state_t state; + +void finish() { // untested + printf("Shutting down...\n"); + SDLNet_TCP_Close(state.sd); + for (int loop = 0; loop < MAX_CLIENTS; loop++) + { + if (state.clientSocket[loop] != NULL) + { + SDLNet_TCP_Close(state.clientSocket[loop]); + state.clientSocket[loop] = NULL; + } + } + SDLNet_Quit(); + +#ifdef __EMSCRIPTEN__ + emscripten_force_exit(0); +#else + exit(0); +#endif +} + +void main_loop() { + char buffer[512]; + IPaddress *remoteIP; + + /* Check the sd if there is a pending connection. + * If there is one, accept that, and open a new socket for communicating */ + SDLNet_CheckSockets(state.socketSet, 20); + int serverSocketActivity = SDLNet_SocketReady(state.sd); + + if (serverSocketActivity) + { + printf("new server socket activity!\n"); + TCPsocket csd = SDLNet_TCP_Accept(state.sd); + /* Now we can communicate with the client using csd socket + * sd will remain opened waiting other connections */ + + /* Get the remote address */ + if ((remoteIP = SDLNet_TCP_GetPeerAddress(csd))) + { + /* Print the address, converting in the host format */ + printf("Host connected: %x %d\n", SDLNet_Read32(&remoteIP->host), SDLNet_Read16(&remoteIP->port)); + } else { + fprintf(stderr, "SDLNet_TCP_GetPeerAddress: %s\n", SDLNet_GetError()); + } + + for (int loop = 0; loop < MAX_CLIENTS; loop++) + { + if (state.clientSocket[loop] == NULL) + { + state.clientSocket[loop] = csd; + SDLNet_TCP_AddSocket(state.socketSet, state.clientSocket[loop]); + break; + } + } + } + for (int clientNumber = 0; clientNumber < MAX_CLIENTS; clientNumber++) + { + int clientSocketActivity = SDLNet_SocketReady(state.clientSocket[clientNumber]); + + if (clientSocketActivity != 0) + { + int recvLen = SDLNet_TCP_Recv(state.clientSocket[clientNumber], buffer, 512); + if (recvLen > 0) + { + assert(buffer[recvLen-1] == '\0'); + + printf("Client %d says: %s\n", clientNumber, buffer); + if (SDLNet_TCP_Send(state.clientSocket[clientNumber], buffer, recvLen) < recvLen) { + printf("Failed to echo message %s\n", buffer); + } else { + printf("Echoed back %d bytes\n", recvLen); + } + + if(strcmp(buffer, "exit") == 0) /* Terminate this connection */ + { + printf("Terminate connection\n"); + SDLNet_TCP_Close(state.clientSocket[clientNumber]); + state.clientSocket[clientNumber] = NULL; + } + if(strcmp(buffer, "quit") == 0) /* Quit the program */ + { + printf("Quit program\n"); + finish(); + } + } else { + printf("Closing client socket\n"); + SDLNet_TCP_Close(state.clientSocket[clientNumber]); + state.clientSocket[clientNumber] = NULL; + } + } + } +} + +int main(int argc, char **argv) +{ + IPaddress ip; + + if (SDLNet_Init() < 0) + { + fprintf(stderr, "SDLNet_Init: %s\n", SDLNet_GetError()); + exit(EXIT_FAILURE); + } + + /* Resolving the host using NULL make network interface to listen */ + if (SDLNet_ResolveHost(&ip, INADDR_ANY, SOCKK) < 0) + { + fprintf(stderr, "SDLNet_ResolveHost: %s\n", SDLNet_GetError()); + exit(EXIT_FAILURE); + } + + state.socketSet = SDLNet_AllocSocketSet(MAX_SOCKETS); + + for (int loop = 0; loop < MAX_CLIENTS; loop++) + { + state.clientSocket[loop] = NULL; + } + /* Open a connection with the IP provided (listen on the host's port) */ + if (!(state.sd = SDLNet_TCP_Open(&ip))) + { + fprintf(stderr, "SDLNet_TCP_Open: %s\n", SDLNet_GetError()); + exit(EXIT_FAILURE); + } + SDLNet_TCP_AddSocket(state.socketSet, state.sd); + + /* Wait for a connection, send data and term */ +#ifdef __EMSCRIPTEN__ + emscripten_set_main_loop(main_loop, 60, 0); +#else + while (1) main_loop(); +#endif + + return EXIT_SUCCESS; +} + diff --git a/tests/sockets/test_sockets_echo_server.c b/tests/sockets/test_sockets_echo_server.c index 4b5b75c68c0f5..980f2aed0e0b8 100644 --- a/tests/sockets/test_sockets_echo_server.c +++ b/tests/sockets/test_sockets_echo_server.c @@ -92,12 +92,12 @@ void main_loop() { int fd = server.fd; #endif if (client.state == MSG_READ) { - socklen_t addrlen; if (!FD_ISSET(fd, &fdr)) { return; } + socklen_t addrlen = sizeof(client.addr); res = do_msg_read(fd, &client.msg, client.read, 0, (struct sockaddr *)&client.addr, &addrlen); if (res == -1) { return; diff --git a/tests/sockets/test_sockets_msg.h b/tests/sockets/test_sockets_msg.h index 0e68803cfb53a..9ada2b78f2b85 100644 --- a/tests/sockets/test_sockets_msg.h +++ b/tests/sockets/test_sockets_msg.h @@ -11,12 +11,13 @@ int do_msg_read(int sockfd, msg_t *msg, int offset, int length, struct sockaddr if (!msg->length) { // read the message length - res = recvfrom(sockfd, &msg->length, sizeof(int), 0, (struct sockaddr *)addr, addrlen); + res = recvfrom(sockfd, &msg->length, sizeof(int), 0, addr, addrlen); if (res == -1) { assert(errno == EAGAIN); return res; + } else if (res == 0) { + return 0; } - assert(res != 0); printf("do_msg_read: allocating %d bytes for message\n", msg->length); @@ -28,7 +29,7 @@ int do_msg_read(int sockfd, msg_t *msg, int offset, int length, struct sockaddr if (length && max > length) { max = length; } - res = recvfrom(sockfd, msg->buffer + offset, max, 0, (struct sockaddr *)addr, addrlen); + res = recvfrom(sockfd, msg->buffer + offset, max, 0, addr, addrlen); if (res == -1) { assert(errno == EAGAIN); return res; @@ -77,4 +78,4 @@ int do_msg_write(int sockfd, msg_t *msg, int offset, int length, struct sockaddr return res; } -#endif \ No newline at end of file +#endif diff --git a/tests/sockets/test_sockets_partial_client.c b/tests/sockets/test_sockets_partial_client.c index 2d930516f9b36..4ddd82f84b5d4 100644 --- a/tests/sockets/test_sockets_partial_client.c +++ b/tests/sockets/test_sockets_partial_client.c @@ -21,8 +21,10 @@ void finish(int result) { close(sockfd); #ifdef __EMSCRIPTEN__ REPORT_RESULT(); -#endif + emscripten_force_exit(result); +#else exit(result); +#endif } void iter() { @@ -54,7 +56,7 @@ void iter() { } if (res != 1) { - perror("should read 1 byte"); + fprintf(stderr, "should read 1 byte, got: %d, sum %d so far\n", res, sum); finish(EXIT_FAILURE); } diff --git a/tests/sockets/test_sockets_partial_server.c b/tests/sockets/test_sockets_partial_server.c index 5c3621a0cc744..b670efcf78d5d 100644 --- a/tests/sockets/test_sockets_partial_server.c +++ b/tests/sockets/test_sockets_partial_server.c @@ -18,15 +18,19 @@ int serverfd = 0; int clientfd = 0; +void cleanup_client() { + if (clientfd) { + close(clientfd); + clientfd = 0; + } +} + void cleanup() { if (serverfd) { close(serverfd); serverfd = 0; } - if (clientfd) { - close(clientfd); - clientfd = 0; - } + cleanup_client(); } void do_send(int sockfd) { @@ -53,12 +57,10 @@ void do_send(int sockfd) { res = send(sockfd, buffer, strlen(buffer), 0); if (res == -1) { perror("send failed"); - exit(EXIT_FAILURE); + return; } printf("sent \"%s\" (%d bytes)\n", buffer, res); } - - exit(EXIT_SUCCESS); } void iter() { @@ -85,6 +87,7 @@ void iter() { if (FD_ISSET(clientfd, &fdw)) { do_send(clientfd); + cleanup_client(); } } diff --git a/tests/sockets/test_sockets_select_server_closes_connection_client_rw.c b/tests/sockets/test_sockets_select_server_closes_connection_client_rw.c index 28a81cd94ba43..4ab0148be427a 100644 --- a/tests/sockets/test_sockets_select_server_closes_connection_client_rw.c +++ b/tests/sockets/test_sockets_select_server_closes_connection_client_rw.c @@ -26,8 +26,10 @@ void finish(int result) { close(sockfd); #ifdef __EMSCRIPTEN__ REPORT_RESULT(); -#endif + emscripten_force_exit(result); +#else exit(result); +#endif } void main_loop() { diff --git a/tests/sockets/test_sockets_select_server_down_client.c b/tests/sockets/test_sockets_select_server_down_client.c index 5b05087053e69..3b5696f844b2c 100644 --- a/tests/sockets/test_sockets_select_server_down_client.c +++ b/tests/sockets/test_sockets_select_server_down_client.c @@ -22,8 +22,10 @@ void finish(int result) { close(sockfd); #ifdef __EMSCRIPTEN__ REPORT_RESULT(); -#endif + emscripten_force_exit(result); +#else exit(result); +#endif } void iter() { diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index 38d1e55fb0732..10ba28ac52e79 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -16,6 +16,8 @@ CORE_BENCHMARKS = True # core benchmarks vs full regression suite +IGNORE_COMPILATION = 0 + class Benchmarker: def __init__(self, name): self.name = name @@ -27,7 +29,10 @@ def bench(self, args, output_parser=None, reps=TEST_REPS): start = time.time() output = self.run(args) if not output_parser: - curr = time.time()-start + if IGNORE_COMPILATION: + curr = float(re.search('took +([\d\.]+) milliseconds', output).group(1)) / 1000 + else: + curr = time.time() - start else: curr = output_parser(output) self.times.append(curr) @@ -58,7 +63,7 @@ def __init__(self, name, cc, cxx, args=['-O2']): self.cxx = cxx self.args = args - def build(self, parent, filename, args, shared_args, emcc_args, native_args, native_exec, lib_builder): + def build(self, parent, filename, args, shared_args, emcc_args, native_args, native_exec, lib_builder, has_output_parser): self.parent = parent if lib_builder: native_args = native_args + lib_builder(self.name, native=True, env_init={ 'CC': self.cc, 'CXX': self.cxx }) if not native_exec: @@ -89,7 +94,7 @@ def __init__(self, name, engine, extra_args=[], env={}): for k, v in env.iteritems(): self.env[k] = v - def build(self, parent, filename, args, shared_args, emcc_args, native_args, native_exec, lib_builder): + def build(self, parent, filename, args, shared_args, emcc_args, native_args, native_exec, lib_builder, has_output_parser): self.filename = filename llvm_root = self.env.get('LLVM') or LLVM_ROOT if lib_builder: emcc_args = emcc_args + lib_builder('js_' + llvm_root, native=False, env_init=self.env) @@ -105,13 +110,15 @@ def process(filename): ''' % str(args[:-1]) # do not hardcode in the last argument, the default arg ) - final = os.path.dirname(filename) + os.path.sep + self.name+'_' + os.path.basename(filename) + '.js' + final = os.path.dirname(filename) + os.path.sep + self.name + ('_' if self.name else '') + os.path.basename(filename) + '.js' + final = final.replace('.cpp', '') try_delete(final) output = Popen([PYTHON, EMCC, filename, #'-O3', '-O3', '-s', 'DOUBLE_MODE=0', '-s', 'PRECISE_I64_MATH=0', '--memory-init-file', '0', '--js-transform', 'python hardcode.py', '-s', 'TOTAL_MEMORY=128*1024*1024', '-s', 'NO_EXIT_RUNTIME=1', + '-s', 'BENCHMARK=%d' % (1 if IGNORE_COMPILATION and not has_output_parser else 0), #'--profiling', #'--closure', '1', '-o', final] + shared_args + emcc_args + self.extra_args, stdout=PIPE, stderr=PIPE, env=self.env).communicate() @@ -143,6 +150,8 @@ def run(self, args): #NativeBenchmarker('clang-3.4', os.path.join(LLVM_3_4, 'clang'), os.path.join(LLVM_3_4, 'clang++')), #NativeBenchmarker('gcc', 'gcc', 'g++'), JSBenchmarker('sm-f32', SPIDERMONKEY_ENGINE, ['-s', 'PRECISE_F32=2']), + #JSBenchmarker('sm-wasm', SPIDERMONKEY_ENGINE, ['-s', 'PRECISE_F32=2''-s', 'BINARYEN=1', '-s', 'BINARYEN_METHOD="native-wasm"', '-s', 'BINARYEN_SCRIPTS="spidermonkify.py"']) + #JSBenchmarker('sm-imprecise', SPIDERMONKEY_ENGINE, ['-s', 'PRECISE_F32=1', '-s', 'BINARYEN=1', '-s', 'BINARYEN_METHOD="native-wasm"', '-s', 'BINARYEN_SCRIPTS="spidermonkify.py"', '-s', 'BINARYEN_IMPRECISE=1']), #JSBenchmarker('sm-f32-si', SPIDERMONKEY_ENGINE, ['--profiling', '-s', 'PRECISE_F32=2', '-s', 'SIMPLIFY_IFS=1']), #JSBenchmarker('sm-f32-aggro', SPIDERMONKEY_ENGINE, ['-s', 'PRECISE_F32=2', '-s', 'AGGRESSIVE_VARIABLE_ELIMINATION=1']), #JSBenchmarker('sm-f32-3.2', SPIDERMONKEY_ENGINE, ['-s', 'PRECISE_F32=2'], env={ 'LLVM': LLVM_3_2 }), @@ -164,7 +173,7 @@ class benchmark(RunnerCore): def setUpClass(self): super(benchmark, self).setUpClass() - fingerprint = [time.asctime()] + fingerprint = ['ignoring compilation' if IGNORE_COMPILATION else 'including compilation', time.asctime()] try: fingerprint.append('em: ' + Popen(['git', 'show'], stdout=PIPE).communicate()[0].split('\n')[0]) except: @@ -206,7 +215,7 @@ def do_benchmark(self, name, src, expected_output='FAIL', args=[], emcc_args=[], print for b in benchmarkers: - b.build(self, filename, args, shared_args, emcc_args, native_args, native_exec, lib_builder) + b.build(self, filename, args, shared_args, emcc_args, native_args, native_exec, lib_builder, has_output_parser=output_parser is not None) b.bench(args, output_parser, reps) b.display(benchmarkers[0]) diff --git a/tests/test_browser.py b/tests/test_browser.py index ae32002417e96..a4205a99cb1ad 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -323,6 +323,58 @@ def make_main(path): self.run_browser('page.html', 'You should see |load me right before|.', '/report_result?1') self.run_browser('page.html', 'You should see |load me right before|.', '/report_result?2') + def test_preload_caching_indexeddb_name(self): + open(os.path.join(self.get_dir(), 'somefile.txt'), 'w').write('''load me right before running the code please''') + def make_main(path): + print path + open(os.path.join(self.get_dir(), 'main.cpp'), 'w').write(self.with_report_result(r''' + #include + #include + #include + + extern "C" { + extern int checkPreloadResults(); + } + + int main(int argc, char** argv) { + FILE *f = fopen("%s", "r"); + char buf[100]; + fread(buf, 1, 20, f); + buf[20] = 0; + fclose(f); + printf("|%%s|\n", buf); + + int result = 0; + + result += !strcmp("load me right before", buf); + result += checkPreloadResults(); + + REPORT_RESULT(); + return 0; + } + ''' % path)) + + open(os.path.join(self.get_dir(), 'test.js'), 'w').write(''' + mergeInto(LibraryManager.library, { + checkPreloadResults: function() { + var cached = 0; + var packages = Object.keys(Module['preloadResults']); + packages.forEach(function(package) { + var fromCache = Module['preloadResults'][package]['fromCache']; + if (fromCache) + ++ cached; + }); + return cached; + } + }); + ''') + + make_main('somefile.txt') + Popen([PYTHON, FILE_PACKAGER, os.path.join(self.get_dir(), 'somefile.data'), '--use-preload-cache', '--indexedDB-name=testdb', '--preload', os.path.join(self.get_dir(), 'somefile.txt'), '--js-output=' + os.path.join(self.get_dir(), 'somefile.js')]).communicate() + Popen([PYTHON, EMCC, os.path.join(self.get_dir(), 'main.cpp'), '--js-library', os.path.join(self.get_dir(), 'test.js'), '--pre-js', 'somefile.js', '-o', 'page.html']).communicate() + self.run_browser('page.html', 'You should see |load me right before|.', '/report_result?1') + self.run_browser('page.html', 'You should see |load me right before|.', '/report_result?2') + def test_multifile(self): # a few files inside a directory self.clear() @@ -600,8 +652,8 @@ def test_sdl_canvas_alpha(self): Module['arguments'] = ['-0']; ''') - self.btest('sdl_canvas_alpha.c', reference='sdl_canvas_alpha.png', reference_slack=11) - self.btest('sdl_canvas_alpha.c', args=['--pre-js', 'flag_0.js'], reference='sdl_canvas_alpha_flag_0.png', reference_slack=11) + self.btest('sdl_canvas_alpha.c', reference='sdl_canvas_alpha.png', reference_slack=12) + self.btest('sdl_canvas_alpha.c', args=['--pre-js', 'flag_0.js'], reference='sdl_canvas_alpha_flag_0.png', reference_slack=12) def test_sdl_key(self): @@ -1478,6 +1530,9 @@ def test_emscripten_fs_api2(self): def test_emscripten_main_loop(self): self.btest('emscripten_main_loop.cpp', '0') + def test_emscripten_main_loop_settimeout(self): + self.btest('emscripten_main_loop_settimeout.cpp', '1') + def test_emscripten_main_loop_and_blocker(self): self.btest('emscripten_main_loop_and_blocker.cpp', '0') @@ -3072,3 +3127,6 @@ def test_split_memory_large_file(self): open('huge.dat', 'w').write(''.join([chr((x*x)&255) for x in range(size*2)])) # larger than a memory chunk self.btest('split_memory_large_file.cpp', expected='1', args=['-s', 'SPLIT_MEMORY=' + str(size), '-s', 'TOTAL_MEMORY=100000000', '-s', 'TOTAL_STACK=10240', '--preload-file', 'huge.dat'], timeout=60) + def test_binaryen(self): + self.btest('browser_test_hello_world.c', expected='0', args=['-s', 'BINARYEN=1', '-s', 'BINARYEN_METHOD="interpret-binary"']) + diff --git a/tests/test_core.py b/tests/test_core.py index c16a2d6351f79..b6c82ae35b9aa 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -23,15 +23,19 @@ def decorated(self): f(self) return decorated +def no_wasm(f): + def decorated(self): + if self.is_wasm(): return self.skip('todo') + f(self) + return decorated + class T(RunnerCore): # Short name, to make it more fun to use manually on the commandline def is_emterpreter(self): return 'EMTERPRETIFY=1' in self.emcc_args def is_split_memory(self): return 'SPLIT_MEMORY=' in str(self.emcc_args) def is_wasm(self): - return 'WASM=1' in self.emcc_args - def is_binaryen(self): - return 'BINARYEN' in str(self.emcc_args) + return 'BINARYEN' in str(self.emcc_args) or LLVM_TARGET == WASM_TARGET def test_hello_world(self): test_path = path_from_root('tests', 'core', 'test_hello_world') @@ -654,7 +658,7 @@ def test(): printf("Alignment: %d addr: 0x%x\n", ((int)&v) % 16, (int)&v); printf("Alignment: %d addr: 0x%x\n", ((int)&m) % 16, (int)&m); } - ''', 'Alignment: 0 addr: 0xa20\nAlignment: 0 addr: 0xa60\n') # hardcoded addresses, just to track if this ever changes by surprise. will need normal updates. + ''', ('Alignment: 0 addr: 0xa20\nAlignment: 0 addr: 0xa60\n', 'Alignment: 0 addr: 0xe10\nAlignment: 0 addr: 0xe50\n')) # hardcoded addresses, for 2 common global_base values. this tracks if this ever changes by surprise. will need normal updates. test() print 'relocatable' @@ -2317,10 +2321,11 @@ def test(): test() - print 'split memory' - Settings.SPLIT_MEMORY = 16*1024*1024 - test() - Settings.SPLIT_MEMORY = 0 + if not self.is_wasm(): + print 'split memory' + Settings.SPLIT_MEMORY = 16*1024*1024 + test() + Settings.SPLIT_MEMORY = 0 def test_ssr(self): # struct self-ref src = ''' @@ -3019,7 +3024,7 @@ def test_runtimelink(self): def can_dlfcn(self): if Settings.ALLOW_MEMORY_GROWTH == 1: return self.skip('no dlfcn with memory growth yet') - if self.is_binaryen(): return self.skip('no shared modules in wasm') + if self.is_wasm(): return self.skip('no shared modules in wasm') return True def prep_dlfcn_lib(self): @@ -5790,10 +5795,11 @@ def test(): self.emcc_args += ['-s', 'EMTERPRETIFY_WHITELIST=["_frexpl"]'] # test double call assertions test() - print 'split memory' - Settings.SPLIT_MEMORY = 8*1024*1024 - test() - Settings.SPLIT_MEMORY = 0 + if not self.is_wasm(): + print 'split memory' + Settings.SPLIT_MEMORY = 8*1024*1024 + test() + Settings.SPLIT_MEMORY = 0 @SIMD def test_sse1(self): @@ -6271,8 +6277,9 @@ def process(filename): Settings.ELIMINATE_DUPLICATE_FUNCTIONS = 1 test() - # Make sure that DFE ends up eliminating more than 200 functions - assert (num_original_funcs - self.count_funcs('src.cpp.o.js')) > 200 + # Make sure that DFE ends up eliminating more than 200 functions (if we can view source) + if not self.is_wasm(): + assert (num_original_funcs - self.count_funcs('src.cpp.o.js')) > 200 break def test_openjpeg(self): @@ -6459,7 +6466,7 @@ def test_cases(self): 'i1282vecnback', # uses simd ]: continue - if self.is_binaryen() and os.path.basename(shortname) in [ + if self.is_wasm() and os.path.basename(shortname) in [ 'i1282vecnback', # uses simd ]: continue @@ -7976,6 +7983,12 @@ def test_stack_overflow_check(self): self.emcc_args = args + ['-s', 'ASSERTIONS=1'] self.do_run(open(path_from_root('tests', 'stack_overflow.cpp'), 'r').read(), 'Stack overflow! Attempted to allocate') + @no_wasm + @no_emterpreter + def test_binaryen(self): + self.emcc_args += ['-s', 'BINARYEN=1', '-s', 'BINARYEN_METHOD="interpret-binary"'] + self.do_run(open(path_from_root('tests', 'hello_world.c')).read(), 'hello, world!') + # Generate tests for everything def make_run(fullname, name=-1, compiler=-1, embetter=0, quantum_size=0, typed_arrays=0, emcc_args=None, env=None): @@ -8044,8 +8057,9 @@ def setUp(self): asm2g = make_run("asm2g", compiler=CLANG, emcc_args=["-O2", "-g", "-s", "ASSERTIONS=1", "-s", "SAFE_HEAP=1"]) asm2i = make_run("asm2i", compiler=CLANG, emcc_args=["-O2", '-s', 'EMTERPRETIFY=1']) #asm2m = make_run("asm2m", compiler=CLANG, emcc_args=["-O2", "--memory-init-file", "0", "-s", "MEM_INIT_METHOD=2", "-s", "ASSERTIONS=1"]) -#binaryen = make_run("binaryen", compiler=CLANG, emcc_args=["-s", "BINARYEN='..path..'"]) -#normalyen = make_run("normalyen", compiler=CLANG, emcc_args=['-s', 'GLOBAL_BASE=1024']) # useful comparison to binaryen +#binaryen = make_run("binaryen", compiler=CLANG, emcc_args=['-O2', '-s', 'BINARYEN=1', '-s', 'BINARYEN_METHOD="interpret-s-expr"']) +#normalyen = make_run("normalyen", compiler=CLANG, emcc_args=['-O0', '-s', 'GLOBAL_BASE=1024']) # useful comparison to binaryen +#spidaryen = make_run("binaryen", compiler=CLANG, emcc_args=['-O0', '-s', 'BINARYEN=1', '-s', 'BINARYEN_SCRIPTS="spidermonkify.py"']) # Legacy test modes - asm2nn = make_run("asm2nn", compiler=CLANG, emcc_args=["-O2"], env={"EMCC_NATIVE_OPTIMIZER": "0"}) diff --git a/tests/test_other.py b/tests/test_other.py index 1ebcd42956d4f..7e9797b93ae56 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -325,9 +325,9 @@ def test_emcc_cache_flag(self): return 0; } ''') - Popen([PYTHON, EMCC, c_file, '--cache', cache_dir_name]).communicate() + subprocess.check_call([PYTHON, EMCC, c_file, '--cache', cache_dir_name]) assert os.path.exists(cache_dir_name), 'The cache directory %s must exist after the build' % cache_dir_name - assert os.path.exists(os.path.join(cache_dir_name, 'libc.bc')), 'The cache directory must contain a built libc' + assert os.path.exists(os.path.join(cache_dir_name, 'asmjs', 'libc.bc')), 'The cache directory must contain a built libc' finally: os.chdir(path_from_root('tests')) # Move away from the directory we are about to remove. shutil.rmtree(tempdirname) @@ -752,7 +752,7 @@ def measure_funcs(filename): }), ]: Building.COMPILER_TEST_OPTS = test_opts - test('zlib', path_from_root('tests', 'zlib', 'example.c'), + test('zlib', path_from_root('tests', 'zlib', 'example.c'), get_zlib_library(self), open(path_from_root('tests', 'zlib', 'ref.txt'), 'r').read(), expected_ranges, @@ -1532,7 +1532,7 @@ def test_libpng(self): def test_bullet(self): Building.emcc(path_from_root('tests','bullet_hello_world.cpp'), ['-s', 'USE_BULLET=1'], output_filename='a.out.js') self.assertContained('BULLET RUNNING', Popen(JS_ENGINES[0] + ['a.out.js'], stdout=PIPE, stderr=PIPE).communicate()[0]) - + def test_vorbis(self): #This will also test if ogg compiles, because vorbis depends on ogg Building.emcc(path_from_root('tests','vorbis_test.c'), ['-s', 'USE_VORBIS=1'], output_filename='a.out.js') @@ -2013,7 +2013,7 @@ def test_embind(self): print args, fail self.clear() try_delete(self.in_dir('a.out.js')) - + testFiles = [ path_from_root('tests', 'embind', 'underscore-1.4.2.js'), path_from_root('tests', 'embind', 'imvu_test_adapter.js'), @@ -2310,6 +2310,28 @@ def test_demangle(self): assert 'one(int)' in output assert 'two(char)' in output + def test_demangle_cpp(self): + open('src.cpp', 'w').write(''' + #include + #include + #include + #include + + int main() { + char out[256]; + int status = 1; + size_t length = 255; + abi::__cxa_demangle("_ZN4Waka1f12a234123412345pointEv", out, &length, &status); + assert(status == 0); + printf("%s\\n", out); + return 0; + } + ''') + + Popen([PYTHON, EMCC, 'src.cpp']).communicate() + output = run_js('a.out.js') + self.assertContained('Waka::f::a23412341234::point()', output) + def test_module_exports_with_closure(self): # This test checks that module.export is retained when JavaScript is minified by compiling with --closure 1 # This is important as if module.export is not present the Module object will not be visible to node.js @@ -3335,7 +3357,7 @@ def test_rename_silly(self): self.assertContained(r'''Failed to rename paths: abc, ; errno=2''', run_js('a.out.js', args=['abc', ''])) def test_readdir_r_silly(self): - open('src.cpp', 'w').write(r''' + open('src.cpp', 'w').write(r''' #include #include #include @@ -4279,7 +4301,6 @@ def do(name, source, moar_opts): if '--closure' in opts: # no EXPORTED_RUNTIME_METHODS makes closure much more effective assert sizes['no_nuthin'] < 0.975*sizes['no_fs'] assert sizes['no_fs_manual'] < sizes['no_fs'] # manual can remove a tiny bit more - assert sizes['no_fs'] < 1.02*sizes['no_fs_manual'] test([], 0.75, 360000) test(['-O1'], 0.66, 210000) test(['-O2'], 0.50, 70000) @@ -5438,9 +5459,9 @@ def test_realpath(self): #include #include #include - + #define TEST_PATH "/boot/README.txt" - + int main(int argc, char **argv) { @@ -6123,9 +6144,9 @@ def test(p1, p2, p3, last, expected): int main() {} ''') out, err = Popen([PYTHON, EMCC, 'src.cpp', '-Oz'], stderr=PIPE).communicate() - assert '___syscall54' in err, 'the failing call should be mentioned' - assert 'ctorEval.js' in err, 'with a stack trace' - assert 'ctor_evaller: not successful' in err, 'with logging' + self.assertContained('___syscall54', err) # the failing call should be mentioned + self.assertContained('ctorEval.js', err) # with a stack trace + self.assertContained('ctor_evaller: not successful', err) # with logging finally: del os.environ['EMCC_DEBUG'] @@ -6334,3 +6355,13 @@ def test_function_eliminator_replace_variable_value_with_hash_info(self): self.function_eliminator_test_helper('test-function-eliminator-replace-variable-value-with-hash-info.js', 'test-function-eliminator-replace-variable-value-output.js', use_hash_info=True) + + def test_cyberdwarf_pointers(self): + check_execute([PYTHON, EMCC, path_from_root('tests', 'debugger', 'test_pointers.cpp'), '-Oz', '-s', 'CYBERDWARF=1', + '-std=c++11', '--pre-js', path_from_root('tests', 'debugger', 'test_preamble.js'), '-o', 'test_pointers.js' ], stderr=PIPE) + run_js('test_pointers.js', engine=NODE_JS) + + def test_cyberdwarf_union(self): + check_execute([PYTHON, EMCC, path_from_root('tests', 'debugger', 'test_union.cpp'), '-Oz', '-s', 'CYBERDWARF=1', + '-std=c++11', '--pre-js', path_from_root('tests', 'debugger', 'test_preamble.js'), '-o', 'test_union.js' ], stderr=PIPE) + run_js('test_union.js', engine=NODE_JS) diff --git a/tests/test_sanity.py b/tests/test_sanity.py index 1987c8be105c8..87b7309d3f5d0 100644 --- a/tests/test_sanity.py +++ b/tests/test_sanity.py @@ -30,6 +30,7 @@ def setUpClass(self): assert os.path.exists(CONFIG_FILE), 'To run these tests, we need a (working!) %s file to already exist' % EM_CONFIG assert not os.environ.get('EMCC_DEBUG'), 'do not run sanity checks in debug mode!' + assert not os.environ.get('EMCC_WASM_BACKEND'), 'do not force wasm backend either way in sanity checks!' @classmethod def tearDownClass(self): @@ -545,6 +546,7 @@ def test_emcc_ports(self): assert 'Available ports:' in out, out assert 'SDL2' in out, out assert 'SDL2_image' in out, out + assert 'SDL2_net' in out, out # using ports @@ -564,7 +566,6 @@ def test_emcc_ports(self): else: self.do([PYTHON, compiler, '--clear-ports']) assert not os.path.exists(PORTS_DIR) - if i == 0: Cache.erase() # test with cache erased and without # Building a file that doesn't need ports should not trigger anything output = self.do([compiler, path_from_root('tests', 'hello_world_sdl.cpp')]) @@ -703,7 +704,6 @@ def test_embuilder(self): ([PYTHON, 'embuilder.py', 'build', 'libcxx_noexcept'], ['success'], True, ['libcxx_noexcept.a']), ([PYTHON, 'embuilder.py', 'build', 'libcxxabi'], ['success'], True, ['libcxxabi.bc']), ([PYTHON, 'embuilder.py', 'build', 'gl'], ['success'], True, ['gl.bc']), - ([PYTHON, 'embuilder.py', 'build', 'struct_info'], ['success'], True, ['struct_info.compiled.json']), ([PYTHON, 'embuilder.py', 'build', 'native_optimizer'], ['success'], True, ['optimizer.2.exe']), ([PYTHON, 'embuilder.py', 'build', 'zlib'], ['building and verifying zlib', 'success'], True, [os.path.join('ports-builds', 'zlib', 'libz.a')]), ([PYTHON, 'embuilder.py', 'build', 'libpng'], ['building and verifying libpng', 'success'], True, [os.path.join('ports-builds', 'libpng', 'libpng.bc')]), @@ -714,6 +714,8 @@ def test_embuilder(self): ([PYTHON, 'embuilder.py', 'build', 'sdl2-image'], ['success'], True, [os.path.join('ports-builds', 'sdl2-image', 'libsdl2_image.bc')]), ([PYTHON, 'embuilder.py', 'build', 'freetype'], ['building and verifying freetype', 'success'], True, [os.path.join('ports-builds', 'freetype', 'libfreetype.a')]), ([PYTHON, 'embuilder.py', 'build', 'sdl2-ttf'], ['building and verifying sdl2-ttf', 'success'], True, [os.path.join('ports-builds', 'sdl2-ttf', 'libsdl2_ttf.bc')]), + ([PYTHON, 'embuilder.py', 'build', 'sdl2-net'], ['building and verifying sdl2-net', 'success'], True, [os.path.join('ports-builds', 'sdl2-net', 'libsdl2_net.bc')]), + ([PYTHON, 'embuilder.py', 'build', 'binaryen'], ['building and verifying binaryen', 'success'], True, []), ]: print command Cache.erase() @@ -794,7 +796,7 @@ def test(): test() try: - print 'wacky env vars, these should not mess our bootstrapping of struct_info etc. up' + print 'wacky env vars, these should not mess our bootstrapping' os.environ['EMCC_FORCE_STDLIBS'] = '1' Cache.erase() build() @@ -802,3 +804,162 @@ def test(): finally: del os.environ['EMCC_FORCE_STDLIBS'] + def test_struct_info(self): + restore() + + struct_info_file = path_from_root('src', 'struct_info.compiled.json') + before = open(struct_info_file).read() + os.remove(struct_info_file) + self.check_working([EMCC, 'tests/hello_world.c'], '') + self.assertContained('hello, world!', run_js('a.out.js')) + assert os.path.exists(struct_info_file), 'removing the struct info file forces a rebuild' + after = open(struct_info_file).read() + assert len(after) == len(before), 'struct info must be already valid, recreating it should not alter anything (checking size, since order might change)' + + def test_vanilla(self): + restore() + Cache.erase() + + try: + os.environ['EMCC_DEBUG'] = '1' + # see that we test vanilla status, and just once + TESTING = 'testing for asm.js target' + self.check_working(EMCC, TESTING) + for i in range(3): + output = self.check_working(EMCC, 'check tells us to use') + assert TESTING not in output + # if env var tells us, do what it says + os.environ['EMCC_WASM_BACKEND'] = '1' + self.check_working(EMCC, 'EMCC_WASM_BACKEND tells us to use wasm backend') + os.environ['EMCC_WASM_BACKEND'] = '0' + self.check_working(EMCC, 'EMCC_WASM_BACKEND tells us to use asm.js backend') + finally: + del os.environ['EMCC_DEBUG'] + if 'EMCC_WASM_BACKEND' in os.environ: + del os.environ['EMCC_WASM_BACKEND'] + + def make_fake(report): + f = open(CONFIG_FILE, 'a') + f.write('LLVM_ROOT = "' + path_from_root('tests', 'fake', 'bin') + '"\n') + f.close() + + f = open(path_from_root('tests', 'fake', 'bin', 'llc'), 'w') + f.write('#!/bin/sh\n') + f.write('echo "llc fake output\nRegistered Targets:\n%s"' % report) + f.close() + os.chmod(path_from_root('tests', 'fake', 'bin', 'llc'), stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC) + + try: + os.environ['EMCC_DEBUG'] = '1' + os.environ['EMCC_WASM_BACKEND'] = '1' + make_fake('wasm32-unknown-unknown') + # see that we request the right backend from llvm + self.check_working([EMCC, 'tests/hello_world.c', '-c'], 'wasm32-unknown-unknown') + os.environ['EMCC_WASM_BACKEND'] = '0' + make_fake('asmjs-unknown-emscripten') + self.check_working([EMCC, 'tests/hello_world.c', '-c'], 'asmjs-unknown-emscripten') + del os.environ['EMCC_WASM_BACKEND'] + # check the current installed one is ok + restore() + self.check_working(EMCC) + output = self.check_working(EMCC, 'check tells us to use') + if 'wasm backend' in output: + self.check_working([EMCC, 'tests/hello_world.c', '-c'], 'wasm32-unknown-unknown') + else: + assert 'asm.js backend' in output + self.check_working([EMCC, 'tests/hello_world.c', '-c'], 'asmjs-unknown-emscripten') + finally: + del os.environ['EMCC_DEBUG'] + if 'EMCC_WASM_BACKEND' in os.environ: + del os.environ['EMCC_WASM_BACKEND'] + + # fake llc output + + try_delete(path_from_root('tests', 'fake')) + os.makedirs(path_from_root('tests', 'fake', 'bin')) + + def test_with_fake(report, expected): + make_fake(report) + try: + os.environ['EMCC_DEBUG'] = '1' + output = self.check_working([EMCC, 'tests/hello_world.c', '-c'], expected) + self.assertContained('config file changed since we checked vanilla', output) + finally: + del os.environ['EMCC_DEBUG'] + + test_with_fake('got js backend! JavaScript (asm.js, emscripten) backend', 'check tells us to use asm.js backend') + test_with_fake('got wasm32 backend! WebAssembly 32-bit', 'check tells us to use wasm backend') + + # use LLVM env var to modify LLVM between vanilla checks + + assert not os.environ.get('LLVM'), 'we need to modify LLVM env var for this' + + f = open(CONFIG_FILE, 'a') + f.write('LLVM_ROOT = os.getenv("LLVM") or "' + path_from_root('tests', 'fake1', 'bin') + '"\n') + f.close() + + safe_ensure_dirs(path_from_root('tests', 'fake1', 'bin')) + f = open(path_from_root('tests', 'fake1', 'bin', 'llc'), 'w') + f.write('#!/bin/sh\n') + f.write('echo "llc fake1 output\nRegistered Targets:\n%s"' % 'got js backend! JavaScript (asm.js, emscripten) backend') + f.close() + os.chmod(path_from_root('tests', 'fake1', 'bin', 'llc'), stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC) + + safe_ensure_dirs(path_from_root('tests', 'fake2', 'bin')) + f = open(path_from_root('tests', 'fake2', 'bin', 'llc'), 'w') + f.write('#!/bin/sh\n') + f.write('echo "llc fake2 output\nRegistered Targets:\n%s"' % 'got wasm32 backend! WebAssembly 32-bit') + f.close() + os.chmod(path_from_root('tests', 'fake2', 'bin', 'llc'), stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC) + + try: + os.environ['EMCC_DEBUG'] = '1' + self.check_working([EMCC, 'tests/hello_world.c', '-c'], 'use asm.js backend') + os.environ['LLVM'] = path_from_root('tests', 'fake2', 'bin') + self.check_working([EMCC, 'tests/hello_world.c', '-c'], 'regenerating vanilla check since other llvm') + finally: + del os.environ['EMCC_DEBUG'] + if os.environ.get('LLVM'): + del os.environ['LLVM'] + + return # TODO: the rest of this + + # check separate cache dirs are used + + restore() + self.check_working([EMCC], '') + + root_cache = os.path.expanduser('~/.emscripten_cache') + if os.path.exists(os.path.join(root_cache, 'asmjs')): + shutil.rmtree(os.path.join(root_cache, 'asmjs')) + if os.path.exists(os.path.join(root_cache, 'wasm')): + shutil.rmtree(os.path.join(root_cache, 'wasm')) + + try: + os.environ['EMCC_WASM_BACKEND'] = '1' + self.check_working([EMCC, 'tests/hello_world.c'], '') + assert os.path.exists(os.path.join(root_cache, 'wasm')) + os.environ['EMCC_WASM_BACKEND'] = '0' + self.check_working([EMCC, 'tests/hello_world.c'], '') + assert os.path.exists(os.path.join(root_cache, 'asmjs')) + shutil.rmtree(os.path.join(root_cache, 'asmjs')) + del os.environ['EMCC_WASM_BACKEND'] + self.check_working([EMCC, 'tests/hello_world.c'], '') + assert os.path.exists(os.path.join(root_cache, 'asmjs')) + finally: + del os.environ['EMCC_WASM_BACKEND'] + + def test_binaryen(self): + import tools.ports.binaryen as binaryen + tag_file = Cache.get_path('binaryen_tag_' + binaryen.TAG + '.txt') + try_delete(tag_file) + # if BINARYEN_ROOT is set, we don't build the port. Check we do built it if not + restore() + config = open(CONFIG_FILE).read() + config = config.replace('BINARYEN_ROOT', '#') + open(CONFIG_FILE, 'w').write(config) + subprocess.check_call([PYTHON, 'embuilder.py', 'build', 'binaryen']) + assert os.path.exists(tag_file) + subprocess.check_call([PYTHON, 'emcc.py', 'tests/hello_world.c', '-s', 'BINARYEN=1']) + self.assertContained('hello, world!', run_js('a.out.js')) + diff --git a/tests/test_sockets.py b/tests/test_sockets.py index bc8ed7cadb341..66a065497a047 100644 --- a/tests/test_sockets.py +++ b/tests/test_sockets.py @@ -1,4 +1,4 @@ -import os, multiprocessing, subprocess +import os, multiprocessing, subprocess, socket, time from runner import BrowserCore, path_from_root from tools.shared import * @@ -30,12 +30,13 @@ def make_relay_server(port1, port2): return proc class WebsockifyServerHarness: - def __init__(self, filename, args, listen_port): + def __init__(self, filename, args, listen_port, do_server_check=True): self.processes = [] self.filename = filename self.listen_port = listen_port self.target_port = listen_port-1 self.args = args or [] + self.do_server_check = do_server_check def __enter__(self): import socket, websockify @@ -44,7 +45,10 @@ def __enter__(self): # NOTE empty filename support is a hack to support # the current test_enet if self.filename: - Popen([CLANG_CC, path_from_root('tests', self.filename), '-o', 'server', '-DSOCKK=%d' % self.target_port] + get_clang_native_args() + self.args, env=get_clang_native_env()).communicate() + sp = Popen([CLANG_CC, path_from_root('tests', self.filename), '-o', 'server', '-DSOCKK=%d' % self.target_port] + get_clang_native_args() + self.args, env=get_clang_native_env(), stdout=PIPE, stderr=PIPE) + out = sp.communicate() + print 'Socket server build: out:', out[0] or '', '/ err:', out[1] or '' + assert sp.returncode == 0 process = Popen([os.path.abspath('server')]) self.processes.append(process) @@ -54,6 +58,21 @@ def __enter__(self): self.websockify = multiprocessing.Process(target=wsp.start_server) self.websockify.start() self.processes.append(self.websockify) + # Make sure both the actual server and the websocket proxy are running + for i in range(10): + try: + if self.do_server_check: + server_sock = socket.create_connection(('localhost', self.target_port), timeout=1) + server_sock.close() + proxy_sock = socket.create_connection(('localhost', self.listen_port), timeout=1) + proxy_sock.close() + break + except: + time.sleep(1) + else: + clean_processes(self.processes) + raise Exception('[Websockify failed to start up in a timely manner]') + print '[Websockify on process %s]' % str(self.processes[-2:]) def __exit__(self, *args, **kwargs): @@ -89,7 +108,11 @@ def __enter__(self): assert child.returncode == 0, 'ws module for Node.js not installed, and automatic installation failed! Please run \'npm install\' from %s' % EMSCRIPTEN_ROOT # compile the server - Popen([PYTHON, EMCC, path_from_root('tests', self.filename), '-o', 'server.js', '-DSOCKK=%d' % self.listen_port] + self.args).communicate() + sp = Popen([PYTHON, EMCC, path_from_root('tests', self.filename), '-o', 'server.js', '-DSOCKK=%d' % self.listen_port] + self.args) + out = sp.communicate() + print 'Socket server build: out:', out[0] or '', '/ err:', out[1] or '' + assert sp.returncode == 0 + process = Popen(NODE_JS + ['server.js']) self.processes.append(process) @@ -100,21 +123,12 @@ def __exit__(self, *args, **kwargs): # always run these tests last # make sure to use different ports in each one because it takes a while for the processes to be cleaned up - # NOTE all datagram tests are temporarily disabled, as - # we can't truly test datagram sockets until we have - # proper listen server support. - -def filter_harnesses(harnesses): - # XXX avoid websockify for now due to intermittent errors. see issue #2700 - return filter(lambda harness: (harness[0].__class__ if type(harness) is tuple else harness.__class__) is not WebsockifyServerHarness, harnesses) - class sockets(BrowserCore): emcc_args = [] @classmethod def setUpClass(self): super(sockets, self).setUpClass() - self.browser_timeout = 30 print print 'Running the socket tests. Make sure the browser allows popups from localhost.' print @@ -335,12 +349,16 @@ def test_sockets_echo(self): # The following forces non-NULL addr and addlen parameters for the accept call (CompiledServerHarness(os.path.join('sockets', 'test_sockets_echo_server.c'), [sockets_include, '-DTEST_DGRAM=0', '-DTEST_ACCEPT_ADDR=1'], 49163), 0) ] - harnesses = filter_harnesses(harnesses) for harness, datagram in harnesses: with harness: self.btest(os.path.join('sockets', 'test_sockets_echo_client.c'), expected='0', args=['-DSOCKK=%d' % harness.listen_port, '-DTEST_DGRAM=%d' % datagram, sockets_include]) + def test_sdl2_sockets_echo(self): + harness = CompiledServerHarness('sdl2_net_server.c', ['-s', 'USE_SDL=2', '-s', 'USE_SDL_NET=2'], 49164) + with harness: + self.btest('sdl2_net_client.c', expected='0', args=['-s', 'USE_SDL=2', '-s', 'USE_SDL_NET=2', '-DSOCKK=%d' % harness.listen_port]) + def test_sockets_async_echo(self): if WINDOWS: return self.skip('This test is Unix-specific.') # Run with ./runner.py sockets.test_sockets_async_echo @@ -354,7 +372,6 @@ def test_sockets_async_echo(self): # The following forces non-NULL addr and addlen parameters for the accept call (CompiledServerHarness(os.path.join('sockets', 'test_sockets_echo_server.c'), [sockets_include, '-DTEST_DGRAM=0', '-DTEST_ACCEPT_ADDR=1', '-DTEST_ASYNC=1'], 49168), 0) ] - #harnesses = filter_harnesses(harnesses) for harness, datagram in harnesses: with harness: @@ -381,7 +398,6 @@ def test_sockets_echo_bigdata(self): (CompiledServerHarness(os.path.join('sockets', 'test_sockets_echo_server.c'), [sockets_include, '-DTEST_DGRAM=0'], 49171), 0), (CompiledServerHarness(os.path.join('sockets', 'test_sockets_echo_server.c'), [sockets_include, '-DTEST_DGRAM=1'], 49172), 1) ] - harnesses = filter_harnesses(harnesses) for harness, datagram in harnesses: with harness: @@ -399,7 +415,7 @@ def test_sockets_partial(self): def test_sockets_select_server_down(self): if WINDOWS: return self.skip('This test is Unix-specific.') for harness in [ - WebsockifyServerHarness(os.path.join('sockets', 'test_sockets_select_server_down_server.c'), [], 49190), + WebsockifyServerHarness(os.path.join('sockets', 'test_sockets_select_server_down_server.c'), [], 49190, do_server_check=False), CompiledServerHarness(os.path.join('sockets', 'test_sockets_select_server_down_server.c'), [], 49191) ]: with harness: @@ -553,7 +569,6 @@ def test_nodejs_sockets_echo(self): (CompiledServerHarness(os.path.join('sockets', 'test_sockets_echo_server.c'), [sockets_include, '-DTEST_DGRAM=0'], 59162), 0), (CompiledServerHarness(os.path.join('sockets', 'test_sockets_echo_server.c'), [sockets_include, '-DTEST_DGRAM=1'], 59164), 1) ] - harnesses = filter_harnesses(harnesses) # Basic test of node client against both a Websockified and compiled echo server. for harness, datagram in harnesses: @@ -567,9 +582,9 @@ def test_nodejs_sockets_echo(self): # server because as long as the subprotocol list contains binary it will configure itself to accept binary # This test also checks that the connect url contains the correct subprotocols. print "\nTesting compile time WebSocket configuration.\n" - for harness in filter_harnesses([ + for harness in [ WebsockifyServerHarness(os.path.join('sockets', 'test_sockets_echo_server.c'), [sockets_include], 59166) - ]): + ]: with harness: Popen([PYTHON, EMCC, path_from_root('tests', 'sockets', 'test_sockets_echo_client.c'), '-o', 'client.js', '-s', 'SOCKET_DEBUG=1', '-s', 'WEBSOCKET_SUBPROTOCOL="base64, binary"', '-DSOCKK=59166', '-DREPORT_RESULT=int dummy'], stdout=PIPE, stderr=PIPE).communicate() @@ -581,9 +596,9 @@ def test_nodejs_sockets_echo(self): # In this test we have *deliberately* used the wrong port '-DSOCKK=12345' to configure the echo_client.c, so # the connection would fail without us specifying a valid WebSocket URL in the configuration. print "\nTesting runtime WebSocket configuration.\n" - for harness in filter_harnesses([ + for harness in [ WebsockifyServerHarness(os.path.join('sockets', 'test_sockets_echo_server.c'), [sockets_include], 59168) - ]): + ]: with harness: open(os.path.join(self.get_dir(), 'websocket_pre.js'), 'w').write(''' var Module = { diff --git a/tests/webidl/output_ALL.txt b/tests/webidl/output_ALL.txt index 80e2649a55045..5a7788263c22b 100644 --- a/tests/webidl/output_ALL.txt +++ b/tests/webidl/output_ALL.txt @@ -62,6 +62,7 @@ object 11 21.12 198 +getAsArray: 24 0 1 34,34 diff --git a/tests/webidl/output_DEFAULT.txt b/tests/webidl/output_DEFAULT.txt index 7fffab8b95664..afc0bb3b2a609 100644 --- a/tests/webidl/output_DEFAULT.txt +++ b/tests/webidl/output_DEFAULT.txt @@ -62,6 +62,7 @@ object 11 21.12 198 +getAsArray: 24 0 1 34,34 diff --git a/tests/webidl/output_FAST.txt b/tests/webidl/output_FAST.txt index 7fffab8b95664..afc0bb3b2a609 100644 --- a/tests/webidl/output_FAST.txt +++ b/tests/webidl/output_FAST.txt @@ -62,6 +62,7 @@ object 11 21.12 198 +getAsArray: 24 0 1 34,34 diff --git a/tests/webidl/post.js b/tests/webidl/post.js index 52bf5b63bb98f..c7b99a23b8e44 100644 --- a/tests/webidl/post.js +++ b/tests/webidl/post.js @@ -122,6 +122,7 @@ TheModule.print(bv2.getCopy().getValue(bv2)); bv2.getAnother().PrintFloat(21.12); TheModule.print(new TheModule.Inner().get()); +TheModule.print('getAsArray: ' + new TheModule.Inner().getAsArray(12)); new TheModule.Inner().mul(2); TheModule.print(TheModule.enum_value1); diff --git a/tests/webidl/test.h b/tests/webidl/test.h index b28b888f2c6ef..dd433b3a54b5d 100644 --- a/tests/webidl/test.h +++ b/tests/webidl/test.h @@ -85,6 +85,7 @@ namespace Space { Inner() {} int get() { return 198; } Inner& operator*=(float x) { return *this; } + int operator[](int x) { return x*2; } }; } diff --git a/tests/webidl/test.idl b/tests/webidl/test.idl index d1d0265b68571..9c44a92788615 100644 --- a/tests/webidl/test.idl +++ b/tests/webidl/test.idl @@ -76,6 +76,7 @@ interface Inner { void Inner(); long get(); [Operator="*=", Ref] Inner mul(float x); + [Operator="[]"] long getAsArray(long x); }; enum AnEnum { diff --git a/third_party/websockify/.gitignore b/third_party/websockify/.gitignore index 3bf91ddd76fa6..2b32f02d34c9d 100644 --- a/third_party/websockify/.gitignore +++ b/third_party/websockify/.gitignore @@ -4,8 +4,10 @@ other/.lein-deps-sum other/classes other/lib -other/node_modules +other/js/node_modules .project .pydevproject target.cfg target.cfg.d +.tox +*.egg-info diff --git a/third_party/websockify/.travis.yml b/third_party/websockify/.travis.yml new file mode 100644 index 0000000000000..4aced3a7df7a1 --- /dev/null +++ b/third_party/websockify/.travis.yml @@ -0,0 +1,10 @@ +language: python +python: + - 2.6 + - 2.7 + - 3.3 + - 3.4 + +install: pip install -r test-requirements.txt + +script: python setup.py nosetests --verbosity=3 diff --git a/third_party/websockify/CHANGES.txt b/third_party/websockify/CHANGES.txt index 0ed52b80fa5f2..10f1ca81f8f17 100644 --- a/third_party/websockify/CHANGES.txt +++ b/third_party/websockify/CHANGES.txt @@ -1,6 +1,74 @@ Changes ======= +0.8.0 +----- + +* Make websockify properly terminate children on SIGTERM (#226) +* Remove logging in signal handlers (this can cause Python to hang under certain conditions) (#219) +* Make it easier to log to a file (#205) +* Add support for IPv6 addresses in tokens in the TokenFile token plugins (#197) +* Improve auth plugin framework to enable better support for HTTP auth (#194, #201) +* Fix bug in JSONTokenAPI token plugin (#192) +* Fix a missing variable in the exception handler (#178) + +0.7.0 +----- + +* Python 3 support fixes (#140, #155, #159) +* Generic token-parsing plugins support (#162) +* Generic authentication plugins support (#172) +* Fixed frame corruption on big-endian systems (#161) +* Support heartbeats (via PING) and automatic responses to PONG (#169) +* Automatically reject unmasked client frames by default (strict mode) (#174) +* Automatically restart interrupted select calls (#175) +* Make 'run' respect environment settings (including virtualenv) (#176) + +0.6.1 - May 11, 2015 +-------------------- + +* **PATCH RELEASE**: Fixes a bug causing file_only to not be passed properly + +0.6.0 - Feb 18, 2014 +-------------------- + +* **NOTE** : 0.6.0 will break existing code that sub-classes WebsocketProxy +* Refactor to use standard SocketServer RequestHandler design +* Fix zombie process bug on certain systems when using multiprocessing +* Add better unit tests +* Log information via python `logging` module + +0.5.1 - Jun 27, 2013 +-------------------- + + * use upstream einaros/ws (>=0.4.27) with websockify.js + * file_only and no_parent security options for WSRequestHandler + * Update build of web-socket-js (c0855c6cae) + * add include/web-socket-js-project submodule to gimite/web-socket-js + for DSFG compliance. + * drop Hixie protocol support + +0.4.1 - Mar 12, 2013 +-------------------- + + * ***NOTE*** : 0.5.0 will drop Hixie protocol support + * add include/ directory and remove some dev files from source + distribution. + +0.4.0 - Mar 12, 2013 +-------------------- + + * ***NOTE*** : 0.5.0 will drop Hixie protocol support + * use Buffer base64 support in Node.js implementation + +0.3.0 - Jan 15, 2013 +-------------------- + + * refactor into modules: websocket, websocketproxy + * switch to web-socket-js that uses IETF 6455 + * change to MPL 2.0 license for include/*.js + * fix session recording + 0.2.1 - Oct 15, 2012 -------------------- diff --git a/third_party/websockify/MANIFEST.in b/third_party/websockify/MANIFEST.in index 2dd46d2563b97..43b458ee5c9b1 100644 --- a/third_party/websockify/MANIFEST.in +++ b/third_party/websockify/MANIFEST.in @@ -1 +1,2 @@ -include CHANGES.txt *.py README.md LICENSE.txt +include CHANGES.txt README.md LICENSE.txt +graft include diff --git a/third_party/websockify/README.md b/third_party/websockify/README.md index 2cfac0f58e15b..539903267e6c5 100644 --- a/third_party/websockify/README.md +++ b/third_party/websockify/README.md @@ -8,41 +8,68 @@ to normal socket traffic. Websockify accepts the WebSockets handshake, parses it, and then begins forwarding traffic between the client and the target in both directions. -### WebSockets binary data +### News/help/contact + +Notable commits, announcements and news are posted to +@noVNC + +If you are a websockify developer/integrator/user (or want to be) +please join the noVNC/websockify +discussion group -Websockify supports all versions of the WebSockets protocol (Hixie and -HyBi). The older Hixie versions of the protocol only support UTF-8 -text payloads. In order to transport binary data over UTF-8 an -encoding must used to encapsulate the data within UTF-8. +Bugs and feature requests can be submitted via [github +issues](https://github.com/kanaka/websockify/issues). -With Hixie clients, Websockify uses base64 to encode all traffic to -and from the client. This does not affect the data between websockify -and the server. +If you want to show appreciation for websockify you could donate to a great +non-profits such as: [Compassion +International](http://www.compassion.com/), [SIL](http://www.sil.org), +[Habitat for Humanity](http://www.habitat.org), [Electronic Frontier +Foundation](https://www.eff.org/), [Against Malaria +Foundation](http://www.againstmalaria.com/), [Nothing But +Nets](http://www.nothingbutnets.net/), etc. Please tweet @noVNC if you do. + +### WebSockets binary data -With HyBi clients, websockify negotiates whether to base64 encode -traffic to and from the client via the subprotocol header -(Sec-WebSocket-Protocol). The valid subprotocol values are 'binary' -and 'base64' and if the client sends both then the server (the python -implementation) will prefer 'binary'. The 'binary' subprotocol -indicates that the data will be sent raw using binary WebSocket -frames. Some HyBi clients (such as the Flash fallback and older Chrome -and iOS versions) do not support binary data which is why the -negotiation is necessary. +Starting with websockify 0.5.0, only the HyBi / IETF +6455 WebSocket protocol is supported. + +Websockify negotiates whether to base64 encode traffic to and from the +client via the subprotocol header (Sec-WebSocket-Protocol). The valid +subprotocol values are 'binary' and 'base64' and if the client sends +both then the server (the python implementation) will prefer 'binary'. +The 'binary' subprotocol indicates that the data will be sent raw +using binary WebSocket frames. Some HyBi clients (such as the Flash +fallback and older Chrome and iOS versions) do not support binary data +which is why the negotiation is necessary. ### Encrypted WebSocket connections (wss://) -To encrypt the traffic using the WebSocket 'wss://' URI scheme you -need to generate a certificate for websockify to load. By default websockify -loads a certificate file name `self.pem` but the `--cert=CERT` option can -override the file name. You can generate a self-signed certificate using -openssl. When asked for the common name, use the hostname of the server where -the proxy will be running: +To encrypt the traffic using the WebSocket 'wss://' URI scheme you need to +generate a certificate and key for Websockify to load. By default, Websockify +loads a certificate file name `self.pem` but the `--cert=CERT` and `--key=KEY` +options can override the file name. You can generate a self-signed certificate +using openssl. When asked for the common name, use the hostname of the server +where the proxy will be running: ``` openssl req -new -x509 -days 365 -nodes -out self.pem -keyout self.pem ``` +For a self-signed certificate to work, you need to make your client/browser +understand it. You can do this by installing it as accepted certificate, or by +using that same certificate for a HTTPS connection to which you navigate first +and approve. Browsers generally don't give you the "trust certificate?" prompt +by opening a WSS socket with invalid certificate, hence you need to have it +acccept it by either of those two methods. + +If you have a commercial/valid SSL certificate with one ore more intermediate +certificates, concat them into one file, server certificate first, then the +intermediate(s) from the CA, etc. Point to this file with the `--cert` option +and then also to the key with `--key`. Finally, use `--ssl-only` as needed. + ### Websock Javascript library @@ -88,11 +115,14 @@ These are not necessary for the basic operation. * Mini-webserver: websockify can detect and respond to normal web requests on the same port as the WebSockets proxy and Flash security - policy. This functionality is activate with the `--web DIR` option + policy. This functionality is activated with the `--web DIR` option where DIR is the root of the web directory to serve. * Wrap a program: see the "Wrap a Program" section below. +* Log files: websockify can save all logging information in a file. + This functionality is activated with the `--log-file FILE` option + where FILE is the file where the logs should be saved. ### Implementations of websockify @@ -123,7 +153,7 @@ new (moved) port of the program. The program wrap mode is invoked by replacing the target with `--` followed by the program command line to wrap. - `./websockify 2023 -- PROGRAM ARGS` + `./run 2023 -- PROGRAM ARGS` The `--wrap-mode` option can be used to indicate what action to take when the wrapped program exits or daemonizes. @@ -132,16 +162,17 @@ Here is an example of using websockify to wrap the vncserver command (which backgrounds itself) for use with [noVNC](https://github.com/kanaka/noVNC): - `./websockify 5901 --wrap-mode=ignore -- vncserver -geometry 1024x768 :1` + `./run 5901 --wrap-mode=ignore -- vncserver -geometry 1024x768 :1` -Here is an example of wrapping telnetd (from krb5-telnetd).telnetd +Here is an example of wrapping telnetd (from krb5-telnetd). telnetd exits after the connection closes so the wrap mode is set to respawn the command: - `sudo ./websockify 2023 --wrap-mode=respawn -- telnetd -debug 2023` + `sudo ./run 2023 --wrap-mode=respawn -- telnetd -debug 2023` The `wstelnet.html` page demonstrates a simple WebSockets based telnet -client. +client (use 'localhost' and '2023' for the host and port +respectively). ### Building the Python ssl module (for python 2.5 and older) @@ -150,11 +181,10 @@ client. `sudo aptitude install python-dev bluetooth-dev` -* Download, build the ssl module and symlink to it: - - `cd websockify/` +* At the top level of the websockify repostory, download, build and + symlink the ssl module: - `wget http://pypi.python.org/packages/source/s/ssl/ssl-1.15.tar.gz` + `wget --no-check-certificate http://pypi.python.org/packages/source/s/ssl/ssl-1.15.tar.gz` `tar xvzf ssl-1.15.tar.gz` diff --git a/third_party/websockify/Windows/Windows Service Readme.txt b/third_party/websockify/Windows/Windows Service Readme.txt index 1ddc2f82f53c0..0e7ea2ccdc9db 100644 --- a/third_party/websockify/Windows/Windows Service Readme.txt +++ b/third_party/websockify/Windows/Windows Service Readme.txt @@ -12,7 +12,7 @@ Installation --------------------------- 1. This service requires websockify.exe be in the same directory. Instructions on how to compile websockify python script as a windows executable can be found here: -https://github.com/kanaka/noVNC/wiki/Compiling-Websockify-to-Windows-Executable +https://github.com/kanaka/websockify/wiki/Compiling-Websockify-as-Windows-Executable 2.To add this service to a Windows PC you need to run the commandline as administrator and then run this line: diff --git a/third_party/websockify/docs/TODO b/third_party/websockify/docs/TODO index cc4b477601d95..7fca9d131ebbe 100644 --- a/third_party/websockify/docs/TODO +++ b/third_party/websockify/docs/TODO @@ -1,9 +1,6 @@ - Go implementation - -- Support multiple targets that are selected by the path line. Some - sort of wildcarding of ports too. - -- Support SSL targets too. - +- Rust implementation +- Add sub-protocol support to upstream einaros/ws module and use that + instead of the patched module. - wstelnet: support CSI L and CSI M diff --git a/third_party/websockify/docs/notes b/third_party/websockify/docs/notes index b3cc0cfb17afa..e340375c5ac79 100644 --- a/third_party/websockify/docs/notes +++ b/third_party/websockify/docs/notes @@ -13,5 +13,12 @@ browser. Building web-socket-js emulator: -cd include/web-socket-js/flash-src -mxmlc -static-link-runtime-shared-libraries WebSocketMain.as + cd include/web-socket-js/flash-src + mxmlc -static-link-runtime-shared-libraries WebSocketMain.as + +Building release tarball: + - not really necessary since tagged revision can be downloaded + from github as tarballs + + git archive --format=tar --prefix=websockify-${WVER}/ v${WVER} > websockify-${WVER}.tar + gzip websockify-${WVER}.tar diff --git a/third_party/websockify/docs/release.txt b/third_party/websockify/docs/release.txt index 5750c88a2957e..1b15c692e19dc 100644 --- a/third_party/websockify/docs/release.txt +++ b/third_party/websockify/docs/release.txt @@ -1,9 +1,13 @@ -- Update setup.py and CHANGES.txt and commit +- Update setup.py, CHANGES.txt and other/package.json and commit - Create version tag and tarball from tag WVER=0.1.0 git tag v${WVER} git push origin master git push origin v${WVER} - git archive --format=tar --prefix=websockify-${WVER}/ v${WVER} > websockify-${WVER}.tar - gzip websockify-${WVER}.tar -- Upload tarball to repo +- Register with pypi.python.org (once): + python setup.py register +- Upload the source distribution to pypi + python setup.py sdist upload +- Register with npmjs.org (once) +- Upload websockify.js npmjs.org package + npm publish other/js diff --git a/third_party/websockify/include/util.js b/third_party/websockify/include/util.js index ab2e1c1e0c213..67d21338f5c31 100644 --- a/third_party/websockify/include/util.js +++ b/third_party/websockify/include/util.js @@ -217,32 +217,52 @@ Util.conf_defaults = function(cfg, api, defaults, arr) { Util.get_include_uri = function() { return (typeof INCLUDE_URI !== "undefined") ? INCLUDE_URI : "include/"; } +Util._loading_scripts = []; Util._pending_scripts = []; Util.load_scripts = function(files) { - var head = document.getElementsByTagName('head')[0], - ps = Util._pending_scripts; + var head = document.getElementsByTagName('head')[0], script, + ls = Util._loading_scripts, ps = Util._pending_scripts; for (var f=0; f 0 && (ls[0].readyState === 'loaded' || + ls[0].readyState === 'complete')) { + // For IE, append the script to trigger execution + var s = ls.shift(); + //console.log("loaded script: " + s.src); + head.appendChild(s); + } + if (!this.readyState || + (Util.Engine.presto && this.readyState === 'loaded') || + this.readyState === 'complete') { if (ps.indexOf(this) >= 0) { - //console.log("loaded script: " + this.src); + this.onload = this.onreadystatechange = null; + //console.log("completed script: " + this.src); ps.splice(ps.indexOf(this), 1); - } - // Call window.onscriptsload after last script loads - if (ps.length === 0 && window.onscriptsload) { - window.onscriptsload(); + + // Call window.onscriptsload after last script loads + if (ps.length === 0 && window.onscriptsload) { + window.onscriptsload(); + } } } + }; + // In-order script execution tricks + if (Util.Engine.trident) { + // For IE wait until readyState is 'loaded' before + // appending it which will trigger execution + // http://wiki.whatwg.org/wiki/Dynamic_Script_Execution_Order + ls.push(script); + } else { + // For webkit and firefox set async=false and append now + // https://developer.mozilla.org/en-US/docs/HTML/Element/script + script.async = false; + head.appendChild(script); } + ps.push(script); } } diff --git a/third_party/websockify/include/web-socket-js/WebSocketMain.swf b/third_party/websockify/include/web-socket-js/WebSocketMain.swf index 8174466912475..f286c81aacc95 100644 Binary files a/third_party/websockify/include/web-socket-js/WebSocketMain.swf and b/third_party/websockify/include/web-socket-js/WebSocketMain.swf differ diff --git a/third_party/websockify/include/websock.js b/third_party/websockify/include/websock.js index 7d12644e424ed..0e4718ab9ddfe 100644 --- a/third_party/websockify/include/websock.js +++ b/third_party/websockify/include/websock.js @@ -262,7 +262,7 @@ function on(evt, handler) { eventHandlers[evt] = handler; } -function init(protocols) { +function init(protocols, ws_schema) { rQ = []; rQi = 0; sQ = []; @@ -277,12 +277,13 @@ function init(protocols) { ('set' in Uint8Array.prototype)) { bt = true; } - - // Check for full binary type support in WebSockets - // TODO: this sucks, the property should exist on the prototype - // but it does not. + // Check for full binary type support in WebSocket + // Inspired by: + // https://github.com/Modernizr/Modernizr/issues/370 + // https://github.com/Modernizr/Modernizr/blob/master/feature-detects/websockets/binary.js try { - if (bt && ('binaryType' in (new WebSocket("ws://localhost:17523")))) { + if (bt && ('binaryType' in WebSocket.prototype || + !!(new WebSocket(ws_schema + '://.').binaryType))) { Util.Info("Detected binaryType support in WebSockets"); wsbt = true; } @@ -325,12 +326,16 @@ function init(protocols) { } function open(uri, protocols) { - protocols = init(protocols); + var ws_schema = uri.match(/^([a-z]+):\/\//)[1]; + protocols = init(protocols, ws_schema); if (test_mode) { websocket = {}; } else { websocket = new WebSocket(uri, protocols); + if (protocols.indexOf('binary') >= 0) { + websocket.binaryType = 'arraybuffer'; + } } websocket.onmessage = recv_message; @@ -343,9 +348,6 @@ function open(uri, protocols) { mode = 'base64'; Util.Error("Server select no sub-protocol!: " + websocket.protocol); } - if (mode === 'binary') { - websocket.binaryType = 'arraybuffer'; - } eventHandlers.open(); Util.Debug("<< WebSock.onopen"); }; diff --git a/third_party/websockify/other/js/README.md b/third_party/websockify/other/js/README.md new file mode 100644 index 0000000000000..8143f984f4162 --- /dev/null +++ b/third_party/websockify/other/js/README.md @@ -0,0 +1,7 @@ +A JavaScript implementation of the websockify WebSocket-to-TCP bridge/proxy. + +Copyright (C) 2013 - Joel Martin (github.com/kanaka) + +Licensed under LGPL-3. + +See http://github.com/kanaka/websockify for more info. diff --git a/third_party/websockify/other/js/package.json b/third_party/websockify/other/js/package.json new file mode 100644 index 0000000000000..cf84539fc2172 --- /dev/null +++ b/third_party/websockify/other/js/package.json @@ -0,0 +1,23 @@ +{ + "author": "Joel Martin (http://github.com/kanaka)", + "name": "websockify", + "description": "websockify is a WebSocket-to-TCP proxy/bridge", + "license": "LGPL-3.0", + "version": "0.8.0", + "repository": { + "type": "git", + "url": "git://github.com/kanaka/websockify.git" + }, + "files": ["../../docs/LICENSE.LGPL-3","websockify.js"], + "bin": { + "websockify": "./websockify.js" + }, + "engines": { + "node": ">=0.8.9" + }, + "dependencies": { + "ws": ">=0.4.27", + "optimist": "latest", + "policyfile": "latest" + } +} diff --git a/third_party/websockify/other/websockify.js b/third_party/websockify/other/js/websockify.js similarity index 88% rename from third_party/websockify/other/websockify.js rename to third_party/websockify/other/js/websockify.js index 104475870e2c4..0f6c65275019d 100755 --- a/third_party/websockify/other/websockify.js +++ b/third_party/websockify/other/js/websockify.js @@ -5,16 +5,8 @@ // Licensed under LGPL version 3 (see docs/LICENSE.LGPL-3) // Known to work with node 0.8.9 -// Requires node modules: ws, base64, optimist and policyfile -// npm install ws base64 optimist policyfile -// -// NOTE: -// This version requires a patched version of einaros/ws that supports -// subprotocol negotiation. You can use the patched version like this: -// -// cd websockify/other -// git clone https://github.com/kanaka/ws -// npm link ./ws +// Requires node modules: ws, optimist and policyfile +// npm install ws optimist policyfile var argv = require('optimist').argv, @@ -26,7 +18,6 @@ var argv = require('optimist').argv, fs = require('fs'), policyfile = require('policyfile'), - base64 = require('base64/build/Release/base64'), Buffer = require('buffer').Buffer, WebSocketServer = require('ws').Server, @@ -38,6 +29,7 @@ var argv = require('optimist').argv, // Handle new WebSocket client new_client = function(client) { var clientAddr = client._socket.remoteAddress, log; + console.log(client.upgradeReq.url); log = function (msg) { console.log(' ' + clientAddr + ': '+ msg); }; @@ -51,7 +43,7 @@ new_client = function(client) { //log("sending message: " + data); try { if (client.protocol === 'base64') { - client.send(base64.encode(new Buffer(data))); + client.send(new Buffer(data).toString('base64')); } else { client.send(data,{binary: true}); } @@ -62,12 +54,18 @@ new_client = function(client) { }); target.on('end', function() { log('target disconnected'); + client.close(); + }); + target.on('error', function() { + log('target connection error'); + target.end(); + client.close(); }); client.on('message', function(msg) { //log('got message: ' + msg); if (client.protocol === 'base64') { - target.write(base64.decode(msg),'binary'); + target.write(new Buffer(msg, 'base64')); } else { target.write(msg,'binary'); } @@ -127,11 +125,9 @@ http_request = function (request, response) { // Select 'binary' or 'base64' subprotocol, preferring 'binary' selectProtocol = function(protocols, callback) { - var plist = protocols ? protocols.split(',') : ""; - var plist = protocols.split(','); - if (plist.indexOf('binary') >= 0) { + if (protocols.indexOf('binary') >= 0) { callback(true, 'binary'); - } else if (plist.indexOf('base64') >= 0) { + } else if (protocols.indexOf('base64') >= 0) { callback(true, 'base64'); } else { console.log("Client must support 'binary' or 'base64' protocol"); diff --git a/third_party/websockify/other/launch.sh b/third_party/websockify/other/launch.sh index 613250c6a5564..1581f17a6024c 100755 --- a/third_party/websockify/other/launch.sh +++ b/third_party/websockify/other/launch.sh @@ -16,6 +16,8 @@ usage() { echo " Default: localhost:5900" echo " --cert CERT Path to combined cert/key file" echo " Default: self.pem" + echo " --web WEB Path to web files (e.g. vnc.html)" + echo " Default: ./" exit 2 } @@ -24,6 +26,7 @@ HERE="$(cd "$(dirname "$0")" && pwd)" PORT="6080" VNC_DEST="localhost:5900" CERT="" +WEB="" proxy_pid="" die() { @@ -50,6 +53,7 @@ while [ "$*" ]; do --listen) PORT="${OPTARG}"; shift ;; --vnc) VNC_DEST="${OPTARG}"; shift ;; --cert) CERT="${OPTARG}"; shift ;; + --web) WEB="${OPTARG}"; shift ;; -h|--help) usage ;; -*) usage "Unknown chrooter option: ${param}" ;; *) break ;; @@ -60,18 +64,24 @@ done which netstat >/dev/null 2>&1 \ || die "Must have netstat installed" -netstat -ltn | grep -qs "${PORT}.*LISTEN" \ +netstat -ltn | grep -qs "${PORT} .*LISTEN" \ && die "Port ${PORT} in use. Try --listen PORT" trap "cleanup" TERM QUIT INT EXIT # Find vnc.html -if [ -e "$(pwd)/vnc.html" ]; then +if [ -n "${WEB}" ]; then + if [ ! -e "${WEB}/vnc.html" ]; then + die "Could not find ${WEB}/vnc.html" + fi +elif [ -e "$(pwd)/vnc.html" ]; then WEB=$(pwd) elif [ -e "${HERE}/../vnc.html" ]; then WEB=${HERE}/../ elif [ -e "${HERE}/vnc.html" ]; then WEB=${HERE} +elif [ -e "${HERE}/../share/novnc/vnc.html" ]; then + WEB=${HERE}/../share/novnc/ else die "Could not find vnc.html" fi @@ -92,7 +102,7 @@ else fi echo "Starting webserver and WebSockets proxy on port ${PORT}" -${HERE}/wsproxy.py --web ${WEB} ${CERT:+--cert ${CERT}} ${PORT} ${VNC_DEST} & +${HERE}/websockify --web ${WEB} ${CERT:+--cert ${CERT}} ${PORT} ${VNC_DEST} & proxy_pid="$!" sleep 1 if ! ps -p ${proxy_pid} >/dev/null; then @@ -101,7 +111,7 @@ if ! ps -p ${proxy_pid} >/dev/null; then exit 1 fi -echo -e "\n\nNavigate to to this URL:\n" +echo -e "\n\nNavigate to this URL:\n" echo -e " http://$(hostname):${PORT}/vnc.html?host=$(hostname)&port=${PORT}\n" echo -e "Press Ctrl-C to exit\n\n" diff --git a/third_party/websockify/other/websocket.c b/third_party/websockify/other/websocket.c index c365409c46099..9755711ae7ebd 100644 --- a/third_party/websockify/other/websocket.c +++ b/third_party/websockify/other/websocket.c @@ -795,7 +795,7 @@ void start_server() { } handler_msg("handler exit\n"); } else { - handler_msg("wsproxy exit\n"); + handler_msg("websockify exit\n"); } } diff --git a/third_party/websockify/other/websocket.rb b/third_party/websockify/other/websocket.rb index 6bf9e63a650a5..bb287ea4b5fb8 100644 --- a/third_party/websockify/other/websocket.rb +++ b/third_party/websockify/other/websocket.rb @@ -9,11 +9,22 @@ # - http://tools.ietf.org/html/draft-ietf-hybi-thewebsocketprotocol-10 require 'gserver' +require 'openssl' require 'stringio' require 'digest/md5' require 'digest/sha1' require 'base64' +unless OpenSSL::SSL::SSLSocket.instance_methods.index("read_nonblock") + module OpenSSL + module SSL + class SSLSocket + alias :read_nonblock :readpartial + end + end + end +end + class EClose < Exception end @@ -44,7 +55,17 @@ def initialize(opts) host = opts['listen_host'] || GServer::DEFAULT_HOST super(port, host) - + msg opts.inspect + if opts['server_cert'] + msg "creating ssl context" + @sslContext = OpenSSL::SSL::SSLContext.new + @sslContext.cert = OpenSSL::X509::Certificate.new(File.open(opts['server_cert'])) + @sslContext.key = OpenSSL::PKey::RSA.new(File.open(opts['server_key'])) + @sslContext.ca_file = opts['server_cert'] + @sslContext.verify_mode = OpenSSL::SSL::VERIFY_NONE + @sslContext.verify_depth = 0 + end + @@client_id = 0 # Track client number total on class @verbose = opts['verbose'] @@ -53,6 +74,18 @@ def initialize(opts) def serve(io) @@client_id += 1 + msg self.inspect + if @sslContext + msg "Enabling SSL context" + ssl = OpenSSL::SSL::SSLSocket.new(io, @sslContext) + #ssl.sync_close = true + #ssl.sync = true + msg "SSL accepting" + ssl.accept + io = ssl # replace the unencrypted handle with the encrypted one + end + + msg "initializing thread" # Initialize per thread state t = Thread.current @@ -61,11 +94,11 @@ def serve(io) t[:recv_part] = nil t[:base64] = nil - puts "in serve, client: #{t[:client].inspect}" + puts "in serve, client: #{t[:my_client_id].inspect}" begin t[:client] = do_handshake(io) - new_client(t[:client]) + new_websocket_client(t[:client]) rescue EClose => e msg "Client closed: #{e.message}" return @@ -85,12 +118,12 @@ def traffic(token) if @verbose then print token; STDOUT.flush; end end - def msg(msg) - puts "% 3d: %s" % [Thread.current[:my_client_id], msg] + def msg(m) + printf("% 3d: %s\n", Thread.current[:my_client_id] || 0, m) end - def vmsg(msg) - if @verbose then msg(msg) end + def vmsg(m) + if @verbose then msg(m) end end # @@ -110,12 +143,11 @@ def gen_md5(h) def unmask(buf, hlen, length) pstart = hlen + 4 - mask = buf[hlen...hlen+4] + mask = buf[hlen...hlen+4].each_byte.map{|b|b} data = buf[pstart...pstart+length] #data = data.bytes.zip(mask.bytes.cycle(length)).map { |d,m| d^m } - for i in (0...data.length) do - data[i] ^= mask[i%4] - end + i=-1 + data = data.each_byte.map{|b| i+=1; (b ^ mask[i % 4]).chr}.join("") return data end @@ -237,7 +269,7 @@ def send_frames(bufs) while t[:send_parts].length > 0 buf = t[:send_parts].shift - sent = t[:client].send(buf, 0) + sent = t[:client].write(buf) if sent == buf.length traffic "<" @@ -257,7 +289,7 @@ def recv_frames() closed = false bufs = [] - buf = t[:client].recv(@@Buffer_size) + buf = t[:client].read_nonblock(@@Buffer_size) if buf.length == 0 return bufs, "Client closed abrubtly" @@ -286,10 +318,10 @@ def recv_frames() end end else - if buf[0...2] == "\xff\x00": + if buf[0...2] == "\xff\x00" closed = "Client sent orderly close frame" break - elsif buf[0...2] == "\x00\xff": + elsif buf[0...2] == "\x00\xff" buf = buf[2...buf.length] continue # No-op frame elsif buf.count("\xff") == 0 @@ -308,7 +340,7 @@ def recv_frames() bufs << frame['payload'] - if frame['left'] > 0: + if frame['left'] > 0 buf = buf[-frame['left']...buf.length] else buf = '' @@ -328,10 +360,10 @@ def send_close(code=nil, reason='') end buf, lenh, lent = encode_hybi(msg, opcode=0x08, base64=false) - t[:client].send(buf, 0) + t[:client].write(buf) elsif t[:version] == "hixie-76" buf = "\xff\x00" - t[:client].send(buf, 0) + t[:client].write(buf) end end @@ -344,16 +376,21 @@ def do_handshake(sock) raise EClose, "ignoring socket not ready" end - handshake = sock.recv(1024, Socket::MSG_PEEK) - #msg "Handshake [#{handshake.inspect}]" + handshake = "" + msg "About to read from sock [#{sock.inspect}]" + handshake = sock.read_nonblock(1024) + msg "Handshake [#{handshake.inspect}]" - if handshake == "" + if handshake == nil or handshake == "" raise(EClose, "ignoring empty handshake") else stype = "Plain non-SSL (ws://)" scheme = "ws" + if sock.class == OpenSSL::SSL::SSLSocket + stype = "SSL (wss://)" + scheme = "wss" + end retsock = sock - sock.recv(1024) end h = t[:headers] = {} @@ -365,7 +402,7 @@ def do_handshake(sock) hsplit = hline.match(/^([^:]+):\s*(.+)$/) h[hsplit[1].strip.downcase] = hsplit[2] end - #puts "Headers: #{h.inspect}" + puts "Headers: #{h.inspect}" unless h.has_key?('upgrade') && h['upgrade'].downcase == 'websocket' @@ -445,7 +482,7 @@ def do_handshake(sock) if t[:path] then msg "Path: '%s'" % [t[:path]] end #puts "sending reponse #{response.inspect}" - retsock.send(response, 0) + retsock.write(response) # Return the WebSocket socket which may be SSL wrapped return retsock diff --git a/third_party/websockify/other/websockify.rb b/third_party/websockify/other/websockify.rb index bdc61f16fd05b..e54d0dd96c3e8 100755 --- a/third_party/websockify/other/websockify.rb +++ b/third_party/websockify/other/websockify.rb @@ -39,7 +39,7 @@ def initialize(opts) end # Echo back whatever is received - def new_client(client) + def new_websocket_client(client) msg "connecting to: %s:%s" % [@target_host, @target_port] tsock = TCPSocket.open(@target_host, @target_port) @@ -92,7 +92,7 @@ def do_proxy(client, target) # Receive target data and queue for the client if ins && ins.include?(target) buf = target.recv(@@Buffer_size) - if buf.length == 0: + if buf.length == 0 raise EClose, "Target closed" end @@ -128,7 +128,7 @@ def do_proxy(client, target) o.parse! end -if ARGV.length < 2: +if ARGV.length < 2 puts "Too few arguments" exit 2 end diff --git a/third_party/websockify/rebind.c b/third_party/websockify/rebind.c index d7b004d693c67..811031c6042e4 100644 --- a/third_party/websockify/rebind.c +++ b/third_party/websockify/rebind.c @@ -7,8 +7,8 @@ * REBIND_PORT_NEW environment variables are set then bind on the new * port (of localhost) instead of the old port. * - * This allows a proxy (such as wsproxy) to run on the old port and translate - * traffic to/from the new port. + * This allows a bridge/proxy (such as websockify) to run on the old port and + * translate traffic to/from the new port. * * Usage: * LD_PRELOAD=./rebind.so \ diff --git a/third_party/websockify/run b/third_party/websockify/run index 2802254fcbe7b..9ad217c04cd87 100755 --- a/third_party/websockify/run +++ b/third_party/websockify/run @@ -1,4 +1,4 @@ -#!/usr/bin/python2 +#!/usr/bin/env python import websockify diff --git a/third_party/websockify/setup.py b/third_party/websockify/setup.py index ebf5e6108eaab..4fa59260fbad4 100644 --- a/third_party/websockify/setup.py +++ b/third_party/websockify/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -version = '0.2.1' +version = '0.8.0' name = 'websockify' long_description = open("README.md").read() + "\n" + \ open("CHANGES.txt").read() + "\n" @@ -11,7 +11,21 @@ long_description=long_description, classifiers=[ "Programming Language :: Python", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.6", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.3", + "Programming Language :: Python :: 3.4" ], + data_files=[('share/websockify/include', + ['include/util.js', + 'include/base64.js', + 'include/websock.js']), + ('share/websockify/include/web-socket-js', + ['include/web-socket-js/WebSocketMain.swf', + 'include/web-socket-js/swfobject.js', + 'include/web-socket-js/web_socket.js'])], keywords='noVNC websockify', license='LGPLv3', url="https://github.com/kanaka/websockify", diff --git a/third_party/websockify/test-requirements.txt b/third_party/websockify/test-requirements.txt new file mode 100644 index 0000000000000..93207c1834a94 --- /dev/null +++ b/third_party/websockify/test-requirements.txt @@ -0,0 +1,2 @@ +mox +nose diff --git a/third_party/websockify/tests/echo.py b/third_party/websockify/tests/echo.py index 878c31ae63ca2..e6a685150634b 100755 --- a/third_party/websockify/tests/echo.py +++ b/third_party/websockify/tests/echo.py @@ -10,17 +10,17 @@ as taken from http://docs.python.org/dev/library/ssl.html#certificates ''' -import os, sys, select, optparse -sys.path.insert(0,os.path.dirname(__file__) + "/../") -from websocket import WebSocketServer +import os, sys, select, optparse, logging +sys.path.insert(0,os.path.join(os.path.dirname(__file__), "..")) +from websockify.websocket import WebSocketServer, WebSocketRequestHandler -class WebSocketEcho(WebSocketServer): +class WebSocketEcho(WebSocketRequestHandler): """ WebSockets server that echos back whatever is received from the client. """ buffer_size = 8096 - def new_client(self): + def new_websocket_client(self): """ Echo back whatever is received. """ @@ -28,28 +28,27 @@ def new_client(self): cqueue = [] c_pend = 0 cpartial = "" - rlist = [self.client] + rlist = [self.request] while True: wlist = [] - if cqueue or c_pend: wlist.append(self.client) + if cqueue or c_pend: wlist.append(self.request) ins, outs, excepts = select.select(rlist, wlist, [], 1) if excepts: raise Exception("Socket exception") - if self.client in outs: + if self.request in outs: # Send queued target data to the client c_pend = self.send_frames(cqueue) cqueue = [] - if self.client in ins: + if self.request in ins: # Receive client data, decode it, and send it back frames, closed = self.recv_frames() cqueue.extend(frames) if closed: self.send_close() - raise self.EClose(closed) if __name__ == '__main__': parser = optparse.OptionParser(usage="%prog [options] listen_port") @@ -69,7 +68,9 @@ def new_client(self): except: parser.error("Invalid arguments") + logging.basicConfig(level=logging.INFO) + opts.web = "." - server = WebSocketEcho(**opts.__dict__) + server = WebSocketServer(WebSocketEcho, **opts.__dict__) server.start_server() diff --git a/third_party/websockify/tests/echo.rb b/third_party/websockify/tests/echo.rb index ea34db5d70bb7..8db7dd39023e6 100755 --- a/third_party/websockify/tests/echo.rb +++ b/third_party/websockify/tests/echo.rb @@ -12,7 +12,7 @@ class WebSocketEcho < WebSocketServer # Echo back whatever is received - def new_client(client) + def new_websocket_client(client) cqueue = [] c_pend = 0 @@ -50,10 +50,17 @@ def new_client(client) end end -port = ARGV[0].to_i +port = ARGV[0].to_i || 8080 puts "Starting server on port #{port}" +server_cert = nil +server_key = nil +if ARGV.length > 2 + server_cert = ARGV[1] + server_key = ARGV[2] +end -server = WebSocketEcho.new('listen_port' => port, 'verbose' => true) +server = WebSocketEcho.new('listen_port' => port, 'verbose' => true, + 'server_cert' => server_cert, 'server_key' => server_key) server.start server.join diff --git a/third_party/websockify/tests/load.py b/third_party/websockify/tests/load.py index 0501f7da8897e..c76feb12df685 100755 --- a/third_party/websockify/tests/load.py +++ b/third_party/websockify/tests/load.py @@ -6,35 +6,37 @@ given a sequence number. Any errors are reported and counted. ''' -import sys, os, select, random, time, optparse -sys.path.insert(0,os.path.dirname(__file__) + "/../") -from websocket import WebSocketServer +import sys, os, select, random, time, optparse, logging +sys.path.insert(0,os.path.join(os.path.dirname(__file__), "..")) +from websockify.websocket import WebSocketServer, WebSocketRequestHandler -class WebSocketLoad(WebSocketServer): +class WebSocketLoadServer(WebSocketServer): - buffer_size = 65536 - - max_packet_size = 10000 recv_cnt = 0 send_cnt = 0 def __init__(self, *args, **kwargs): - self.errors = 0 self.delay = kwargs.pop('delay') + WebSocketServer.__init__(self, *args, **kwargs) + + +class WebSocketLoad(WebSocketRequestHandler): + + max_packet_size = 10000 + + def new_websocket_client(self): print "Prepopulating random array" self.rand_array = [] for i in range(0, self.max_packet_size): self.rand_array.append(random.randint(0, 9)) - WebSocketServer.__init__(self, *args, **kwargs) - - def new_client(self): + self.errors = 0 self.send_cnt = 0 self.recv_cnt = 0 try: - self.responder(self.client) + self.responder(self.request) except: print "accumulated errors:", self.errors self.errors = 0 @@ -61,14 +63,13 @@ def responder(self, client): if closed: self.send_close() - raise self.EClose(closed) now = time.time() * 1000 if client in outs: if c_pend: last_send = now c_pend = self.send_frames() - elif now > (last_send + self.delay): + elif now > (last_send + self.server.delay): last_send = now c_pend = self.send_frames([self.generate()]) @@ -161,7 +162,9 @@ def check(self, frames): except: parser.error("Invalid arguments") + logging.basicConfig(level=logging.INFO) + opts.web = "." - server = WebSocketLoad(**opts.__dict__) + server = WebSocketLoadServer(WebSocketLoad, **opts.__dict__) server.start_server() diff --git a/third_party/websockify/tests/test_auth_plugins.py b/third_party/websockify/tests/test_auth_plugins.py new file mode 100644 index 0000000000000..4b3bfb5c44f50 --- /dev/null +++ b/third_party/websockify/tests/test_auth_plugins.py @@ -0,0 +1,28 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +""" Unit tests for Authentication plugins""" + +from websockify.auth_plugins import BasicHTTPAuth, AuthenticationError +import unittest + + +class BasicHTTPAuthTestCase(unittest.TestCase): + + def setUp(self): + self.plugin = BasicHTTPAuth('Aladdin:open sesame') + + def test_no_auth(self): + headers = {} + self.assertRaises(AuthenticationError, self.plugin.authenticate, headers, 'localhost', '1234') + + def test_invalid_password(self): + headers = {'Authorization': 'Basic QWxhZGRpbjpzZXNhbWUgc3RyZWV0'} + self.assertRaises(AuthenticationError, self.plugin.authenticate, headers, 'localhost', '1234') + + def test_valid_password(self): + headers = {'Authorization': 'Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ=='} + self.plugin.authenticate(headers, 'localhost', '1234') + + def test_garbage_auth(self): + headers = {'Authorization': 'Basic xxxxxxxxxxxxxxxxxxxxxxxxxxxx'} + self.assertRaises(AuthenticationError, self.plugin.authenticate, headers, 'localhost', '1234') diff --git a/third_party/websockify/tests/test_websocket.py b/third_party/websockify/tests/test_websocket.py new file mode 100644 index 0000000000000..9d538db1b7de0 --- /dev/null +++ b/third_party/websockify/tests/test_websocket.py @@ -0,0 +1,430 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright(c)2013 NTT corp. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" Unit tests for websocket """ +import errno +import os +import logging +import select +import shutil +import socket +import ssl +import stubout +import sys +import tempfile +import unittest +import socket +import signal +from websockify import websocket + +try: + from SimpleHTTPServer import SimpleHTTPRequestHandler +except ImportError: + from http.server import SimpleHTTPRequestHandler + +try: + from StringIO import StringIO + BytesIO = StringIO +except ImportError: + from io import StringIO + from io import BytesIO + + + + +def raise_oserror(*args, **kwargs): + raise OSError('fake error') + + +class FakeSocket(object): + def __init__(self, data=''): + if isinstance(data, bytes): + self._data = data + else: + self._data = data.encode('latin_1') + + def recv(self, amt, flags=None): + res = self._data[0:amt] + if not (flags & socket.MSG_PEEK): + self._data = self._data[amt:] + + return res + + def makefile(self, mode='r', buffsize=None): + if 'b' in mode: + return BytesIO(self._data) + else: + return StringIO(self._data.decode('latin_1')) + + +class WebSocketRequestHandlerTestCase(unittest.TestCase): + def setUp(self): + super(WebSocketRequestHandlerTestCase, self).setUp() + self.stubs = stubout.StubOutForTesting() + self.tmpdir = tempfile.mkdtemp('-websockify-tests') + # Mock this out cause it screws tests up + self.stubs.Set(os, 'chdir', lambda *args, **kwargs: None) + self.stubs.Set(SimpleHTTPRequestHandler, 'send_response', + lambda *args, **kwargs: None) + + def tearDown(self): + """Called automatically after each test.""" + self.stubs.UnsetAll() + os.rmdir(self.tmpdir) + super(WebSocketRequestHandlerTestCase, self).tearDown() + + def _get_server(self, handler_class=websocket.WebSocketRequestHandler, + **kwargs): + web = kwargs.pop('web', self.tmpdir) + return websocket.WebSocketServer( + handler_class, listen_host='localhost', + listen_port=80, key=self.tmpdir, web=web, + record=self.tmpdir, daemon=False, ssl_only=0, idle_timeout=1, + **kwargs) + + def test_normal_get_with_only_upgrade_returns_error(self): + server = self._get_server(web=None) + handler = websocket.WebSocketRequestHandler( + FakeSocket('GET /tmp.txt HTTP/1.1'), '127.0.0.1', server) + + def fake_send_response(self, code, message=None): + self.last_code = code + + self.stubs.Set(SimpleHTTPRequestHandler, 'send_response', + fake_send_response) + + handler.do_GET() + self.assertEqual(handler.last_code, 405) + + def test_list_dir_with_file_only_returns_error(self): + server = self._get_server(file_only=True) + handler = websocket.WebSocketRequestHandler( + FakeSocket('GET / HTTP/1.1'), '127.0.0.1', server) + + def fake_send_response(self, code, message=None): + self.last_code = code + + self.stubs.Set(SimpleHTTPRequestHandler, 'send_response', + fake_send_response) + + handler.path = '/' + handler.do_GET() + self.assertEqual(handler.last_code, 404) + + +class WebSocketServerTestCase(unittest.TestCase): + def setUp(self): + super(WebSocketServerTestCase, self).setUp() + self.stubs = stubout.StubOutForTesting() + self.tmpdir = tempfile.mkdtemp('-websockify-tests') + # Mock this out cause it screws tests up + self.stubs.Set(os, 'chdir', lambda *args, **kwargs: None) + + def tearDown(self): + """Called automatically after each test.""" + self.stubs.UnsetAll() + os.rmdir(self.tmpdir) + super(WebSocketServerTestCase, self).tearDown() + + def _get_server(self, handler_class=websocket.WebSocketRequestHandler, + **kwargs): + return websocket.WebSocketServer( + handler_class, listen_host='localhost', + listen_port=80, key=self.tmpdir, web=self.tmpdir, + record=self.tmpdir, **kwargs) + + def test_daemonize_raises_error_while_closing_fds(self): + server = self._get_server(daemon=True, ssl_only=1, idle_timeout=1) + self.stubs.Set(os, 'fork', lambda *args: 0) + self.stubs.Set(signal, 'signal', lambda *args: None) + self.stubs.Set(os, 'setsid', lambda *args: None) + self.stubs.Set(os, 'close', raise_oserror) + self.assertRaises(OSError, server.daemonize, keepfd=None, chdir='./') + + def test_daemonize_ignores_ebadf_error_while_closing_fds(self): + def raise_oserror_ebadf(fd): + raise OSError(errno.EBADF, 'fake error') + + server = self._get_server(daemon=True, ssl_only=1, idle_timeout=1) + self.stubs.Set(os, 'fork', lambda *args: 0) + self.stubs.Set(os, 'setsid', lambda *args: None) + self.stubs.Set(signal, 'signal', lambda *args: None) + self.stubs.Set(os, 'close', raise_oserror_ebadf) + self.stubs.Set(os, 'open', raise_oserror) + self.assertRaises(OSError, server.daemonize, keepfd=None, chdir='./') + + def test_handshake_fails_on_not_ready(self): + server = self._get_server(daemon=True, ssl_only=0, idle_timeout=1) + + def fake_select(rlist, wlist, xlist, timeout=None): + return ([], [], []) + + self.stubs.Set(select, 'select', fake_select) + self.assertRaises( + websocket.WebSocketServer.EClose, server.do_handshake, + FakeSocket(), '127.0.0.1') + + def test_empty_handshake_fails(self): + server = self._get_server(daemon=True, ssl_only=0, idle_timeout=1) + + sock = FakeSocket('') + + def fake_select(rlist, wlist, xlist, timeout=None): + return ([sock], [], []) + + self.stubs.Set(select, 'select', fake_select) + self.assertRaises( + websocket.WebSocketServer.EClose, server.do_handshake, + sock, '127.0.0.1') + + def test_handshake_policy_request(self): + # TODO(directxman12): implement + pass + + def test_handshake_ssl_only_without_ssl_raises_error(self): + server = self._get_server(daemon=True, ssl_only=1, idle_timeout=1) + + sock = FakeSocket('some initial data') + + def fake_select(rlist, wlist, xlist, timeout=None): + return ([sock], [], []) + + self.stubs.Set(select, 'select', fake_select) + self.assertRaises( + websocket.WebSocketServer.EClose, server.do_handshake, + sock, '127.0.0.1') + + def test_do_handshake_no_ssl(self): + class FakeHandler(object): + CALLED = False + def __init__(self, *args, **kwargs): + type(self).CALLED = True + + FakeHandler.CALLED = False + + server = self._get_server( + handler_class=FakeHandler, daemon=True, + ssl_only=0, idle_timeout=1) + + sock = FakeSocket('some initial data') + + def fake_select(rlist, wlist, xlist, timeout=None): + return ([sock], [], []) + + self.stubs.Set(select, 'select', fake_select) + self.assertEqual(server.do_handshake(sock, '127.0.0.1'), sock) + self.assertTrue(FakeHandler.CALLED, True) + + def test_do_handshake_ssl(self): + # TODO(directxman12): implement this + pass + + def test_do_handshake_ssl_without_ssl_raises_error(self): + # TODO(directxman12): implement this + pass + + def test_do_handshake_ssl_without_cert_raises_error(self): + server = self._get_server(daemon=True, ssl_only=0, idle_timeout=1, + cert='afdsfasdafdsafdsafdsafdas') + + sock = FakeSocket("\x16some ssl data") + + def fake_select(rlist, wlist, xlist, timeout=None): + return ([sock], [], []) + + self.stubs.Set(select, 'select', fake_select) + self.assertRaises( + websocket.WebSocketServer.EClose, server.do_handshake, + sock, '127.0.0.1') + + def test_do_handshake_ssl_error_eof_raises_close_error(self): + server = self._get_server(daemon=True, ssl_only=0, idle_timeout=1) + + sock = FakeSocket("\x16some ssl data") + + def fake_select(rlist, wlist, xlist, timeout=None): + return ([sock], [], []) + + def fake_wrap_socket(*args, **kwargs): + raise ssl.SSLError(ssl.SSL_ERROR_EOF) + + self.stubs.Set(select, 'select', fake_select) + self.stubs.Set(ssl, 'wrap_socket', fake_wrap_socket) + self.assertRaises( + websocket.WebSocketServer.EClose, server.do_handshake, + sock, '127.0.0.1') + + def test_fallback_sigchld_handler(self): + # TODO(directxman12): implement this + pass + + def test_start_server_error(self): + server = self._get_server(daemon=False, ssl_only=1, idle_timeout=1) + sock = server.socket('localhost') + + def fake_select(rlist, wlist, xlist, timeout=None): + raise Exception("fake error") + + self.stubs.Set(websocket.WebSocketServer, 'socket', + lambda *args, **kwargs: sock) + self.stubs.Set(websocket.WebSocketServer, 'daemonize', + lambda *args, **kwargs: None) + self.stubs.Set(select, 'select', fake_select) + server.start_server() + + def test_start_server_keyboardinterrupt(self): + server = self._get_server(daemon=False, ssl_only=0, idle_timeout=1) + sock = server.socket('localhost') + + def fake_select(rlist, wlist, xlist, timeout=None): + raise KeyboardInterrupt + + self.stubs.Set(websocket.WebSocketServer, 'socket', + lambda *args, **kwargs: sock) + self.stubs.Set(websocket.WebSocketServer, 'daemonize', + lambda *args, **kwargs: None) + self.stubs.Set(select, 'select', fake_select) + server.start_server() + + def test_start_server_systemexit(self): + server = self._get_server(daemon=False, ssl_only=0, idle_timeout=1) + sock = server.socket('localhost') + + def fake_select(rlist, wlist, xlist, timeout=None): + sys.exit() + + self.stubs.Set(websocket.WebSocketServer, 'socket', + lambda *args, **kwargs: sock) + self.stubs.Set(websocket.WebSocketServer, 'daemonize', + lambda *args, **kwargs: None) + self.stubs.Set(select, 'select', fake_select) + server.start_server() + + def test_socket_set_keepalive_options(self): + keepcnt = 12 + keepidle = 34 + keepintvl = 56 + + server = self._get_server(daemon=False, ssl_only=0, idle_timeout=1) + sock = server.socket('localhost', + tcp_keepcnt=keepcnt, + tcp_keepidle=keepidle, + tcp_keepintvl=keepintvl) + + self.assertEqual(sock.getsockopt(socket.SOL_TCP, + socket.TCP_KEEPCNT), keepcnt) + self.assertEqual(sock.getsockopt(socket.SOL_TCP, + socket.TCP_KEEPIDLE), keepidle) + self.assertEqual(sock.getsockopt(socket.SOL_TCP, + socket.TCP_KEEPINTVL), keepintvl) + + sock = server.socket('localhost', + tcp_keepalive=False, + tcp_keepcnt=keepcnt, + tcp_keepidle=keepidle, + tcp_keepintvl=keepintvl) + + self.assertNotEqual(sock.getsockopt(socket.SOL_TCP, + socket.TCP_KEEPCNT), keepcnt) + self.assertNotEqual(sock.getsockopt(socket.SOL_TCP, + socket.TCP_KEEPIDLE), keepidle) + self.assertNotEqual(sock.getsockopt(socket.SOL_TCP, + socket.TCP_KEEPINTVL), keepintvl) + + +class HyBiEncodeDecodeTestCase(unittest.TestCase): + def test_decode_hybi_text(self): + buf = b'\x81\x85\x37\xfa\x21\x3d\x7f\x9f\x4d\x51\x58' + res = websocket.WebSocketRequestHandler.decode_hybi(buf) + + self.assertEqual(res['fin'], 1) + self.assertEqual(res['opcode'], 0x1) + self.assertEqual(res['masked'], True) + self.assertEqual(res['length'], 5) + self.assertEqual(res['payload'], b'Hello') + self.assertEqual(res['left'], 0) + + def test_decode_hybi_binary(self): + buf = b'\x82\x04\x01\x02\x03\x04' + res = websocket.WebSocketRequestHandler.decode_hybi(buf, strict=False) + + self.assertEqual(res['fin'], 1) + self.assertEqual(res['opcode'], 0x2) + self.assertEqual(res['length'], 4) + self.assertEqual(res['payload'], b'\x01\x02\x03\x04') + self.assertEqual(res['left'], 0) + + def test_decode_hybi_extended_16bit_binary(self): + data = (b'\x01\x02\x03\x04' * 65) # len > 126 -- len == 260 + buf = b'\x82\x7e\x01\x04' + data + res = websocket.WebSocketRequestHandler.decode_hybi(buf, strict=False) + + self.assertEqual(res['fin'], 1) + self.assertEqual(res['opcode'], 0x2) + self.assertEqual(res['length'], 260) + self.assertEqual(res['payload'], data) + self.assertEqual(res['left'], 0) + + def test_decode_hybi_extended_64bit_binary(self): + data = (b'\x01\x02\x03\x04' * 65) # len > 126 -- len == 260 + buf = b'\x82\x7f\x00\x00\x00\x00\x00\x00\x01\x04' + data + res = websocket.WebSocketRequestHandler.decode_hybi(buf, strict=False) + + self.assertEqual(res['fin'], 1) + self.assertEqual(res['opcode'], 0x2) + self.assertEqual(res['length'], 260) + self.assertEqual(res['payload'], data) + self.assertEqual(res['left'], 0) + + def test_decode_hybi_multi(self): + buf1 = b'\x01\x03\x48\x65\x6c' + buf2 = b'\x80\x02\x6c\x6f' + + res1 = websocket.WebSocketRequestHandler.decode_hybi(buf1, strict=False) + self.assertEqual(res1['fin'], 0) + self.assertEqual(res1['opcode'], 0x1) + self.assertEqual(res1['length'], 3) + self.assertEqual(res1['payload'], b'Hel') + self.assertEqual(res1['left'], 0) + + res2 = websocket.WebSocketRequestHandler.decode_hybi(buf2, strict=False) + self.assertEqual(res2['fin'], 1) + self.assertEqual(res2['opcode'], 0x0) + self.assertEqual(res2['length'], 2) + self.assertEqual(res2['payload'], b'lo') + self.assertEqual(res2['left'], 0) + + def test_encode_hybi_basic(self): + res = websocket.WebSocketRequestHandler.encode_hybi(b'Hello', 0x1) + expected = (b'\x81\x05\x48\x65\x6c\x6c\x6f', 2, 0) + + self.assertEqual(res, expected) + + def test_strict_mode_refuses_unmasked_client_frames(self): + buf = b'\x81\x05\x48\x65\x6c\x6c\x6f' + self.assertRaises(websocket.WebSocketRequestHandler.CClose, + websocket.WebSocketRequestHandler.decode_hybi, + buf) + + def test_no_strict_mode_accepts_unmasked_client_frames(self): + buf = b'\x81\x05\x48\x65\x6c\x6c\x6f' + res = websocket.WebSocketRequestHandler.decode_hybi(buf, strict=False) + + self.assertEqual(res['fin'], 1) + self.assertEqual(res['opcode'], 0x1) + self.assertEqual(res['masked'], False) + self.assertEqual(res['length'], 5) + self.assertEqual(res['payload'], b'Hello') diff --git a/third_party/websockify/tests/test_websocketproxy.py b/third_party/websockify/tests/test_websocketproxy.py new file mode 100644 index 0000000000000..92fd5dbe45025 --- /dev/null +++ b/third_party/websockify/tests/test_websocketproxy.py @@ -0,0 +1,136 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright(c) 2015 Red Hat, Inc All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" Unit tests for websocketproxy """ + +import unittest +import unittest +import socket + +import stubout + +from websockify import websocket +from websockify import websocketproxy +from websockify import token_plugins +from websockify import auth_plugins + +try: + from StringIO import StringIO + BytesIO = StringIO +except ImportError: + from io import StringIO + from io import BytesIO + + +class FakeSocket(object): + def __init__(self, data=''): + if isinstance(data, bytes): + self._data = data + else: + self._data = data.encode('latin_1') + + def recv(self, amt, flags=None): + res = self._data[0:amt] + if not (flags & socket.MSG_PEEK): + self._data = self._data[amt:] + + return res + + def makefile(self, mode='r', buffsize=None): + if 'b' in mode: + return BytesIO(self._data) + else: + return StringIO(self._data.decode('latin_1')) + + +class FakeServer(object): + class EClose(Exception): + pass + + def __init__(self): + self.token_plugin = None + self.auth_plugin = None + self.wrap_cmd = None + self.ssl_target = None + self.unix_target = None + +class ProxyRequestHandlerTestCase(unittest.TestCase): + def setUp(self): + super(ProxyRequestHandlerTestCase, self).setUp() + self.stubs = stubout.StubOutForTesting() + self.handler = websocketproxy.ProxyRequestHandler( + FakeSocket(''), "127.0.0.1", FakeServer()) + self.handler.path = "https://localhost:6080/websockify?token=blah" + self.handler.headers = None + self.stubs.Set(websocket.WebSocketServer, 'socket', + staticmethod(lambda *args, **kwargs: None)) + + def tearDown(self): + self.stubs.UnsetAll() + super(ProxyRequestHandlerTestCase, self).tearDown() + + def test_get_target(self): + class TestPlugin(token_plugins.BasePlugin): + def lookup(self, token): + return ("some host", "some port") + + host, port = self.handler.get_target( + TestPlugin(None), self.handler.path) + + self.assertEqual(host, "some host") + self.assertEqual(port, "some port") + + def test_get_target_raises_error_on_unknown_token(self): + class TestPlugin(token_plugins.BasePlugin): + def lookup(self, token): + return None + + self.assertRaises(FakeServer.EClose, self.handler.get_target, + TestPlugin(None), "https://localhost:6080/websockify?token=blah") + + def test_token_plugin(self): + class TestPlugin(token_plugins.BasePlugin): + def lookup(self, token): + return (self.source + token).split(',') + + self.stubs.Set(websocketproxy.ProxyRequestHandler, 'send_auth_error', + staticmethod(lambda *args, **kwargs: None)) + + self.handler.server.token_plugin = TestPlugin("somehost,") + self.handler.validate_connection() + + self.assertEqual(self.handler.server.target_host, "somehost") + self.assertEqual(self.handler.server.target_port, "blah") + + def test_auth_plugin(self): + class TestPlugin(auth_plugins.BasePlugin): + def authenticate(self, headers, target_host, target_port): + if target_host == self.source: + raise auth_plugins.AuthenticationError(response_msg="some_error") + + self.stubs.Set(websocketproxy.ProxyRequestHandler, 'send_auth_error', + staticmethod(lambda *args, **kwargs: None)) + + self.handler.server.auth_plugin = TestPlugin("somehost") + self.handler.server.target_host = "somehost" + self.handler.server.target_port = "someport" + + self.assertRaises(auth_plugins.AuthenticationError, + self.handler.validate_connection) + + self.handler.server.target_host = "someotherhost" + self.handler.validate_connection() + diff --git a/third_party/websockify/tests/utf8-list.py b/third_party/websockify/tests/utf8-list.py index 5a36da0904e8c..77bd4309b8072 100755 --- a/third_party/websockify/tests/utf8-list.py +++ b/third_party/websockify/tests/utf8-list.py @@ -5,18 +5,15 @@ import sys, os, socket, ssl, time, traceback from select import select - -sys.path.insert(0,os.path.dirname(__file__) + "/../") -from websocket import WebSocketServer +sys.path.insert(0,os.path.join(os.path.dirname(__file__), "..")) +from websockify.websocket import WebSocketServer if __name__ == '__main__': - print "val: hixie | hybi_base64 | hybi_binary" + print "val: hybi_base64 | hybi_binary" for c in range(0, 256): - hixie = WebSocketServer.encode_hixie(chr(c)) hybi_base64 = WebSocketServer.encode_hybi(chr(c), opcode=1, base64=True) hybi_binary = WebSocketServer.encode_hybi(chr(c), opcode=2, base64=False) - print "%d: %s | %s | %s" % (c, repr(hixie), repr(hybi_base64), - repr(hybi_binary)) + print "%d: %s | %s" % (c, repr(hybi_base64), repr(hybi_binary)) diff --git a/third_party/websockify/tox.ini b/third_party/websockify/tox.ini new file mode 100644 index 0000000000000..79f7201db0877 --- /dev/null +++ b/third_party/websockify/tox.ini @@ -0,0 +1,17 @@ +# Tox (http://tox.testrun.org/) is a tool for running tests +# in multiple virtualenvs. This configuration file will run the +# test suite on all supported python versions. To use it, "pip install tox" +# and then run "tox" from this directory. + +[tox] +envlist = py24,py26,py27,py33,py34 + +[testenv] +commands = nosetests {posargs} +deps = -r{toxinidir}/test-requirements.txt + +# At some point we should enable this since tox expects it to exist but +# the code will need pep8ising first. +#[testenv:pep8] +#commands = flake8 +#dep = flake8 diff --git a/third_party/websockify/websockify/__init__.py b/third_party/websockify/websockify/__init__.py index e8f2eaf339b37..37a6f47b4524d 100644 --- a/third_party/websockify/websockify/__init__.py +++ b/third_party/websockify/websockify/__init__.py @@ -1,2 +1,2 @@ -from websocket import * -from websocketproxy import * +from websockify.websocket import * +from websockify.websocketproxy import * diff --git a/third_party/websockify/websockify/auth_plugins.py b/third_party/websockify/websockify/auth_plugins.py new file mode 100644 index 0000000000000..93f1385504f09 --- /dev/null +++ b/third_party/websockify/websockify/auth_plugins.py @@ -0,0 +1,83 @@ +class BasePlugin(object): + def __init__(self, src=None): + self.source = src + + def authenticate(self, headers, target_host, target_port): + pass + + +class AuthenticationError(Exception): + def __init__(self, log_msg=None, response_code=403, response_headers={}, response_msg=None): + self.code = response_code + self.headers = response_headers + self.msg = response_msg + + if log_msg is None: + log_msg = response_msg + + super(AuthenticationError, self).__init__('%s %s' % (self.code, log_msg)) + + +class InvalidOriginError(AuthenticationError): + def __init__(self, expected, actual): + self.expected_origin = expected + self.actual_origin = actual + + super(InvalidOriginError, self).__init__( + response_msg='Invalid Origin', + log_msg="Invalid Origin Header: Expected one of " + "%s, got '%s'" % (expected, actual)) + + +class BasicHTTPAuth(object): + """Verifies Basic Auth headers. Specify src as username:password""" + + def __init__(self, src=None): + self.src = src + + def authenticate(self, headers, target_host, target_port): + import base64 + auth_header = headers.get('Authorization') + if auth_header: + if not auth_header.startswith('Basic '): + raise AuthenticationError(response_code=403) + + try: + user_pass_raw = base64.b64decode(auth_header[6:]) + except TypeError: + raise AuthenticationError(response_code=403) + + try: + # http://stackoverflow.com/questions/7242316/what-encoding-should-i-use-for-http-basic-authentication + user_pass_as_text = user_pass_raw.decode('ISO-8859-1') + except UnicodeDecodeError: + raise AuthenticationError(response_code=403) + + user_pass = user_pass_as_text.split(':', 1) + if len(user_pass) != 2: + raise AuthenticationError(response_code=403) + + if not self.validate_creds(*user_pass): + raise AuthenticationError(response_code=403) + + else: + raise AuthenticationError(response_code=401, + response_headers={'WWW-Authenticate': 'Basic realm="Websockify"'}) + + def validate_creds(self, username, password): + if '%s:%s' % (username, password) == self.src: + return True + else: + return False + +class ExpectOrigin(object): + def __init__(self, src=None): + if src is None: + self.source = [] + else: + self.source = src.split() + + def authenticate(self, headers, target_host, target_port): + origin = headers.get('Origin', None) + if origin is None or origin not in self.source: + raise InvalidOriginError(expected=self.source, actual=origin) diff --git a/third_party/websockify/websockify/token_plugins.py b/third_party/websockify/websockify/token_plugins.py new file mode 100644 index 0000000000000..92494eb03ff0e --- /dev/null +++ b/third_party/websockify/websockify/token_plugins.py @@ -0,0 +1,83 @@ +import os + +class BasePlugin(object): + def __init__(self, src): + self.source = src + + def lookup(self, token): + return None + + +class ReadOnlyTokenFile(BasePlugin): + # source is a token file with lines like + # token: host:port + # or a directory of such files + def __init__(self, *args, **kwargs): + super(ReadOnlyTokenFile, self).__init__(*args, **kwargs) + self._targets = None + + def _load_targets(self): + if os.path.isdir(self.source): + cfg_files = [os.path.join(self.source, f) for + f in os.listdir(self.source)] + else: + cfg_files = [self.source] + + self._targets = {} + for f in cfg_files: + for line in [l.strip() for l in open(f).readlines()]: + if line and not line.startswith('#'): + tok, target = line.split(': ') + self._targets[tok] = target.strip().rsplit(':', 1) + + def lookup(self, token): + if self._targets is None: + self._load_targets() + + if token in self._targets: + return self._targets[token] + else: + return None + + +# the above one is probably more efficient, but this one is +# more backwards compatible (although in most cases +# ReadOnlyTokenFile should suffice) +class TokenFile(ReadOnlyTokenFile): + # source is a token file with lines like + # token: host:port + # or a directory of such files + def lookup(self, token): + self._load_targets() + + return super(TokenFile, self).lookup(token) + + +class BaseTokenAPI(BasePlugin): + # source is a url with a '%s' in it where the token + # should go + + # we import things on demand so that other plugins + # in this file can be used w/o unecessary dependencies + + def process_result(self, resp): + return resp.text.split(':') + + def lookup(self, token): + import requests + + resp = requests.get(self.source % token) + + if resp.ok: + return self.process_result(resp) + else: + return None + + +class JSONTokenApi(BaseTokenAPI): + # source is a url with a '%s' in it where the token + # should go + + def process_result(self, resp): + resp_json = resp.json() + return (resp_json['host'], resp_json['port']) diff --git a/third_party/websockify/websockify/websocket.py b/third_party/websockify/websockify/websocket.py index 9ce13159787a3..16e1dc493b65c 100644 --- a/third_party/websockify/websockify/websocket.py +++ b/third_party/websockify/websockify/websocket.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python ''' Python WebSocket library with support for "wss://" encryption. @@ -6,9 +6,9 @@ Licensed under LGPL version 3 (see docs/LICENSE.LGPL-3) Supports following protocol versions: - - http://tools.ietf.org/html/draft-hixie-thewebsocketprotocol-75 - - http://tools.ietf.org/html/draft-hixie-thewebsocketprotocol-76 + - http://tools.ietf.org/html/draft-ietf-hybi-thewebsocketprotocol-07 - http://tools.ietf.org/html/draft-ietf-hybi-thewebsocketprotocol-10 + - http://tools.ietf.org/html/rfc6455 You can make a cert/key with openssl using: openssl req -new -x509 -days 365 -nodes -out self.pem -keyout self.pem @@ -16,7 +16,7 @@ ''' -import os, sys, time, errno, signal, socket, traceback, select +import os, sys, time, errno, signal, socket, select, logging import array, struct from base64 import b64encode, b64decode @@ -37,8 +37,8 @@ except: from SimpleHTTPServer import SimpleHTTPRequestHandler # python 2.6 differences -try: from hashlib import md5, sha1 -except: from md5 import md5; from sha import sha as sha1 +try: from hashlib import sha1 +except: from sha import sha as sha1 # python 2.5 differences try: @@ -50,194 +50,71 @@ def unpack_from(fmt, buf, offset=0): return struct.unpack(fmt, slice) # Degraded functionality if these imports are missing -for mod, sup in [('numpy', 'HyBi protocol'), ('ssl', 'TLS/SSL/wss'), - ('multiprocessing', 'Multi-Processing'), - ('resource', 'daemonizing')]: +for mod, msg in [('numpy', 'HyBi protocol will be slower'), + ('ssl', 'TLS/SSL/wss is disabled'), + ('multiprocessing', 'Multi-Processing is disabled'), + ('resource', 'daemonizing is disabled')]: try: globals()[mod] = __import__(mod) except ImportError: globals()[mod] = None - print("WARNING: no '%s' module, %s is slower or disabled" % ( - mod, sup)) + print("WARNING: no '%s' module, %s" % (mod, msg)) + if multiprocessing and sys.platform == 'win32': # make sockets pickle-able/inheritable import multiprocessing.reduction -class WebSocketServer(object): +# HTTP handler with WebSocket upgrade support +class WebSocketRequestHandler(SimpleHTTPRequestHandler): """ - WebSockets server class. - Must be sub-classed with new_client method definition. + WebSocket Request Handler Class, derived from SimpleHTTPRequestHandler. + Must be sub-classed with new_websocket_client method definition. + The request handler can be configured by setting optional + attributes on the server object: + + * only_upgrade: If true, SimpleHTTPRequestHandler will not be enabled, + only websocket is allowed. + * verbose: If true, verbose logging is activated. + * daemon: Running as daemon, do not write to console etc + * record: Record raw frame data as JavaScript array into specified filename + * run_once: Handle a single request + * handler_id: A sequence number for this connection, appended to record filename """ - buffer_size = 65536 - - server_handshake_hixie = """HTTP/1.1 101 Web Socket Protocol Handshake\r -Upgrade: WebSocket\r -Connection: Upgrade\r -%sWebSocket-Origin: %s\r -%sWebSocket-Location: %s://%s%s\r -""" - - server_handshake_hybi = """HTTP/1.1 101 Switching Protocols\r -Upgrade: websocket\r -Connection: Upgrade\r -Sec-WebSocket-Accept: %s\r -""" - GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11" - policy_response = """\n""" + server_version = "WebSockify" - # An exception before the WebSocket connection was established - class EClose(Exception): - pass + protocol_version = "HTTP/1.1" # An exception while the WebSocket client was connected class CClose(Exception): pass - def __init__(self, listen_host='', listen_port=None, source_is_ipv6=False, - verbose=False, cert='', key='', ssl_only=None, - daemon=False, record='', web='', - run_once=False, timeout=0, idle_timeout=0): - - # settings - self.verbose = verbose - self.listen_host = listen_host - self.listen_port = listen_port - self.prefer_ipv6 = source_is_ipv6 - self.ssl_only = ssl_only - self.daemon = daemon - self.run_once = run_once - self.timeout = timeout - self.idle_timeout = idle_timeout - - self.launch_time = time.time() - self.ws_connection = False - self.handler_id = 1 - - # Make paths settings absolute - self.cert = os.path.abspath(cert) - self.key = self.web = self.record = '' - if key: - self.key = os.path.abspath(key) - if web: - self.web = os.path.abspath(web) - if record: - self.record = os.path.abspath(record) - - if self.web: - os.chdir(self.web) - - # Sanity checks - if not ssl and self.ssl_only: - raise Exception("No 'ssl' module and SSL-only specified") - if self.daemon and not resource: - raise Exception("Module 'resource' required to daemonize") - - # Show configuration - print("WebSocket server settings:") - print(" - Listen on %s:%s" % ( - self.listen_host, self.listen_port)) - print(" - Flash security policy server") - if self.web: - print(" - Web server. Web root: %s" % self.web) - if ssl: - if os.path.exists(self.cert): - print(" - SSL/TLS support") - if self.ssl_only: - print(" - Deny non-SSL/TLS connections") - else: - print(" - No SSL/TLS support (no cert file)") - else: - print(" - No SSL/TLS support (no 'ssl' module)") - if self.daemon: - print(" - Backgrounding (daemon)") - if self.record: - print(" - Recording to '%s.*'" % self.record) - - # - # WebSocketServer static methods - # - - @staticmethod - def socket(host, port=None, connect=False, prefer_ipv6=False, unix_socket=None, use_ssl=False): - """ Resolve a host (and optional port) to an IPv4 or IPv6 - address. Create a socket. Bind to it if listen is set, - otherwise connect to it. Return the socket. - """ - flags = 0 - if host == '': - host = None - if connect and not (port or unix_socket): - raise Exception("Connect mode requires a port") - if use_ssl and not ssl: - raise Exception("SSL socket requested but Python SSL module not loaded."); - if not connect and use_ssl: - raise Exception("SSL only supported in connect mode (for now)") - if not connect: - flags = flags | socket.AI_PASSIVE - - if not unix_socket: - addrs = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM, - socket.IPPROTO_TCP, flags) - if not addrs: - raise Exception("Could not resolve host '%s'" % host) - addrs.sort(key=lambda x: x[0]) - if prefer_ipv6: - addrs.reverse() - sock = socket.socket(addrs[0][0], addrs[0][1]) - if connect: - sock.connect(addrs[0][4]) - if use_ssl: - sock = ssl.wrap_socket(sock) - else: - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.bind(addrs[0][4]) - sock.listen(100) - else: - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - sock.connect(unix_socket) - - return sock - - @staticmethod - def daemonize(keepfd=None, chdir='/'): - os.umask(0) - if chdir: - os.chdir(chdir) - else: - os.chdir('/') - os.setgid(os.getgid()) # relinquish elevations - os.setuid(os.getuid()) # relinquish elevations - - # Double fork to daemonize - if os.fork() > 0: os._exit(0) # Parent exits - os.setsid() # Obtain new process group - if os.fork() > 0: os._exit(0) # Parent exits + def __init__(self, req, addr, server): + # Retrieve a few configuration variables from the server + self.only_upgrade = getattr(server, "only_upgrade", False) + self.verbose = getattr(server, "verbose", False) + self.daemon = getattr(server, "daemon", False) + self.record = getattr(server, "record", False) + self.run_once = getattr(server, "run_once", False) + self.rec = None + self.handler_id = getattr(server, "handler_id", False) + self.file_only = getattr(server, "file_only", False) + self.traffic = getattr(server, "traffic", False) + self.auto_pong = getattr(server, "auto_pong", False) + self.strict_mode = getattr(server, "strict_mode", True) - # Signal handling - def terminate(a,b): os._exit(0) - signal.signal(signal.SIGTERM, terminate) - signal.signal(signal.SIGINT, signal.SIG_IGN) + self.logger = getattr(server, "logger", None) + if self.logger is None: + self.logger = WebSocketServer.get_logger() - # Close open files - maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1] - if maxfd == resource.RLIM_INFINITY: maxfd = 256 - for fd in reversed(range(maxfd)): - try: - if fd != keepfd: - os.close(fd) - except OSError: - _, exc, _ = sys.exc_info() - if exc.errno != errno.EBADF: raise + SimpleHTTPRequestHandler.__init__(self, req, addr, server) - # Redirect I/O to /dev/null - os.dup2(os.open(os.devnull, os.O_RDWR), sys.stdin.fileno()) - os.dup2(os.open(os.devnull, os.O_RDWR), sys.stdout.fileno()) - os.dup2(os.open(os.devnull, os.O_RDWR), sys.stderr.fileno()) + def log_message(self, format, *args): + self.logger.info("%s - - [%s] %s" % (self.address_string(), self.log_date_time_string(), format % args)) @staticmethod def unmask(buf, hlen, plen): @@ -246,20 +123,24 @@ def unmask(buf, hlen, plen): if numpy: b = c = s2b('') if plen >= 4: - mask = numpy.frombuffer(buf, dtype=numpy.dtype('') + mask = numpy.frombuffer(buf, dtype, offset=hlen, count=1) + data = numpy.frombuffer(buf, dtype, offset=pstart, + count=int(plen / 4)) #b = numpy.bitwise_xor(data, mask).data b = numpy.bitwise_xor(data, mask).tostring() if plen % 4: - #print("Partial unmask") - mask = numpy.frombuffer(buf, dtype=numpy.dtype('B'), - offset=hlen, count=(plen % 4)) - data = numpy.frombuffer(buf, dtype=numpy.dtype('B'), - offset=pend - (plen % 4), + #self.msg("Partial unmask") + dtype=numpy.dtype('B') + if sys.byteorder == 'big': + dtype = dtype.newbyteorder('>') + mask = numpy.frombuffer(buf, dtype, offset=hlen, count=(plen % 4)) + data = numpy.frombuffer(buf, dtype, + offset=pend - (plen % 4), count=(plen % 4)) c = numpy.bitwise_xor(data, mask).tostring() return b + c else: @@ -295,12 +176,12 @@ def encode_hybi(buf, opcode, base64=False): elif payload_len >= 65536: header = pack('>BBQ', b1, 127, payload_len) - #print("Encoded: %s" % repr(header + buf)) + #self.msg("Encoded: %s", repr(header + buf)) return header + buf, len(header), 0 @staticmethod - def decode_hybi(buf, base64=False): + def decode_hybi(buf, base64=False, logger=None, strict=True): """ Decode HyBi style WebSocket packets. Returns: {'fin' : 0_or_1, @@ -324,6 +205,9 @@ def decode_hybi(buf, base64=False): 'close_code' : 1000, 'close_reason' : ''} + if logger is None: + logger = WebSocketServer.get_logger() + blen = len(buf) f['left'] = blen @@ -359,18 +243,22 @@ def decode_hybi(buf, base64=False): # Process 1 frame if f['masked']: # unmask payload - f['payload'] = WebSocketServer.unmask(buf, f['hlen'], + f['payload'] = WebSocketRequestHandler.unmask(buf, f['hlen'], f['length']) else: - print("Unmasked frame: %s" % repr(buf)) + logger.debug("Unmasked frame: %s" % repr(buf)) + + if strict: + raise WebSocketRequestHandler.CClose(1002, "The client sent an unmasked frame.") + f['payload'] = buf[(f['hlen'] + f['masked'] * 4):full_len] if base64 and f['opcode'] in [1, 2]: try: f['payload'] = b64decode(f['payload']) except: - print("Exception while b64decoding buffer: %s" % - repr(buf)) + logger.exception("Exception while b64decoding buffer: %s" % + (repr(buf))) raise if f['opcode'] == 0x08: @@ -381,56 +269,34 @@ def decode_hybi(buf, base64=False): return f - @staticmethod - def encode_hixie(buf): - return s2b("\x00" + b2s(b64encode(buf)) + "\xff"), 1, 1 - - @staticmethod - def decode_hixie(buf): - end = buf.find(s2b('\xff')) - return {'payload': b64decode(buf[1:end]), - 'hlen': 1, - 'masked': False, - 'length': end - 1, - 'left': len(buf) - (end + 1)} - - - @staticmethod - def gen_md5(keys): - """ Generate hash value for WebSockets hixie-76. """ - key1 = keys['Sec-WebSocket-Key1'] - key2 = keys['Sec-WebSocket-Key2'] - key3 = keys['key3'] - spaces1 = key1.count(" ") - spaces2 = key2.count(" ") - num1 = int("".join([c for c in key1 if c.isdigit()])) / spaces1 - num2 = int("".join([c for c in key2 if c.isdigit()])) / spaces2 - - return b2s(md5(pack('>II8s', - int(num1), int(num2), key3)).digest()) # - # WebSocketServer logging/output functions + # WebSocketRequestHandler logging/output functions # - def traffic(self, token="."): - """ Show traffic flow in verbose mode. """ - if self.verbose and not self.daemon: + def print_traffic(self, token="."): + """ Show traffic flow mode. """ + if self.traffic: sys.stdout.write(token) sys.stdout.flush() - def msg(self, msg): + def msg(self, msg, *args, **kwargs): """ Output message with handler_id prefix. """ - if not self.daemon: - print("% 3d: %s" % (self.handler_id, msg)) + prefix = "% 3d: " % self.handler_id + self.logger.log(logging.INFO, "%s%s" % (prefix, msg), *args, **kwargs) - def vmsg(self, msg): - """ Same as msg() but only if verbose. """ - if self.verbose: - self.msg(msg) + def vmsg(self, msg, *args, **kwargs): + """ Same as msg() but as debug. """ + prefix = "% 3d: " % self.handler_id + self.logger.log(logging.DEBUG, "%s%s" % (prefix, msg), *args, **kwargs) + + def warn(self, msg, *args, **kwargs): + """ Same as msg() but as warning. """ + prefix = "% 3d: " % self.handler_id + self.logger.log(logging.WARN, "%s%s" % (prefix, msg), *args, **kwargs) # - # Main WebSocketServer methods + # Main WebSocketRequestHandler methods # def send_frames(self, bufs=None): """ Encode and send WebSocket frames. Any frames already @@ -444,16 +310,10 @@ def send_frames(self, bufs=None): if bufs: for buf in bufs: - if self.version.startswith("hybi"): - if self.base64: - encbuf, lenhead, lentail = self.encode_hybi( - buf, opcode=1, base64=True) - else: - encbuf, lenhead, lentail = self.encode_hybi( - buf, opcode=2, base64=False) - + if self.base64: + encbuf, lenhead, lentail = self.encode_hybi(buf, opcode=1, base64=True) else: - encbuf, lenhead, lentail = self.encode_hixie(buf) + encbuf, lenhead, lentail = self.encode_hybi(buf, opcode=2, base64=False) if self.rec: self.rec.write("%s,\n" % @@ -465,12 +325,12 @@ def send_frames(self, bufs=None): while self.send_parts: # Send pending frames buf = self.send_parts.pop(0) - sent = self.client.send(buf) + sent = self.request.send(buf) if sent == len(buf): - self.traffic("<") + self.print_traffic("<") else: - self.traffic("<.") + self.print_traffic("<.") self.send_parts.insert(0, buf[sent:]) break @@ -487,7 +347,7 @@ def recv_frames(self): bufs = [] tdelta = int(time.time()*1000) - self.start_time - buf = self.client.recv(self.buffer_size) + buf = self.request.recv(self.buffer_size) if len(buf) == 0: closed = {'code': 1000, 'reason': "Client closed abruptly"} return bufs, closed @@ -498,48 +358,39 @@ def recv_frames(self): self.recv_part = None while buf: - if self.version.startswith("hybi"): - - frame = self.decode_hybi(buf, base64=self.base64) - #print("Received buf: %s, frame: %s" % (repr(buf), frame)) - - if frame['payload'] == None: - # Incomplete/partial frame - self.traffic("}.") - if frame['left'] > 0: - self.recv_part = buf[-frame['left']:] - break - else: - if frame['opcode'] == 0x8: # connection close - closed = {'code': frame['close_code'], - 'reason': frame['close_reason']} - break - + frame = self.decode_hybi(buf, base64=self.base64, + logger=self.logger, + strict=self.strict_mode) + #self.msg("Received buf: %s, frame: %s", repr(buf), frame) + + if frame['payload'] == None: + # Incomplete/partial frame + self.print_traffic("}.") + if frame['left'] > 0: + self.recv_part = buf[-frame['left']:] + break else: - if buf[0:2] == s2b('\xff\x00'): - closed = {'code': 1000, - 'reason': "Client sent orderly close frame"} - break - - elif buf[0:2] == s2b('\x00\xff'): - buf = buf[2:] - continue # No-op - - elif buf.count(s2b('\xff')) == 0: - # Partial frame - self.traffic("}.") - self.recv_part = buf + if frame['opcode'] == 0x8: # connection close + closed = {'code': frame['close_code'], + 'reason': frame['close_reason']} break - - frame = self.decode_hixie(buf) - - self.traffic("}") + elif self.auto_pong and frame['opcode'] == 0x9: # ping + self.print_traffic("} ping %s\n" % + repr(frame['payload'])) + self.send_pong(frame['payload']) + return [], False + elif frame['opcode'] == 0xA: # pong + self.print_traffic("} pong %s\n" % + repr(frame['payload'])) + return [], False + + self.print_traffic("}") if self.rec: start = frame['hlen'] end = frame['hlen'] + frame['length'] if frame['masked']: - recbuf = WebSocketServer.unmask(buf, frame['hlen'], + recbuf = WebSocketRequestHandler.unmask(buf, frame['hlen'], frame['length']) else: recbuf = buf[frame['hlen']:frame['hlen'] + @@ -560,21 +411,22 @@ def recv_frames(self): def send_close(self, code=1000, reason=''): """ Send a WebSocket orderly close frame. """ - if self.version.startswith("hybi"): - msg = pack(">H%ds" % len(reason), code, reason) - - buf, h, t = self.encode_hybi(msg, opcode=0x08, base64=False) - self.client.send(buf) + msg = pack(">H%ds" % len(reason), code, s2b(reason)) + buf, h, t = self.encode_hybi(msg, opcode=0x08, base64=False) + self.request.send(buf) - elif self.version == "hixie-76": - buf = s2b('\xff\x00') - self.client.send(buf) + def send_pong(self, data=''): + """ Send a WebSocket pong frame. """ + buf, h, t = self.encode_hybi(s2b(data), opcode=0x0A, base64=False) + self.request.send(buf) - # No orderly close for 75 + def send_ping(self, data=''): + """ Send a WebSocket ping frame. """ + buf, h, t = self.encode_hybi(s2b(data), opcode=0x09, base64=False) + self.request.send(buf) - def do_websocket_handshake(self, headers, path): - h = self.headers = headers - self.path = path + def do_websocket_handshake(self): + h = self.headers prot = 'WebSocket-Protocol' protocols = h.get('Sec-'+prot, h.get(prot, '')).split(',') @@ -589,7 +441,8 @@ def do_websocket_handshake(self, headers, path): if ver in ['7', '8', '13']: self.version = "hybi-%02d" % int(ver) else: - raise self.EClose('Unsupported protocol version %s' % ver) + self.send_error(400, "Unsupported protocol version %s" % ver) + return False key = h['Sec-WebSocket-Key'] @@ -599,44 +452,338 @@ def do_websocket_handshake(self, headers, path): elif 'base64' in protocols: self.base64 = True else: - raise self.EClose("Client must support 'binary' or 'base64' protocol") + self.send_error(400, "Client must support 'binary' or 'base64' protocol") + return False # Generate the hash value for the accept header accept = b64encode(sha1(s2b(key + self.GUID)).digest()) - response = self.server_handshake_hybi % b2s(accept) + self.send_response(101, "Switching Protocols") + self.send_header("Upgrade", "websocket") + self.send_header("Connection", "Upgrade") + self.send_header("Sec-WebSocket-Accept", b2s(accept)) if self.base64: - response += "Sec-WebSocket-Protocol: base64\r\n" + self.send_header("Sec-WebSocket-Protocol", "base64") + else: + self.send_header("Sec-WebSocket-Protocol", "binary") + self.end_headers() + return True + else: + self.send_error(400, "Missing Sec-WebSocket-Version header. Hixie protocols not supported.") + + return False + + def handle_websocket(self): + """Upgrade a connection to Websocket, if requested. If this succeeds, + new_websocket_client() will be called. Otherwise, False is returned. + """ + + if (self.headers.get('upgrade') and + self.headers.get('upgrade').lower() == 'websocket'): + + # ensure connection is authorized, and determine the target + self.validate_connection() + + if not self.do_websocket_handshake(): + return False + + # Indicate to server that a Websocket upgrade was done + self.server.ws_connection = True + # Initialize per client settings + self.send_parts = [] + self.recv_part = None + self.start_time = int(time.time()*1000) + + # client_address is empty with, say, UNIX domain sockets + client_addr = "" + is_ssl = False + try: + client_addr = self.client_address[0] + is_ssl = self.client_address[2] + except IndexError: + pass + + if is_ssl: + self.stype = "SSL/TLS (wss://)" else: - response += "Sec-WebSocket-Protocol: binary\r\n" - response += "\r\n" + self.stype = "Plain non-SSL (ws://)" + + self.log_message("%s: %s WebSocket connection", client_addr, + self.stype) + self.log_message("%s: Version %s, base64: '%s'", client_addr, + self.version, self.base64) + if self.path != '/': + self.log_message("%s: Path: '%s'", client_addr, self.path) + + if self.record: + # Record raw frame data as JavaScript array + fname = "%s.%s" % (self.record, + self.handler_id) + self.log_message("opening record file: %s", fname) + self.rec = open(fname, 'w+') + encoding = "binary" + if self.base64: encoding = "base64" + self.rec.write("var VNC_frame_encoding = '%s';\n" + % encoding) + self.rec.write("var VNC_frame_data = [\n") + try: + self.new_websocket_client() + except self.CClose: + # Close the client + _, exc, _ = sys.exc_info() + self.send_close(exc.args[0], exc.args[1]) + return True else: - # Hixie version of the protocol (75 or 76) + return False - if h.get('key3'): - trailer = self.gen_md5(h) - pre = "Sec-" - self.version = "hixie-76" + def do_GET(self): + """Handle GET request. Calls handle_websocket(). If unsuccessful, + and web server is enabled, SimpleHTTPRequestHandler.do_GET will be called.""" + if not self.handle_websocket(): + if self.only_upgrade: + self.send_error(405, "Method Not Allowed") else: - trailer = "" - pre = "" - self.version = "hixie-75" + SimpleHTTPRequestHandler.do_GET(self) + + def list_directory(self, path): + if self.file_only: + self.send_error(404, "No such file") + else: + return SimpleHTTPRequestHandler.list_directory(self, path) + + def new_websocket_client(self): + """ Do something with a WebSockets client connection. """ + raise Exception("WebSocketRequestHandler.new_websocket_client() must be overloaded") - # We only support base64 in Hixie era - self.base64 = True + def validate_connection(self): + """ Ensure that the connection is a valid connection, and set the target. """ + pass + + def do_HEAD(self): + if self.only_upgrade: + self.send_error(405, "Method Not Allowed") + else: + SimpleHTTPRequestHandler.do_HEAD(self) + + def finish(self): + if self.rec: + self.rec.write("'EOF'];\n") + self.rec.close() + + def handle(self): + # When using run_once, we have a single process, so + # we cannot loop in BaseHTTPRequestHandler.handle; we + # must return and handle new connections + if self.run_once: + self.handle_one_request() + else: + SimpleHTTPRequestHandler.handle(self) + + def log_request(self, code='-', size='-'): + if self.verbose: + SimpleHTTPRequestHandler.log_request(self, code, size) - response = self.server_handshake_hixie % (pre, - h['Origin'], pre, self.scheme, h['Host'], path) - if 'base64' in protocols: - response += "%sWebSocket-Protocol: base64\r\n" % pre +class WebSocketServer(object): + """ + WebSockets server class. + As an alternative, the standard library SocketServer can be used + """ + + policy_response = """\n""" + log_prefix = "websocket" + + # An exception before the WebSocket connection was established + class EClose(Exception): + pass + + class Terminate(Exception): + pass + + def __init__(self, RequestHandlerClass, listen_host='', + listen_port=None, source_is_ipv6=False, + verbose=False, cert='', key='', ssl_only=None, + daemon=False, record='', web='', + file_only=False, + run_once=False, timeout=0, idle_timeout=0, traffic=False, + tcp_keepalive=True, tcp_keepcnt=None, tcp_keepidle=None, + tcp_keepintvl=None, auto_pong=False, strict_mode=True): + + # settings + self.RequestHandlerClass = RequestHandlerClass + self.verbose = verbose + self.listen_host = listen_host + self.listen_port = listen_port + self.prefer_ipv6 = source_is_ipv6 + self.ssl_only = ssl_only + self.daemon = daemon + self.run_once = run_once + self.timeout = timeout + self.idle_timeout = idle_timeout + self.traffic = traffic + self.file_only = file_only + self.strict_mode = strict_mode + + self.launch_time = time.time() + self.ws_connection = False + self.handler_id = 1 + + self.logger = self.get_logger() + self.tcp_keepalive = tcp_keepalive + self.tcp_keepcnt = tcp_keepcnt + self.tcp_keepidle = tcp_keepidle + self.tcp_keepintvl = tcp_keepintvl + + self.auto_pong = auto_pong + # Make paths settings absolute + self.cert = os.path.abspath(cert) + self.key = self.web = self.record = '' + if key: + self.key = os.path.abspath(key) + if web: + self.web = os.path.abspath(web) + if record: + self.record = os.path.abspath(record) + + if self.web: + os.chdir(self.web) + self.only_upgrade = not self.web + + # Sanity checks + if not ssl and self.ssl_only: + raise Exception("No 'ssl' module and SSL-only specified") + if self.daemon and not resource: + raise Exception("Module 'resource' required to daemonize") + + # Show configuration + self.msg("WebSocket server settings:") + self.msg(" - Listen on %s:%s", + self.listen_host, self.listen_port) + self.msg(" - Flash security policy server") + if self.web: + if self.file_only: + self.msg(" - Web server (no directory listings). Web root: %s", self.web) + else: + self.msg(" - Web server. Web root: %s", self.web) + if ssl: + if os.path.exists(self.cert): + self.msg(" - SSL/TLS support") + if self.ssl_only: + self.msg(" - Deny non-SSL/TLS connections") else: - self.msg("Warning: client does not report 'base64' protocol support") - response += "\r\n" + trailer + self.msg(" - No SSL/TLS support (no cert file)") + else: + self.msg(" - No SSL/TLS support (no 'ssl' module)") + if self.daemon: + self.msg(" - Backgrounding (daemon)") + if self.record: + self.msg(" - Recording to '%s.*'", self.record) + + # + # WebSocketServer static methods + # + + @staticmethod + def get_logger(): + return logging.getLogger("%s.%s" % ( + WebSocketServer.log_prefix, + WebSocketServer.__class__.__name__)) + + @staticmethod + def socket(host, port=None, connect=False, prefer_ipv6=False, + unix_socket=None, use_ssl=False, tcp_keepalive=True, + tcp_keepcnt=None, tcp_keepidle=None, tcp_keepintvl=None): + """ Resolve a host (and optional port) to an IPv4 or IPv6 + address. Create a socket. Bind to it if listen is set, + otherwise connect to it. Return the socket. + """ + flags = 0 + if host == '': + host = None + if connect and not (port or unix_socket): + raise Exception("Connect mode requires a port") + if use_ssl and not ssl: + raise Exception("SSL socket requested but Python SSL module not loaded."); + if not connect and use_ssl: + raise Exception("SSL only supported in connect mode (for now)") + if not connect: + flags = flags | socket.AI_PASSIVE - return response + if not unix_socket: + addrs = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM, + socket.IPPROTO_TCP, flags) + if not addrs: + raise Exception("Could not resolve host '%s'" % host) + addrs.sort(key=lambda x: x[0]) + if prefer_ipv6: + addrs.reverse() + sock = socket.socket(addrs[0][0], addrs[0][1]) + if tcp_keepalive: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + if tcp_keepcnt: + sock.setsockopt(socket.SOL_TCP, socket.TCP_KEEPCNT, + tcp_keepcnt) + if tcp_keepidle: + sock.setsockopt(socket.SOL_TCP, socket.TCP_KEEPIDLE, + tcp_keepidle) + if tcp_keepintvl: + sock.setsockopt(socket.SOL_TCP, socket.TCP_KEEPINTVL, + tcp_keepintvl) + + if connect: + sock.connect(addrs[0][4]) + if use_ssl: + sock = ssl.wrap_socket(sock) + else: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind(addrs[0][4]) + sock.listen(100) + else: + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.connect(unix_socket) + + return sock + + @staticmethod + def daemonize(keepfd=None, chdir='/'): + + if keepfd is None: + keepfd = [] + + os.umask(0) + if chdir: + os.chdir(chdir) + else: + os.chdir('/') + os.setgid(os.getgid()) # relinquish elevations + os.setuid(os.getuid()) # relinquish elevations + + # Double fork to daemonize + if os.fork() > 0: os._exit(0) # Parent exits + os.setsid() # Obtain new process group + if os.fork() > 0: os._exit(0) # Parent exits + + # Signal handling + signal.signal(signal.SIGTERM, signal.SIG_IGN) + signal.signal(signal.SIGINT, signal.SIG_IGN) + + # Close open files + maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1] + if maxfd == resource.RLIM_INFINITY: maxfd = 256 + for fd in reversed(range(maxfd)): + try: + if fd not in keepfd: + os.close(fd) + except OSError: + _, exc, _ = sys.exc_info() + if exc.errno != errno.EBADF: raise + + # Redirect I/O to /dev/null + os.dup2(os.open(os.devnull, os.O_RDWR), sys.stdin.fileno()) + os.dup2(os.open(os.devnull, os.O_RDWR), sys.stdout.fileno()) + os.dup2(os.open(os.devnull, os.O_RDWR), sys.stderr.fileno()) def do_handshake(self, sock, address): """ @@ -655,10 +802,9 @@ def do_handshake(self, sock, address): - Send a WebSockets handshake server response. - Return the socket for this WebSocket client. """ - stype = "" ready = select.select([sock], [], [], 3)[0] - + if not ready: raise self.EClose("ignoring socket not ready") # Peek, but do not read the data so that we have a opportunity @@ -666,7 +812,7 @@ def do_handshake(self, sock, address): handshake = sock.recv(1024, socket.MSG_PEEK) #self.msg("Handshake [%s]" % handshake) - if handshake == "": + if not handshake: raise self.EClose("ignoring empty handshake") elif handshake.startswith(s2b("")): @@ -699,45 +845,38 @@ def do_handshake(self, sock, address): else: raise - self.scheme = "wss" - stype = "SSL/TLS (wss://)" - elif self.ssl_only: raise self.EClose("non-SSL connection received but disallowed") else: retsock = sock - self.scheme = "ws" - stype = "Plain non-SSL (ws://)" - wsh = WSRequestHandler(retsock, address, not self.web) - if wsh.last_code == 101: - # Continue on to handle WebSocket upgrade - pass - elif wsh.last_code == 405: - raise self.EClose("Normal web request received but disallowed") - elif wsh.last_code < 200 or wsh.last_code >= 300: - raise self.EClose(wsh.last_message) - elif self.verbose: - raise self.EClose(wsh.last_message) - else: - raise self.EClose("") + # If the address is like (host, port), we are extending it + # with a flag indicating SSL. Not many other options + # available... + if len(address) == 2: + address = (address[0], address[1], (retsock != sock)) - response = self.do_websocket_handshake(wsh.headers, wsh.path) + self.RequestHandlerClass(retsock, address, self) - self.msg("%s: %s WebSocket connection" % (address[0], stype)) - self.msg("%s: Version %s, base64: '%s'" % (address[0], - self.version, self.base64)) - if self.path != '/': - self.msg("%s: Path: '%s'" % (address[0], self.path)) + # Return the WebSockets socket which may be SSL wrapped + return retsock + # + # WebSocketServer logging/output functions + # - # Send server WebSockets handshake response - #self.msg("sending response [%s]" % response) - retsock.send(s2b(response)) + def msg(self, *args, **kwargs): + """ Output message as info """ + self.logger.log(logging.INFO, *args, **kwargs) - # Return the WebSockets socket which may be SSL wrapped - return retsock + def vmsg(self, *args, **kwargs): + """ Same as msg() but as debug. """ + self.logger.log(logging.DEBUG, *args, **kwargs) + + def warn(self, *args, **kwargs): + """ Same as msg() but as warning. """ + self.logger.log(logging.WARN, *args, **kwargs) # @@ -752,9 +891,18 @@ def poll(self): #self.vmsg("Running poll()") pass + def terminate(self): + raise self.Terminate() + + def multiprocessing_SIGCHLD(self, sig, stack): + # TODO: figure out a way to actually log this information without + # calling `log` in the signal handlers + multiprocessing.active_children() + def fallback_SIGCHLD(self, sig, stack): # Reap zombies when using os.fork() (python 2.4) - self.vmsg("Got SIGCHLD, reaping zombies") + # TODO: figure out a way to actually log this information without + # calling `log` in the signal handlers try: result = os.waitpid(-1, os.WNOHANG) while result[0]: @@ -764,219 +912,198 @@ def fallback_SIGCHLD(self, sig, stack): pass def do_SIGINT(self, sig, stack): - self.msg("Got SIGINT, exiting") - sys.exit(0) + # TODO: figure out a way to actually log this information without + # calling `log` in the signal handlers + self.terminate() + + def do_SIGTERM(self, sig, stack): + # TODO: figure out a way to actually log this information without + # calling `log` in the signal handlers + self.terminate() def top_new_client(self, startsock, address): """ Do something with a WebSockets client connection. """ - # Initialize per client settings - self.send_parts = [] - self.recv_part = None - self.base64 = False - self.rec = None - self.start_time = int(time.time()*1000) - - # handler process + # handler process + client = None try: try: - self.client = self.do_handshake(startsock, address) - - if self.record: - # Record raw frame data as JavaScript array - fname = "%s.%s" % (self.record, - self.handler_id) - self.msg("opening record file: %s" % fname) - self.rec = open(fname, 'w+') - encoding = "binary" - if self.base64: encoding = "base64" - self.rec.write("var VNC_frame_encoding = '%s';\n" - % encoding) - self.rec.write("var VNC_frame_data = [\n") - - self.ws_connection = True - self.new_client() - except self.CClose: - # Close the client - _, exc, _ = sys.exc_info() - if self.client: - self.send_close(exc.args[0], exc.args[1]) + client = self.do_handshake(startsock, address) except self.EClose: _, exc, _ = sys.exc_info() # Connection was not a WebSockets connection if exc.args[0]: self.msg("%s: %s" % (address[0], exc.args[0])) + except WebSocketServer.Terminate: + raise except Exception: _, exc, _ = sys.exc_info() self.msg("handler exception: %s" % str(exc)) - if self.verbose: - self.msg(traceback.format_exc()) + self.vmsg("exception", exc_info=True) finally: - if self.rec: - self.rec.write("'EOF'];\n") - self.rec.close() - if self.client and self.client != startsock: + if client and client != startsock: # Close the SSL wrapped socket # Original socket closed by caller - self.client.close() + client.close() - def new_client(self): - """ Do something with a WebSockets client connection. """ - raise("WebSocketServer.new_client() must be overloaded") + def get_log_fd(self): + """ + Get file descriptors for the loggers. + They should not be closed when the process is forked. + """ + descriptors = [] + for handler in self.logger.parent.handlers: + if isinstance(handler, logging.FileHandler): + descriptors.append(handler.stream.fileno()) + + return descriptors def start_server(self): """ Daemonize if requested. Listen for for connections. Run do_handshake() method for each connection. If the connection - is a WebSockets client then call new_client() method (which must + is a WebSockets client then call new_websocket_client() method (which must be overridden) for each new client connection. """ - lsock = self.socket(self.listen_host, self.listen_port, False, self.prefer_ipv6) + lsock = self.socket(self.listen_host, self.listen_port, False, + self.prefer_ipv6, + tcp_keepalive=self.tcp_keepalive, + tcp_keepcnt=self.tcp_keepcnt, + tcp_keepidle=self.tcp_keepidle, + tcp_keepintvl=self.tcp_keepintvl) if self.daemon: - self.daemonize(keepfd=lsock.fileno(), chdir=self.web) + keepfd = self.get_log_fd() + keepfd.append(lsock.fileno()) + self.daemonize(keepfd=keepfd, chdir=self.web) self.started() # Some things need to happen after daemonizing - # Allow override of SIGINT + # Allow override of signals + original_signals = { + signal.SIGINT: signal.getsignal(signal.SIGINT), + signal.SIGTERM: signal.getsignal(signal.SIGTERM), + signal.SIGCHLD: signal.getsignal(signal.SIGCHLD), + } signal.signal(signal.SIGINT, self.do_SIGINT) + signal.signal(signal.SIGTERM, self.do_SIGTERM) if not multiprocessing: # os.fork() (python 2.4) child reaper signal.signal(signal.SIGCHLD, self.fallback_SIGCHLD) + else: + # make sure that _cleanup is called when children die + # by calling active_children on SIGCHLD + signal.signal(signal.SIGCHLD, self.multiprocessing_SIGCHLD) last_active_time = self.launch_time - while True: - try: + try: + while True: try: - self.client = None - startsock = None - pid = err = 0 - child_count = 0 - - if multiprocessing and self.idle_timeout: - child_count = len(multiprocessing.active_children()) - - time_elapsed = time.time() - self.launch_time - if self.timeout and time_elapsed > self.timeout: - self.msg('listener exit due to --timeout %s' - % self.timeout) - break - - if self.idle_timeout: - idle_time = 0 - if child_count == 0: - idle_time = time.time() - last_active_time - else: - idle_time = 0 - last_active_time = time.time() - - if idle_time > self.idle_timeout and child_count == 0: - self.msg('listener exit due to --idle-timeout %s' - % self.idle_timeout) - break - try: - self.poll() + startsock = None + pid = err = 0 + child_count = 0 + + if multiprocessing: + # Collect zombie child processes + child_count = len(multiprocessing.active_children()) + + time_elapsed = time.time() - self.launch_time + if self.timeout and time_elapsed > self.timeout: + self.msg('listener exit due to --timeout %s' + % self.timeout) + break - ready = select.select([lsock], [], [], 1)[0] - if lsock in ready: - startsock, address = lsock.accept() - else: - continue - except Exception: - _, exc, _ = sys.exc_info() - if hasattr(exc, 'errno'): - err = exc.errno - elif hasattr(exc, 'args'): - err = exc.args[0] - else: - err = exc[0] - if err == errno.EINTR: - self.vmsg("Ignoring interrupted syscall") - continue - else: + if self.idle_timeout: + idle_time = 0 + if child_count == 0: + idle_time = time.time() - last_active_time + else: + idle_time = 0 + last_active_time = time.time() + + if idle_time > self.idle_timeout and child_count == 0: + self.msg('listener exit due to --idle-timeout %s' + % self.idle_timeout) + break + + try: + self.poll() + + ready = select.select([lsock], [], [], 1)[0] + if lsock in ready: + startsock, address = lsock.accept() + else: + continue + except self.Terminate: raise - - if self.run_once: - # Run in same process if run_once - self.top_new_client(startsock, address) - if self.ws_connection : - self.msg('%s: exiting due to --run-once' - % address[0]) - break - elif multiprocessing: - self.vmsg('%s: new handler Process' % address[0]) - p = multiprocessing.Process( - target=self.top_new_client, - args=(startsock, address)) - p.start() - # child will not return - else: - # python 2.4 - self.vmsg('%s: forking handler' % address[0]) - pid = os.fork() - if pid == 0: - # child handler process + except Exception: + _, exc, _ = sys.exc_info() + if hasattr(exc, 'errno'): + err = exc.errno + elif hasattr(exc, 'args'): + err = exc.args[0] + else: + err = exc[0] + if err == errno.EINTR: + self.vmsg("Ignoring interrupted syscall") + continue + else: + raise + + if self.run_once: + # Run in same process if run_once self.top_new_client(startsock, address) - break # child process exits - - # parent process - self.handler_id += 1 - - except KeyboardInterrupt: - _, exc, _ = sys.exc_info() - print("In KeyboardInterrupt") - pass - except SystemExit: - _, exc, _ = sys.exc_info() - print("In SystemExit") - break - except Exception: - _, exc, _ = sys.exc_info() - self.msg("handler exception: %s" % str(exc)) - if self.verbose: - self.msg(traceback.format_exc()) - - finally: - if startsock: - startsock.close() - - # Close listen port - self.vmsg("Closing socket listening at %s:%s" - % (self.listen_host, self.listen_port)) - lsock.close() + if self.ws_connection : + self.msg('%s: exiting due to --run-once' + % address[0]) + break + elif multiprocessing: + self.vmsg('%s: new handler Process' % address[0]) + p = multiprocessing.Process( + target=self.top_new_client, + args=(startsock, address)) + p.start() + # child will not return + else: + # python 2.4 + self.vmsg('%s: forking handler' % address[0]) + pid = os.fork() + if pid == 0: + # child handler process + self.top_new_client(startsock, address) + break # child process exits + + # parent process + self.handler_id += 1 + + except (self.Terminate, SystemExit, KeyboardInterrupt): + self.msg("In exit") + # terminate all child processes + if multiprocessing and not self.run_once: + children = multiprocessing.active_children() + + for child in children: + self.msg("Terminating child %s" % child.pid) + child.terminate() + break + except Exception: + exc = sys.exc_info()[1] + self.msg("handler exception: %s", str(exc)) + self.vmsg("exception", exc_info=True) -# HTTP handler with WebSocket upgrade support -class WSRequestHandler(SimpleHTTPRequestHandler): - def __init__(self, req, addr, only_upgrade=False): - self.only_upgrade = only_upgrade # only allow upgrades - SimpleHTTPRequestHandler.__init__(self, req, addr, object()) + finally: + if startsock: + startsock.close() + finally: + # Close listen port + self.vmsg("Closing socket listening at %s:%s", + self.listen_host, self.listen_port) + lsock.close() - def do_GET(self): - if (self.headers.get('upgrade') and - self.headers.get('upgrade').lower() == 'websocket'): - - if (self.headers.get('sec-websocket-key1') or - self.headers.get('websocket-key1')): - # For Hixie-76 read out the key hash - self.headers.__setitem__('key3', self.rfile.read(8)) - - # Just indicate that an WebSocket upgrade is needed - self.last_code = 101 - self.last_message = "101 Switching Protocols" - elif self.only_upgrade: - # Normal web request responses are disabled - self.last_code = 405 - self.last_message = "405 Method Not Allowed" - else: - SimpleHTTPRequestHandler.do_GET(self) + # Restore signals + for sig, func in original_signals.items(): + signal.signal(sig, func) - def send_response(self, code, message=None): - # Save the status code - self.last_code = code - SimpleHTTPRequestHandler.send_response(self, code, message) - def log_message(self, f, *args): - # Save instead of printing - self.last_message = f % args diff --git a/third_party/websockify/websockify/websocketproxy.py b/third_party/websockify/websockify/websocketproxy.py index 18d0a28447e89..c098023e60f92 100755 --- a/third_party/websockify/websockify/websocketproxy.py +++ b/third_party/websockify/websockify/websocketproxy.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python ''' A WebSocket to TCP socket proxy with support for "wss://" encryption. @@ -11,24 +11,21 @@ ''' -import signal, socket, optparse, time, os, sys, subprocess -from select import select -import websocket +import signal, socket, optparse, time, os, sys, subprocess, logging, errno +try: from socketserver import ForkingMixIn +except: from SocketServer import ForkingMixIn +try: from http.server import HTTPServer +except: from BaseHTTPServer import HTTPServer +import select +from websockify import websocket +from websockify import auth_plugins as auth try: from urllib.parse import parse_qs, urlparse except: from cgi import parse_qs from urlparse import urlparse -class WebSocketProxy(websocket.WebSocketServer): - """ - Proxy traffic to and from a WebSockets client to a normal TCP - socket server target. All traffic to/from the client is base64 - encoded/decoded to allow binary data to be sent/received to/from - the target. - """ - - buffer_size = 65536 +class ProxyRequestHandler(websocket.WebSocketRequestHandler): traffic_legend = """ Traffic Legend: @@ -41,8 +38,186 @@ class WebSocketProxy(websocket.WebSocketServer): < - Client send <. - Client send partial """ + + def send_auth_error(self, ex): + self.send_response(ex.code, ex.msg) + self.send_header('Content-Type', 'text/html') + for name, val in ex.headers.items(): + self.send_header(name, val) + + self.end_headers() + + def validate_connection(self): + if self.server.token_plugin: + (self.server.target_host, self.server.target_port) = self.get_target(self.server.token_plugin, self.path) + + if self.server.auth_plugin: + try: + self.server.auth_plugin.authenticate( + headers=self.headers, target_host=self.server.target_host, + target_port=self.server.target_port) + except auth.AuthenticationError: + ex = sys.exc_info()[1] + self.send_auth_error(ex) + raise + + def new_websocket_client(self): + """ + Called after a new WebSocket connection has been established. + """ + # Checking for a token is done in validate_connection() + + # Connect to the target + if self.server.wrap_cmd: + msg = "connecting to command: '%s' (port %s)" % (" ".join(self.server.wrap_cmd), self.server.target_port) + elif self.server.unix_target: + msg = "connecting to unix socket: %s" % self.server.unix_target + else: + msg = "connecting to: %s:%s" % ( + self.server.target_host, self.server.target_port) + + if self.server.ssl_target: + msg += " (using SSL)" + self.log_message(msg) + + tsock = websocket.WebSocketServer.socket(self.server.target_host, + self.server.target_port, + connect=True, use_ssl=self.server.ssl_target, unix_socket=self.server.unix_target) + + self.print_traffic(self.traffic_legend) + + # Start proxying + try: + self.do_proxy(tsock) + except: + if tsock: + tsock.shutdown(socket.SHUT_RDWR) + tsock.close() + if self.verbose: + self.log_message("%s:%s: Closed target", + self.server.target_host, self.server.target_port) + raise + + def get_target(self, target_plugin, path): + """ + Parses the path, extracts a token, and looks up a target + for that token using the token plugin. Sets + target_host and target_port if successful + """ + # The files in targets contain the lines + # in the form of token: host:port + + # Extract the token parameter from url + args = parse_qs(urlparse(path)[4]) # 4 is the query from url + + if not 'token' in args or not len(args['token']): + raise self.server.EClose("Token not present") + + token = args['token'][0].rstrip('\n') + + result_pair = target_plugin.lookup(token) + + if result_pair is not None: + return result_pair + else: + raise self.server.EClose("Token '%s' not found" % token) + + def do_proxy(self, target): + """ + Proxy client WebSocket to normal target socket. + """ + cqueue = [] + c_pend = 0 + tqueue = [] + rlist = [self.request, target] + + if self.server.heartbeat: + now = time.time() + self.heartbeat = now + self.server.heartbeat + else: + self.heartbeat = None + + while True: + wlist = [] + + if self.heartbeat is not None: + now = time.time() + if now > self.heartbeat: + self.heartbeat = now + self.server.heartbeat + self.send_ping() + + if tqueue: wlist.append(target) + if cqueue or c_pend: wlist.append(self.request) + try: + ins, outs, excepts = select.select(rlist, wlist, [], 1) + except (select.error, OSError): + exc = sys.exc_info()[1] + if hasattr(exc, 'errno'): + err = exc.errno + else: + err = exc[0] - def __init__(self, *args, **kwargs): + if err != errno.EINTR: + raise + else: + continue + + if excepts: raise Exception("Socket exception") + + if self.request in outs: + # Send queued target data to the client + c_pend = self.send_frames(cqueue) + + cqueue = [] + + if self.request in ins: + # Receive client data, decode it, and queue for target + bufs, closed = self.recv_frames() + tqueue.extend(bufs) + + if closed: + # TODO: What about blocking on client socket? + if self.verbose: + self.log_message("%s:%s: Client closed connection", + self.server.target_host, self.server.target_port) + raise self.CClose(closed['code'], closed['reason']) + + + if target in outs: + # Send queued client data to the target + dat = tqueue.pop(0) + sent = target.send(dat) + if sent == len(dat): + self.print_traffic(">") + else: + # requeue the remaining data + tqueue.insert(0, dat[sent:]) + self.print_traffic(".>") + + + if target in ins: + # Receive target data, encode it and queue for client + buf = target.recv(self.buffer_size) + if len(buf) == 0: + if self.verbose: + self.log_message("%s:%s: Target closed connection", + self.server.target_host, self.server.target_port) + raise self.CClose(1000, "Target closed") + + cqueue.append(buf) + self.print_traffic("{") + +class WebSocketProxy(websocket.WebSocketServer): + """ + Proxy traffic to and from a WebSockets client to a normal TCP + socket server target. All traffic to/from the client is base64 + encoded/decoded to allow binary data to be sent/received to/from + the target. + """ + + buffer_size = 65536 + + def __init__(self, RequestHandlerClass=ProxyRequestHandler, *args, **kwargs): # Save off proxy specific options self.target_host = kwargs.pop('target_host', None) self.target_port = kwargs.pop('target_port', None) @@ -50,12 +225,19 @@ def __init__(self, *args, **kwargs): self.wrap_mode = kwargs.pop('wrap_mode', None) self.unix_target = kwargs.pop('unix_target', None) self.ssl_target = kwargs.pop('ssl_target', None) - self.target_cfg = kwargs.pop('target_cfg', None) + self.heartbeat = kwargs.pop('heartbeat', None) + + self.token_plugin = kwargs.pop('token_plugin', None) + self.auth_plugin = kwargs.pop('auth_plugin', None) + # Last 3 timestamps command was run self.wrap_times = [0, 0, 0] if self.wrap_cmd: - rebinder_path = ['./', os.path.dirname(sys.argv[0])] + wsdir = os.path.dirname(sys.argv[0]) + rebinder_path = [os.path.join(wsdir, "..", "lib"), + os.path.join(wsdir, "..", "lib", "websockify"), + wsdir] self.rebinder = None for rdir in rebinder_path: @@ -80,13 +262,10 @@ def __init__(self, *args, **kwargs): "REBIND_OLD_PORT": str(kwargs['listen_port']), "REBIND_NEW_PORT": str(self.target_port)}) - if self.target_cfg: - self.target_cfg = os.path.abspath(self.target_cfg) - - websocket.WebSocketServer.__init__(self, *args, **kwargs) + websocket.WebSocketServer.__init__(self, RequestHandlerClass, *args, **kwargs) def run_wrap_cmd(self): - print("Starting '%s'" % " ".join(self.wrap_cmd)) + self.msg("Starting '%s'", " ".join(self.wrap_cmd)) self.wrap_times.append(time.time()) self.wrap_times.pop(0) self.cmd = subprocess.Popen( @@ -106,9 +285,9 @@ def started(self): else: dst_string = "%s:%s" % (self.target_host, self.target_port) - if self.target_cfg: - msg = " - proxying from %s:%s to targets in %s" % ( - self.listen_host, self.listen_port, self.target_cfg) + if self.token_plugin: + msg = " - proxying from %s:%s to targets generated by %s" % ( + self.listen_host, self.listen_port, type(self.token_plugin).__name__) else: msg = " - proxying from %s:%s to %s" % ( self.listen_host, self.listen_port, dst_string) @@ -116,7 +295,7 @@ def started(self): if self.ssl_target: msg += " (using SSL)" - print(msg + "\n") + self.msg("%s", msg) if self.wrap_cmd: self.run_wrap_cmd() @@ -142,158 +321,11 @@ def poll(self): if (now - avg) < 10: # 3 times in the last 10 seconds if self.spawn_message: - print("Command respawning too fast") + self.warn("Command respawning too fast") self.spawn_message = False else: self.run_wrap_cmd() - # - # Routines above this point are run in the master listener - # process. - # - - # - # Routines below this point are connection handler routines and - # will be run in a separate forked process for each connection. - # - - def new_client(self): - """ - Called after a new WebSocket connection has been established. - """ - # Checks if we receive a token, and look - # for a valid target for it then - if self.target_cfg: - (self.target_host, self.target_port) = self.get_target(self.target_cfg, self.path) - - # Connect to the target - if self.wrap_cmd: - msg = "connecting to command: '%s' (port %s)" % (" ".join(self.wrap_cmd), self.target_port) - elif self.unix_target: - msg = "connecting to unix socket: %s" % self.unix_target - else: - msg = "connecting to: %s:%s" % ( - self.target_host, self.target_port) - - if self.ssl_target: - msg += " (using SSL)" - self.msg(msg) - - tsock = self.socket(self.target_host, self.target_port, - connect=True, use_ssl=self.ssl_target, unix_socket=self.unix_target) - - if self.verbose and not self.daemon: - print(self.traffic_legend) - - # Start proxying - try: - self.do_proxy(tsock) - except: - if tsock: - tsock.shutdown(socket.SHUT_RDWR) - tsock.close() - self.vmsg("%s:%s: Closed target" %( - self.target_host, self.target_port)) - raise - - def get_target(self, target_cfg, path): - """ - Parses the path, extracts a token, and looks for a valid - target for that token in the configuration file(s). Sets - target_host and target_port if successful - """ - # The files in targets contain the lines - # in the form of token: host:port - - # Extract the token parameter from url - args = parse_qs(urlparse(path)[4]) # 4 is the query from url - - if not args.has_key('token') or not len(args['token']): - raise self.EClose("Token not present") - - token = args['token'][0].rstrip('\n') - - # target_cfg can be a single config file or directory of - # config files - if os.path.isdir(target_cfg): - cfg_files = [os.path.join(target_cfg, f) - for f in os.listdir(target_cfg)] - else: - cfg_files = [target_cfg] - - targets = {} - for f in cfg_files: - for line in [l.strip() for l in file(f).readlines()]: - if line and not line.startswith('#'): - ttoken, target = line.split(': ') - targets[ttoken] = target.strip() - - self.vmsg("Target config: %s" % repr(targets)) - - if targets.has_key(token): - return targets[token].split(':') - else: - raise self.EClose("Token '%s' not found" % token) - - def do_proxy(self, target): - """ - Proxy client WebSocket to normal target socket. - """ - cqueue = [] - c_pend = 0 - tqueue = [] - rlist = [self.client, target] - - while True: - wlist = [] - - if tqueue: wlist.append(target) - if cqueue or c_pend: wlist.append(self.client) - ins, outs, excepts = select(rlist, wlist, [], 1) - if excepts: raise Exception("Socket exception") - - if self.client in outs: - # Send queued target data to the client - c_pend = self.send_frames(cqueue) - - cqueue = [] - - if self.client in ins: - # Receive client data, decode it, and queue for target - bufs, closed = self.recv_frames() - tqueue.extend(bufs) - - if closed: - # TODO: What about blocking on client socket? - self.vmsg("%s:%s: Client closed connection" %( - self.target_host, self.target_port)) - raise self.CClose(closed['code'], closed['reason']) - - - if target in outs: - # Send queued client data to the target - dat = tqueue.pop(0) - sent = target.send(dat) - if sent == len(dat): - self.traffic(">") - else: - # requeue the remaining data - tqueue.insert(0, dat[sent:]) - self.traffic(".>") - - - if target in ins: - # Receive target data, encode it and queue for client - buf = target.recv(self.buffer_size) - if len(buf) == 0: - self.vmsg("%s:%s: Target closed connection" %( - self.target_host, self.target_port)) - raise self.CClose(1000, "Target closed") - - cqueue.append(buf) - self.traffic("{") - - def _subprocess_setup(): # Python installs a SIGPIPE handler by default. This is usually not what @@ -301,14 +333,28 @@ def _subprocess_setup(): signal.signal(signal.SIGPIPE, signal.SIG_DFL) +def logger_init(): + logger = logging.getLogger(WebSocketProxy.log_prefix) + logger.propagate = False + logger.setLevel(logging.INFO) + h = logging.StreamHandler() + h.setLevel(logging.DEBUG) + h.setFormatter(logging.Formatter("%(message)s")) + logger.addHandler(h) + + def websockify_init(): + logger_init() + usage = "\n %prog [options]" usage += " [source_addr:]source_port [target_addr:target_port]" usage += "\n %prog [options]" usage += " [source_addr:]source_port -- WRAP_COMMAND_LINE" parser = optparse.OptionParser(usage=usage) parser.add_option("--verbose", "-v", action="store_true", - help="verbose messages and per frame traffic") + help="verbose messages") + parser.add_option("--traffic", action="store_true", + help="per frame traffic") parser.add_option("--record", help="record sessions to FILE.[session_number]", metavar="FILE") parser.add_option("--daemon", "-D", @@ -340,15 +386,69 @@ def websockify_init(): parser.add_option("--prefer-ipv6", "-6", action="store_true", dest="source_is_ipv6", help="prefer IPv6 when resolving source_addr") + parser.add_option("--libserver", action="store_true", + help="use Python library SocketServer engine") parser.add_option("--target-config", metavar="FILE", dest="target_cfg", help="Configuration file containing valid targets " "in the form 'token: host:port' or, alternatively, a " - "directory containing configuration files of this form") + "directory containing configuration files of this form " + "(DEPRECATED: use `--token-plugin TokenFile --token-source " + " path/to/token/file` instead)") + parser.add_option("--token-plugin", default=None, metavar="PLUGIN", + help="use the given Python class to process tokens " + "into host:port pairs") + parser.add_option("--token-source", default=None, metavar="ARG", + help="an argument to be passed to the token plugin" + "on instantiation") + parser.add_option("--auth-plugin", default=None, metavar="PLUGIN", + help="use the given Python class to determine if " + "a connection is allowed") + parser.add_option("--auth-source", default=None, metavar="ARG", + help="an argument to be passed to the auth plugin" + "on instantiation") + parser.add_option("--auto-pong", action="store_true", + help="Automatically respond to ping frames with a pong") + parser.add_option("--heartbeat", type=int, default=0, + help="send a ping to the client every HEARTBEAT seconds") + parser.add_option("--log-file", metavar="FILE", + dest="log_file", + help="File where logs will be saved") + + (opts, args) = parser.parse_args() + if opts.log_file: + opts.log_file = os.path.abspath(opts.log_file) + handler = logging.FileHandler(opts.log_file) + handler.setLevel(logging.DEBUG) + handler.setFormatter(logging.Formatter("%(message)s")) + logging.getLogger(WebSocketProxy.log_prefix).addHandler(handler) + + del opts.log_file + + if opts.verbose: + logging.getLogger(WebSocketProxy.log_prefix).setLevel(logging.DEBUG) + + if opts.token_source and not opts.token_plugin: + parser.error("You must use --token-plugin to use --token-source") + + if opts.auth_source and not opts.auth_plugin: + parser.error("You must use --auth-plugin to use --auth-source") + + + # Transform to absolute path as daemon may chdir + if opts.target_cfg: + opts.target_cfg = os.path.abspath(opts.target_cfg) + + if opts.target_cfg: + opts.token_plugin = 'TokenFile' + opts.token_source = opts.target_cfg + + del opts.target_cfg + # Sanity checks - if len(args) < 2 and not (opts.target_cfg or opts.unix_target): + if len(args) < 2 and not (opts.token_plugin or opts.unix_target): parser.error("Too few arguments") if sys.argv.count('--'): opts.wrap_cmd = args[1:] @@ -373,7 +473,7 @@ def websockify_init(): try: opts.listen_port = int(opts.listen_port) except: parser.error("Error parsing listen port") - if opts.wrap_cmd or opts.unix_target or opts.target_cfg: + if opts.wrap_cmd or opts.unix_target or opts.token_plugin: opts.target_host = None opts.target_port = None else: @@ -385,9 +485,97 @@ def websockify_init(): try: opts.target_port = int(opts.target_port) except: parser.error("Error parsing target port") + if opts.token_plugin is not None: + if '.' not in opts.token_plugin: + opts.token_plugin = ( + 'websockify.token_plugins.%s' % opts.token_plugin) + + token_plugin_module, token_plugin_cls = opts.token_plugin.rsplit('.', 1) + + __import__(token_plugin_module) + token_plugin_cls = getattr(sys.modules[token_plugin_module], token_plugin_cls) + + opts.token_plugin = token_plugin_cls(opts.token_source) + + del opts.token_source + + if opts.auth_plugin is not None: + if '.' not in opts.auth_plugin: + opts.auth_plugin = 'websockify.auth_plugins.%s' % opts.auth_plugin + + auth_plugin_module, auth_plugin_cls = opts.auth_plugin.rsplit('.', 1) + + __import__(auth_plugin_module) + auth_plugin_cls = getattr(sys.modules[auth_plugin_module], auth_plugin_cls) + + opts.auth_plugin = auth_plugin_cls(opts.auth_source) + + del opts.auth_source + # Create and start the WebSockets proxy - server = WebSocketProxy(**opts.__dict__) - server.start_server() + libserver = opts.libserver + del opts.libserver + if libserver: + # Use standard Python SocketServer framework + server = LibProxyServer(**opts.__dict__) + server.serve_forever() + else: + # Use internal service framework + server = WebSocketProxy(**opts.__dict__) + server.start_server() + + +class LibProxyServer(ForkingMixIn, HTTPServer): + """ + Just like WebSocketProxy, but uses standard Python SocketServer + framework. + """ + + def __init__(self, RequestHandlerClass=ProxyRequestHandler, **kwargs): + # Save off proxy specific options + self.target_host = kwargs.pop('target_host', None) + self.target_port = kwargs.pop('target_port', None) + self.wrap_cmd = kwargs.pop('wrap_cmd', None) + self.wrap_mode = kwargs.pop('wrap_mode', None) + self.unix_target = kwargs.pop('unix_target', None) + self.ssl_target = kwargs.pop('ssl_target', None) + self.token_plugin = kwargs.pop('token_plugin', None) + self.auth_plugin = kwargs.pop('auth_plugin', None) + self.heartbeat = kwargs.pop('heartbeat', None) + + self.token_plugin = None + self.auth_plugin = None + self.daemon = False + + # Server configuration + listen_host = kwargs.pop('listen_host', '') + listen_port = kwargs.pop('listen_port', None) + web = kwargs.pop('web', '') + + # Configuration affecting base request handler + self.only_upgrade = not web + self.verbose = kwargs.pop('verbose', False) + record = kwargs.pop('record', '') + if record: + self.record = os.path.abspath(record) + self.run_once = kwargs.pop('run_once', False) + self.handler_id = 0 + + for arg in kwargs.keys(): + print("warning: option %s ignored when using --libserver" % arg) + + if web: + os.chdir(web) + + HTTPServer.__init__(self, (listen_host, listen_port), + RequestHandlerClass) + + + def process_request(self, request, client_address): + """Override process_request to implement a counter""" + self.handler_id += 1 + ForkingMixIn.process_request(self, request, client_address) + if __name__ == '__main__': websockify_init() diff --git a/tools/cache.py b/tools/cache.py index 0bd6e0696d3a6..d1e22fba31977 100644 --- a/tools/cache.py +++ b/tools/cache.py @@ -4,11 +4,16 @@ # Permanent cache for dlmalloc and stdlibc++ class Cache: - def __init__(self, dirname=None, debug=False): + def __init__(self, dirname=None, debug=False, use_subdir=True): if dirname is None: dirname = os.environ.get('EM_CACHE') if not dirname: dirname = os.path.expanduser(os.path.join('~', '.emscripten_cache')) + if use_subdir: + if os.environ.get('EMCC_WASM_BACKEND') and os.environ.get('EMCC_WASM_BACKEND') != '0': + dirname = os.path.join(dirname, 'wasm') + else: + dirname = os.path.join(dirname, 'asmjs') self.dirname = dirname self.debug = debug @@ -27,10 +32,10 @@ def get_path(self, shortname): # Request a cached file. If it isn't in the cache, it will be created with # the given creator function - def get(self, shortname, creator, extension='.bc', what=None): + def get(self, shortname, creator, extension='.bc', what=None, force=False): if not shortname.endswith(extension): shortname += extension cachename = os.path.join(self.dirname, shortname) - if os.path.exists(cachename): + if os.path.exists(cachename) and not force: return cachename if what is None: if shortname.endswith(('.bc', '.so', '.a')): what = 'system library' diff --git a/tools/ctor_evaller.py b/tools/ctor_evaller.py index 618afa62c6106..603511cde666b 100644 --- a/tools/ctor_evaller.py +++ b/tools/ctor_evaller.py @@ -18,7 +18,8 @@ config = shared.Configuration() if shared.DEBUG: - temp_file = '/tmp/emscripten_temp/ctorEval.js' + temp_file = os.path.join(shared.CANONICAL_TEMP_DIR, 'ctorEval.js') + shared.safe_ensure_dirs(shared.CANONICAL_TEMP_DIR) else: temp_file = config.get_temp_files().get('.ctorEval.js').name @@ -230,7 +231,7 @@ def add_func(asm, func): num_successful, mem_init_raw, atexits = json.loads(open(out_file).read()) mem_init = ''.join(map(chr, mem_init_raw)) if num_successful < total_ctors: - shared.logging.debug('not all ctors could be evalled:\n' + open(err_file).read()) + shared.logging.debug('not all ctors could be evalled, something was used that was not safe (and therefore was not defined, and caused an error):\n========\n' + open(err_file).read() + '========') # Remove the evalled ctors, add a new one for atexits if needed, and write that out if len(ctors) == total_ctors and len(atexits) == 0: new_ctors = '' @@ -245,65 +246,65 @@ def add_func(asm, func): return (num_successful, js, mem_init, ctors) # main +if __name__ == '__main__': + js = open(js_file).read() + ctors_start, ctors_end = find_ctors(js) + if ctors_start < 0: + shared.logging.debug('ctor_evaller: no ctors') + sys.exit(0) -js = open(js_file).read() -ctors_start, ctors_end = find_ctors(js) -if ctors_start < 0: - shared.logging.debug('ctor_evaller: no ctors') - sys.exit(0) - -ctors_text = js[ctors_start:ctors_end]; -if ctors_text.count('(') == 1: - shared.logging.debug('ctor_evaller: push, but no ctors') - sys.exit(0) + ctors_text = js[ctors_start:ctors_end]; + if ctors_text.count('(') == 1: + shared.logging.debug('ctor_evaller: push, but no ctors') + sys.exit(0) -num_ctors = ctors_text.count(',') + 1 -shared.logging.debug('ctor_evaller: %d ctors' % num_ctors) + num_ctors = ctors_text.count('function()') + shared.logging.debug('ctor_evaller: %d ctors, from |%s|' % (num_ctors, ctors_text)) -if os.path.exists(mem_init_file): - mem_init = json.dumps(map(ord, open(mem_init_file, 'rb').read())) -else: - mem_init = [] + if os.path.exists(mem_init_file): + mem_init = json.dumps(map(ord, open(mem_init_file, 'rb').read())) + else: + mem_init = [] -# find how many ctors we can remove, by bisection (if there are hundreds, running them sequentially is silly slow) + # find how many ctors we can remove, by bisection (if there are hundreds, running them sequentially is silly slow) -shared.logging.debug('ctor_evaller: trying to eval %d global constructors' % num_ctors) -num_successful, new_js, new_mem_init, removed = eval_ctors(js, mem_init, num_ctors) -if num_successful == 0: - shared.logging.debug('ctor_evaller: not successful') - sys.exit(0) + shared.logging.debug('ctor_evaller: trying to eval %d global constructors' % num_ctors) + num_successful, new_js, new_mem_init, removed = eval_ctors(js, mem_init, num_ctors) + if num_successful == 0: + shared.logging.debug('ctor_evaller: not successful') + sys.exit(0) -shared.logging.debug('ctor_evaller: we managed to remove %d ctors' % num_successful) -if num_successful == num_ctors: - js = new_js - mem_init = new_mem_init -else: - shared.logging.debug('ctor_evaller: final execution') - check, js, mem_init, removed = eval_ctors(js, mem_init, num_successful) - assert check == num_successful -open(js_file, 'w').write(js) -open(mem_init_file, 'wb').write(mem_init) - -# Dead function elimination can help us - -shared.logging.debug('ctor_evaller: eliminate no longer needed functions after ctor elimination') -# find exports -asm = get_asm(open(js_file).read()) -exports_start = asm.find('return {') -exports_end = asm.find('};', exports_start) -exports_text = asm[asm.find('{', exports_start) + 1 : exports_end] -exports = map(lambda x: x.split(':')[1].strip(), exports_text.replace(' ', '').split(',')) -for r in removed: - assert r in exports, 'global ctors were exported' -exports = filter(lambda e: e not in removed, exports) -# fix up the exports -js = open(js_file).read() -absolute_exports_start = js.find(exports_text) -js = js[:absolute_exports_start] + ', '.join(map(lambda e: e + ': ' + e, exports)) + js[absolute_exports_start + len(exports_text):] -open(js_file, 'w').write(js) -# find unreachable methods and remove them -reachable = shared.Building.calculate_reachable_functions(js_file, exports, can_reach=False)['reachable'] -for r in removed: - assert r not in reachable, 'removed ctors must NOT be reachable' -shared.Building.js_optimizer(js_file, ['removeFuncs'], extra_info={ 'keep': reachable }, output_filename=js_file) + shared.logging.debug('ctor_evaller: we managed to remove %d ctors' % num_successful) + if num_successful == num_ctors: + js = new_js + mem_init = new_mem_init + else: + shared.logging.debug('ctor_evaller: final execution') + check, js, mem_init, removed = eval_ctors(js, mem_init, num_successful) + assert check == num_successful + open(js_file, 'w').write(js) + open(mem_init_file, 'wb').write(mem_init) + + # Dead function elimination can help us + + shared.logging.debug('ctor_evaller: eliminate no longer needed functions after ctor elimination') + # find exports + asm = get_asm(open(js_file).read()) + exports_start = asm.find('return {') + exports_end = asm.find('};', exports_start) + exports_text = asm[asm.find('{', exports_start) + 1 : exports_end] + exports = map(lambda x: x.split(':')[1].strip(), exports_text.replace(' ', '').split(',')) + for r in removed: + assert r in exports, 'global ctors were exported' + exports = filter(lambda e: e not in removed, exports) + # fix up the exports + js = open(js_file).read() + absolute_exports_start = js.find(exports_text) + js = js[:absolute_exports_start] + ', '.join(map(lambda e: e + ': ' + e, exports)) + js[absolute_exports_start + len(exports_text):] + open(js_file, 'w').write(js) + # find unreachable methods and remove them + reachable = shared.Building.calculate_reachable_functions(js_file, exports, can_reach=False)['reachable'] + for r in removed: + assert r not in reachable, 'removed ctors must NOT be reachable' + shared.Building.js_optimizer(js_file, ['removeFuncs'], extra_info={ 'keep': reachable }, output_filename=js_file) diff --git a/tools/duplicate_function_eliminator.py b/tools/duplicate_function_eliminator.py index 1a673136e2cad..ae1387b9450be 100644 --- a/tools/duplicate_function_eliminator.py +++ b/tools/duplicate_function_eliminator.py @@ -247,6 +247,8 @@ def write_chunk(chunk, i): if DEBUG: print >> sys.stderr, 'splitting up js optimization into %d chunks, using %d cores (total: %.2f MB)' % (len(chunks), cores, total_size/(1024*1024.)) pool = multiprocessing.Pool(processes=cores) filenames = pool.map(run_on_chunk, commands, chunksize=1) + pool.terminate() + pool.join() else: # We can't parallize, but still break into chunks to avoid uglify/node memory issues if len(chunks) > 1 and DEBUG: print >> sys.stderr, 'splitting up js optimization into %d chunks' % (len(chunks)) diff --git a/tools/emdebug_cd_merger.py b/tools/emdebug_cd_merger.py new file mode 100755 index 0000000000000..e687df2db27bc --- /dev/null +++ b/tools/emdebug_cd_merger.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python2 +# -*- Mode: python -*- + +import logging, sys, json + +def run(): + args = sys.argv[1:] + assert len(args) == 2 + + with open(args[0]) as cd_f: + cd_data = json.load(cd_f) + + try: + with open(args[1]) as symbol_f: + symbol_list = {x[0]:x[2] for x in [z.strip().partition(":") for z in symbol_f.readlines() if len(z) > 0]} + except IOError as e: + # If there's no symbol file, use an empty one + symbol_list = {} + + cd_data['cyberdwarf']['function_name_map'] = symbol_list + + with open(args[0],"w") as cd_f: + json.dump(cd_data, cd_f, separators=(',',':')) + + +if __name__ == '__main__': + run() diff --git a/tools/emterpretify.py b/tools/emterpretify.py index 797fa86094f52..86a29f853cd08 100755 --- a/tools/emterpretify.py +++ b/tools/emterpretify.py @@ -57,7 +57,7 @@ def handle_arg(arg): # consts -BLACKLIST = set(['_malloc', '_free', '_memcpy', '_memmove', '_memset', 'copyTempDouble', 'copyTempFloat', '_strlen', 'stackAlloc', 'setThrew', 'stackRestore', 'setTempRet0', 'getTempRet0', 'stackSave', '_emscripten_autodebug_double', '_emscripten_autodebug_float', '_emscripten_autodebug_i8', '_emscripten_autodebug_i16', '_emscripten_autodebug_i32', '_emscripten_autodebug_i64', '_strncpy', '_strcpy', '_strcat', '_saveSetjmp', '_testSetjmp', '_emscripten_replace_memory', '_bitshift64Shl', '_bitshift64Ashr', '_bitshift64Lshr', 'setAsyncState', 'emtStackSave', 'emtStackRestore']) +BLACKLIST = set(['_malloc', '_free', '_memcpy', '_memmove', '_memset', '_strlen', 'stackAlloc', 'setThrew', 'stackRestore', 'setTempRet0', 'getTempRet0', 'stackSave', '_emscripten_autodebug_double', '_emscripten_autodebug_float', '_emscripten_autodebug_i8', '_emscripten_autodebug_i16', '_emscripten_autodebug_i32', '_emscripten_autodebug_i64', '_strncpy', '_strcpy', '_strcat', '_saveSetjmp', '_testSetjmp', '_emscripten_replace_memory', '_bitshift64Shl', '_bitshift64Ashr', '_bitshift64Lshr', 'setAsyncState', 'emtStackSave', 'emtStackRestore']) WHITELIST = [] SYNC_FUNCS = set(['_emscripten_sleep', '_emscripten_sleep_with_yield', '_emscripten_wget_data', '_emscripten_idb_load', '_emscripten_idb_store', '_emscripten_idb_delete']) diff --git a/tools/file_packager.py b/tools/file_packager.py index 832ceef59b581..114cbf614a660 100644 --- a/tools/file_packager.py +++ b/tools/file_packager.py @@ -11,7 +11,7 @@ Usage: - file_packager.py TARGET [--preload A [B..]] [--embed C [D..]] [--exclude E [F..]] [--crunch[=X]] [--js-output=OUTPUT.js] [--no-force] [--use-preload-cache] [--no-heap-copy] [--separate-metadata] [--lz4] [--use-preload-plugins] + file_packager.py TARGET [--preload A [B..]] [--embed C [D..]] [--exclude E [F..]] [--crunch[=X]] [--js-output=OUTPUT.js] [--no-force] [--use-preload-cache] [--indexedDB-name=EM_PRELOAD_CACHE] [--no-heap-copy] [--separate-metadata] [--lz4] [--use-preload-plugins] --preload , --embed See emcc --help for more details on those options. @@ -34,6 +34,8 @@ --use-preload-cache Stores package in IndexedDB so that subsequent loads don't need to do XHR. Checks package version. + --indexedDB-name Use specified IndexedDB database name (Default: 'EM_PRELOAD_CACHE') + --no-heap-copy If specified, the preloaded filesystem is not copied inside the Emscripten HEAP, but kept in a separate typed array outside it. The default, if this is not specified, is to embed the VFS inside the HEAP, so that mmap()ing files in it is a no-op. Passing this flag optimizes for fread() usage, omitting it optimizes for mmap() usage. @@ -96,6 +98,7 @@ # If set to True, IndexedDB (IDBFS in library_idbfs.js) is used to locally cache VFS XHR so that subsequent # page loads can read the data from the offline cache instead. use_preload_cache = False +indexeddb_name = 'EM_PRELOAD_CACHE' # If set to True, the blob received from XHR is moved to the Emscripten HEAP, optimizing for mmap() performance. # If set to False, the XHR blob is kept intact, and fread()s etc. are performed directly to that data. This optimizes for minimal memory usage and fread() performance. no_heap_copy = True @@ -119,6 +122,9 @@ elif arg == '--use-preload-cache': use_preload_cache = True leading = '' + elif arg.startswith('--indexedDB-name'): + indexeddb_name = arg.split('=')[1] if '=' in arg else None + leading = '' elif arg == '--no-heap-copy': no_heap_copy = False leading = '' @@ -226,7 +232,7 @@ def should_ignore(fullname): # Returns the given string with escapes added so that it can safely be placed inside a string in JS code. def escape_for_js_string(s): - s = s.replace("'", "\\'").replace('"', '\\"') + s = s.replace("'", "\\'").replace('"', '\\"').replace('\\', '/') return s # Expand directories into individual files @@ -515,7 +521,7 @@ def was_seen(name): DataRequest.prototype.requests[files[i].filename].onload(); } ''' - use_data += " Module['removeRunDependency']('datafile_%s');\n" % data_target + use_data += " Module['removeRunDependency']('datafile_%s');\n" % escape_for_js_string(data_target) else: # LZ4FS usage @@ -529,7 +535,7 @@ def was_seen(name): assert(typeof LZ4 === 'object', 'LZ4 not present - was your app build with -s LZ4=1 ?'); LZ4.loadPackage({ 'metadata': metadata, 'compressedData': compressedData }); Module['removeRunDependency']('datafile_%s'); - ''' % (meta, data_target) + ''' % (meta, escape_for_js_string(data_target)) package_uuid = uuid.uuid4(); package_name = data_target @@ -568,7 +574,7 @@ def was_seen(name): var indexedDB = window.indexedDB || window.mozIndexedDB || window.webkitIndexedDB || window.msIndexedDB; var IDB_RO = "readonly"; var IDB_RW = "readwrite"; - var DB_NAME = 'EM_PRELOAD_CACHE'; + var DB_NAME = "''' + indexeddb_name + '''"; var DB_VERSION = 1; var METADATA_STORE_NAME = 'METADATA'; var PACKAGE_STORE_NAME = 'PACKAGES'; @@ -605,7 +611,7 @@ def was_seen(name): var transaction = db.transaction([METADATA_STORE_NAME], IDB_RO); var metadata = transaction.objectStore(METADATA_STORE_NAME); - var getRequest = metadata.get(packageName); + var getRequest = metadata.get("metadata/" + packageName); getRequest.onsuccess = function(event) { var result = event.target.result; if (!result) { @@ -623,7 +629,7 @@ def was_seen(name): var transaction = db.transaction([PACKAGE_STORE_NAME], IDB_RO); var packages = transaction.objectStore(PACKAGE_STORE_NAME); - var getRequest = packages.get(packageName); + var getRequest = packages.get("package/" + packageName); getRequest.onsuccess = function(event) { var result = event.target.result; callback(result); @@ -634,13 +640,14 @@ def was_seen(name): }; function cacheRemotePackage(db, packageName, packageData, packageMeta, callback, errback) { - var transaction = db.transaction([PACKAGE_STORE_NAME, METADATA_STORE_NAME], IDB_RW); - var packages = transaction.objectStore(PACKAGE_STORE_NAME); - var metadata = transaction.objectStore(METADATA_STORE_NAME); + var transaction_packages = db.transaction([PACKAGE_STORE_NAME], IDB_RW); + var packages = transaction_packages.objectStore(PACKAGE_STORE_NAME); - var putPackageRequest = packages.put(packageData, packageName); + var putPackageRequest = packages.put(packageData, "package/" + packageName); putPackageRequest.onsuccess = function(event) { - var putMetadataRequest = metadata.put(packageMeta, packageName); + var transaction_metadata = db.transaction([METADATA_STORE_NAME], IDB_RW); + var metadata = transaction_metadata.objectStore(METADATA_STORE_NAME); + var putMetadataRequest = metadata.put(packageMeta, "metadata/" + packageName); putMetadataRequest.onsuccess = function(event) { callback(packageData); }; @@ -718,7 +725,7 @@ def was_seen(name): %s }; Module['addRunDependency']('datafile_%s'); - ''' % (use_data, data_target) # use basename because from the browser's point of view, we need to find the datafile in the same dir as the html file + ''' % (use_data, escape_for_js_string(data_target)) # use basename because from the browser's point of view, we need to find the datafile in the same dir as the html file code += r''' if (!Module.preloadResults) Module.preloadResults = {}; diff --git a/tools/gen_struct_info.py b/tools/gen_struct_info.py index 65a2153879d12..de8287a4ac016 100644 --- a/tools/gen_struct_info.py +++ b/tools/gen_struct_info.py @@ -375,22 +375,31 @@ def inspect_code(headers, cpp_opts, structs, defines): for opt in ['EMCC_FORCE_STDLIBS', 'EMCC_ONLY_FORCED_STDLIBS']: if opt in safe_env: del safe_env[opt] - + + # Use binaryen, if necessary + binaryen = os.environ.get('EMCC_WASM_BACKEND_BINARYEN') + if binaryen: + cpp_opts += ['-s', 'BINARYEN=1'] + info = [] + try: - # Compile the program. - show('Compiling generated code...') - subprocess.check_call([shared.PYTHON, shared.EMCC] + cpp_opts + ['-o', js_file[1], src_file[1], '-s', 'BOOTSTRAPPING_STRUCT_INFO=1', '-s', 'WARN_ON_UNDEFINED_SYMBOLS=0', '-Oz', '--js-opts', '0', '--memory-init-file', '0'], env=safe_env) # -Oz optimizes enough to avoid warnings on code size/num locals + try: + # Compile the program. + show('Compiling generated code...') + subprocess.check_call([shared.PYTHON, shared.EMCC] + cpp_opts + ['-o', js_file[1], src_file[1], '-s', 'BOOTSTRAPPING_STRUCT_INFO=1', '-s', 'WARN_ON_UNDEFINED_SYMBOLS=0', '-Oz', '--js-opts', '0', '--memory-init-file', '0'], env=safe_env) # -Oz optimizes enough to avoid warnings on code size/num locals + except: + sys.stderr.write('FAIL: Compilation failed!\n') + sys.exit(1) + # Run the compiled program. show('Calling generated program...') - info = shared.run_js(js_file[1]).splitlines() - except subprocess.CalledProcessError: - if os.path.isfile(js_file[1]): + try: + info = shared.run_js(js_file[1]).splitlines() + except subprocess.CalledProcessError: sys.stderr.write('FAIL: Running the generated program failed!\n') - else: - sys.stderr.write('FAIL: Compilation failed!\n') - - sys.exit(1) + sys.exit(1) + finally: # Remove all temporary files. os.unlink(src_file[1]) diff --git a/tools/js-optimizer.js b/tools/js-optimizer.js index cd20db37f235f..96b1ba93ecbec 100644 --- a/tools/js-optimizer.js +++ b/tools/js-optimizer.js @@ -2234,7 +2234,7 @@ function getStackBumpSize(ast) { // Name minification -var RESERVED = set('do', 'if', 'in', 'for', 'new', 'try', 'var', 'env', 'let'); +var RESERVED = set('do', 'if', 'in', 'for', 'new', 'try', 'var', 'env', 'let', 'case', 'else', 'enum', 'void', 'this', 'void', 'with'); var VALID_MIN_INITS = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_$'; var VALID_MIN_LATERS = VALID_MIN_INITS + '0123456789'; @@ -7940,15 +7940,7 @@ if (arguments_.indexOf('receiveJSON') < 0) { var emitAst = true; arguments_.slice(1).forEach(function(arg) { - //traverse(ast, function(node) { - // if (node[0] === 'defun' && node[1] === 'copyTempFloat') printErr('pre ' + JSON.stringify(node, null, ' ')); - //}); passes[arg](ast); - //var func; - //traverse(ast, function(node) { - // if (node[0] === 'defun') func = node; - // if (isEmptyNode(node)) throw 'empty node after ' + arg + ', in ' + func[1]; - //}); }); if (asm && last) { prepDotZero(ast); diff --git a/tools/js_optimizer.py b/tools/js_optimizer.py index be42d06bb5caf..1fefa81164b49 100644 --- a/tools/js_optimizer.py +++ b/tools/js_optimizer.py @@ -71,9 +71,13 @@ def get_native_optimizer(): sys.exit(1) # Allow users to override the location of the optimizer executable by setting an environment variable EMSCRIPTEN_NATIVE_OPTIMIZER=/path/to/optimizer(.exe) - if os.environ.get('EMSCRIPTEN_NATIVE_OPTIMIZER') and len(os.environ.get('EMSCRIPTEN_NATIVE_OPTIMIZER')) > 0: return os.environ.get('EMSCRIPTEN_NATIVE_OPTIMIZER') + if os.environ.get('EMSCRIPTEN_NATIVE_OPTIMIZER') and len(os.environ.get('EMSCRIPTEN_NATIVE_OPTIMIZER')) > 0: + logging.debug('env forcing native optimizer at ' + os.environ.get('EMSCRIPTEN_NATIVE_OPTIMIZER')) + return os.environ.get('EMSCRIPTEN_NATIVE_OPTIMIZER') # Also, allow specifying the location of the optimizer in .emscripten configuration file under EMSCRIPTEN_NATIVE_OPTIMIZER='/path/to/optimizer' - if hasattr(shared, 'EMSCRIPTEN_NATIVE_OPTIMIZER') and len(shared.EMSCRIPTEN_NATIVE_OPTIMIZER) > 0: return shared.EMSCRIPTEN_NATIVE_OPTIMIZER + if hasattr(shared, 'EMSCRIPTEN_NATIVE_OPTIMIZER') and len(shared.EMSCRIPTEN_NATIVE_OPTIMIZER) > 0: + logging.debug('config forcing native optimizer at ' + shared.EMSCRIPTEN_NATIVE_OPTIMIZER) + return shared.EMSCRIPTEN_NATIVE_OPTIMIZER FAIL_MARKER = shared.Cache.get_path('optimizer.building_failed') if os.path.exists(FAIL_MARKER): @@ -440,6 +444,8 @@ def write_chunk(chunk, i): if DEBUG: print >> sys.stderr, 'splitting up js optimization into %d chunks, using %d cores (total: %.2f MB)' % (len(chunks), cores, total_size/(1024*1024.)) pool = multiprocessing.Pool(processes=cores) filenames = pool.map(run_on_chunk, commands, chunksize=1) + pool.terminate() + pool.join() else: # We can't parallize, but still break into chunks to avoid uglify/node memory issues if len(chunks) > 1 and DEBUG: print >> sys.stderr, 'splitting up js optimization into %d chunks' % (len(chunks)) diff --git a/tools/optimizer/optimizer.cpp b/tools/optimizer/optimizer.cpp index 2f20a0c970f16..d9fb6ee854f1d 100644 --- a/tools/optimizer/optimizer.cpp +++ b/tools/optimizer/optimizer.cpp @@ -3572,7 +3572,7 @@ void registerizeHarder(Ref ast) { // end registerizeHarder // minified names generation -StringSet RESERVED("do if in for new try var env let"); +StringSet RESERVED("do if in for new try var env let case else enum this void with"); const char *VALID_MIN_INITS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_$"; const char *VALID_MIN_LATERS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_$0123456789"; diff --git a/tools/ports/__init__.py b/tools/ports/__init__.py index 13fa135f38dfe..fc73b892dc426 100644 --- a/tools/ports/__init__.py +++ b/tools/ports/__init__.py @@ -1,4 +1,4 @@ -import bullet, freetype, libpng, ogg, sdl, sdl_image, sdl_ttf, vorbis, zlib +import binaryen, bullet, freetype, libpng, ogg, sdl, sdl_image, sdl_ttf, sdl_net, vorbis, zlib # If port A depends on port B, then A should be _after_ B -ports = [zlib, libpng, sdl, sdl_image, ogg, vorbis, bullet, freetype, sdl_ttf] +ports = [zlib, libpng, sdl, sdl_image, ogg, vorbis, bullet, freetype, sdl_ttf, sdl_net, binaryen] diff --git a/tools/ports/binaryen.py b/tools/ports/binaryen.py new file mode 100644 index 0000000000000..a171799be192b --- /dev/null +++ b/tools/ports/binaryen.py @@ -0,0 +1,39 @@ +import os, shutil, logging + +TAG = 'version_7' + +def needed(settings, shared, ports): + if not settings.BINARYEN: return False + try: + if shared.BINARYEN_ROOT: # if defined, and not falsey, we don't need the port + logging.debug('binaryen root already set to ' + shared.BINARYEN_ROOT) + settings.BINARYEN_ROOT = shared.BINARYEN_ROOT + return False + except: + pass + settings.BINARYEN_ROOT = os.path.join(ports.get_dir(), 'binaryen', 'binaryen-' + TAG) + logging.debug('setting binaryen root to ' + settings.BINARYEN_ROOT) + return True + +def get(ports, settings, shared): + if not needed(settings, shared, ports): + return [] + ports.fetch_project('binaryen', 'https://github.com/WebAssembly/binaryen/archive/' + TAG + '.zip', 'binaryen-' + TAG) + def create(): + logging.warning('building port: binaryen') + ports.build_native(os.path.join(ports.get_dir(), 'binaryen', 'binaryen-' + TAG)) + # the "output" of this port build is a tag file, saying which port we have + tag_file = os.path.join(ports.get_dir(), 'binaryen', 'tag.txt') + open(tag_file, 'w').write(TAG) + return tag_file + return [shared.Cache.get('binaryen_tag_' + TAG, create, what='port', extension='.txt')] + +def process_args(ports, args, settings, shared): + if not needed(settings, shared, ports): + return args + get(ports, settings, shared) + return args + +def show(): + return 'Binaryen (Apache 2.0 license)' + diff --git a/tools/ports/sdl_net.py b/tools/ports/sdl_net.py new file mode 100644 index 0000000000000..ef8ab2cc0fd0f --- /dev/null +++ b/tools/ports/sdl_net.py @@ -0,0 +1,37 @@ +import os, shutil, logging + +TAG = 'version_2' + +def get(ports, settings, shared): + if settings.USE_SDL_NET == 2: + sdl_build = os.path.join(ports.get_build_dir(), 'sdl2') + assert os.path.exists(sdl_build), 'You must use SDL2 to use SDL2_net' + ports.fetch_project('sdl2-net', 'https://github.com/emscripten-ports/SDL2_net/archive/' + TAG + '.zip', 'SDL2_net-' + TAG) + def create(): + logging.warning('building port: sdl2-net') + shutil.copyfile(os.path.join(ports.get_dir(), 'sdl2-net', 'SDL2_net-' + TAG, 'SDL_net.h'), os.path.join(ports.get_build_dir(), 'sdl2', 'include', 'SDL_net.h')) + shutil.copyfile(os.path.join(ports.get_dir(), 'sdl2-net', 'SDL2_net-' + TAG, 'SDL_net.h'), os.path.join(ports.get_build_dir(), 'sdl2', 'include', 'SDL2', 'SDL_net.h')) + srcs = 'SDLnet.c SDLnetselect.c SDLnetTCP.c SDLnetUDP.c'.split(' ') + commands = [] + o_s = [] + for src in srcs: + o = os.path.join(ports.get_build_dir(), 'sdl2-net', src + '.o') + commands.append([shared.PYTHON, shared.EMCC, os.path.join(ports.get_dir(), 'sdl2-net', 'SDL2_net-' + TAG, src), '-O2', '-s', 'USE_SDL=2', '-o', o, '-w']) + o_s.append(o) + shared.safe_ensure_dirs(os.path.dirname(o_s[0])) + ports.run_commands(commands) + final = os.path.join(ports.get_build_dir(), 'sdl2-net', 'libsdl2_net.bc') + shared.Building.link(o_s, final) + return final + return [shared.Cache.get('sdl2-net', create, what='port')] + else: + return [] + +def process_args(ports, args, settings, shared): + if settings.USE_SDL_NET == 2: + get(ports, settings, shared) + return args + +def show(): + return 'SDL2_net (zlib license)' + diff --git a/tools/separate_asm.py b/tools/separate_asm.py index f0225ac09f417..625c30dc3286f 100644 --- a/tools/separate_asm.py +++ b/tools/separate_asm.py @@ -19,12 +19,11 @@ everything = everything.replace(module, 'Module["asm"]') else: # closure compiler removes |var Module|, we need to find the closured name - evil = everything.find('eval(') - evil = everything.rfind('=', 0, evil) - start = evil - while everything[start] in [' ', '=']: start -= 1 - while everything[start] not in [' ', ',', '(']: start -= 1 - closured_name = everything[start+1:evil].strip() + # seek a pattern like (e.ENVIRONMENT), which is in the shell.js if-cascade for the ENVIRONMENT override + import re + m = re.search('\((\w+)\.ENVIRONMENT\)', everything) + assert m, 'cannot figure out the closured name of Module statically' + closured_name = m.group(1) everything = everything.replace(module, closured_name + '["asm"]') o = open(asmfile, 'w') diff --git a/tools/shared.py b/tools/shared.py index 6ed3ae4584c5f..cab549c3991e2 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -168,10 +168,11 @@ def new(*args): WINDOWS = sys.platform.startswith('win') OSX = sys.platform == 'darwin' -if WINDOWS: - logging.StreamHandler.emit = add_coloring_to_emit_windows(logging.StreamHandler.emit) -else: - logging.StreamHandler.emit = add_coloring_to_emit_ansi(logging.StreamHandler.emit) +if sys.stderr.isatty(): + if WINDOWS: + logging.StreamHandler.emit = add_coloring_to_emit_windows(logging.StreamHandler.emit) + else: + logging.StreamHandler.emit = add_coloring_to_emit_ansi(logging.StreamHandler.emit) # Emscripten configuration is done through the --em-config command line option or # the EM_CONFIG environment variable. If the specified string value contains newline @@ -361,43 +362,66 @@ def get_fastcomp_src_dir(): d = os.path.dirname(d) return None -def check_fastcomp(): +def get_llc_targets(): try: llc_version_info = Popen([LLVM_COMPILER, '--version'], stdout=PIPE).communicate()[0] pre, targets = llc_version_info.split('Registered Targets:') - if 'js' not in targets or 'JavaScript (asm.js, emscripten) backend' not in targets: - logging.critical('fastcomp in use, but LLVM has not been built with the JavaScript backend as a target, llc reports:') - print >> sys.stderr, '===========================================================================' - print >> sys.stderr, llc_version_info, - print >> sys.stderr, '===========================================================================' - logging.critical('you can fall back to the older (pre-fastcomp) compiler core, although that is not recommended, see http://kripken.github.io/emscripten-site/docs/building_from_source/LLVM-Backend.html') - return False + return targets + except Exception, e: + return '(no targets could be identified: ' + str(e) + ')' - # check repo versions - d = get_fastcomp_src_dir() - shown_repo_version_error = False - if d is not None: - llvm_version = get_emscripten_version(os.path.join(d, 'emscripten-version.txt')) - if os.path.exists(os.path.join(d, 'tools', 'clang', 'emscripten-version.txt')): - clang_version = get_emscripten_version(os.path.join(d, 'tools', 'clang', 'emscripten-version.txt')) - elif os.path.exists(os.path.join(d, 'tools', 'clang')): - clang_version = '?' # Looks like the LLVM compiler tree has an old checkout from the time before it contained a version.txt: Should update! - else: - clang_version = llvm_version # This LLVM compiler tree does not have a tools/clang, so it's probably an out-of-source build directory. No need for separate versioning. - if EMSCRIPTEN_VERSION != llvm_version or EMSCRIPTEN_VERSION != clang_version: - logging.error('Emscripten, llvm and clang repo versions do not match, this is dangerous (%s, %s, %s)', EMSCRIPTEN_VERSION, llvm_version, clang_version) - logging.error('Make sure to use the same branch in each repo, and to be up-to-date on each. See http://kripken.github.io/emscripten-site/docs/building_from_source/LLVM-Backend.html') - shown_repo_version_error = True +def has_asm_js_target(targets): + return 'js' in targets and 'JavaScript (asm.js, emscripten) backend' in targets + +def has_wasm_target(targets): + return 'wasm32' in targets and 'WebAssembly 32-bit' in targets + +def check_fastcomp(): + try: + targets = get_llc_targets() + if get_llvm_target() == ASM_JS_TARGET: + if not has_asm_js_target(targets): + logging.critical('fastcomp in use, but LLVM has not been built with the JavaScript backend as a target, llc reports:') + print >> sys.stderr, '===========================================================================' + print >> sys.stderr, targets + print >> sys.stderr, '===========================================================================' + logging.critical('you can fall back to the older (pre-fastcomp) compiler core, although that is not recommended, see http://kripken.github.io/emscripten-site/docs/building_from_source/LLVM-Backend.html') + return False else: - logging.warning('did not see a source tree above or next to the LLVM root directory (guessing based on directory of %s), could not verify version numbers match' % LLVM_COMPILER) + assert get_llvm_target() == WASM_TARGET + if not has_wasm_target(targets): + logging.critical('WebAssembly set as target, but LLVM has not been built with the WebAssembly backend, llc reports:') + print >> sys.stderr, '===========================================================================' + print >> sys.stderr, targets + print >> sys.stderr, '===========================================================================' + return False + + if get_llvm_target() == ASM_JS_TARGET: + # check repo versions + d = get_fastcomp_src_dir() + shown_repo_version_error = False + if d is not None: + llvm_version = get_emscripten_version(os.path.join(d, 'emscripten-version.txt')) + if os.path.exists(os.path.join(d, 'tools', 'clang', 'emscripten-version.txt')): + clang_version = get_emscripten_version(os.path.join(d, 'tools', 'clang', 'emscripten-version.txt')) + elif os.path.exists(os.path.join(d, 'tools', 'clang')): + clang_version = '?' # Looks like the LLVM compiler tree has an old checkout from the time before it contained a version.txt: Should update! + else: + clang_version = llvm_version # This LLVM compiler tree does not have a tools/clang, so it's probably an out-of-source build directory. No need for separate versioning. + if EMSCRIPTEN_VERSION != llvm_version or EMSCRIPTEN_VERSION != clang_version: + logging.error('Emscripten, llvm and clang repo versions do not match, this is dangerous (%s, %s, %s)', EMSCRIPTEN_VERSION, llvm_version, clang_version) + logging.error('Make sure to use the same branch in each repo, and to be up-to-date on each. See http://kripken.github.io/emscripten-site/docs/building_from_source/LLVM-Backend.html') + shown_repo_version_error = True + else: + logging.warning('did not see a source tree above or next to the LLVM root directory (guessing based on directory of %s), could not verify version numbers match' % LLVM_COMPILER) - # check build versions. don't show it if the repos are wrong, user should fix that first - if not shown_repo_version_error: - clang_v = Popen([CLANG, '--version'], stdout=PIPE).communicate()[0] - llvm_build_version, clang_build_version = clang_v.split('(emscripten ')[1].split(')')[0].split(' : ') - if EMSCRIPTEN_VERSION != llvm_build_version or EMSCRIPTEN_VERSION != clang_build_version: - logging.error('Emscripten, llvm and clang build versions do not match, this is dangerous (%s, %s, %s)', EMSCRIPTEN_VERSION, llvm_build_version, clang_build_version) - logging.error('Make sure to rebuild llvm and clang after updating repos') + # check build versions. don't show it if the repos are wrong, user should fix that first + if not shown_repo_version_error: + clang_v = Popen([CLANG, '--version'], stdout=PIPE).communicate()[0] + llvm_build_version, clang_build_version = clang_v.split('(emscripten ')[1].split(')')[0].split(' : ') + if EMSCRIPTEN_VERSION != llvm_build_version or EMSCRIPTEN_VERSION != clang_build_version: + logging.error('Emscripten, llvm and clang build versions do not match, this is dangerous (%s, %s, %s)', EMSCRIPTEN_VERSION, llvm_build_version, clang_build_version) + logging.error('Make sure to rebuild llvm and clang after updating repos') return True except Exception, e: @@ -471,7 +495,7 @@ def get_emscripten_version(path): EMSCRIPTEN_VERSION = 'unknown' def generate_sanity(): - return EMSCRIPTEN_VERSION + '|' + get_llvm_target() + '|' + LLVM_ROOT + '|' + get_clang_version() + return EMSCRIPTEN_VERSION + '|' + LLVM_ROOT + '|' + get_clang_version() + ('_wasm' if get_llvm_target() == WASM_TARGET else '') def check_sanity(force=False): try: @@ -483,13 +507,15 @@ def check_sanity(force=False): else: settings_mtime = os.stat(CONFIG_FILE).st_mtime sanity_file = CONFIG_FILE + '_sanity' + if get_llvm_target() == WASM_TARGET: + sanity_file += '_wasm' if os.path.exists(sanity_file): try: sanity_mtime = os.stat(sanity_file).st_mtime if sanity_mtime <= settings_mtime: reason = 'settings file has changed' else: - sanity_data = open(sanity_file).read() + sanity_data = open(sanity_file).read().rstrip('\n\r') # workaround weird bug with read() that appends new line char in some old python version if sanity_data != generate_sanity(): reason = 'system change: %s vs %s' % (generate_sanity(), sanity_data) else: @@ -788,10 +814,63 @@ def set_logging(): # Additional compiler options -# Target choice. Must be synced with src/settings.js (TARGET_*) +# Target choice. +ASM_JS_TARGET = 'asmjs-unknown-emscripten' +WASM_TARGET = 'wasm32-unknown-unknown' + +def check_vanilla(): + global LLVM_TARGET + # if the env var tells us what to do, do that + if 'EMCC_WASM_BACKEND' in os.environ: + if os.environ['EMCC_WASM_BACKEND'] != '0': + logging.debug('EMCC_WASM_BACKEND tells us to use wasm backend') + LLVM_TARGET = WASM_TARGET + else: + logging.debug('EMCC_WASM_BACKEND tells us to use asm.js backend') + LLVM_TARGET = ASM_JS_TARGET + else: + # if we are using vanilla LLVM, i.e. we don't have our asm.js backend, then we + # must use wasm (or at least try to). to know that, we have to run llc to + # see which backends it has. we cache this result. + temp_cache = cache.Cache(use_subdir=False) + def check_vanilla(): + logging.debug('testing for asm.js target, because if not present (i.e. this is plain vanilla llvm, not emscripten fastcomp), we will use the wasm target instead (set EMCC_WASM_BACKEND to skip this check)') + targets = get_llc_targets() + return has_wasm_target(targets) and not has_asm_js_target(targets) + def get_vanilla_file(): + saved_file = os.path.join(temp_cache.dirname, 'is_vanilla.txt') + open(saved_file, 'w').write(('1' if check_vanilla() else '0') + ':' + LLVM_ROOT) + return saved_file + is_vanilla_file = temp_cache.get('is_vanilla', get_vanilla_file, extension='.txt') + if CONFIG_FILE and os.stat(CONFIG_FILE).st_mtime > os.stat(is_vanilla_file).st_mtime: + logging.debug('config file changed since we checked vanilla; re-checking') + is_vanilla_file = temp_cache.get('is_vanilla', get_vanilla_file, extension='.txt', force=True) + try: + contents = open(is_vanilla_file).read() + is_vanilla, llvm_used = contents.split(':') + is_vanilla = int(is_vanilla) + if llvm_used != LLVM_ROOT: + logging.debug('regenerating vanilla check since other llvm') + temp_cache.get('is_vanilla', get_vanilla_file, extension='.txt', force=True) + is_vanilla = check_vanilla() + except Exception, e: + logging.debug('failed to use vanilla file, will re-check: ' + str(e)) + is_vanilla = check_vanilla() + temp_cache = None + if is_vanilla: + logging.debug('check tells us to use wasm backend') + LLVM_TARGET = WASM_TARGET + os.environ['EMCC_WASM_BACKEND'] = '1' + else: + logging.debug('check tells us to use asm.js backend') + LLVM_TARGET = ASM_JS_TARGET + +check_vanilla() + def get_llvm_target(): - return 'asmjs-unknown-emscripten' -LLVM_TARGET = get_llvm_target() + global LLVM_TARGET + assert LLVM_TARGET is not None + return LLVM_TARGET # COMPILER_OPTS: options passed to clang when generating bitcode for us try: @@ -799,19 +878,24 @@ def get_llvm_target(): except: COMPILER_OPTS = [] COMPILER_OPTS = COMPILER_OPTS + [#'-fno-threadsafe-statics', # disabled due to issue 1289 - '-target', LLVM_TARGET, + '-target', get_llvm_target(), '-D__EMSCRIPTEN_major__=' + str(EMSCRIPTEN_VERSION_MAJOR), '-D__EMSCRIPTEN_minor__=' + str(EMSCRIPTEN_VERSION_MINOR), '-D__EMSCRIPTEN_tiny__=' + str(EMSCRIPTEN_VERSION_TINY)] +if LLVM_TARGET == WASM_TARGET: + # wasm target does not automatically define emscripten stuff, so do it here + COMPILER_OPTS = COMPILER_OPTS + ['-DEMSCRIPTEN', + '-D__EMSCRIPTEN__'] + # Changes to default clang behavior -if LLVM_TARGET == 'asmjs-unknown-emscripten': - # Implicit functions can cause horribly confusing asm.js function pointer type errors, see #2175 - # If your codebase really needs them - very unrecommended! - you can disable the error with - # -Wno-error=implicit-function-declaration - # or disable even a warning about it with - # -Wno-implicit-function-declaration - COMPILER_OPTS += ['-Werror=implicit-function-declaration'] + +# Implicit functions can cause horribly confusing function pointer type errors, see #2175 +# If your codebase really needs them - very unrecommended! - you can disable the error with +# -Wno-error=implicit-function-declaration +# or disable even a warning about it with +# -Wno-implicit-function-declaration +COMPILER_OPTS += ['-Werror=implicit-function-declaration'] USE_EMSDK = not os.environ.get('EMMAKEN_NO_SDK') @@ -826,7 +910,8 @@ def get_llvm_target(): path_from_root('system', 'lib', 'libc', 'musl', 'arch', 'emscripten') ] - CXX_INCLUDE_PATHS = [path_from_root('system', 'include', 'libcxx') + CXX_INCLUDE_PATHS = [path_from_root('system', 'include', 'libcxx'), + path_from_root('system', 'lib', 'libcxxabi', 'include') ] C_OPTS = ['-nostdinc', '-Xclang', '-nobuiltininc', '-Xclang', '-nostdsysteminc', @@ -846,9 +931,6 @@ def include_directive(paths): EMSDK_OPTS = [] EMSDK_CXX_OPTS = [] -#print >> sys.stderr, 'SDK opts', ' '.join(EMSDK_OPTS) -#print >> sys.stderr, 'Compiler opts', ' '.join(COMPILER_OPTS) - # Engine tweaks try: diff --git a/tools/system_libs.py b/tools/system_libs.py index ca09e6aa3402e..069811c4b13e4 100644 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -1,6 +1,7 @@ import os, json, logging, zipfile, glob import shared from subprocess import Popen, CalledProcessError +import subprocess, multiprocessing, re import multiprocessing from tools.shared import check_call @@ -49,6 +50,7 @@ def read_symbols(path, exclude=None): gl_symbols = read_symbols(shared.path_from_root('system', 'lib', 'gl.symbols')) compiler_rt_symbols = read_symbols(shared.path_from_root('system', 'lib', 'compiler-rt.symbols')) pthreads_symbols = read_symbols(shared.path_from_root('system', 'lib', 'pthreads.symbols')) + wasm_libc_symbols = read_symbols(shared.path_from_root('system', 'lib', 'wasm-libc.symbols')) # XXX we should disable EMCC_DEBUG when building libs, just like in the relooper @@ -58,7 +60,7 @@ def build_libc(lib_filename, files, lib_opts): commands = [] # Hide several musl warnings that produce a lot of spam to unit test build server logs. # TODO: When updating musl the next time, feel free to recheck which of their warnings might have been fixed, and which ones of these could be cleaned up. - c_opts = ['-Wno-dangling-else', '-Wno-unknown-pragmas', '-Wno-shift-op-parentheses', '-Wno-string-plus-int', '-Wno-logical-op-parentheses', '-Wno-bitwise-op-parentheses', '-Wno-visibility', '-Wno-pointer-sign'] + c_opts = ['-Wno-return-type', '-Wno-parentheses', '-Wno-ignored-attributes', '-Wno-shift-count-overflow', '-Wno-shift-negative-value', '-Wno-dangling-else', '-Wno-unknown-pragmas', '-Wno-shift-op-parentheses', '-Wno-string-plus-int', '-Wno-logical-op-parentheses', '-Wno-bitwise-op-parentheses', '-Wno-visibility', '-Wno-pointer-sign'] for src in files: o = in_temp(os.path.basename(src) + '.o') commands.append([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', src), '-o', o] + musl_internal_includes + default_opts + c_opts + lib_opts) @@ -125,6 +127,15 @@ def create_pthreads(libname): pthreads_files += [shared.path_from_root('system', 'lib', 'libc', 'musl', 'src', 'thread', x) for x in ('pthread_attr_destroy.c', 'pthread_condattr_setpshared.c', 'pthread_mutex_lock.c', 'pthread_spin_destroy.c', 'pthread_attr_get.c', 'pthread_cond_broadcast.c', 'pthread_mutex_setprioceiling.c', 'pthread_spin_init.c', 'pthread_attr_init.c', 'pthread_cond_destroy.c', 'pthread_mutex_timedlock.c', 'pthread_spin_lock.c', 'pthread_attr_setdetachstate.c', 'pthread_cond_init.c', 'pthread_mutex_trylock.c', 'pthread_spin_trylock.c', 'pthread_attr_setguardsize.c', 'pthread_cond_signal.c', 'pthread_mutex_unlock.c', 'pthread_spin_unlock.c', 'pthread_attr_setinheritsched.c', 'pthread_cond_timedwait.c', 'pthread_once.c', 'sem_destroy.c', 'pthread_attr_setschedparam.c', 'pthread_cond_wait.c', 'pthread_rwlockattr_destroy.c', 'sem_getvalue.c', 'pthread_attr_setschedpolicy.c', 'pthread_equal.c', 'pthread_rwlockattr_init.c', 'sem_init.c', 'pthread_attr_setscope.c', 'pthread_getspecific.c', 'pthread_rwlockattr_setpshared.c', 'sem_open.c', 'pthread_attr_setstack.c', 'pthread_key_create.c', 'pthread_rwlock_destroy.c', 'sem_post.c', 'pthread_attr_setstacksize.c', 'pthread_mutexattr_destroy.c', 'pthread_rwlock_init.c', 'sem_timedwait.c', 'pthread_barrierattr_destroy.c', 'pthread_mutexattr_init.c', 'pthread_rwlock_rdlock.c', 'sem_trywait.c', 'pthread_barrierattr_init.c', 'pthread_mutexattr_setprotocol.c', 'pthread_rwlock_timedrdlock.c', 'sem_unlink.c', 'pthread_barrierattr_setpshared.c', 'pthread_mutexattr_setpshared.c', 'pthread_rwlock_timedwrlock.c', 'sem_wait.c', 'pthread_barrier_destroy.c', 'pthread_mutexattr_setrobust.c', 'pthread_rwlock_tryrdlock.c', '__timedwait.c', 'pthread_barrier_init.c', 'pthread_mutexattr_settype.c', 'pthread_rwlock_trywrlock.c', 'vmlock.c', 'pthread_barrier_wait.c', 'pthread_mutex_consistent.c', 'pthread_rwlock_unlock.c', '__wait.c', 'pthread_condattr_destroy.c', 'pthread_mutex_destroy.c', 'pthread_rwlock_wrlock.c', 'pthread_condattr_init.c', 'pthread_mutex_getprioceiling.c', 'pthread_setcanceltype.c', 'pthread_condattr_setclock.c', 'pthread_mutex_init.c', 'pthread_setspecific.c')] return build_libc(libname, pthreads_files, ['-O2', '-s', 'USE_PTHREADS=1']) + def create_wasm_libc(libname): + # in asm.js we just use Math.sin etc., which is good for code size. But wasm doesn't have such builtins, so + # we need to bundle in more code + # we also build in musl versions of things that we have hand-optimized asm.js for + files = ([shared.path_from_root('system', 'lib', 'libc', 'musl', 'src', 'math', x) for x in ('cos.c', 'cosf.c', 'cosl.c', 'sin.c', 'sinf.c', 'sinl.c', 'tan.c', 'tanf.c', 'tanl.c', 'acos.c', 'acosf.c', 'acosl.c', 'asin.c', 'asinf.c', 'asinl.c', 'atan.c', 'atanf.c', 'atanl.c', 'atan2.c', 'atan2f.c', 'atan2l.c', 'exp.c', 'expf.c', 'expl.c', 'log.c', 'logf.c', 'logl.c', 'pow.c', 'powf.c', 'powl.c')] + + [shared.path_from_root('system', 'lib', 'libc', 'musl', 'src', 'string', x) for x in ('memcpy.c', 'memset.c', 'memmove.c')]) + + return build_libc(libname, files, ['-O2']) + # libcxx def create_libcxx(libname): logging.debug('building libcxx for cache') @@ -181,7 +192,7 @@ def create_gl(libname): # libname is ignored, this is just one .o file return o def create_compiler_rt(libname): - srcdir = shared.path_from_root('system', 'lib', 'compiler-rt') + srcdir = shared.path_from_root('system', 'lib', 'compiler-rt', 'lib', 'builtins') filenames = ['divdc3.c', 'divsc3.c', 'muldc3.c', 'mulsc3.c'] files = (os.path.join(srcdir, f) for f in filenames) @@ -192,7 +203,7 @@ def create_compiler_rt(libname): commands.append([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', src), '-O2', '-o', o]) o_s.append(o) run_commands(commands) - shared.Building.link(o_s, in_temp(libname)) + shared.Building.emar('cr', in_temp(libname), o_s) return in_temp(libname) def create_dlmalloc(out_name, clflags): @@ -291,7 +302,7 @@ class Dummy: system_libs = [('libcxx', 'a', create_libcxx, libcxx_symbols, ['libcxxabi'], True), ('libcxxabi', 'bc', create_libcxxabi, libcxxabi_symbols, ['libc'], False), ('gl', 'bc', create_gl, gl_symbols, ['libc'], False), - ('compiler-rt', 'bc', create_compiler_rt, compiler_rt_symbols, ['libc'], False)] + ('compiler-rt', 'a', create_compiler_rt, compiler_rt_symbols, ['libc'], False)] # malloc dependency is force-added, so when using pthreads, it must be force-added # as well, since malloc needs to be thread-safe, so it depends on mutexes. @@ -319,6 +330,17 @@ class Dummy: system_libs += [('dlmalloc', 'bc', create_dlmalloc_singlethreaded, [], [], False)] force.add('dlmalloc') + # if building to wasm, we need more math code, since we have less builtins + if shared.Settings.BINARYEN: + system_libs += [('wasm-libc', 'bc', create_wasm_libc, wasm_libc_symbols, [], False)] + # if libc is included, we definitely must be, as it might need us + for data in system_libs: + if data[3] == libc_symbols: + data[4].append('wasm-libc') + break + else: + raise Exception('did not find libc?') + # Go over libraries to figure out which we must include def maybe_noexcept(name): if shared.Settings.DISABLE_EXCEPTION_CATCHING: @@ -528,6 +550,32 @@ def clear_project_build(name): shared.try_delete(os.path.join(Ports.get_build_dir(), name)) shared.try_delete(shared.Cache.get_path(name + '.bc')) + @staticmethod + def build_native(subdir): + old = os.getcwd() + + try: + os.chdir(subdir) + + cmake_build_type = 'Release' + + # Configure + subprocess.check_call(['cmake', '-DCMAKE_BUILD_TYPE=' + cmake_build_type, '.']) + + # Check which CMake generator CMake used so we know which form to pass parameters to make/msbuild/etc. build tool. + generator = re.search('CMAKE_GENERATOR:INTERNAL=(.*)$', open('CMakeCache.txt', 'r').read(), re.MULTILINE).group(1) + + # Make variants support '-jX' for number of cores to build, MSBuild does /maxcpucount:X + num_cores = os.environ.get('EMCC_CORES') or str(multiprocessing.cpu_count()) + make_args = [] + if 'Makefiles' in generator: make_args = ['--', '-j', num_cores] + elif 'Visual Studio' in generator: make_args = ['--config', cmake_build_type, '--', '/maxcpucount:' + num_cores] + + # Kick off the build. + subprocess.check_call(['cmake', '--build', '.'] + make_args) + finally: + os.chdir(old) + def get_ports(settings): ret = [] @@ -535,11 +583,12 @@ def get_ports(settings): try: process_dependencies(settings) for port in ports.ports: - ret += port.get(Ports, settings, shared) + # ports return their output files, which will be linked, or a txt file + ret += filter(lambda f: not f.endswith('.txt'), port.get(Ports, settings, shared)) ok = True finally: if not ok: - logging.error('a problem occurred when using an emscripten-ports library. try to run emcc --clear-cache --clear-ports and then run this command again') + logging.error('a problem occurred when using an emscripten-ports library. try to run emcc --clear-ports and then run this command again') ret.reverse() return ret @@ -559,4 +608,3 @@ def show_ports(): print 'Available ports:' for port in ports.ports: print ' ', port.show() - diff --git a/tools/webidl_binder.py b/tools/webidl_binder.py index e2a2bc8df9336..72171ee5bf241 100644 --- a/tools/webidl_binder.py +++ b/tools/webidl_binder.py @@ -465,12 +465,16 @@ def render_function(class_name, func_name, sigs, return_type, non_pointer, copy, call += '(' + call_args + ')' if operator: - assert '=' in operator, 'can only do += *= etc. for now, all with "="' cast_self = 'self' if class_name != func_scope: # this function comes from an ancestor class; for operators, we must cast it cast_self = 'dynamic_cast<' + type_to_c(func_scope) + '>(' + cast_self + ')' - call = '(*%s %s %sarg0)' % (cast_self, operator, '*' if sig[0] in interfaces else '') + if '=' in operator: + call = '(*%s %s %sarg0)' % (cast_self, operator, '*' if sig[0] in interfaces else '') + elif operator == '[]': + call = '((*%s)[%sarg0])' % (cast_self, '*' if sig[0] in interfaces else '') + else: + raise Exception('unfamiliar operator ' + operator) pre = ''