From 93e1119b3ff3d487c8ffd0c63b4247ea9c918e72 Mon Sep 17 00:00:00 2001 From: Bruce Mitchener Date: Fri, 13 Sep 2013 11:07:18 +0700 Subject: [PATCH 01/26] Typo fix. --- emcc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/emcc b/emcc index 4a49cc120c328..4b4273ca76787 100755 --- a/emcc +++ b/emcc @@ -669,7 +669,7 @@ if '-M' in sys.argv or '-MM' in sys.argv: if '-E' in sys.argv: # Just run the preprocessor cmd = [CC] + sys.argv[1:] - logging.debug('just preprocssor ' + ' '.join(cmd)) + logging.debug('just preprocessor ' + ' '.join(cmd)) exit(subprocess.call(cmd)) # Check if a target is specified From 5404c47c1e715d8710bf0aac6889037b164691d5 Mon Sep 17 00:00:00 2001 From: Bruce Mitchener Date: Fri, 13 Sep 2013 11:08:12 +0700 Subject: [PATCH 02/26] Fix using emcc -E. We don't want to pass this extra flag to emcc as it breaks when doing EMCONFIGURE_JS=1 emconfigure ./configure ... --- emcc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/emcc b/emcc index 4a49cc120c328..3014640035fd2 100755 --- a/emcc +++ b/emcc @@ -556,6 +556,8 @@ if CONFIGURE_CONFIG or CMAKE_CONFIG: elif arg.endswith('.s'): if debug_configure: open(tempout, 'a').write('(compiling .s assembly, must use clang\n') use_js = 0 + elif arg == '-E': + use_js = 0 if src: if 'fopen' in src and '"w"' in src: From 8c53e6a8edc021654de1353d9d37bb80806f4870 Mon Sep 17 00:00:00 2001 From: Bruce Mitchener Date: Fri, 13 Sep 2013 22:04:11 +0700 Subject: [PATCH 03/26] Expand on the instructions for building Python. --- tests/python/readme.md | 59 +++++++++++++++++++++++++++++++++++++++++ tests/python/readme.txt | 46 -------------------------------- 2 files changed, 59 insertions(+), 46 deletions(-) create mode 100644 tests/python/readme.md delete mode 100644 tests/python/readme.txt diff --git a/tests/python/readme.md b/tests/python/readme.md new file mode 100644 index 0000000000000..e728dce3d1ac8 --- /dev/null +++ b/tests/python/readme.md @@ -0,0 +1,59 @@ +Building Python with Emscripten +=============================== + +These directions should work for Python 2.7.x (last tested with 2.7.5). + +First, uncompress Python into two separate directories, one for native +and one for JavaScript. + +In the JavaScript directory, do: + +```` + EMCONFIGURE_JS=1 emconfigure ./configure --without-threads --without-pymalloc --enable-shared --disable-ipv6 +```` + +If you are on Mac OS X, you will also want ``disable-toolbox-glue``. +If you are on an older version of Python (such as 2.7.2), you may +not need the ``--disable-ipv6`` option. + +If you are on Python 2.7.4 or later, you will need to edit the +``Makefile`` generated and remove the ``MULTIARCH=`` line(s). +You will also need to edit ``pyconfig.h`` and remove defines +for ``DOUBLE_IS_BIG_ENDIAN_IEEE754``, ``DOUBLE_IS_ARM_MIXED_ENDIAN_IEEE754``, +and ``HAVE_GCC_ASM_FOR_X87``. + +On Python 2.7.2, you will need to edit ``pyconfig.h`` and remove +``HAVE_GCC_ASM_FOR_X87``, ``HAVE_SIG*`` except for ``SIGNAL_H`` +and *add* ``#define PY_NO_SHORT_FLOAT_REPR``. + +Now, you can run ``make``. It will fail trying to run ``pgen``. + +At this point, go to your native directory and run: + +```` +./configure --without-threads --without-pymalloc --enable-shared --disable-ipv6 +```` + +Now, run ``make`` in the native directory and then copy the generated ``Parser/pgen`` +to your JavaScript directory. Back in your JavaScript directory, be sure to flag +the ``pgen`` executable as executable: + +```` +chmod +x Parser/pgen +```` + +Now, run ``make`` again. + +You will get an error about trying to run ``python`` or ``python.exe``. This +can be ignored. + +Now, you can link the bitcode file that you need: + +```` +llvm-link libpython2.7.so Modules/python.o -o python.bc +```` + +If you are on Mac OS X, you will want to look for ``libpython2.7.dylib`` +instead of ``libpython2.7.so``. + +Thanks to rasjidw and everyone else who has helped with this! diff --git a/tests/python/readme.txt b/tests/python/readme.txt deleted file mode 100644 index 31b0ff45e2398..0000000000000 --- a/tests/python/readme.txt +++ /dev/null @@ -1,46 +0,0 @@ -Updates to Python 2.7.4 and emscripten of Arp 29 2013 with le32 - -Go to js dir, run EMCONFIGURE_JS=1 ~/Dev/emscripten/emconfigure ./configure --without-threads --without-pymalloc --enable-shared --disable-ipv6 -clean out MULTIARCH= in Makefile -make, until error on pgen -Go to native, run ./configure --without-threads --without-pymalloc --enable-shared --disable-ipv6 -cp Parser/pgen ../JS_DIR/Parser -return to JS -chmod +x Parser/pgen -remove #defines of DOUBLE_IS_BIG_ENDIAN_IEEE754 and DOUBLE_IS_ARM_MIXED_ENDIAN_IEEE754 and HAVE_GCC_ASM_FOR_X87 in pyconfig.h -make -link libpython2.7.so with Modules/python.o to get the bitcode file you want - -========================= - - -This is Python 2.7.2, compiled to .bc as follows: - -Uncompress Python into two separate directories, one for native and one for JS. - -In the JS one, do - emconfigure ./configure --without-threads --without-pymalloc --enable-shared - EDIT pyconfig.h, remove - HAVE_GCC_ASM_FOR_X87 - HAVE_SIG* except SIGNAL_H - and *add* - #define PY_NO_SHORT_FLOAT_REPR - make -It will fail on lack of permissions to run Parser/pgen. - -Go to the native one, do - ./configure --without-threads --without-pymalloc - make - (Note: you don't need to let it complete, just enough for Parse/pgen is sufficient.) - cp Parser/pgen ../YOUR_JS_DIR/Parser/ - -Return to the JS one, do - chmod +x Parser/pgen - make -You will get an error on lack of permissions to run ./python. Ignore that, and do - llvm-link libpython2.7.so python -o python.bc - -That's it! - -Thanks to rasjidw for helping with this! - From 7b583fbae80ffaa472416c16cc36f767ed00f5c3 Mon Sep 17 00:00:00 2001 From: Bruce Mitchener Date: Sat, 31 Aug 2013 11:06:25 +0700 Subject: [PATCH 04/26] Emscripten requires different #s for stdio fds. --- system/include/libc/unistd.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/system/include/libc/unistd.h b/system/include/libc/unistd.h index a00a9c4f3b040..19158b67bbe47 100644 --- a/system/include/libc/unistd.h +++ b/system/include/libc/unistd.h @@ -7,9 +7,9 @@ extern "C" { #include -#define STDIN_FILENO 0 -#define STDOUT_FILENO 1 -#define STDERR_FILENO 2 +#define STDIN_FILENO 1 +#define STDOUT_FILENO 2 +#define STDERR_FILENO 3 #define SEEK_SET 0 #define SEEK_CUR 1 From 1421ef98bf35b3383077a7d9e250ce28102f4cce Mon Sep 17 00:00:00 2001 From: Bruce Mitchener Date: Wed, 4 Sep 2013 11:43:46 +0700 Subject: [PATCH 05/26] Handle fpclassify and signbit for long double. --- src/library.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/library.js b/src/library.js index d947ebfe5d232..da3662be45245 100644 --- a/src/library.js +++ b/src/library.js @@ -4870,8 +4870,9 @@ LibraryManager.library = { // for negative zero (once copysign supports that). return _copysign(1.0, x) < 0; }, - __signbitf: '__signbit', __signbitd: '__signbit', + __signbitf: '__signbit', + __signbitl: '__signbit', hypot: function(a, b) { return Math.sqrt(a*a + b*b); }, @@ -5045,6 +5046,7 @@ LibraryManager.library = { }, __fpclassifyd: '__fpclassify', // Needed by tests/python/python.le32.bc __fpclassifyf: '__fpclassify', + __fpclassifyl: '__fpclassify', // ========================================================================== // sys/utsname.h From 2a60fa0c6f107959b754824a525a3d69294b9696 Mon Sep 17 00:00:00 2001 From: Bruce Mitchener Date: Wed, 4 Sep 2013 16:40:07 +0700 Subject: [PATCH 06/26] Fix sockaddr struct definitions for musl. --- src/library.js | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/library.js b/src/library.js index da3662be45245..466dc5190e1d6 100644 --- a/src/library.js +++ b/src/library.js @@ -7723,14 +7723,13 @@ LibraryManager.library = { 0x0600000a, 0x0700000a, 0x0800000a, 0x0900000a, 0x0a00000a, 0x0b00000a, 0x0c00000a, 0x0d00000a, 0x0e00000a], /* 0x0100000a is reserved */ sockaddr_in_layout: Runtime.generateStructInfo([ - ['i32', 'sin_family'], + ['i16', 'sin_family'], ['i16', 'sin_port'], ['i32', 'sin_addr'], - ['i32', 'sin_zero'], - ['i16', 'sin_zero_b'], + ['b8', 'sin_zero'], ]), sockaddr_in6_layout: Runtime.generateStructInfo([ - ['i32', 'sin6_family'], + ['i16', 'sin6_family'], ['i16', 'sin6_port'], ['i32', 'sin6_flowinfo'], ['b16', 'sin6_addr'], @@ -8139,7 +8138,7 @@ LibraryManager.library = { _read_sockaddr__deps: ['$Sockets', '_inet_ntop4_raw', '_inet_ntop6_raw'], _read_sockaddr: function (sa, salen) { // family / port offsets are common to both sockaddr_in and sockaddr_in6 - var family = {{{ makeGetValue('sa', 'Sockets.sockaddr_in_layout.sin_family', 'i32') }}}; + var family = {{{ makeGetValue('sa', 'Sockets.sockaddr_in_layout.sin_family', 'i16') }}}; var port = _ntohs({{{ makeGetValue('sa', 'Sockets.sockaddr_in_layout.sin_port', 'i16') }}}); var addr; @@ -8174,7 +8173,7 @@ LibraryManager.library = { switch (family) { case {{{ cDefine('AF_INET') }}}: addr = __inet_pton4_raw(addr); - {{{ makeSetValue('sa', 'Sockets.sockaddr_in_layout.sin_family', 'family', 'i32') }}}; + {{{ makeSetValue('sa', 'Sockets.sockaddr_in_layout.sin_family', 'family', 'i16') }}}; {{{ makeSetValue('sa', 'Sockets.sockaddr_in_layout.sin_addr', 'addr', 'i32') }}}; {{{ makeSetValue('sa', 'Sockets.sockaddr_in_layout.sin_port', '_htons(port)', 'i16') }}}; break; From b9c8b95795849264c58be7d1496da3c4078df96c Mon Sep 17 00:00:00 2001 From: Bruce Mitchener Date: Thu, 5 Sep 2013 09:26:42 +0700 Subject: [PATCH 07/26] gethostbyname_r update. * gethostbyname_r is now the 6 arg version. * Make enet use the right code path (this should be upstreamed). * Add a compat header to make these declarations visible to all without extra compilation flags. --- src/library.js | 5 +++-- system/include/compat/netdb.h | 22 ++++++++++++++++++++++ tests/enet/unix.c | 2 +- tests/sockets/test_gethostbyname.c | 9 +++++++-- 4 files changed, 33 insertions(+), 5 deletions(-) create mode 100644 system/include/compat/netdb.h diff --git a/src/library.js b/src/library.js index 466dc5190e1d6..af0e8c97d28b0 100644 --- a/src/library.js +++ b/src/library.js @@ -7480,12 +7480,13 @@ LibraryManager.library = { }, gethostbyname_r__deps: ['gethostbyname'], - gethostbyname_r: function(name, ret, buf, buflen, err) { + gethostbyname_r: function(name, ret, buf, buflen, out, err) { var data = _gethostbyname(name); _memcpy(ret, data, ___hostent_struct_layout.__size__); _free(data); {{{ makeSetValue('err', '0', '0', 'i32') }}}; - return ret; + {{{ makeSetValue('out', '0', 'ret', '*') }}}; + return 0; }, getaddrinfo__deps: ['$Sockets', '$DNS', '_addrinfo_layout', '_inet_pton4_raw', '_inet_ntop4_raw', '_inet_pton6_raw', '_inet_ntop6_raw', '_write_sockaddr', 'htonl'], diff --git a/system/include/compat/netdb.h b/system/include/compat/netdb.h new file mode 100644 index 0000000000000..2aa1e9506396c --- /dev/null +++ b/system/include/compat/netdb.h @@ -0,0 +1,22 @@ +#ifndef _COMPAT_NETDB_H_ +#define _COMPAT_NETDB_H_ + +#include_next + +#ifdef __cplusplus +extern "C" { +#endif + +/* The musl includes only define these things for old sources or + when certain flags are activated. We want these available + all of the time for now. */ +struct hostent *gethostbyname (const char *); +struct hostent *gethostbyaddr (const void *, socklen_t, int); + +int gethostbyname_r(const char *, struct hostent *, char *, size_t, struct hostent **, int *); + +#ifdef __cplusplus +} +#endif + +#endif /* _COMPAT_NETDB_H_ */ diff --git a/tests/enet/unix.c b/tests/enet/unix.c index a225b57d69b89..67d4a8b82916e 100644 --- a/tests/enet/unix.c +++ b/tests/enet/unix.c @@ -80,7 +80,7 @@ enet_address_set_host (ENetAddress * address, const char * name) char buffer [2048]; int errnum; -#if defined(linux) || defined(__linux) || defined(__linux__) || defined(__FreeBSD__) || defined(__FreeBSD_kernel__) +#if defined(linux) || defined(__linux) || defined(__linux__) || defined(__FreeBSD__) || defined(__FreeBSD_kernel__) || defined(__EMSCRIPTEN__) gethostbyname_r (name, & hostData, buffer, sizeof (buffer), & hostEntry, & errnum); #else hostEntry = gethostbyname_r (name, & hostData, buffer, sizeof (buffer), & errnum); diff --git a/tests/sockets/test_gethostbyname.c b/tests/sockets/test_gethostbyname.c index de7da7060041f..459c6b9840cb7 100644 --- a/tests/sockets/test_gethostbyname.c +++ b/tests/sockets/test_gethostbyname.c @@ -13,12 +13,17 @@ int main() { char str[INET_ADDRSTRLEN]; + struct hostent *host = NULL; + struct hostent hostData; struct in_addr addr; const char *res; + char buffer[2048]; int err; - // resolve the hostname ot an actual address - struct hostent *host = gethostbyname("slashdot.org"); + // gethostbyname_r calls the same stuff as gethostbyname, so we'll test the + // more complicated one. + // resolve the hostname to an actual address + gethostbyname_r("slashdot.org", &hostData, buffer, sizeof(buffer), &host, &err); assert(host->h_addrtype == AF_INET); assert(host->h_length == sizeof(uint32_t)); From 595d7eeae698d985ea2da9d3b98d27af9e035f6c Mon Sep 17 00:00:00 2001 From: Bruce Mitchener Date: Fri, 6 Sep 2013 16:18:01 +0700 Subject: [PATCH 08/26] Change various 64 bit typedefs over to 32 bit. This also removes all of the changes that were needed for those typedefs to be 64 bit. --- src/library.js | 85 ++++++++++++----------------- system/include/libc/bits/alltypes.h | 12 ++-- system/include/libc/sys/resource.h | 2 +- tests/zlib/ref.txt | 2 +- 4 files changed, 44 insertions(+), 57 deletions(-) diff --git a/src/library.js b/src/library.js index af0e8c97d28b0..6f731dbf646aa 100644 --- a/src/library.js +++ b/src/library.js @@ -29,8 +29,8 @@ LibraryManager.library = { // ========================================================================== __dirent_struct_layout: Runtime.generateStructInfo([ - ['i64', 'd_ino'], - ['i64', 'd_off'], + ['i32', 'd_ino'], + ['i32', 'd_off'], ['i16', 'd_reclen'], ['i8', 'd_type'], ['b256', 'd_name']]), @@ -82,7 +82,7 @@ LibraryManager.library = { seekdir: function(dirp, loc) { // void seekdir(DIR *dirp, long int loc); // http://pubs.opengroup.org/onlinepubs/007908799/xsh/seekdir.html - _lseek(dirp, loc, 0, {{{ cDefine('SEEK_SET') }}}); + _lseek(dirp, loc, {{{ cDefine('SEEK_SET') }}}); }, rewinddir__deps: ['seekdir'], rewinddir: function(dirp) { @@ -254,27 +254,25 @@ LibraryManager.library = { // ========================================================================== __stat_struct_layout: Runtime.generateStructInfo([ - ['i64', 'st_dev'], + ['i32', 'st_dev'], ['i32', '__st_dev_padding'], ['i32', '__st_ino_truncated'], ['i32', 'st_mode'], ['i32', 'st_nlink'], ['i32', 'st_uid'], ['i32', 'st_gid'], - ['i64', 'st_rdev'], + ['i32', 'st_rdev'], ['i32', '__st_rdev_padding'], - ['i32', '__st_rdev_padding2'], - ['i64', 'st_size'], + ['i32', 'st_size'], ['i32', 'st_blksize'], - ['i32', '__st_blksize_padding'], - ['i64', 'st_blocks'], + ['i32', 'st_blocks'], ['i32', 'st_atim_secs'], ['i32', 'st_atim_nsecs'], ['i32', 'st_mtim_secs'], ['i32', 'st_mtim_nsecs'], ['i32', 'st_ctim_secs'], ['i32', 'st_ctim_nsecs'], - ['i64', 'st_ino']]), + ['i32', 'st_ino']]), stat__deps: ['$FS', '__stat_struct_layout'], stat: function(path, buf, dontResolveLastLink) { // http://pubs.opengroup.org/onlinepubs/7908799/xsh/stat.html @@ -293,10 +291,8 @@ LibraryManager.library = { {{{ makeSetValue('buf', '___stat_struct_layout.st_gid', 'stat.gid', 'i32') }}} {{{ makeSetValue('buf', '___stat_struct_layout.st_rdev', 'stat.rdev', 'i32') }}} {{{ makeSetValue('buf', '___stat_struct_layout.__st_rdev_padding', '0', 'i32') }}}; - {{{ makeSetValue('buf', '___stat_struct_layout.__st_rdev_padding2', '0', 'i32') }}}; {{{ makeSetValue('buf', '___stat_struct_layout.st_size', 'stat.size', 'i32') }}} {{{ makeSetValue('buf', '___stat_struct_layout.st_blksize', '4096', 'i32') }}} - {{{ makeSetValue('buf', '___stat_struct_layout.__st_blksize_padding', '0', 'i32') }}} {{{ makeSetValue('buf', '___stat_struct_layout.st_blocks', 'stat.blocks', 'i32') }}} {{{ makeSetValue('buf', '___stat_struct_layout.st_atim_secs', 'Math.floor(stat.atime.getTime() / 1000)', 'i32') }}} {{{ makeSetValue('buf', '___stat_struct_layout.st_atim_nsecs', '0', 'i32') }}} @@ -329,7 +325,7 @@ LibraryManager.library = { return _stat(stream.path, buf); }, mknod__deps: ['$FS', '__setErrNo', '$ERRNO_CODES'], - mknod: function(path, mode, dev_lo, dev_hi) { + mknod: function(path, mode, dev) { // int mknod(const char *path, mode_t mode, dev_t dev); // http://pubs.opengroup.org/onlinepubs/7908799/xsh/mknod.html path = Pointer_stringify(path); @@ -347,7 +343,7 @@ LibraryManager.library = { return -1; } try { - FS.mknod(path, mode, dev_lo); + FS.mknod(path, mode, dev); return 0; } catch (e) { FS.handleFSError(e); @@ -443,12 +439,12 @@ LibraryManager.library = { __statvfs_struct_layout: Runtime.generateStructInfo([ ['i32', 'f_bsize'], ['i32', 'f_frsize'], - ['i64', 'f_blocks'], - ['i64', 'f_bfree'], - ['i64', 'f_bavail'], - ['i64', 'f_files'], - ['i64', 'f_ffree'], - ['i64', 'f_favail'], + ['i32', 'f_blocks'], + ['i32', 'f_bfree'], + ['i32', 'f_bavail'], + ['i32', 'f_files'], + ['i32', 'f_ffree'], + ['i32', 'f_favail'], ['i32', 'f_fsid'], ['i32', '__padding'], ['i32', 'f_flag'], @@ -497,8 +493,7 @@ LibraryManager.library = { ['i16', 'l_whence'], ['i32', 'l_start'], ['i32', 'l_len'], - ['i16', 'l_pid'], - ['i16', 'l_xxx']]), + ['i16', 'l_pid']]), open__deps: ['$FS', '__setErrNo', '$ERRNO_CODES', '__dirent_struct_layout'], open: function(path, oflag, varargs) { // int open(const char *path, int oflag, ...); @@ -586,7 +581,7 @@ LibraryManager.library = { // Should never be reached. Only to silence strict warnings. return -1; }, - posix_fadvise: function(fd, offset_lo, offset_hi, len_lo, len_hi, advice) { + posix_fadvise: function(fd, offset, len, advice) { // int posix_fadvise(int fd, off_t offset, off_t len, int advice); // http://pubs.opengroup.org/onlinepubs/009695399/functions/posix_fadvise.html // Advise as much as you wish. We don't care. @@ -594,7 +589,7 @@ LibraryManager.library = { }, posix_madvise: 'posix_fadvise', posix_fallocate__deps: ['$FS', '__setErrNo', '$ERRNO_CODES'], - posix_fallocate: function(fd, offset_lo, offset_hi, len_lo, len_hi) { + posix_fallocate: function(fd, offset, len) { // int posix_fallocate(int fd, off_t offset, off_t len); // http://pubs.opengroup.org/onlinepubs/009695399/functions/posix_fallocate.html var stream = FS.getStream(fd); @@ -603,7 +598,7 @@ LibraryManager.library = { return -1; } try { - FS.allocate(stream, offset_lo, len_lo); + FS.allocate(stream, offset, len); return 0; } catch (e) { FS.handleFSError(e); @@ -882,13 +877,13 @@ LibraryManager.library = { }, fdatasync: 'fsync', truncate__deps: ['$FS', '__setErrNo', '$ERRNO_CODES'], - truncate: function(path, length_lo, length_hi) { + truncate: function(path, length) { // int truncate(const char *path, off_t length); // http://pubs.opengroup.org/onlinepubs/000095399/functions/truncate.html // NOTE: The path argument may be a string, to simplify ftruncate(). if (typeof path !== 'string') path = Pointer_stringify(path); try { - FS.truncate(path, length_lo); + FS.truncate(path, length); return 0; } catch (e) { FS.handleFSError(e); @@ -896,11 +891,11 @@ LibraryManager.library = { } }, ftruncate__deps: ['$FS', '__setErrNo', '$ERRNO_CODES', 'truncate'], - ftruncate: function(fildes, length_lo, length_hi) { + ftruncate: function(fildes, length) { // int ftruncate(int fildes, off_t length); // http://pubs.opengroup.org/onlinepubs/000095399/functions/ftruncate.html try { - FS.ftruncate(fildes, length_lo); + FS.ftruncate(fildes, length); return 0; } catch (e) { FS.handleFSError(e); @@ -962,7 +957,7 @@ LibraryManager.library = { return -1; }, lockf__deps: ['$FS', '__setErrNo', '$ERRNO_CODES'], - lockf: function(fildes, func, size_lo, size_hi) { + lockf: function(fildes, func, size) { // int lockf(int fildes, int function, off_t size); // http://pubs.opengroup.org/onlinepubs/000095399/functions/lockf.html var stream = FS.getStream(fildes); @@ -976,17 +971,16 @@ LibraryManager.library = { } }, lseek__deps: ['$FS', '__setErrNo', '$ERRNO_CODES'], - lseek: function(fildes, offset_lo, offset_hi, whence) { + lseek: function(fildes, offset, whence) { // off_t lseek(int fildes, off_t offset, int whence); // http://pubs.opengroup.org/onlinepubs/000095399/functions/lseek.html var stream = FS.getStream(fildes); - tempRet0 = 0; if (!stream) { ___setErrNo(ERRNO_CODES.EBADF); return -1; } try { - return FS.llseek(stream, offset_lo, whence); + return FS.llseek(stream, offset, whence); } catch (e) { FS.handleFSError(e); return -1; @@ -1002,7 +996,7 @@ LibraryManager.library = { return -1; }, pread__deps: ['$FS', '__setErrNo', '$ERRNO_CODES'], - pread: function(fildes, buf, nbyte, offset_lo, offset_hi) { + pread: function(fildes, buf, nbyte, offset) { // ssize_t pread(int fildes, void *buf, size_t nbyte, off_t offset); // http://pubs.opengroup.org/onlinepubs/000095399/functions/read.html var stream = FS.getStream(fildes); @@ -1017,7 +1011,7 @@ LibraryManager.library = { SAFE_HEAP_FILL_HISTORY(buf, buf+nbyte, 'i8'); // VFS does not use makeSetValues, so we need to do it manually #endif #endif - return FS.read(stream, slab, buf, nbyte, offset_lo); + return FS.read(stream, slab, buf, nbyte, offset); } catch (e) { FS.handleFSError(e); return -1; @@ -1137,7 +1131,7 @@ LibraryManager.library = { return str.length; }, pwrite__deps: ['$FS', '__setErrNo', '$ERRNO_CODES'], - pwrite: function(fildes, buf, nbyte, offset_lo, offset_hi) { + pwrite: function(fildes, buf, nbyte, offset) { // ssize_t pwrite(int fildes, const void *buf, size_t nbyte, off_t offset); // http://pubs.opengroup.org/onlinepubs/000095399/functions/write.html var stream = FS.getStream(fildes); @@ -1152,7 +1146,7 @@ LibraryManager.library = { SAFE_HEAP_FILL_HISTORY(buf, buf+nbyte, 'i8'); // VFS does not use makeSetValues, so we need to do it manually #endif #endif - return FS.write(stream, slab, buf, nbyte, offset_lo); + return FS.write(stream, slab, buf, nbyte, offset); } catch (e) { FS.handleFSError(e); return -1; @@ -2555,7 +2549,7 @@ LibraryManager.library = { fseek: function(stream, offset, whence) { // int fseek(FILE *stream, long offset, int whence); // http://pubs.opengroup.org/onlinepubs/000095399/functions/fseek.html - var ret = _lseek(stream, offset, 0, whence); + var ret = _lseek(stream, offset, whence); if (ret == -1) { return -1; } @@ -2563,8 +2557,8 @@ LibraryManager.library = { stream.eof = false; return 0; }, - fseeko: 'lseek', - fseeko64: 'lseek', + fseeko: 'fseek', + fseeko64: 'fseek', fsetpos__deps: ['$FS', 'lseek', '__setErrNo', '$ERRNO_CODES'], fsetpos: function(stream, pos) { // int fsetpos(FILE *stream, const fpos_t *pos); @@ -2600,15 +2594,8 @@ LibraryManager.library = { return stream.position; } }, - ftello__deps: ['$FS', '__setErrNo', '$ERRNO_CODES'], - ftello: function(stream) { - // off_t ftello(FILE *stream); - // http://pubs.opengroup.org/onlinepubs/000095399/functions/ftello.html - var result = ftell(stream); - tempRet0 = result << 32; - return 0; - }, - ftello64: 'ftello', + ftello: 'ftell', + ftello64: 'ftell', fwrite__deps: ['$FS', 'write'], fwrite: function(ptr, size, nitems, stream) { // size_t fwrite(const void *restrict ptr, size_t size, size_t nitems, FILE *restrict stream); diff --git a/system/include/libc/bits/alltypes.h b/system/include/libc/bits/alltypes.h index e7f3bdc792eda..c4eb1f84dda55 100644 --- a/system/include/libc/bits/alltypes.h +++ b/system/include/libc/bits/alltypes.h @@ -210,17 +210,17 @@ typedef unsigned _Reg nlink_t; #endif #if defined(__NEED_off_t) && !defined(__DEFINED_off_t) -typedef _Int64 off_t; +typedef int off_t; #define __DEFINED_off_t #endif #if defined(__NEED_ino_t) && !defined(__DEFINED_ino_t) -typedef unsigned _Int64 ino_t; +typedef unsigned int ino_t; #define __DEFINED_ino_t #endif #if defined(__NEED_dev_t) && !defined(__DEFINED_dev_t) -typedef unsigned _Int64 dev_t; +typedef unsigned int dev_t; #define __DEFINED_dev_t #endif @@ -230,17 +230,17 @@ typedef long blksize_t; #endif #if defined(__NEED_blkcnt_t) && !defined(__DEFINED_blkcnt_t) -typedef _Int64 blkcnt_t; +typedef int blkcnt_t; #define __DEFINED_blkcnt_t #endif #if defined(__NEED_fsblkcnt_t) && !defined(__DEFINED_fsblkcnt_t) -typedef unsigned _Int64 fsblkcnt_t; +typedef unsigned int fsblkcnt_t; #define __DEFINED_fsblkcnt_t #endif #if defined(__NEED_fsfilcnt_t) && !defined(__DEFINED_fsfilcnt_t) -typedef unsigned _Int64 fsfilcnt_t; +typedef unsigned int fsfilcnt_t; #define __DEFINED_fsfilcnt_t #endif diff --git a/system/include/libc/sys/resource.h b/system/include/libc/sys/resource.h index 0cfbcf4434901..22ff2f3b24081 100644 --- a/system/include/libc/sys/resource.h +++ b/system/include/libc/sys/resource.h @@ -16,7 +16,7 @@ extern "C" { #include -typedef unsigned long long rlim_t; +typedef unsigned long rlim_t; struct rlimit { diff --git a/tests/zlib/ref.txt b/tests/zlib/ref.txt index bdbc4c1f2ecda..7211459eec6be 100644 --- a/tests/zlib/ref.txt +++ b/tests/zlib/ref.txt @@ -1,4 +1,4 @@ -zlib version 1.2.5 = 4688, compile flags = 149 +zlib version 1.2.5 = 4688, compile flags = 85 uncompress(): hello, hello! inflate(): hello, hello! large_inflate(): OK From 4c3c7df30af6dc9ba6309e527a5173d9078c56b9 Mon Sep 17 00:00:00 2001 From: Bruce Mitchener Date: Fri, 6 Sep 2013 17:42:51 +0700 Subject: [PATCH 09/26] Correctly note musl provenance and differences. --- system/lib/libc/musl/readme.txt | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/system/lib/libc/musl/readme.txt b/system/lib/libc/musl/readme.txt index 0df3429d84eea..64509ccdf68e4 100644 --- a/system/lib/libc/musl/readme.txt +++ b/system/lib/libc/musl/readme.txt @@ -1 +1,8 @@ -These sources were downloaded from the musl-0.9.10 release on April 14, 2003. +These sources were downloaded from the musl-0.9.12 release on July 29, 2013, +along with some updates from git to fix various issues that were found. + +Differences from upstream musl include: + +* file descriptor numbers for stdio are 1-3 rather than 0-2. +* various 64 bit types are 32 bit instead including off_t, + ino_t, dev_t, blkcnt_t, fsblkcnt_t, fsfilcnt_t, rlim_t. From 733e6877d7b1146ecb5c796f4d249843101a7b5f Mon Sep 17 00:00:00 2001 From: Bruce Mitchener Date: Fri, 6 Sep 2013 17:48:28 +0700 Subject: [PATCH 10/26] Update to musl-0.9.13 headers. --- system/include/libc/netinet/tcp.h | 35 +++++++++++++++ system/include/libc/pthread.h | 3 ++ system/include/libc/resolv.h | 2 +- system/include/libc/sched.h | 61 +++++++++++++++++++++++++++ system/include/libc/stdlib.h | 8 +++- system/include/libc/sys/msg.h | 1 + system/include/libc/sys/personality.h | 40 ++++++++++++++++++ system/include/libc/sys/un.h | 19 +++++++++ system/include/libc/sys/wait.h | 4 +- system/lib/libc/musl/readme.txt | 3 +- 10 files changed, 169 insertions(+), 7 deletions(-) diff --git a/system/include/libc/netinet/tcp.h b/system/include/libc/netinet/tcp.h index b7828a502dafe..8266f21daabe6 100644 --- a/system/include/libc/netinet/tcp.h +++ b/system/include/libc/netinet/tcp.h @@ -33,4 +33,39 @@ #include #endif +#ifdef _GNU_SOURCE +#include +struct tcphdr +{ + u_int16_t source; + u_int16_t dest; + u_int32_t seq; + u_int32_t ack_seq; +#if __BYTE_ORDER == __LITTLE_ENDIAN + u_int16_t res1:4; + u_int16_t doff:4; + u_int16_t fin:1; + u_int16_t syn:1; + u_int16_t rst:1; + u_int16_t psh:1; + u_int16_t ack:1; + u_int16_t urg:1; + u_int16_t res2:2; +#else + u_int16_t doff:4; + u_int16_t res1:4; + u_int16_t res2:2; + u_int16_t urg:1; + u_int16_t ack:1; + u_int16_t psh:1; + u_int16_t rst:1; + u_int16_t syn:1; + u_int16_t fin:1; +#endif + u_int16_t window; + u_int16_t check; + u_int16_t urg_ptr; +}; +#endif + #endif diff --git a/system/include/libc/pthread.h b/system/include/libc/pthread.h index 731bce385306a..f7c9568c80bb3 100644 --- a/system/include/libc/pthread.h +++ b/system/include/libc/pthread.h @@ -210,6 +210,9 @@ void _pthread_cleanup_pop(struct __ptcb *, int); #define pthread_cleanup_pop(r) _pthread_cleanup_pop(&__cb, (r)); } while(0) #ifdef _GNU_SOURCE +struct cpu_set_t; +int pthread_getaffinity_np(pthread_t, size_t, struct cpu_set_t *); +int pthread_setaffinity_np(pthread_t, size_t, const struct cpu_set_t *); int pthread_getattr_np(pthread_t, pthread_attr_t *); #endif diff --git a/system/include/libc/resolv.h b/system/include/libc/resolv.h index cdedd274d04d5..259e4bc1ee47a 100644 --- a/system/include/libc/resolv.h +++ b/system/include/libc/resolv.h @@ -134,7 +134,7 @@ int res_search(const char *, int, int, unsigned char *, int); int res_mkquery(int, const char *, int, int, char *, int, struct rrec *, char *, int); int res_send(const char *, int, char *, int); int dn_comp(unsigned char *, unsigned char *, int, unsigned char **, unsigned char *, unsigned char **); -int dn_expand(unsigned char *, unsigned char *, unsigned char *, unsigned char *, int); +int dn_expand(const unsigned char *, const unsigned char *, const unsigned char *, char *, int); int dn_skipname(const unsigned char *, const unsigned char *); #ifdef __cplusplus diff --git a/system/include/libc/sched.h b/system/include/libc/sched.h index 994260d097f48..6a6b2fcb4333a 100644 --- a/system/include/libc/sched.h +++ b/system/include/libc/sched.h @@ -10,6 +10,10 @@ extern "C" { #define __NEED_pid_t #define __NEED_time_t +#ifdef _GNU_SOURCE +#define __NEED_size_t +#endif + #include struct sched_param { @@ -63,6 +67,63 @@ int sched_yield(void); int clone (int (*)(void *), void *, int, void *, ...); int unshare(int); int setns(int, int); + +void *memcpy(void *__restrict, const void *__restrict, size_t); +int memcmp(const void *, const void *, size_t); +void *calloc(size_t, size_t); +void free(void *); + +typedef struct cpu_set_t { unsigned long __bits[128/sizeof(long)]; } cpu_set_t; +int __sched_cpucount(size_t, const cpu_set_t *); +int sched_getaffinity(pid_t, size_t, cpu_set_t *); +int sched_setaffinity(pid_t, size_t, const cpu_set_t *); + +#define __CPU_op_S(i, size, set, op) ( (i)/8 >= (size) ? 0 : \ + ((set)->__bits[(i)/8/sizeof(long)] op (1UL<<((i)%(8*sizeof(long))))) ) + +#define CPU_SET_S(i, size, set) __CPU_op_S(i, size, set, |=) +#define CPU_CLR_S(i, size, set) __CPU_op_S(i, size, set, &=~) +#define CPU_ISSET_S(i, size, set) __CPU_op_S(i, size, set, &) + +#define __CPU_op_func_S(func, op) \ +static __inline void __CPU_##func##_S(size_t __size, cpu_set_t *__dest, \ + const cpu_set_t *__src1, const cpu_set_t *__src2) \ +{ \ + size_t __i; \ + for (__i=0; __i<__size/sizeof(long); __i++) \ + __dest->__bits[__i] = __src1->__bits[__i] \ + op __src2->__bits[__i] ; \ +} + +__CPU_op_func_S(AND, &) +__CPU_op_func_S(OR, |) +__CPU_op_func_S(XOR, ^) + +#define CPU_AND_S(a,b,c,d) __CPU_AND_S(a,b,c,d) +#define CPU_OR_S(a,b,c,d) __CPU_OR_S(a,b,c,d) +#define CPU_XOR_S(a,b,c,d) __CPU_XOR_S(a,b,c,d) + +#define CPU_COUNT_S(size,set) __sched_cpucount(size,set) +#define CPU_ZERO_S(size,set) memset(set,0,size) +#define CPU_EQUAL_S(size,set1,set2) (!memcmp(set1,set2,size)) + +#define CPU_ALLOC_SIZE(n) (sizeof(long) * ( (n)/(8*sizeof(long)) \ + + ((n)%(8*sizeof(long)) + 8*sizeof(long)-1)/(8*sizeof(long)) ) ) +#define CPU_ALLOC(n) ((cpu_set_t *)calloc(1,CPU_ALLOC_SIZE(n))) +#define CPU_FREE(set) free(set) + +#define CPU_SETSIZE 128 + +#define CPU_SET(i, set) CPU_SET_S(i,sizeof(cpu_set_t),set) +#define CPU_CLR(i, set) CPU_CLR_S(i,sizeof(cpu_set_t),set) +#define CPU_ISSET(i, set) CPU_ISSET_S(i,sizeof(cpu_set_t),set) +#define CPU_AND(d,s1,s2) CPU_AND_S(sizeof(cpu_set_t),d,s1,s2) +#define CPU_OR(d,s1,s2) CPU_OR_S(sizeof(cpu_set_t),d,s1,s2) +#define CPU_XOR(d,s1,s2) CPU_XOR_S(sizeof(cpu_set_t),d,s1,s2) +#define CPU_COUNT(set) CPU_COUNT_S(sizeof(cpu_set_t),set) +#define CPU_ZERO(set) CPU_ZERO_S(sizeof(cpu_set_t),set) +#define CPU_EQUAL(set) CPU_EQUAL_S(sizeof(cpu_set_t),set) + #endif #ifdef __cplusplus diff --git a/system/include/libc/stdlib.h b/system/include/libc/stdlib.h index 0bcc9f4f91abd..bca1fb4194c43 100644 --- a/system/include/libc/stdlib.h +++ b/system/include/libc/stdlib.h @@ -88,8 +88,8 @@ size_t wcstombs (char *__restrict, const wchar_t *__restrict, size_t); #define WTERMSIG(s) ((s) & 0x7f) #define WSTOPSIG(s) WEXITSTATUS(s) #define WIFEXITED(s) (!WTERMSIG(s)) -#define WIFSTOPPED(s) (((s) & 0xff) == 0x7f) -#define WIFSIGNALED(s) (((signed char) (((s) & 0x7f) + 1) >> 1) > 0) +#define WIFSTOPPED(s) ((short)((((s)&0xffff)*0x10001)>>8) > 0x7f00) +#define WIFSIGNALED(s) (((s)&0xffff)-1 < 0xffu) int posix_memalign (void **, size_t, size_t); int setenv (const char *, const char *, int); @@ -149,6 +149,10 @@ int ptsname_r(int, char *, size_t); char *ecvt(double, int, int *, int *); char *fcvt(double, int, int *, int *); char *gcvt(double, int, char *); +struct __locale_struct; +float strtof_l(const char *__restrict, char **__restrict, struct __locale_struct *); +double strtod_l(const char *__restrict, char **__restrict, struct __locale_struct *); +long double strtold_l(const char *__restrict, char **__restrict, struct __locale_struct *); #endif #if defined(_LARGEFILE64_SOURCE) || defined(_GNU_SOURCE) diff --git a/system/include/libc/sys/msg.h b/system/include/libc/sys/msg.h index ceedd1c6254c6..139f22b7008fa 100644 --- a/system/include/libc/sys/msg.h +++ b/system/include/libc/sys/msg.h @@ -23,6 +23,7 @@ typedef unsigned long msglen_t; #define __msg_cbytes msg_cbytes #define MSG_NOERROR 010000 +#define MSG_EXCEPT 020000 #define MSG_STAT 11 #define MSG_INFO 12 diff --git a/system/include/libc/sys/personality.h b/system/include/libc/sys/personality.h index 852c0248abdc5..31d43dfe13ea4 100644 --- a/system/include/libc/sys/personality.h +++ b/system/include/libc/sys/personality.h @@ -1,6 +1,46 @@ #ifndef _PERSONALITY_H #define _PERSONALITY_H +#ifdef __cplusplus +extern "C" { +#endif + +#define ADDR_NO_RANDOMIZE 0x0040000 +#define MMAP_PAGE_ZERO 0x0100000 +#define ADDR_COMPAT_LAYOUT 0x0200000 +#define READ_IMPLIES_EXEC 0x0400000 +#define ADDR_LIMIT_32BIT 0x0800000 +#define SHORT_INODE 0x1000000 +#define WHOLE_SECONDS 0x2000000 +#define STICKY_TIMEOUTS 0x4000000 +#define ADDR_LIMIT_3GB 0x8000000 + +#define PER_LINUX 0 +#define PER_LINUX_32BIT ADDR_LIMIT_32BIT +#define PER_SVR4 (1 | STICKY_TIMEOUTS | MMAP_PAGE_ZERO) +#define PER_SVR3 (2 | STICKY_TIMEOUTS | SHORT_INODE) +#define PER_SCOSVR3 (3 | STICKY_TIMEOUTS | WHOLE_SECONDS | SHORT_INODE) +#define PER_OSR5 (3 | STICKY_TIMEOUTS | WHOLE_SECONDS) +#define PER_WYSEV386 (4 | STICKY_TIMEOUTS | SHORT_INODE) +#define PER_ISCR4 (5 | STICKY_TIMEOUTS) +#define PER_BSD 6 +#define PER_SUNOS (6 | STICKY_TIMEOUTS) +#define PER_XENIX (7 | STICKY_TIMEOUTS | SHORT_INODE) +#define PER_LINUX32 8 +#define PER_LINUX32_3GB (8 | ADDR_LIMIT_3GB) +#define PER_IRIX32 (9 | STICKY_TIMEOUTS) +#define PER_IRIXN32 (0xa | STICKY_TIMEOUTS) +#define PER_IRIX64 (0x0b | STICKY_TIMEOUTS) +#define PER_RISCOS 0xc +#define PER_SOLARIS (0xd | STICKY_TIMEOUTS) +#define PER_UW7 (0xe | STICKY_TIMEOUTS | MMAP_PAGE_ZERO) +#define PER_OSF4 0xf +#define PER_HPUX 0x10 +#define PER_MASK 0xff + int personality(unsigned long); +#ifdef __cplusplus +} +#endif #endif diff --git a/system/include/libc/sys/un.h b/system/include/libc/sys/un.h index 769dac6b53263..7494f1a382ed0 100644 --- a/system/include/libc/sys/un.h +++ b/system/include/libc/sys/un.h @@ -1,7 +1,17 @@ #ifndef _SYS_UN_H #define _SYS_UN_H +#ifdef __cplusplus +extern "C" { +#endif + +#include + #define __NEED_sa_family_t +#if defined(_GNU_SOURCE) || defined(_BSD_SOURCE) +#define __NEED_size_t +#endif + #include struct sockaddr_un @@ -10,4 +20,13 @@ struct sockaddr_un char sun_path[108]; }; +#if defined(_GNU_SOURCE) || defined(_BSD_SOURCE) +size_t strlen(const char *); +#define SUN_LEN(s) (2+strlen((s)->sun_path)) +#endif + +#ifdef __cplusplus +} +#endif + #endif diff --git a/system/include/libc/sys/wait.h b/system/include/libc/sys/wait.h index b6dfe01d6ff66..a7ad7cd15a94b 100644 --- a/system/include/libc/sys/wait.h +++ b/system/include/libc/sys/wait.h @@ -45,8 +45,8 @@ pid_t wait4 (pid_t, int *, int, struct rusage *); #define WSTOPSIG(s) WEXITSTATUS(s) #define WCOREDUMP(s) ((s) & 0x80) #define WIFEXITED(s) (!WTERMSIG(s)) -#define WIFSTOPPED(s) (((s) & 0xff) == 0x7f) -#define WIFSIGNALED(s) (((signed char) (((s) & 0x7f) + 1) >> 1) > 0) +#define WIFSTOPPED(s) ((short)((((s)&0xffff)*0x10001)>>8) > 0x7f00) +#define WIFSIGNALED(s) (((s)&0xffff)-1 < 0xffu) #define WIFCONTINUED(s) ((s) == 0xffff) #ifdef __cplusplus diff --git a/system/lib/libc/musl/readme.txt b/system/lib/libc/musl/readme.txt index 64509ccdf68e4..4d1c49982731c 100644 --- a/system/lib/libc/musl/readme.txt +++ b/system/lib/libc/musl/readme.txt @@ -1,5 +1,4 @@ -These sources were downloaded from the musl-0.9.12 release on July 29, 2013, -along with some updates from git to fix various issues that were found. +These sources were downloaded from the musl-0.9.13 release on August 30, 2013. Differences from upstream musl include: From 8f1db82f5ab8c2fdfcdeccd7e502f1854da5d696 Mon Sep 17 00:00:00 2001 From: Bruce Mitchener Date: Fri, 6 Sep 2013 21:32:39 +0700 Subject: [PATCH 11/26] Incorrect return type on a compat header. This was exposed by the update to musl 0.9.13 which now has a prototype for this function. --- system/include/compat/xlocale.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system/include/compat/xlocale.h b/system/include/compat/xlocale.h index c0091164dc00b..52b9b26a05f89 100644 --- a/system/include/compat/xlocale.h +++ b/system/include/compat/xlocale.h @@ -9,7 +9,7 @@ extern "C" { long long strtoll_l(const char *start, char **end, int base, locale_t loc); unsigned long long strtoull_l(const char *start, char **end, int base, locale_t loc); -double strtold_l(const char *start, char **end, locale_t loc); +long double strtold_l(const char *start, char **end, locale_t loc); #ifdef __cplusplus } From 03a74d3600462fbcaf44168b170a9486ca0178c2 Mon Sep 17 00:00:00 2001 From: Bruce Mitchener Date: Tue, 10 Sep 2013 19:38:48 +0700 Subject: [PATCH 12/26] Skip test_math unless using ta2 isfinite and friends require ta2 with musl headers. --- tests/test_core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_core.py b/tests/test_core.py index d02e6778a0661..dd3b7c44aceeb 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1431,6 +1431,7 @@ def test_globaldoubles(self): self.do_run(src, 'BUG?\nDisplay: Vu=465.100000 Vv=465.200000 Wu=160.300000 Wv=111.400000') def test_math(self): + if Settings.USE_TYPED_ARRAYS != 2: return self.skip('requires ta2') src = ''' #include #include From 1dc5a8a5cddab263bd455c3df11813a15c198a77 Mon Sep 17 00:00:00 2001 From: Bruce Mitchener Date: Thu, 12 Sep 2013 11:33:04 +0700 Subject: [PATCH 13/26] Don't define _POSIX_SHARED_MEMORY_OBJECTS. This isn't implemented, so don't define it. --- system/include/libc/unistd.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system/include/libc/unistd.h b/system/include/libc/unistd.h index 19158b67bbe47..ef6ba3b33d628 100644 --- a/system/include/libc/unistd.h +++ b/system/include/libc/unistd.h @@ -239,7 +239,7 @@ int eaccess(const char *, int); #define _POSIX_READER_WRITER_LOCKS _POSIX_VERSION #define _POSIX_ASYNCHRONOUS_IO _POSIX_VERSION #define _POSIX_SEMAPHORES _POSIX_VERSION -#define _POSIX_SHARED_MEMORY_OBJECTS _POSIX_VERSION +#undef _POSIX_SHARED_MEMORY_OBJECTS #define _POSIX2_C_BIND _POSIX_VERSION From ed9596d393b585ba21983ab585ff5bb7c488fe0d Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 12 Sep 2013 16:30:38 -0700 Subject: [PATCH 14/26] headers readme --- system/include/libc/readme.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 system/include/libc/readme.txt diff --git a/system/include/libc/readme.txt b/system/include/libc/readme.txt new file mode 100644 index 0000000000000..1ac552e4d2d3c --- /dev/null +++ b/system/include/libc/readme.txt @@ -0,0 +1,3 @@ + +These are musl headers, see system/lib/libc/musl/ for details and copyright info + From 25c16aeb17d59dd84fe2c8d551c6080cc650ac15 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 12 Sep 2013 16:43:35 -0700 Subject: [PATCH 15/26] update sanity.test_jcache --- tests/test_sanity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_sanity.py b/tests/test_sanity.py index 4188afff81d6d..6fdf5ddd2be61 100644 --- a/tests/test_sanity.py +++ b/tests/test_sanity.py @@ -478,7 +478,7 @@ def test_jcache(self): (['--jcache'], 'hello_malloc.cpp', False, True, False, True, False, True, []), ([], 'hello_malloc.cpp', False, False, False, False, False, False, []), # new, huge file - ([], 'hello_libcxx.cpp', False, False, False, False, False, False, ('3 chunks',)), + ([], 'hello_libcxx.cpp', False, False, False, False, False, False, ('4 chunks',)), (['--jcache'], 'hello_libcxx.cpp', True, False, True, False, True, False, []), (['--jcache'], 'hello_libcxx.cpp', False, True, False, True, False, True, []), ([], 'hello_libcxx.cpp', False, False, False, False, False, False, []), From 8fda4dc52d87cb5e4ac7bfffdbb15bd2d0e1c958 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 12 Sep 2013 18:23:31 -0700 Subject: [PATCH 16/26] simplify other.test_llvm_nativizer to not rely on libc stuff --- tests/test_other.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_other.py b/tests/test_other.py index 64be60fac64f9..b11f22e5287b1 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -1755,7 +1755,7 @@ def test_llvm_nativizer(self): other=ay file... seeked= file. ''', output[0]) - self.assertIdentical('texte\n', output[1]) + self.assertContained('texte\n', output[1]) def test_emconfig(self): output = Popen([PYTHON, EMCONFIG, 'LLVM_ROOT'], stdout=PIPE, stderr=PIPE).communicate()[0].strip() From 2c5899b2df1a349d3bd5bf349ba767fce4df7680 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 13 Sep 2013 10:02:55 -0700 Subject: [PATCH 17/26] add some stubs for libc needed by libc++ --- src/library.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/library.js b/src/library.js index 6f731dbf646aa..f414a05ff1f2b 100644 --- a/src/library.js +++ b/src/library.js @@ -8800,7 +8800,7 @@ function autoAddDeps(object, name) { } // Add aborting stubs for various libc stuff needed by libc++ -['pthread_cond_signal', 'pthread_equal', 'wcstol', 'wcstoll', 'wcstoul', 'wcstoull', 'wcstof', 'wcstod', 'wcstold', 'swprintf', 'pthread_join', 'pthread_detach', 'strcoll_l', 'strxfrm_l', 'wcscoll_l', 'toupper_l', 'tolower_l', 'iswspace_l', 'iswprint_l', 'iswcntrl_l', 'iswupper_l', 'iswlower_l', 'iswalpha_l', 'iswdigit_l', 'iswpunct_l', 'iswxdigit_l', 'iswblank_l', 'wcsxfrm_l', 'towupper_l', 'towlower_l'].forEach(function(aborter) { +['pthread_cond_signal', 'pthread_equal', 'wcstol', 'wcstoll', 'wcstoul', 'wcstoull', 'wcstof', 'wcstod', 'wcstold', 'swprintf', 'pthread_join', 'pthread_detach', 'strcoll_l', 'strxfrm_l', 'wcscoll_l', 'toupper_l', 'tolower_l', 'iswspace_l', 'iswprint_l', 'iswcntrl_l', 'iswupper_l', 'iswlower_l', 'iswalpha_l', 'iswdigit_l', 'iswpunct_l', 'iswxdigit_l', 'iswblank_l', 'wcsxfrm_l', 'towupper_l', 'towlower_l', 'catgets', 'catopen', 'catclose'].forEach(function(aborter) { LibraryManager.library[aborter] = function() { throw 'TODO: ' + aborter }; }); From bf349b84d900eb403cef64ab5e9469c9bd672ec6 Mon Sep 17 00:00:00 2001 From: Bruce Mitchener Date: Thu, 12 Sep 2013 11:33:04 +0700 Subject: [PATCH 18/26] Don't define _POSIX_SHARED_MEMORY_OBJECTS. This isn't implemented, so don't define it. --- system/lib/libc/musl/readme.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/system/lib/libc/musl/readme.txt b/system/lib/libc/musl/readme.txt index 4d1c49982731c..16c584237c538 100644 --- a/system/lib/libc/musl/readme.txt +++ b/system/lib/libc/musl/readme.txt @@ -5,3 +5,4 @@ Differences from upstream musl include: * file descriptor numbers for stdio are 1-3 rather than 0-2. * various 64 bit types are 32 bit instead including off_t, ino_t, dev_t, blkcnt_t, fsblkcnt_t, fsfilcnt_t, rlim_t. +* We don't define _POSIX_SHARED_MEMORY_OBJECTS. From 61c818d8da85668f2697d032f118db4243d7cafe Mon Sep 17 00:00:00 2001 From: Bruce Mitchener Date: Fri, 13 Sep 2013 11:06:15 +0700 Subject: [PATCH 19/26] Add compat sys/socketvar.h This is required by xmlvm's hysock which is needed for the benchmark.test_zzz_java_nbody test. --- system/include/compat/sys/socketvar.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 system/include/compat/sys/socketvar.h diff --git a/system/include/compat/sys/socketvar.h b/system/include/compat/sys/socketvar.h new file mode 100644 index 0000000000000..1e60ac2f9fa60 --- /dev/null +++ b/system/include/compat/sys/socketvar.h @@ -0,0 +1,14 @@ +#ifndef _COMPAT_SOCKETVAR_H +#define _COMPAT_SOCKETVAR_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#ifdef __cplusplus +} +#endif + +#endif From 5a37214c5bdf39641d480bb4dd67bd5fc26427a0 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 13 Sep 2013 10:52:48 -0700 Subject: [PATCH 20/26] second enet fix to handle musl headers in emscripten --- tests/enet/unix.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/enet/unix.c b/tests/enet/unix.c index 67d4a8b82916e..292eb29b29289 100644 --- a/tests/enet/unix.c +++ b/tests/enet/unix.c @@ -133,7 +133,7 @@ enet_address_get_host (const ENetAddress * address, char * name, size_t nameLeng in.s_addr = address -> host; -#if defined(linux) || defined(__linux) || defined(__linux__) || defined(__FreeBSD__) || defined(__FreeBSD_kernel__) +#if defined(linux) || defined(__linux) || defined(__linux__) || defined(__FreeBSD__) || defined(__FreeBSD_kernel__) || defined(__EMSCRIPTEN__) gethostbyaddr_r ((char *) & in, sizeof (struct in_addr), AF_INET, & hostData, buffer, sizeof (buffer), & hostEntry, & errnum); #else hostEntry = gethostbyaddr_r ((char *) & in, sizeof (struct in_addr), AF_INET, & hostData, buffer, sizeof (buffer), & errnum); From 1c8a94f63981b557026997a7f481d5842de0cf84 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 13 Sep 2013 14:32:18 -0700 Subject: [PATCH 21/26] remove unneeded var --- src/preamble.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/preamble.js b/src/preamble.js index abcd1c673589f..2f4e25475a39e 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -942,7 +942,7 @@ Math.toFloat32 = Math['toFloat32']; // the dependencies are met. var runDependencies = 0; var runDependencyTracking = {}; -var calledInit = false, calledRun = false; +var calledRun = false; var runDependencyWatcher = null; function addRunDependency(id) { runDependencies++; From 8f3bb14fddb65ef6f47c0ca2b602079eba3bbb44 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 13 Sep 2013 14:35:40 -0700 Subject: [PATCH 22/26] refactor code to handle when run dependencies reach 0 --- src/postamble.js | 6 ++++++ src/preamble.js | 8 ++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/postamble.js b/src/postamble.js index 94b60288cc51e..88986dea34603 100644 --- a/src/postamble.js +++ b/src/postamble.js @@ -12,6 +12,12 @@ ExitStatus.prototype.constructor = ExitStatus; var initialStackTop; var preloadStartTime = null; var calledMain = false; +var calledRun = false; + +dependenciesFulfilled = function() { + // If run has never been called, and we should call run (INVOKE_RUN is true, and Module.noInitialRun is not false) + if (!calledRun && shouldRunNow) run(); +} Module['callMain'] = Module.callMain = function callMain(args) { assert(runDependencies == 0, 'cannot call main when async dependencies remain! (listen on __ATMAIN__)'); diff --git a/src/preamble.js b/src/preamble.js index 2f4e25475a39e..9501f8dc519f1 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -942,8 +942,9 @@ Math.toFloat32 = Math['toFloat32']; // the dependencies are met. var runDependencies = 0; var runDependencyTracking = {}; -var calledRun = false; var runDependencyWatcher = null; +var dependenciesFulfilled = null; // overridden to take different actions when all run dependencies are fulfilled + function addRunDependency(id) { runDependencies++; if (Module['monitorRunDependencies']) { @@ -990,9 +991,8 @@ function removeRunDependency(id) { if (runDependencyWatcher !== null) { clearInterval(runDependencyWatcher); runDependencyWatcher = null; - } - // If run has never been called, and we should call run (INVOKE_RUN is true, and Module.noInitialRun is not false) - if (!calledRun && shouldRunNow) run(); + } + dependenciesFulfilled(); } } Module['removeRunDependency'] = removeRunDependency; From ee51b2bd829447fb83c5ebdbc3f8d157355c0fdc Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 13 Sep 2013 14:57:05 -0700 Subject: [PATCH 23/26] clear dependenciesFulfilled after use --- src/preamble.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/preamble.js b/src/preamble.js index 9501f8dc519f1..579e3065861a8 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -992,7 +992,10 @@ function removeRunDependency(id) { clearInterval(runDependencyWatcher); runDependencyWatcher = null; } - dependenciesFulfilled(); + if (dependenciesFulfilled) { + dependenciesFulfilled(); + dependenciesFulfilled = null; + } } } Module['removeRunDependency'] = removeRunDependency; From 1fc6762e3176843ffc378c8b4f43dd73fed25ebe Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 13 Sep 2013 15:13:30 -0700 Subject: [PATCH 24/26] emscripten_async_load_script --- src/library_browser.js | 19 +++++++++ system/include/emscripten/emscripten.h | 18 +++++++++ tests/emscripten_api_browser2.cpp | 53 ++++++++++++++++++++++++++ tests/test_browser.py | 11 ++++++ 4 files changed, 101 insertions(+) create mode 100644 tests/emscripten_api_browser2.cpp diff --git a/src/library_browser.js b/src/library_browser.js index e4966e15e9853..cba8ecdf78606 100644 --- a/src/library_browser.js +++ b/src/library_browser.js @@ -688,6 +688,25 @@ mergeInto(LibraryManager.library, { }, millis); }, + emscripten_async_load_script: function(url, onload, onerror) { + Module['noExitRuntime'] = true; + + onload = Runtime.getFuncWrapper(onload, 'v'); + + assert(runDependencies === 0, 'async_load_script must be run when no other dependencies are active'); + var script = document.createElement('script'); + script.onload = function() { + if (runDependencies > 0) { + dependenciesFulfilled = onload; + } else { + onload(); + } + }; + script.onerror = onerror; + script.src = Pointer_stringify(url); + document.body.appendChild(script); + }, + emscripten_set_main_loop: function(func, fps, simulateInfiniteLoop) { Module['noExitRuntime'] = true; diff --git a/system/include/emscripten/emscripten.h b/system/include/emscripten/emscripten.h index 1b9a8f25a90b1..430fbc1c7784b 100644 --- a/system/include/emscripten/emscripten.h +++ b/system/include/emscripten/emscripten.h @@ -45,8 +45,26 @@ extern "C" { extern void emscripten_run_script(const char *script); extern int emscripten_run_script_int(const char *script); extern char *emscripten_run_script_string(const char *script); // uses a single buffer - shared between calls! + +/* + * Asynchronously run a script, after a specified amount of + * time. + */ extern void emscripten_async_run_script(const char *script, int millis); +/* + * Asynchronously loads a script from a URL. + * + * This integrates with the run dependencies system, so your + * script can call addRunDependency multiple times, prepare + * various asynchronous tasks, and call removeRunDependency + * on them; when all are complete (or there were no run + * dependencies to begin with), onload is called. An example use + * for this is to load an asset module, that is, the output of the + * file packager. + */ +extern void emscripten_async_load_script(const char *script, void (*onload)(), void (*onerror)()); + /* * Set a C function as the main event loop. The JS environment * will call that function at a specified number of frames per diff --git a/tests/emscripten_api_browser2.cpp b/tests/emscripten_api_browser2.cpp new file mode 100644 index 0000000000000..569ed710e80fb --- /dev/null +++ b/tests/emscripten_api_browser2.cpp @@ -0,0 +1,53 @@ +#include +#include +#include + +#include + +int value = 0; + +extern "C" { + void set(int x) { + printf("set! %d\n", x); + value = x; + } +} + +void load2() { + printf("load2\n"); + + char buffer[10]; + memset(buffer, 0, 10); + FILE *f = fopen("file1.txt", "r"); + fread(buffer, 1, 5, f); + fclose(f); + assert(strcmp(buffer, "first") == 0); + + memset(buffer, 0, 10); + f = fopen("file2.txt", "r"); + fread(buffer, 1, 6, f); + fclose(f); + assert(strcmp(buffer, "second") == 0); + + int result = 1; + REPORT_RESULT(); +} +void error2() { + printf("fail2\n"); +} + +void load1() { + printf("load1\n"); + assert(value == 456); + emscripten_async_load_script("script2.js", load2, error2); +} +void error1() { + printf("fail1\n"); +} + +int main() { + emscripten_async_load_script("script1.js", load1, error1); + + return 1; +} + diff --git a/tests/test_browser.py b/tests/test_browser.py index e6fd6544cbee1..6a23b41c65e04 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -1230,6 +1230,17 @@ def test_gles2_emulation(self): def test_emscripten_api(self): self.btest('emscripten_api_browser.cpp', '1', args=['-s', '''EXPORTED_FUNCTIONS=['_main', '_third']''']) + def test_emscripten_api2(self): + open('script1.js', 'w').write(''' + Module._set(456); + ''') + + open('file1.txt', 'w').write('first'); + open('file2.txt', 'w').write('second'); + Popen([PYTHON, FILE_PACKAGER, 'test.data', '--preload', 'file1.txt', 'file2.txt'], stdout=open('script2.js', 'w')).communicate() + + self.btest('emscripten_api_browser2.cpp', '1', args=['-s', '''EXPORTED_FUNCTIONS=['_main', '_set']''']) + def test_emscripten_api_infloop(self): self.btest('emscripten_api_browser_infloop.cpp', '7') From 73579052dae38b12067b350be0a11a74090d0dfe Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 13 Sep 2013 16:15:15 -0700 Subject: [PATCH 25/26] refactor memory initializer code to use run dependencies --- emcc | 2 +- src/postamble.js | 21 +++++++++++++++++++++ src/preamble.js | 23 +---------------------- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/emcc b/emcc index 641e1d6a053ef..9a687bd8cd36b 100755 --- a/emcc +++ b/emcc @@ -1620,7 +1620,7 @@ try: temp_memfile = os.path.join(shared.EMSCRIPTEN_TEMP_DIR, os.path.basename(memfile)) if os.path.abspath(memfile) != os.path.abspath(memfile): shutil.copyfile(memfile, temp_memfile) - return 'loadMemoryInitializer("%s");' % os.path.basename(memfile) + return 'var memoryInitializer = "%s";' % os.path.basename(memfile) src = re.sub(shared.JS.memory_initializer_pattern, repl, open(final).read(), count=1) open(final + '.mem.js', 'w').write(src) final += '.mem.js' diff --git a/src/postamble.js b/src/postamble.js index 88986dea34603..8f585b86f276e 100644 --- a/src/postamble.js +++ b/src/postamble.js @@ -1,6 +1,27 @@ // === Auto-generated postamble setup entry stuff === +if (memoryInitializer) { + function applyData(data) { +#if USE_TYPED_ARRAYS == 2 + HEAPU8.set(data, STATIC_BASE); +#else + allocate(data, 'i8', ALLOC_NONE, STATIC_BASE); +#endif + } + if (ENVIRONMENT_IS_NODE || ENVIRONMENT_IS_SHELL) { + applyData(Module['readBinary'](memoryInitializer)); + } else { + addRunDependency('memory initializer'); + Browser.asyncLoad(memoryInitializer, function(data) { + applyData(data); + removeRunDependency('memory initializer'); + }, function(data) { + throw 'could not load memory initializer ' + memoryInitializer; + }); + } +} + function ExitStatus(status) { this.name = "ExitStatus"; this.message = "Program terminated with exit(" + status + ")"; diff --git a/src/preamble.js b/src/preamble.js index 579e3065861a8..02935f8f56f79 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -1020,28 +1020,7 @@ __ATEXIT__.push({ func: function() { PGOMonitor.dump() } }); addOnPreRun(function() { addRunDependency('pgo') }); #endif -function loadMemoryInitializer(filename) { - function applyData(data) { -#if USE_TYPED_ARRAYS == 2 - HEAPU8.set(data, STATIC_BASE); -#else - allocate(data, 'i8', ALLOC_NONE, STATIC_BASE); -#endif - } - - // always do this asynchronously, to keep shell and web as similar as possible - addOnPreRun(function() { - if (ENVIRONMENT_IS_NODE || ENVIRONMENT_IS_SHELL) { - applyData(Module['readBinary'](filename)); - } else { - Browser.asyncLoad(filename, function(data) { - applyData(data); - }, function(data) { - throw 'could not load memory initializer ' + filename; - }); - } - }); -} +var memoryInitializer = null; // === Body === From eda32b68ea4d04a7467fe1ece23813b8db5250a9 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 13 Sep 2013 17:20:59 -0700 Subject: [PATCH 26/26] RELOOPER_BUFFER_SIZE to change the relooper buffer size easily --- src/relooper/emscripten/glue.js | 12 ++++++------ src/settings.js | 2 ++ tools/shared.py | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/relooper/emscripten/glue.js b/src/relooper/emscripten/glue.js index 40ddabec6f35a..92c5050061b60 100644 --- a/src/relooper/emscripten/glue.js +++ b/src/relooper/emscripten/glue.js @@ -1,9 +1,9 @@ - var RBUFFER_SIZE = 20*1024*1024; + var RBUFFER_SIZE = RELOOPER_BUFFER_SIZE; var rbuffer = _malloc(RBUFFER_SIZE); _rl_set_output_buffer(rbuffer, RBUFFER_SIZE); - var TBUFFER_SIZE = 10*1024*1024; + var TBUFFER_SIZE = RELOOPER_BUFFER_SIZE/2; var tbuffer = _malloc(TBUFFER_SIZE); var VBUFFER_SIZE = 256; @@ -15,10 +15,10 @@ }, RelooperGlue['addBlock'] = function(text, branchVar) { assert(this.r); - assert(text.length+1 < TBUFFER_SIZE); + assert(text.length+1 < TBUFFER_SIZE, 'buffer too small, increase RELOOPER_BUFFER_SIZE'); writeStringToMemory(text, tbuffer); if (branchVar) { - assert(branchVar.length+1 < VBUFFER_SIZE); + assert(branchVar.length+1 < VBUFFER_SIZE, 'buffer too small, increase RELOOPER_BUFFER_SIZE'); writeStringToMemory(branchVar, vbuffer); } var b = _rl_new_block(tbuffer, branchVar ? vbuffer : 0); @@ -28,14 +28,14 @@ RelooperGlue['addBranch'] = function(from, to, condition, code) { assert(this.r); if (condition) { - assert(condition.length+1 < TBUFFER_SIZE/2); + assert(condition.length+1 < TBUFFER_SIZE/2, 'buffer too small, increase RELOOPER_BUFFER_SIZE'); writeStringToMemory(condition, tbuffer); condition = tbuffer; } else { condition = 0; // allow undefined, null, etc. as inputs } if (code) { - assert(code.length+1 < TBUFFER_SIZE/2); + assert(code.length+1 < TBUFFER_SIZE/2, 'buffer too small, increase RELOOPER_BUFFER_SIZE'); writeStringToMemory(code, tbuffer + TBUFFER_SIZE/2); code = tbuffer + TBUFFER_SIZE/2; } else { diff --git a/src/settings.js b/src/settings.js index e00f4e597dc70..15bca4db5251c 100644 --- a/src/settings.js +++ b/src/settings.js @@ -70,6 +70,8 @@ var MAX_SETJMPS = 20; // size of setjmp table allocated in each function invocat var MICRO_OPTS = 1; // Various micro-optimizations, like nativizing variables var RELOOP = 0; // Recreate js native loops from llvm data var RELOOPER = 'relooper.js'; // Loads the relooper from this path relative to compiler.js +var RELOOPER_BUFFER_SIZE = 20*1024*1024; // The internal relooper buffer size. Increase if you see assertions + // on OutputBuffer. var USE_TYPED_ARRAYS = 2; // Use typed arrays for the heap. See https://github.com/kripken/emscripten/wiki/Code-Generation-Modes/ // 0 means no typed arrays are used. This mode disallows LLVM optimizations diff --git a/tools/shared.py b/tools/shared.py index 853b064d11564..45551fec46bcb 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -304,7 +304,7 @@ def find_temp_directory(): # we re-check sanity when the settings are changed) # We also re-check sanity and clear the cache when the version changes -EMSCRIPTEN_VERSION = '1.5.7' +EMSCRIPTEN_VERSION = '1.5.8' def generate_sanity(): return EMSCRIPTEN_VERSION + '|' + get_llvm_target() + '|' + LLVM_ROOT