From ebdf753564c540e4047b5ed15b29a584bed26999 Mon Sep 17 00:00:00 2001 From: Prof Gra Date: Thu, 15 Dec 2016 20:52:52 +0100 Subject: [PATCH 01/74] Remove - Date: Sun, 18 Dec 2016 23:28:23 +0100 Subject: [PATCH 02/74] AUTHORS addition: Ch Gragnic --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index c17ac199be99e..3247bbcfaceac 100644 --- a/AUTHORS +++ b/AUTHORS @@ -237,3 +237,4 @@ a license to everyone to use it as detailed in LICENSE.) * Charles Vaughn (copyright owned by Tableau Software, Inc.) * Pierre Krieger * Jakob Stoklund Olesen +* Christophe Gragnic From f27001aeed5ad81425b8fa09c47b93c9e2e603ee Mon Sep 17 00:00:00 2001 From: junji hashimoto Date: Sat, 24 Dec 2016 09:18:33 +0900 Subject: [PATCH 03/74] Add ability to mount an existing filesystem (#4737) * Add PROXYFS --- src/library_proxyfs.js | 212 +++++++++++++++++++++++++++++++++++ src/modules.js | 1 + tests/test_other.py | 245 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 458 insertions(+) create mode 100644 src/library_proxyfs.js diff --git a/src/library_proxyfs.js b/src/library_proxyfs.js new file mode 100644 index 0000000000000..1fa0737fd8785 --- /dev/null +++ b/src/library_proxyfs.js @@ -0,0 +1,212 @@ +mergeInto(LibraryManager.library, { + $PROXYFS__deps: ['$FS', '$PATH'], + $PROXYFS: { + mount: function (mount) { + return PROXYFS.createNode(null, '/', mount.opts.fs.lstat(mount.opts.root).mode, 0); + }, + createNode: function (parent, name, mode, dev) { + if (!FS.isDir(mode) && !FS.isFile(mode) && !FS.isLink(mode)) { + throw new FS.ErrnoError(ERRNO_CODES.EINVAL); + } + var node = FS.createNode(parent, name, mode); + node.node_ops = PROXYFS.node_ops; + node.stream_ops = PROXYFS.stream_ops; + return node; + }, + realPath: function (node) { + var parts = []; + while (node.parent !== node) { + parts.push(node.name); + node = node.parent; + } + parts.push(node.mount.opts.root); + parts.reverse(); + return PATH.join.apply(null, parts); + }, + node_ops: { + getattr: function(node) { + var path = PROXYFS.realPath(node); + var stat; + try { + stat = node.mount.opts.fs.lstat(path); + } catch (e) { + if (!e.code) throw e; + throw new FS.ErrnoError(ERRNO_CODES[e.code]); + } + return { + dev: stat.dev, + ino: stat.ino, + mode: stat.mode, + nlink: stat.nlink, + uid: stat.uid, + gid: stat.gid, + rdev: stat.rdev, + size: stat.size, + atime: stat.atime, + mtime: stat.mtime, + ctime: stat.ctime, + blksize: stat.blksize, + blocks: stat.blocks + }; + }, + setattr: function(node, attr) { + var path = PROXYFS.realPath(node); + try { + if (attr.mode !== undefined) { + node.mount.opts.fs.chmod(path, attr.mode); + // update the common node structure mode as well + node.mode = attr.mode; + } + if (attr.timestamp !== undefined) { + var date = new Date(attr.timestamp); + node.mount.opts.fs.utime(path, date, date); + } + if (attr.size !== undefined) { + node.mount.opts.fs.truncate(path, attr.size); + } + } catch (e) { + if (!e.code) throw e; + throw new FS.ErrnoError(ERRNO_CODES[e.code]); + } + }, + lookup: function (parent, name) { + try { + var path = PATH.join2(PROXYFS.realPath(parent), name); + var mode = parent.mount.opts.fs.lstat(path).mode; + var node = PROXYFS.createNode(parent, name, mode); + return node; + } catch(e) { + if (!e.code) throw e; + throw new FS.ErrnoError(ERRNO_CODES[e.code]); + } + }, + mknod: function (parent, name, mode, dev) { + var node = PROXYFS.createNode(parent, name, mode, dev); + // create the backing node for this in the fs root as well + var path = PROXYFS.realPath(node); + try { + if (FS.isDir(node.mode)) { + node.mount.opts.fs.mkdir(path, node.mode); + } else { + node.mount.opts.fs.writeFile(path, '', { mode: node.mode }); + } + } catch (e) { + if (!e.code) throw e; + throw new FS.ErrnoError(ERRNO_CODES[e.code]); + } + return node; + }, + rename: function (oldNode, newDir, newName) { + var oldPath = PROXYFS.realPath(oldNode); + var newPath = PATH.join2(PROXYFS.realPath(newDir), newName); + try { + oldNode.mount.opts.fs.rename(oldPath, newPath); + } catch(e) { + if (!e.code) throw e; + throw new FS.ErrnoError(ERRNO_CODES[e.code]); + } + }, + unlink: function(parent, name) { + var path = PATH.join2(PROXYFS.realPath(parent), name); + try { + parent.mount.opts.fs.unlink(path); + } catch(e) { + if (!e.code) throw e; + throw new FS.ErrnoError(ERRNO_CODES[e.code]); + } + }, + rmdir: function(parent, name) { + var path = PATH.join2(PROXYFS.realPath(parent), name); + try { + parent.mount.opts.fs.rmdir(path); + } catch(e) { + if (!e.code) throw e; + throw new FS.ErrnoError(ERRNO_CODES[e.code]); + } + }, + readdir: function(node) { + var path = PROXYFS.realPath(node); + try { + return node.mount.opts.fs.readdir(path); + } catch(e) { + if (!e.code) throw e; + throw new FS.ErrnoError(ERRNO_CODES[e.code]); + } + }, + symlink: function(parent, newName, oldPath) { + var newPath = PATH.join2(PROXYFS.realPath(parent), newName); + try { + parent.mount.opts.fs.symlink(oldPath, newPath); + } catch(e) { + if (!e.code) throw e; + throw new FS.ErrnoError(ERRNO_CODES[e.code]); + } + }, + readlink: function(node) { + var path = PROXYFS.realPath(node); + try { + return node.mount.opts.fs.readlink(path); + } catch(e) { + if (!e.code) throw e; + throw new FS.ErrnoError(ERRNO_CODES[e.code]); + } + }, + }, + stream_ops: { + open: function (stream) { + var path = PROXYFS.realPath(stream.node); + try { + stream.nfd = stream.node.mount.opts.fs.open(path,stream.flags); + } catch(e) { + if (!e.code) throw e; + throw new FS.ErrnoError(ERRNO_CODES[e.code]); + } + }, + close: function (stream) { + try { + stream.node.mount.opts.fs.close(stream.nfd); + } catch(e) { + if (!e.code) throw e; + throw new FS.ErrnoError(ERRNO_CODES[e.code]); + } + }, + read: function (stream, buffer, offset, length, position) { + try { + return stream.node.mount.opts.fs.read(stream.nfd, buffer, offset, length, position); + } catch(e) { + if (!e.code) throw e; + throw new FS.ErrnoError(ERRNO_CODES[e.code]); + } + }, + write: function (stream, buffer, offset, length, position) { + try { + return stream.node.mount.opts.fs.write(stream.nfd, buffer, offset, length, position); + } catch(e) { + if (!e.code) throw e; + throw new FS.ErrnoError(ERRNO_CODES[e.code]); + } + }, + llseek: function (stream, offset, whence) { + var position = offset; + if (whence === 1) { // SEEK_CUR. + position += stream.position; + } else if (whence === 2) { // SEEK_END. + if (FS.isFile(stream.node.mode)) { + try { + var stat = stream.node.mount.opts.fs.fstat(stream.nfd); + position += stat.size; + } catch (e) { + throw new FS.ErrnoError(ERRNO_CODES[e.code]); + } + } + } + + if (position < 0) { + throw new FS.ErrnoError(ERRNO_CODES.EINVAL); + } + + return position; + } + } + } +}); diff --git a/src/modules.js b/src/modules.js index 927f12c8d48f2..1bb6d9eae6cd0 100644 --- a/src/modules.js +++ b/src/modules.js @@ -120,6 +120,7 @@ var LibraryManager = { libraries = libraries.concat([ 'library_idbfs.js', 'library_nodefs.js', + 'library_proxyfs.js', 'library_sockfs.js', 'library_workerfs.js', 'library_lz4.js', diff --git a/tests/test_other.py b/tests/test_other.py index ff804942d3a7f..5fba610c6cbb4 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -2669,6 +2669,251 @@ def test_fs_stream_proto(self): out = run_js('a.out.js', engine=engine, stderr=PIPE, full_output=True) self.assertContained('File size: 724', out) + def test_proxyfs(self): + # This test supposes that 3 different programs share the same directory and files. + # The same JS object is not used for each of them + # But 'require' function caches JS objects. + # If we just load same js-file multiple times like following code, + # these programs (m0,m1,m2) share the same JS object. + # + # var m0 = require('./proxyfs_test.js'); + # var m1 = require('./proxyfs_test.js'); + # var m2 = require('./proxyfs_test.js'); + # + # To separate js-objects for each of them, following 'require' use different js-files. + # + # var m0 = require('./proxyfs_test.js'); + # var m1 = require('./proxyfs_test1.js'); + # var m2 = require('./proxyfs_test2.js'); + # + open('proxyfs_test_main.js', 'w').write(r''' +var m0 = require('./proxyfs_test.js'); +var m1 = require('./proxyfs_test1.js'); +var m2 = require('./proxyfs_test2.js'); + +var section; +function print(str){ + process.stdout.write(section+":"+str+":"); +} + +m0.FS.mkdir('/working'); +m0.FS.mount(m0.PROXYFS,{root:'/',fs:m1.FS},'/working'); +m0.FS.mkdir('/working2'); +m0.FS.mount(m0.PROXYFS,{root:'/',fs:m2.FS},'/working2'); + +section = "child m1 reads and writes local file."; +print("m1 read embed"); +m1.ccall('myreade','number',[],[]); +print("m1 write");console.log(""); +m1.ccall('mywrite0','number',['number'],[1]); +print("m1 read"); +m1.ccall('myread0','number',[],[]); + + +section = "child m2 reads and writes local file."; +print("m2 read embed"); +m2.ccall('myreade','number',[],[]); +print("m2 write");console.log(""); +m2.ccall('mywrite0','number',['number'],[2]); +print("m2 read"); +m2.ccall('myread0','number',[],[]); + +section = "child m1 reads local file."; +print("m1 read"); +m1.ccall('myread0','number',[],[]); + +section = "parent m0 reads and writes local and children's file."; +print("m0 read embed"); +m0.ccall('myreade','number',[],[]); +print("m0 read m1"); +m0.ccall('myread1','number',[],[]); +print("m0 read m2"); +m0.ccall('myread2','number',[],[]); + +section = "m0,m1 and m2 verify local files."; +print("m0 write");console.log(""); +m0.ccall('mywrite0','number',['number'],[0]); +print("m0 read"); +m0.ccall('myread0','number',[],[]); +print("m1 read"); +m1.ccall('myread0','number',[],[]); +print("m2 read"); +m2.ccall('myread0','number',[],[]); + +print("m0 read embed"); +m0.ccall('myreade','number',[],[]); +print("m1 read embed"); +m1.ccall('myreade','number',[],[]); +print("m2 read embed"); +m2.ccall('myreade','number',[],[]); + +section = "parent m0 writes and reads children's files."; +print("m0 write m1");console.log(""); +m0.ccall('mywrite1','number',[],[]); +print("m0 read m1"); +m0.ccall('myread1','number',[],[]); +print("m0 write m2");console.log(""); +m0.ccall('mywrite2','number',[],[]); +print("m0 read m2"); +m0.ccall('myread2','number',[],[]); +print("m1 read"); +m1.ccall('myread0','number',[],[]); +print("m2 read"); +m2.ccall('myread0','number',[],[]); +print("m0 read m0"); +m0.ccall('myread0','number',[],[]); +''') + + open('proxyfs_pre.js', 'w').write(r''' +if (typeof Module === 'undefined') Module = {}; +Module["noInitialRun"]=true; +Module["noExitRuntime"]=true; +''') + + open('proxyfs_embed.txt', 'w').write(r'''test +''') + + open('proxyfs_test.c', 'w').write(r''' +#include + +int +mywrite1(){ + FILE* out = fopen("/working/hoge.txt","w"); + fprintf(out,"test1\n"); + fclose(out); + return 0; +} + +int +myread1(){ + FILE* in = fopen("/working/hoge.txt","r"); + char buf[1024]; + int len; + if(in==NULL) + printf("open failed\n"); + + while(! feof(in)){ + if(fgets(buf,sizeof(buf),in)==buf){ + printf("%s",buf); + } + } + fclose(in); + return 0; +} +int +mywrite2(){ + FILE* out = fopen("/working2/hoge.txt","w"); + fprintf(out,"test2\n"); + fclose(out); + return 0; +} + +int +myread2(){ + { + FILE* in = fopen("/working2/hoge.txt","r"); + char buf[1024]; + int len; + if(in==NULL) + printf("open failed\n"); + + while(! feof(in)){ + if(fgets(buf,sizeof(buf),in)==buf){ + printf("%s",buf); + } + } + fclose(in); + } + return 0; +} + +int +mywrite0(int i){ + FILE* out = fopen("hoge.txt","w"); + fprintf(out,"test0_%d\n",i); + fclose(out); + return 0; +} + +int +myread0(){ + { + FILE* in = fopen("hoge.txt","r"); + char buf[1024]; + int len; + if(in==NULL) + printf("open failed\n"); + + while(! feof(in)){ + if(fgets(buf,sizeof(buf),in)==buf){ + printf("%s",buf); + } + } + fclose(in); + } + return 0; +} + +int +myreade(){ + { + FILE* in = fopen("proxyfs_embed.txt","r"); + char buf[1024]; + int len; + if(in==NULL) + printf("open failed\n"); + + while(! feof(in)){ + if(fgets(buf,sizeof(buf),in)==buf){ + printf("%s",buf); + } + } + fclose(in); + } + return 0; +} +''') + + Popen([PYTHON, EMCC, + '-o', 'proxyfs_test.js', 'proxyfs_test.c', + '--embed-file', 'proxyfs_embed.txt', '--pre-js', 'proxyfs_pre.js', + '-s', 'MAIN_MODULE=1']).communicate() + # Following shutil.copyfile just prevent 'require' of node.js from caching js-object. + # See https://nodejs.org/api/modules.html + shutil.copyfile('proxyfs_test.js', 'proxyfs_test1.js') + shutil.copyfile('proxyfs_test.js', 'proxyfs_test2.js') + out = run_js('proxyfs_test_main.js') + section="child m1 reads and writes local file." + self.assertContained(section+":m1 read embed:test", out) + self.assertContained(section+":m1 write:", out) + self.assertContained(section+":m1 read:test0_1", out) + section="child m2 reads and writes local file." + self.assertContained(section+":m2 read embed:test", out) + self.assertContained(section+":m2 write:", out) + self.assertContained(section+":m2 read:test0_2", out) + section="child m1 reads local file." + self.assertContained(section+":m1 read:test0_1", out) + section="parent m0 reads and writes local and children's file." + self.assertContained(section+":m0 read embed:test", out) + self.assertContained(section+":m0 read m1:test0_1", out) + self.assertContained(section+":m0 read m2:test0_2", out) + section="m0,m1 and m2 verify local files." + self.assertContained(section+":m0 write:", out) + self.assertContained(section+":m0 read:test0_0", out) + self.assertContained(section+":m1 read:test0_1", out) + self.assertContained(section+":m2 read:test0_2", out) + self.assertContained(section+":m0 read embed:test", out) + self.assertContained(section+":m1 read embed:test", out) + self.assertContained(section+":m2 read embed:test", out) + section="parent m0 writes and reads children's files." + self.assertContained(section+":m0 write m1:", out) + self.assertContained(section+":m0 read m1:test1", out) + self.assertContained(section+":m0 write m2:", out) + self.assertContained(section+":m0 read m2:test2", out) + self.assertContained(section+":m1 read:test1", out) + self.assertContained(section+":m2 read:test2", out) + self.assertContained(section+":m0 read m0:test0_0", out) + def check_simd(self, expected_simds, expected_out): if SPIDERMONKEY_ENGINE in JS_ENGINES: out = run_js('a.out.js', engine=SPIDERMONKEY_ENGINE, stderr=PIPE, full_output=True) From 38e893c77ae7d3da294b94eefa11a7757a85a681 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 23 Dec 2016 16:19:06 -0800 Subject: [PATCH 04/74] add test for ccall typed array input, and an assert on valid array input (#4797) * add test for ccall typed array input, and an assert on valid array input * clarify typed arrays in ccall docs * mention ccall array type in interacting with code docs --- site/source/docs/api_reference/preamble.js.rst | 8 +++++--- .../Interacting-with-code.rst | 5 +++-- src/preamble.js | 5 ++++- tests/core/test_ccall.out | 2 ++ tests/test_core.py | 3 ++- 5 files changed, 16 insertions(+), 7 deletions(-) diff --git a/site/source/docs/api_reference/preamble.js.rst b/site/source/docs/api_reference/preamble.js.rst index 2b5d0ce92fa7b..17c9105f65a92 100644 --- a/site/source/docs/api_reference/preamble.js.rst +++ b/site/source/docs/api_reference/preamble.js.rst @@ -23,7 +23,9 @@ Calling compiled C functions from JavaScript Call a compiled C function from JavaScript. The function executes a compiled C function from JavaScript and returns the result. C++ name mangling means that "normal" C++ functions cannot be called; the function must either be defined in a **.c** file or be a C++ function defined with ``extern "C"``. - + + ``returnType`` and ``argTypes`` let you specify the types of parameters and the return value. The possible types are ``"number"``, ``"string"`` or ``"array"``, which correspond to the appropriate JavaScript types. Use ``"number"`` for any numeric type or C pointer, ``string`` for C ``char*`` that represent strings, and ``"array"`` for JavaScript arrays and typed arrays; for typed arrays, it must be a Uint8Array or Int8Array. + .. code-block:: javascript // Call C from JavaScript @@ -51,11 +53,11 @@ Calling compiled C functions from JavaScript :param ident: The name of the C function to be called. - :param returnType: The return type of the function. This can be ``"number"``, ``"string"`` or ``"array"``, which correspond to the appropriate JavaScript types (use ``"number"`` for any C pointer, and ``"array"`` for JavaScript arrays and typed arrays; note that arrays are 8-bit), or for a void function it can be ``null`` (note: the JavaScript ``null`` value, not a string containing the word "null"). + :param returnType: The return type of the function. Note that ``array`` is not supported as there is no way for us to know the length of the array. For a void function this can be ``null`` (note: the JavaScript ``null`` value, not a string containing the word "null"). .. note:: 64-bit integers become two 32-bit parameters, for the low and high bits (since 64-bit integers cannot be represented in JavaScript numbers). - :param argTypes: An array of the types of arguments for the function (if there are no arguments, this can be omitted). Types are as in ``returnType``, except that ``array`` is not supported as there is no way for us to know the length of the array). + :param argTypes: An array of the types of arguments for the function (if there are no arguments, this can be omitted). :param args: An array of the arguments to the function, as native JavaScript values (as in ``returnType``). Note that string arguments will be stored on the stack (the JavaScript string will become a C string on the stack). :returns: The result of the function call as a native JavaScript value (as in ``returnType``). :opts: An optional options object. It can contain the following properties: diff --git a/site/source/docs/porting/connecting_cpp_and_javascript/Interacting-with-code.rst b/site/source/docs/porting/connecting_cpp_and_javascript/Interacting-with-code.rst index efef3e7ca6d7e..160ac96666b2f 100644 --- a/site/source/docs/porting/connecting_cpp_and_javascript/Interacting-with-code.rst +++ b/site/source/docs/porting/connecting_cpp_and_javascript/Interacting-with-code.rst @@ -88,8 +88,9 @@ following JavaScript:: The first parameter is the name of the function to be wrapped, the second is the return type of the function (or a JavaScript `null` value if there isn't one), and the third is an array of parameter types (which may be omitted if there are no parameters). The types are -native JavaScript types, "number" (for a C integer, float, or general -pointer) or "string" (for a C ``char*`` that represents a string). +"number" (for a JavaScript number corresponding to a C integer, float, or general +pointer), "string" (for a JavaScript string that corresponds to a C ``char*`` that represents a string) or +"array" (for a JavaScript array or typed array that corresponds to a C array; for typed arrays, it must be a Uint8Array or Int8Array). You can run this yourself by first opening the generated page **function.html** in a web browser (nothing will happen on page diff --git a/src/preamble.js b/src/preamble.js index c68b11266414a..4cfe89064e477 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -1713,7 +1713,10 @@ function writeStringToMemory(string, buffer, dontAddNull) { {{{ maybeExport('writeStringToMemory') }}} function writeArrayToMemory(array, buffer) { - HEAP8.set(array, buffer); +#if ASSERTIONS + assert(array.length >= 0, 'writeArrayToMemory array must have a length (should be an array or typed array)') +#endif + HEAP8.set(array, buffer); } {{{ maybeExport('writeArrayToMemory') }}} diff --git a/tests/core/test_ccall.out b/tests/core/test_ccall.out index 85b5162b065df..4dca7681bbb7e 100644 --- a/tests/core/test_ccall.out +++ b/tests/core/test_ccall.out @@ -10,6 +10,8 @@ cheez undefined arr-ay undefined +arr-ay +undefined more number,10 650 diff --git a/tests/test_core.py b/tests/test_core.py index 9c658f84088d5..94e5711a27958 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -5766,7 +5766,8 @@ def process(filename): ret = ccall('print_int', null, ['number'], [12]); Module.print(typeof ret); ret = ccall('print_float', null, ['number'], [14.56]); Module.print(typeof ret); ret = ccall('print_string', null, ['string'], ["cheez"]); Module.print(typeof ret); - ret = ccall('print_string', null, ['array'], [[97, 114, 114, 45, 97, 121, 0]]); Module.print(typeof ret); + ret = ccall('print_string', null, ['array'], [[97, 114, 114, 45, 97, 121, 0]]); Module.print(typeof ret); // JS array + ret = ccall('print_string', null, ['array'], [new Uint8Array([97, 114, 114, 45, 97, 121, 0])]); Module.print(typeof ret); // typed array ret = ccall('multi', 'number', ['number', 'number', 'number', 'string'], [2, 1.4, 3, 'more']); Module.print([typeof ret, ret].join(',')); var p = ccall('malloc', 'pointer', ['number'], [4]); setValue(p, 650, 'i32'); From 786f2fed0b825a0d4404dc06f8f38a85c836ede3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Tue, 31 May 2016 22:19:29 +0300 Subject: [PATCH 05/74] Experimenting with how emscripten_fetch api could look like. --- src/library_fetch.js | 41 ++++++++++ system/include/emscripten/fetch.h | 111 ++++++++++++++++++++++++++ system/lib/fetch/emscripten_fetch.cpp | 18 +++++ tests/fetch/cached_xhr.cpp | 50 ++++++++++++ tests/fetch/stream_file.cpp | 27 +++++++ tests/fetch/to_memory.cpp | 26 ++++++ tests/test_browser.py | 10 +++ 7 files changed, 283 insertions(+) create mode 100644 src/library_fetch.js create mode 100644 system/include/emscripten/fetch.h create mode 100644 system/lib/fetch/emscripten_fetch.cpp create mode 100644 tests/fetch/cached_xhr.cpp create mode 100644 tests/fetch/stream_file.cpp create mode 100644 tests/fetch/to_memory.cpp diff --git a/src/library_fetch.js b/src/library_fetch.js new file mode 100644 index 0000000000000..db79f7a8839ef --- /dev/null +++ b/src/library_fetch.js @@ -0,0 +1,41 @@ +var LibraryFetch = { + $Fetch: { + xhrs: [], + }, + + emscripten_fetch__deps: ['$Fetch'], + emscripten_fetch: function(fetch_attr, url) { + var xhr = new XMLHttpRequest(); + var requestType = Pointer_stringify(fetch_attr); + var url_ = Pointer_stringify(url); + xhr.responseType = 'arraybuffer'; + console.log('xhr.open(requestType=' + requestType + ', url: ' + url_ +')'); + xhr.open(requestType, url_, true); + Fetch.xhrs.push(xhr); + var id = Fetch.xhrs.length; + var data = null; // TODO: Support user to pass data to request. + // TODO: Support specifying custom headers to the request. + + var onsuccess = {{{ makeGetValue('fetch_attr', 36/*TODO:structs_info.jsonify this*/, 'i32') }}}; + var onerror = {{{ makeGetValue('fetch_attr', 40/*TODO:structs_info.jsonify this*/, 'i32') }}}; + var onprogress = {{{ makeGetValue('fetch_attr', 44/*TODO:structs_info.jsonify this*/, 'i32') }}}; + + xhr.onload = function(e) { + if (onsuccess) Runtime.dynCall('vi', onsuccess, [0]); + } + xhr.onerror = function(e) { + if (onerror) Runtime.dynCall('vi', onerror, [0]); // TODO: pass error reason + } + xhr.onprogress = function(e) { + if (onprogress) Runtime.dynCall('vi', onprogress, [0]); + } + console.log('xhr.send(data=' + data + ')'); + try { + xhr.send(data); + } catch(e) { + if (onerror) Runtime.dynCall('vi', onerror, [0]); // TODO: pass error reason + } + } +}; + +mergeInto(LibraryManager.library, LibraryFetch); diff --git a/system/include/emscripten/fetch.h b/system/include/emscripten/fetch.h new file mode 100644 index 0000000000000..76a1d59ed75aa --- /dev/null +++ b/system/include/emscripten/fetch.h @@ -0,0 +1,111 @@ +#ifndef __emscripten_fetch_h__ +#define __emscripten_fetch_h__ + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct emscripten_fetch_t +{ + // Unique identifier for this fetch in progress. + unsigned int id; + + // Custom data that can be tagged along the process. + void *userData; + + // The remote URL that is being downloaded. + const char *url; + + // In onsuccess() handler: + // - If the EMSCRIPTEN_FETCH_LOAD_TO_MEMORY attribute was specified for the transfer, this points to the + // body of the downloaded data. Otherwise this will be null. + // In onprogress() handler: + // - If the EMSCRIPTEN_FETCH_STREAM_DATA attribute was specified for the transfer, this points to a partial + // chunk of bytes related to the transfer. Otherwise this will be null. + const char *data; + + // Specifies the length of the above data block in bytes. When the download finishes, this field will be valid even if + // EMSCRIPTEN_FETCH_LOAD_TO_MEMORY was not specified. + uint64_t numBytes; + + // If EMSCRIPTEN_FETCH_STREAM_DATA is being performed, this indicates the byte offset from the start of the stream + // that the data block specifies. (for onprogress() streaming XHR transfer, the number of bytes downloaded so far before this chunk) + uint64_t dataOffset; + + // Specifies the total number of bytes that the response body will be. + uint64_t totalBytes; +}; + +// Emscripten fetch attributes: +// If passed, the body of the request will be present in full in the onsuccess() handler. +#define EMSCRIPTEN_FETCH_LOAD_TO_MEMORY 1 + +// If passed, the intermediate streamed bytes will be passed in to the onprogress() handler. If not specified, the +// onprogress() handler will still be called, but without data bytes. +#define EMSCRIPTEN_FETCH_STREAM_DATA 2 + +// If passed, the final download will be stored in IndexedDB. If not specified, the file will only reside in browser memory. +#define EMSCRIPTEN_FETCH_PERSIST_FILE 4 + +// If the file already exists in IndexedDB, it is returned without redownload. If a partial transfer exists in IndexedDB, +// the download will resume from where it left off and run to completion. +// EMSCRIPTEN_FETCH_APPEND, EMSCRIPTEN_FETCH_REPLACE and EMSCRIPTEN_FETCH_NO_DOWNLOAD are mutually exclusive. +#define EMSCRIPTEN_FETCH_APPEND 8 + +// If the file already exists in IndexedDB, the old file will be deleted and a new download is started. +// EMSCRIPTEN_FETCH_APPEND, EMSCRIPTEN_FETCH_REPLACE and EMSCRIPTEN_FETCH_NO_DOWNLOAD are mutually exclusive. +#define EMSCRIPTEN_FETCH_REPLACE 16 + +// If specified, the file will only be looked up in IndexedDB, but if it does not exist, it is not attempted to be downloaded +// over the network but an error is raised. +// EMSCRIPTEN_FETCH_APPEND, EMSCRIPTEN_FETCH_REPLACE and EMSCRIPTEN_FETCH_NO_DOWNLOAD are mutually exclusive. +#define EMSCRIPTEN_FETCH_NO_DOWNLOAD 32 + +// Specifies the parameters for a newly initiated fetch operation. +struct emscripten_fetch_attr_t +{ + // 'POST', 'GET', etc. + char requestType[32]; + + // Custom data that can be tagged along the process. + void *userData; + + void (*onsuccess)(emscripten_fetch_t *fetch); + void (*onerror)(emscripten_fetch_t *fetch); + void (*onprogress)(emscripten_fetch_t *fetch); + + // Specifies the destination path in IndexedDB where to store the downloaded content body. If this is empty, the transfer + // is not stored to IndexedDB at all. + char destinationPath[PATH_MAX]; + + // EMSCRIPTEN_FETCH_* attributes + uint32_t attributes; +}; + +// Clears the fields of an emscripten_fetch_attr_t structure to their default values in a future-compatible manner. +void emscripten_fetch_attr_init(emscripten_fetch_attr_t *fetch_attr); + +// Initiates a new Emscripten fetch operation, which downloads data from the given URL or from IndexedDB database. +emscripten_fetch_t *emscripten_fetch(emscripten_fetch_attr_t *fetch_attr, const char *url); + +// Synchronously blocks to wait for the given fetch operation to complete. This operation is not allowed in the main browser +// thread, in which case it will return EMSCRIPTEN_RESULT_NOT_SUPPORTED. Pass timeoutMSecs=infinite to wait indefinitely. If +// the wait times out, the return value will be EMSCRIPTEN_RESULT_TIMEOUT. +// The onsuccess()/onerror()/onprogress() handlers will be called in the calling thread from within this function before +// this function returns. +EMSCRIPTEN_RESULT emscripten_fetch_wait(emscripten_fetch_t *fetch, double timeoutMsecs); + +// Closes a finished or an executing fetch operation and frees up all memory. If the fetch operation was still executing, the +// onerror() handler will be called in the calling thread before this function returns. +EMSCRIPTEN_RESULT emscripten_fetch_close(emscripten_fetch_t *fetch); + +#ifdef __cplusplus +} +#endif + +// ~__emscripten_fetch_h__ +#endif diff --git a/system/lib/fetch/emscripten_fetch.cpp b/system/lib/fetch/emscripten_fetch.cpp new file mode 100644 index 0000000000000..0b072ad1cd508 --- /dev/null +++ b/system/lib/fetch/emscripten_fetch.cpp @@ -0,0 +1,18 @@ +#include +#include +#include + +void emscripten_fetch_attr_init(emscripten_fetch_attr_t *fetch_attr) +{ + memset(fetch_attr, 0, sizeof(emscripten_fetch_attr_t)); +} + +EMSCRIPTEN_RESULT emscripten_fetch_wait(emscripten_fetch_t *fetch, double timeoutMsecs) +{ + return EMSCRIPTEN_RESULT_SUCCESS; +} + +EMSCRIPTEN_RESULT emscripten_fetch_close(emscripten_fetch_t *fetch) +{ + return EMSCRIPTEN_RESULT_SUCCESS; +} diff --git a/tests/fetch/cached_xhr.cpp b/tests/fetch/cached_xhr.cpp new file mode 100644 index 0000000000000..a6a4ff4660ebe --- /dev/null +++ b/tests/fetch/cached_xhr.cpp @@ -0,0 +1,50 @@ +#include +#include +#include +#include +#include + +// Fetch file without XHRing. +void fetchFromIndexedDB() +{ + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestType, "GET"); + attr.onsuccess = [](emscripten_fetch_t *fetch) { + assert(fetch->numBytes == fetch->totalBytes); + assert(fetch->data != 0); + printf("Finished downloading %llu bytes\n", fetch->numBytes); + emscripten_fetch_close(fetch); + }; + attr.onprogress = [](emscripten_fetch_t *fetch) { + printf("Downloading.. %.2f%% complete.\n", (fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes); + }; + attr.attributes = EMSCRIPTEN_FETCH_APPEND | EMSCRIPTEN_FETCH_NO_DOWNLOAD; + emscripten_fetch_t *fetch = emscripten_fetch(&attr, "myfile.dat"); + + if (synchronous) { + emscripten_fetch_wait(fetch, INFINITY); + } +} + +// XHR and store to cache. +int main() +{ + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestType, "GET"); + attr.onsuccess = [](emscripten_fetch_t *fetch) { + assert(fetch->numBytes == fetch->totalBytes); + assert(fetch->data != 0); + printf("Finished downloading %llu bytes\n", fetch->numBytes); + emscripten_fetch_close(fetch); + + // Test that the file now exists: + fetchFromIndexedDB(); + }; + attr.onprogress = [](emscripten_fetch_t *fetch) { + printf("Downloading.. %.2f%% complete.\n", (fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes); + }; + attr.attributes = EMSCRIPTEN_FETCH_APPEND | EMSCRIPTEN_FETCH_LOAD_TO_MEMORY | EMSCRIPTEN_FETCH_PERSIST_FILE; + emscripten_fetch_t *fetch = emscripten_fetch(&attr, "myfile.dat"); +} diff --git a/tests/fetch/stream_file.cpp b/tests/fetch/stream_file.cpp new file mode 100644 index 0000000000000..a1ba758d23955 --- /dev/null +++ b/tests/fetch/stream_file.cpp @@ -0,0 +1,27 @@ +#include +#include +#include +#include +#include + +int main() +{ + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestType, "GET"); + attr.onsuccess = [](emscripten_fetch_t *fetch) { + assert(fetch->data == 0); // The data was streamed via onprogress, no bytes available here. + assert(fetch->numBytes == 0); + assert(fetch->totalBytes > 0); + printf("Finished downloading %llu bytes\n", fetch->totalBytes); + emscripten_fetch_close(fetch); + }; + attr.onprogress = [](emscripten_fetch_t *fetch) { + assert(fetch->data != 0); + assert(fetch->numBytes > 0); + assert(fetch->numBytes <= fetch->totalBytes); + printf("Downloading.. %.2f%% complete.\n", (fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes); + }; + attr.attributes = EMSCRIPTEN_FETCH_APPEND | EMSCRIPTEN_FETCH_STREAM_DATA; + emscripten_fetch_t *fetch = emscripten_fetch(&attr, "myfile.dat"); +} diff --git a/tests/fetch/to_memory.cpp b/tests/fetch/to_memory.cpp new file mode 100644 index 0000000000000..2e6a124759e26 --- /dev/null +++ b/tests/fetch/to_memory.cpp @@ -0,0 +1,26 @@ +#include +#include +#include +#include +#include + +int main() +{ + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestType, "GET"); + attr.onsuccess = [](emscripten_fetch_t *fetch) { + assert(fetch->numBytes == fetch->totalBytes); + assert(fetch->data != 0); + printf("Finished downloading %llu bytes\n", fetch->numBytes); + emscripten_fetch_close(fetch); + }; + attr.onprogress = [](emscripten_fetch_t *fetch) { + printf("Downloading.. %.2f%% complete.\n", (fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes); + }; + attr.onerror = [](emscripten_fetch_t *fetch) { + printf("Download failed!\n"); + }; + attr.attributes = EMSCRIPTEN_FETCH_APPEND | EMSCRIPTEN_FETCH_LOAD_TO_MEMORY; + emscripten_fetch_t *fetch = emscripten_fetch(&attr, "myfile.dat"); +} diff --git a/tests/test_browser.py b/tests/test_browser.py index 11e42c0b07110..829ac62665088 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -3292,3 +3292,13 @@ def test_webgl_offscreen_canvas_in_mainthread_after_pthread(self): # In this build mode, the -s TOTAL_MEMORY=xxx option will be ignored. def test_preallocated_heap(self): self.btest('test_preallocated_heap.cpp', expected='1', args=['-s', 'TOTAL_MEMORY='+str(16*1024*1024), '-s', 'ABORTING_MALLOC=0', '--shell-file', path_from_root('tests', 'test_preallocated_heap_shell.html')]) + + def test_fetch_to_memory(self): + # TODO: emscripten_fetch.cpp and library_fetch.js pulled in as system libs + self.btest('fetch/to_memory.cpp', expected='0', args=[path_from_root('system/lib/fetch/emscripten_fetch.cpp'), '--std=c++11', '--js-library', 'src/library_fetch.js']) + + def test_fetch_stream_file(self): + self.btest('fetch/stream_file.cpp', expected='0', args=[path_from_root('system/lib/fetch/stream_file.cpp'), '--std=c++11', '--js-library', 'src/library_fetch.js']) + + def test_fetch_cached_xhr(self): + self.btest('fetch/cached_xhr.cpp', expected='0', args=[path_from_root('system/lib/fetch/cached_xhr.cpp'), '--std=c++11', '--js-library', 'src/library_fetch.js']) From 66cfb4e945c78468c539928e379b3ea83fc14afc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Wed, 1 Jun 2016 20:18:01 +0300 Subject: [PATCH 06/74] More basic fetch functionality. --- src/library_fetch.js | 43 +++++++++++++++++++++++++++++++++++---- tests/fetch/to_memory.cpp | 28 +++++++++++++++++++++++-- tests/test_browser.py | 7 ++++--- 3 files changed, 69 insertions(+), 9 deletions(-) diff --git a/src/library_fetch.js b/src/library_fetch.js index db79f7a8839ef..b4aa181af3b3e 100644 --- a/src/library_fetch.js +++ b/src/library_fetch.js @@ -5,6 +5,8 @@ var LibraryFetch = { emscripten_fetch__deps: ['$Fetch'], emscripten_fetch: function(fetch_attr, url) { + Module['noExitRuntime'] = true; + var xhr = new XMLHttpRequest(); var requestType = Pointer_stringify(fetch_attr); var url_ = Pointer_stringify(url); @@ -12,7 +14,14 @@ var LibraryFetch = { console.log('xhr.open(requestType=' + requestType + ', url: ' + url_ +')'); xhr.open(requestType, url_, true); Fetch.xhrs.push(xhr); + var fetch = _malloc(1000 /*TODO:structs_info.sizeof(fetch)*/); + var urlString = _malloc(url_.length /*TODO*/); + {{{ makeSetValue('fetch', 8/*TODO:jsonify*/, 'urlString', 'i32')}}}; + writeStringToMemory(url_, urlString); var id = Fetch.xhrs.length; + {{{ makeSetValue('fetch', 0/*TODO:jsonify*/, 'id', 'i32')}}}; + var userData = {{{ makeGetValue('fetch_attr', 32/*TODO:structs_info.jsonify this*/, 'i32') }}}; + {{{ makeSetValue('fetch', 4/*TODO:jsonify*/, 'userData', 'i32')}}}; var data = null; // TODO: Support user to pass data to request. // TODO: Support specifying custom headers to the request. @@ -21,19 +30,45 @@ var LibraryFetch = { var onprogress = {{{ makeGetValue('fetch_attr', 44/*TODO:structs_info.jsonify this*/, 'i32') }}}; xhr.onload = function(e) { - if (onsuccess) Runtime.dynCall('vi', onsuccess, [0]); + var len = xhr.response.byteLength; + var ptr = _malloc(len); // TODO: make emscripten_fetch_close() free() this data. + xhrRequest = xhr; + /* + console.log('xhr.response.byteLength: ' + xhr.response.byteLength); + var ta32 = new Uint32Array(xhr.response, 0, xhr.response.byteLength >> 2); + console.log('ta32.length: ' + ta32.length); + for (var i = 0; i < ta32.length; ++i) { + {{{ makeSetValue('ptr', 'i', 'ta32[i]', 'i32')}}}; + } + if ((xhr.response.byteLength & 3) != 0) { + + } + */ + HEAPU8.set(new Uint8Array(xhr.response), ptr); + {{{ makeSetValue('fetch', 12/*TODO:jsonify*/, 'ptr', 'i64')}}}; + {{{ makeSetValue('fetch', 16/*TODO:jsonify*/, 'len', 'i64')}}}; + {{{ makeSetValue('fetch', 24/*TODO:jsonify*/, '0', 'i64')}}}; + {{{ makeSetValue('fetch', 32/*TODO:jsonify*/, 'len', 'i64')}}}; + if (xhr.status == 200 || xhr.status == 0) { + if (onsuccess) Runtime.dynCall('vi', onsuccess, [fetch]); + } else { + if (onerror) Runtime.dynCall('vi', onerror, [fetch]); // TODO: pass error reason + } } xhr.onerror = function(e) { - if (onerror) Runtime.dynCall('vi', onerror, [0]); // TODO: pass error reason + if (onerror) Runtime.dynCall('vi', onerror, [fetch]); // TODO: pass error reason } xhr.onprogress = function(e) { - if (onprogress) Runtime.dynCall('vi', onprogress, [0]); + {{{ makeSetValue('fetch', 16/*TODO:jsonify*/, '0', 'i64')}}}; + {{{ makeSetValue('fetch', 24/*TODO:jsonify*/, 'e.loaded', 'i64')}}}; + {{{ makeSetValue('fetch', 32/*TODO:jsonify*/, 'e.total', 'i64')}}}; + if (onprogress) Runtime.dynCall('vi', onprogress, [fetch]); } console.log('xhr.send(data=' + data + ')'); try { xhr.send(data); } catch(e) { - if (onerror) Runtime.dynCall('vi', onerror, [0]); // TODO: pass error reason + if (onerror) Runtime.dynCall('vi', onerror, [fetch]); // TODO: pass error reason } } }; diff --git a/tests/fetch/to_memory.cpp b/tests/fetch/to_memory.cpp index 2e6a124759e26..a017637f94259 100644 --- a/tests/fetch/to_memory.cpp +++ b/tests/fetch/to_memory.cpp @@ -9,18 +9,42 @@ int main() emscripten_fetch_attr_t attr; emscripten_fetch_attr_init(&attr); strcpy(attr.requestType, "GET"); + attr.userData = (void*)0x12345678; attr.onsuccess = [](emscripten_fetch_t *fetch) { + assert(fetch); + printf("Finished downloading %llu bytes\n", fetch->numBytes); + assert(fetch->url); + assert(!strcmp(fetch->url, "gears.png")); + assert(fetch->id != 0); + assert((uintptr_t)fetch->userData == 0x12345678); + assert(fetch->totalBytes == 6407); assert(fetch->numBytes == fetch->totalBytes); assert(fetch->data != 0); - printf("Finished downloading %llu bytes\n", fetch->numBytes); + // Compute rudimentary checksum of data + uint8_t checksum = 0; + for(int i = 0; i < fetch->numBytes; ++i) + checksum ^= fetch->data[i]; + printf("Data checksum: %02X\n", checksum); + assert(checksum == 0x08); emscripten_fetch_close(fetch); }; attr.onprogress = [](emscripten_fetch_t *fetch) { + assert(fetch); printf("Downloading.. %.2f%% complete.\n", (fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes); + assert(fetch->url); + assert(!strcmp(fetch->url, "gears.png")); + assert(fetch->id != 0); + assert((uintptr_t)fetch->userData == 0x12345678); }; attr.onerror = [](emscripten_fetch_t *fetch) { printf("Download failed!\n"); + assert(false); // The download is not supposed to fail in this test. + + assert(fetch); + assert(fetch->id != 0); + assert(!strcmp(fetch->url, "gears.png")); + assert((uintptr_t)fetch->userData == 0x12345678); }; attr.attributes = EMSCRIPTEN_FETCH_APPEND | EMSCRIPTEN_FETCH_LOAD_TO_MEMORY; - emscripten_fetch_t *fetch = emscripten_fetch(&attr, "myfile.dat"); + emscripten_fetch_t *fetch = emscripten_fetch(&attr, "gears.png"); } diff --git a/tests/test_browser.py b/tests/test_browser.py index 829ac62665088..0f9202647359e 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -3294,11 +3294,12 @@ def test_preallocated_heap(self): self.btest('test_preallocated_heap.cpp', expected='1', args=['-s', 'TOTAL_MEMORY='+str(16*1024*1024), '-s', 'ABORTING_MALLOC=0', '--shell-file', path_from_root('tests', 'test_preallocated_heap_shell.html')]) def test_fetch_to_memory(self): + shutil.copyfile(path_from_root('tests', 'gears.png'), os.path.join(self.get_dir(), 'gears.png')) # TODO: emscripten_fetch.cpp and library_fetch.js pulled in as system libs - self.btest('fetch/to_memory.cpp', expected='0', args=[path_from_root('system/lib/fetch/emscripten_fetch.cpp'), '--std=c++11', '--js-library', 'src/library_fetch.js']) + self.btest('fetch/to_memory.cpp', expected='0', args=[path_from_root('system/lib/fetch/emscripten_fetch.cpp'), '--std=c++11', '--js-library', path_from_root('src/library_fetch.js')]) def test_fetch_stream_file(self): - self.btest('fetch/stream_file.cpp', expected='0', args=[path_from_root('system/lib/fetch/stream_file.cpp'), '--std=c++11', '--js-library', 'src/library_fetch.js']) + self.btest('fetch/stream_file.cpp', expected='0', args=[path_from_root('system/lib/fetch/stream_file.cpp'), '--std=c++11', '--js-library', path_from_root('src/library_fetch.js')]) def test_fetch_cached_xhr(self): - self.btest('fetch/cached_xhr.cpp', expected='0', args=[path_from_root('system/lib/fetch/cached_xhr.cpp'), '--std=c++11', '--js-library', 'src/library_fetch.js']) + self.btest('fetch/cached_xhr.cpp', expected='0', args=[path_from_root('system/lib/fetch/cached_xhr.cpp'), '--std=c++11', '--js-library', path_from_root('src/library_fetch.js')]) From 9c310811c79c91543e283513b392e808b6e931e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Tue, 7 Jun 2016 17:51:54 +0300 Subject: [PATCH 07/74] Fetch update --- src/library_fetch.js | 85 ++++++++++++++++++++++++------- src/settings.js | 3 ++ system/include/emscripten/fetch.h | 60 +++++++++++++++++++--- tests/fetch/cached_xhr.cpp | 12 ++--- tests/fetch/to_memory.cpp | 45 ++++++++++++++-- tests/test_browser.py | 14 +++-- 6 files changed, 178 insertions(+), 41 deletions(-) diff --git a/src/library_fetch.js b/src/library_fetch.js index b4aa181af3b3e..c483f4a50347d 100644 --- a/src/library_fetch.js +++ b/src/library_fetch.js @@ -3,18 +3,52 @@ var LibraryFetch = { xhrs: [], }, - emscripten_fetch__deps: ['$Fetch'], - emscripten_fetch: function(fetch_attr, url) { - Module['noExitRuntime'] = true; - + _emscripten_fetch_xhr: function(fetch_attr, url) { var xhr = new XMLHttpRequest(); - var requestType = Pointer_stringify(fetch_attr); + var requestMethod = Pointer_stringify(fetch_attr); var url_ = Pointer_stringify(url); xhr.responseType = 'arraybuffer'; - console.log('xhr.open(requestType=' + requestType + ', url: ' + url_ +')'); - xhr.open(requestType, url_, true); + xhr.timeout = {{{ makeGetValue('fetch_attr', 52/*TODO*/, 'i32') }}}; + xhr.withCredentials = !!({{{ makeGetValue('fetch_attr', 56/*TODO*/, 'i32') }}}); + var synchronousRequest = !!({{{ makeGetValue('fetch_attr', 60/*TODO*/, 'i32') }}}); + var destinationPath = {{{ makeGetValue('fetch_attr', 64/*TODO*/, 'i32') }}}; + var userName = {{{ makeGetValue('fetch_attr', 68/*TODO*/, 'i32') }}}; + var userNameStr = userName ? Pointer_stringify(userName) : undefined; + var password = {{{ makeGetValue('fetch_attr', 72/*TODO*/, 'i32') }}}; + var passwordStr = password ? Pointer_stringify(password) : undefined; + var requestHeaders = {{{ makeGetValue('fetch_attr', 76/*TODO*/, 'i32') }}}; + var overriddenMimeType = {{{ makeGetValue('fetch_attr', 80/*TODO*/, 'i32') }}}; + var overriddenMimeTypeStr = overriddenMimeType ? Pointer_stringify(overriddenMimeType) : undefined; +#if FETCH_DEBUG + console.log('fetch: xhr.timeout: ' + xhr.timeout + ', xhr.withCredentials: ' + xhr.withCredentials); + console.log('fetch: xhr.open(requestMethod="' + requestMethod + '", url: "' + url_ +'", userName: ' + userNameStr + ', password: ' + passwordStr + ');'); +#endif + xhr.open(requestMethod, url_, !synchronousRequest, userNameStr, passwordStr); + + if (overriddenMimeType) { +#if FETCH_DEBUG + console.log('fetch: xhr.overrideMimeType("' + overriddenMimeTypeStr + '");'); +#endif + xhr.overrideMimeType(overriddenMimeTypeStr); + } + if (requestHeaders) { + for(;;) { + var key = {{{ makeGetValue('requestHeaders', 0, 'i32') }}}; + if (!key) break; + var value = {{{ makeGetValue('requestHeaders', 4, 'i32') }}}; + if (!value) break; + requestHeaders += 8; + var keyStr = Pointer_stringify(key); + var valueStr = Pointer_stringify(value); +#if FETCH_DEBUG + console.log('fetch: xhr.setRequestHeader("' + keyStr + '", "' + valueStr + '");'); +#endif + xhr.setRequestHeader(keyStr, valueStr); + } + } Fetch.xhrs.push(xhr); var fetch = _malloc(1000 /*TODO:structs_info.sizeof(fetch)*/); + _memset(fetch, 0, 1000/*TODO:structs_info.sizeof(fetch)*/); var urlString = _malloc(url_.length /*TODO*/); {{{ makeSetValue('fetch', 8/*TODO:jsonify*/, 'urlString', 'i32')}}}; writeStringToMemory(url_, urlString); @@ -32,18 +66,6 @@ var LibraryFetch = { xhr.onload = function(e) { var len = xhr.response.byteLength; var ptr = _malloc(len); // TODO: make emscripten_fetch_close() free() this data. - xhrRequest = xhr; - /* - console.log('xhr.response.byteLength: ' + xhr.response.byteLength); - var ta32 = new Uint32Array(xhr.response, 0, xhr.response.byteLength >> 2); - console.log('ta32.length: ' + ta32.length); - for (var i = 0; i < ta32.length; ++i) { - {{{ makeSetValue('ptr', 'i', 'ta32[i]', 'i32')}}}; - } - if ((xhr.response.byteLength & 3) != 0) { - - } - */ HEAPU8.set(new Uint8Array(xhr.response), ptr); {{{ makeSetValue('fetch', 12/*TODO:jsonify*/, 'ptr', 'i64')}}}; {{{ makeSetValue('fetch', 16/*TODO:jsonify*/, 'len', 'i64')}}}; @@ -58,7 +80,13 @@ var LibraryFetch = { xhr.onerror = function(e) { if (onerror) Runtime.dynCall('vi', onerror, [fetch]); // TODO: pass error reason } + xhr.ontimeout = function(e) { + if (onerror) Runtime.dynCall('vi', onerror, [fetch]); // TODO: pass error reason + } xhr.onprogress = function(e) { + Module.print(xhr.status); + Module.print(xhr.readyState); + Module.print(xhr.statusText); {{{ makeSetValue('fetch', 16/*TODO:jsonify*/, '0', 'i64')}}}; {{{ makeSetValue('fetch', 24/*TODO:jsonify*/, 'e.loaded', 'i64')}}}; {{{ makeSetValue('fetch', 32/*TODO:jsonify*/, 'e.total', 'i64')}}}; @@ -70,6 +98,25 @@ var LibraryFetch = { } catch(e) { if (onerror) Runtime.dynCall('vi', onerror, [fetch]); // TODO: pass error reason } + }, + + emscripten_fetch__deps: ['$Fetch', '_emscripten_fetch_xhr'], + emscripten_fetch: function(fetch_attr, url) { + Module['noExitRuntime'] = true; + + var fetchAttributes = {{{ makeGetValue('fetch_attr', 48/*TODO*/, 'i32') }}}; + var fetchAttrLoadToMemory = !!(fetchAttributes & 1/*EMSCRIPTEN_FETCH_LOAD_TO_MEMORY*/); + var fetchAttrStreamData = !!(fetchAttributes & 2/*EMSCRIPTEN_FETCH_STREAM_DATA*/); + var fetchAttrPersistFile = !!(fetchAttributes & 4/*EMSCRIPTEN_FETCH_PERSIST_FILE*/); + var fetchAttrAppend = !!(fetchAttributes & 8/*EMSCRIPTEN_FETCH_APPEND*/); + var fetchAttrReplace = !!(fetchAttributes & 16/*EMSCRIPTEN_FETCH_REPLACE*/); + var fetchAttrNoDownload = !!(fetchAttributes & 32/*EMSCRIPTEN_FETCH_NO_DOWNLOAD*/); + + if (!fetchAttrNoDownload) { + __emscripten_fetch_xhr(fetch_attr, url); + } else { + throw 'not implemented!'; + } } }; diff --git a/src/settings.js b/src/settings.js index badd9e527a3c5..c3995a7163b99 100644 --- a/src/settings.js +++ b/src/settings.js @@ -795,8 +795,11 @@ var BUNDLED_CD_DEBUG_FILE = ""; // Path to the CyberDWARF debug file passed to t var TEXTDECODER = 1; // Is enabled, use the JavaScript TextDecoder API for string marshalling. // Enabled by default, set this to 0 to disable. + var OFFSCREENCANVAS_SUPPORT = 0; // If set to 1, enables support for transferring canvases to pthreads and creating WebGL contexts in them, // as well as explicit swap control for GL contexts. This needs browser support for the OffscreenCanvas // specification. +var FETCH_DEBUG = 0; // If nonzero, prints out debugging information in library_fetch.js + // Reserved: variables containing POINTER_MASKING. diff --git a/system/include/emscripten/fetch.h b/system/include/emscripten/fetch.h index 76a1d59ed75aa..7403e06b5ecdf 100644 --- a/system/include/emscripten/fetch.h +++ b/system/include/emscripten/fetch.h @@ -38,6 +38,23 @@ struct emscripten_fetch_t // Specifies the total number of bytes that the response body will be. uint64_t totalBytes; + + + + // Specifies the readyState of the XHR request: + // 0: UNSENT: request not sent yet + // 1: OPENED: emscripten_fetch has been called. + // 2: HEADERS_RECEIVED: emscripten_fetch has been called, and headers and status are available. + // 3: LOADING: download in progress. + // 4: DONE: download finished. + // See https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/readyState + unsigned short readyState; + + // Specifies the status code of the response. + unsigned short status; + + // Specifies a human-readable form of the status code. + char statusText[64]; }; // Emscripten fetch attributes: @@ -69,7 +86,7 @@ struct emscripten_fetch_t struct emscripten_fetch_attr_t { // 'POST', 'GET', etc. - char requestType[32]; + char requestMethod[32]; // Custom data that can be tagged along the process. void *userData; @@ -78,12 +95,43 @@ struct emscripten_fetch_attr_t void (*onerror)(emscripten_fetch_t *fetch); void (*onprogress)(emscripten_fetch_t *fetch); - // Specifies the destination path in IndexedDB where to store the downloaded content body. If this is empty, the transfer - // is not stored to IndexedDB at all. - char destinationPath[PATH_MAX]; - // EMSCRIPTEN_FETCH_* attributes uint32_t attributes; + + // Specifies the amount of time the request can take before failing due to a timeout. + unsigned long timeoutMSecs; + + // Indicates whether cross-site access control requests should be made using credentials. + EM_BOOL withCredentials; + + // If true, performs a synchronous blocking XHR. The emscripten_fetch() function call does not return until + // the request has completed. Setting this to true in the main browser thread will fail with + // EMSCRIPTEN_RESULT_NOT_SUPPORTED. + EM_BOOL synchronousRequest; + + // Specifies the destination path in IndexedDB where to store the downloaded content body. If this is empty, the transfer + // is not stored to IndexedDB at all. + // Note that this struct does not contain space to hold this string, it only carries a pointer. + // Calling emscripten_fetch() will make an internal copy of this string. + const char *destinationPath; + + // Specifies the authentication username to use for the request, if necessary. + // Note that this struct does not contain space to hold this string, it only carries a pointer. + // Calling emscripten_fetch() will make an internal copy of this string. + const char *userName; + + // Specifies the authentication username to use for the request, if necessary. + // Note that this struct does not contain space to hold this string, it only carries a pointer. + // Calling emscripten_fetch() will make an internal copy of this string. + const char *password; + + // Points to an array of strings to pass custom headers to the request. This array takes the form + // {"key1", "value1", "key2", "value2", "key3", "value3", ..., 0 }; Note especially that the array + // needs to be terminated with a null pointer. + const char * const *requestHeaders; + + // Pass a custom MIME type here to force the browser to treat the received data with the given type. + const char *overriddenMimeType; }; // Clears the fields of an emscripten_fetch_attr_t structure to their default values in a future-compatible manner. @@ -97,7 +145,7 @@ emscripten_fetch_t *emscripten_fetch(emscripten_fetch_attr_t *fetch_attr, const // the wait times out, the return value will be EMSCRIPTEN_RESULT_TIMEOUT. // The onsuccess()/onerror()/onprogress() handlers will be called in the calling thread from within this function before // this function returns. -EMSCRIPTEN_RESULT emscripten_fetch_wait(emscripten_fetch_t *fetch, double timeoutMsecs); +EMSCRIPTEN_RESULT emscripten_fetch_wait(emscripten_fetch_t *fetch, double timeoutMSecs); // Closes a finished or an executing fetch operation and frees up all memory. If the fetch operation was still executing, the // onerror() handler will be called in the calling thread before this function returns. diff --git a/tests/fetch/cached_xhr.cpp b/tests/fetch/cached_xhr.cpp index a6a4ff4660ebe..6e3cdffd66aed 100644 --- a/tests/fetch/cached_xhr.cpp +++ b/tests/fetch/cached_xhr.cpp @@ -9,7 +9,7 @@ void fetchFromIndexedDB() { emscripten_fetch_attr_t attr; emscripten_fetch_attr_init(&attr); - strcpy(attr.requestType, "GET"); + strcpy(attr.requestMethod, "GET"); attr.onsuccess = [](emscripten_fetch_t *fetch) { assert(fetch->numBytes == fetch->totalBytes); assert(fetch->data != 0); @@ -20,11 +20,7 @@ void fetchFromIndexedDB() printf("Downloading.. %.2f%% complete.\n", (fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes); }; attr.attributes = EMSCRIPTEN_FETCH_APPEND | EMSCRIPTEN_FETCH_NO_DOWNLOAD; - emscripten_fetch_t *fetch = emscripten_fetch(&attr, "myfile.dat"); - - if (synchronous) { - emscripten_fetch_wait(fetch, INFINITY); - } + emscripten_fetch_t *fetch = emscripten_fetch(&attr, "gears.png"); } // XHR and store to cache. @@ -32,7 +28,7 @@ int main() { emscripten_fetch_attr_t attr; emscripten_fetch_attr_init(&attr); - strcpy(attr.requestType, "GET"); + strcpy(attr.requestMethod, "GET"); attr.onsuccess = [](emscripten_fetch_t *fetch) { assert(fetch->numBytes == fetch->totalBytes); assert(fetch->data != 0); @@ -46,5 +42,5 @@ int main() printf("Downloading.. %.2f%% complete.\n", (fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes); }; attr.attributes = EMSCRIPTEN_FETCH_APPEND | EMSCRIPTEN_FETCH_LOAD_TO_MEMORY | EMSCRIPTEN_FETCH_PERSIST_FILE; - emscripten_fetch_t *fetch = emscripten_fetch(&attr, "myfile.dat"); + emscripten_fetch_t *fetch = emscripten_fetch(&attr, "gears.png"); } diff --git a/tests/fetch/to_memory.cpp b/tests/fetch/to_memory.cpp index a017637f94259..9b6bc19a0e18c 100644 --- a/tests/fetch/to_memory.cpp +++ b/tests/fetch/to_memory.cpp @@ -4,13 +4,24 @@ #include #include +int result = 0; + +// This test is run in two modes: if FILE_DOES_NOT_EXIST defined, +// then testing an XHR of a missing file. +// #define FILE_DOES_NOT_EXIST + int main() { emscripten_fetch_attr_t attr; emscripten_fetch_attr_init(&attr); - strcpy(attr.requestType, "GET"); + strcpy(attr.requestMethod, "GET"); attr.userData = (void*)0x12345678; + attr.attributes = EMSCRIPTEN_FETCH_APPEND | EMSCRIPTEN_FETCH_LOAD_TO_MEMORY; + attr.onsuccess = [](emscripten_fetch_t *fetch) { +#if FILE_DOES_NOT_EXIST + assert(false && "onsuccess handler called, but the file shouldn't exist"); // Shouldn't reach here if the file doesn't exist +#endif assert(fetch); printf("Finished downloading %llu bytes\n", fetch->numBytes); assert(fetch->url); @@ -27,24 +38,50 @@ int main() printf("Data checksum: %02X\n", checksum); assert(checksum == 0x08); emscripten_fetch_close(fetch); + +#ifdef REPORT_RESULT +#ifndef FILE_DOES_NOT_EXIST + result = 1; +#endif + REPORT_RESULT(); +#endif }; + attr.onprogress = [](emscripten_fetch_t *fetch) { assert(fetch); + if (fetch->status != 200) return; + printf("onprogress: dataOffset: %llu, numBytes: %llu, totalBytes: %llu\n", fetch->dataOffset, fetch->numBytes, fetch->totalBytes); printf("Downloading.. %.2f%% complete.\n", (fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes); +#ifdef FILE_DOES_NOT_EXIST + assert(false && "onprogress handler called, but the file should not exist"); // We should not receive progress reports if the file doesn't exist. +#endif + // We must receive a call to the onprogress handler with 100% completion. + if (fetch->dataOffset + fetch->numBytes == fetch->totalBytes) ++result; + assert(fetch->dataOffset + fetch->numBytes <= fetch->totalBytes); assert(fetch->url); assert(!strcmp(fetch->url, "gears.png")); assert(fetch->id != 0); assert((uintptr_t)fetch->userData == 0x12345678); }; + attr.onerror = [](emscripten_fetch_t *fetch) { printf("Download failed!\n"); - assert(false); // The download is not supposed to fail in this test. - +#ifndef FILE_DOES_NOT_EXIST + assert(false && "onerror handler called, but the transfer should have succeeded!"); // The file exists, shouldn't reach here. +#endif assert(fetch); assert(fetch->id != 0); assert(!strcmp(fetch->url, "gears.png")); assert((uintptr_t)fetch->userData == 0x12345678); + +#ifdef REPORT_RESULT +#ifdef FILE_DOES_NOT_EXIST + result = 1; +#endif + REPORT_RESULT(); +#endif + }; - attr.attributes = EMSCRIPTEN_FETCH_APPEND | EMSCRIPTEN_FETCH_LOAD_TO_MEMORY; + emscripten_fetch_t *fetch = emscripten_fetch(&attr, "gears.png"); } diff --git a/tests/test_browser.py b/tests/test_browser.py index 0f9202647359e..a0fa8ccbe2514 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -3294,12 +3294,18 @@ def test_preallocated_heap(self): self.btest('test_preallocated_heap.cpp', expected='1', args=['-s', 'TOTAL_MEMORY='+str(16*1024*1024), '-s', 'ABORTING_MALLOC=0', '--shell-file', path_from_root('tests', 'test_preallocated_heap_shell.html')]) def test_fetch_to_memory(self): + temp_args = [path_from_root('system/lib/fetch/emscripten_fetch.cpp'), '--js-library', path_from_root('src/library_fetch.js')] # TODO: Emscripten system libs, pass these e.g. with -lfetch + self.btest('fetch/to_memory.cpp', expected='1', args=['--std=c++11', '-s', 'FETCH_DEBUG=1', '-DFILE_DOES_NOT_EXIST'] + temp_args) + shutil.copyfile(path_from_root('tests', 'gears.png'), os.path.join(self.get_dir(), 'gears.png')) - # TODO: emscripten_fetch.cpp and library_fetch.js pulled in as system libs - self.btest('fetch/to_memory.cpp', expected='0', args=[path_from_root('system/lib/fetch/emscripten_fetch.cpp'), '--std=c++11', '--js-library', path_from_root('src/library_fetch.js')]) + self.btest('fetch/to_memory.cpp', expected='1', args=['--std=c++11', '-s', 'FETCH_DEBUG=1'] + temp_args) def test_fetch_stream_file(self): - self.btest('fetch/stream_file.cpp', expected='0', args=[path_from_root('system/lib/fetch/stream_file.cpp'), '--std=c++11', '--js-library', path_from_root('src/library_fetch.js')]) + temp_args = [path_from_root('system/lib/fetch/emscripten_fetch.cpp'), '--js-library', path_from_root('src/library_fetch.js')] # TODO: Emscripten system libs, pass these e.g. with -lfetch + self.btest('fetch/stream_file.cpp', expected='0', args=['--std=c++11', '-s', 'FETCH_DEBUG=1'] + temp_args) def test_fetch_cached_xhr(self): - self.btest('fetch/cached_xhr.cpp', expected='0', args=[path_from_root('system/lib/fetch/cached_xhr.cpp'), '--std=c++11', '--js-library', path_from_root('src/library_fetch.js')]) + temp_args = [path_from_root('system/lib/fetch/emscripten_fetch.cpp'), '--js-library', path_from_root('src/library_fetch.js')] # TODO: Emscripten system libs, pass these e.g. with -lfetch + + shutil.copyfile(path_from_root('tests', 'gears.png'), os.path.join(self.get_dir(), 'gears.png')) + self.btest('fetch/cached_xhr.cpp', expected='0', args=['--std=c++11', '-s', 'FETCH_DEBUG=1'] + temp_args) From 41351cce7bc770b5a96356586ca13f710318da23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Wed, 8 Jun 2016 01:17:43 +0300 Subject: [PATCH 08/74] Add IndexedDB support and streaming usage to emscripten_fetch() and other updates. --- src/library_fetch.js | 348 ++++++++++++++++++++++---- system/include/emscripten/fetch.h | 101 ++++---- system/lib/fetch/emscripten_fetch.cpp | 28 +++ tests/fetch/cached_xhr.cpp | 12 +- tests/fetch/stream_file.cpp | 31 ++- tests/fetch/to_memory.cpp | 5 +- tests/test_browser.py | 26 +- 7 files changed, 446 insertions(+), 105 deletions(-) diff --git a/src/library_fetch.js b/src/library_fetch.js index c483f4a50347d..166c1788600cf 100644 --- a/src/library_fetch.js +++ b/src/library_fetch.js @@ -1,24 +1,184 @@ var LibraryFetch = { + $Fetch__postset: 'Fetch.staticInit();', $Fetch: { xhrs: [], + // Specifies an instance to the IndexedDB database. The database is opened + // as a preload step before the Emscripten application starts. + dbInstance: undefined, + + openDatabase: function(dbname, dbversion, onsuccess, onerror) { + try { +#if FETCH_DEBUG + console.log('fetch: indexedDB.open(dbname="' + dbname + '", dbversion="' + dbversion + '");') +#endif + var openRequest = indexedDB.open(dbname, dbversion); + } catch (e) { return onerror(e); } + + openRequest.onupgradeneeded = function(event) { +#if FETCH_DEBUG + console.log('fetch: IndexedDB upgrade needed. Clearing database.'); +#endif + var db = event.target.result; + if (db.objectStoreNames.contains('FILES')) db.deleteObjectStore('FILES'); + db.createObjectStore('FILES'); + }; + openRequest.onsuccess = function(event) { onsuccess(event.target.result); }; + openRequest.onerror = function(error) { onerror(error); }; + }, + + staticInit: function() { + var onsuccess = function(db) { +#if FETCH_DEBUG + console.log('fetch: IndexedDB successfully opened.'); +#endif + Fetch.dbInstance = db; + removeRunDependency('library_fetch_init'); + }; + var onerror = function() { +#if FETCH_DEBUG + console.error('fetch: IndexedDB open failed.'); +#endif + Fetch.dbInstance = false; + removeRunDependency('library_fetch_init'); + }; + Fetch.openDatabase('emscripten_filesystem', 1, onsuccess, onerror); + addRunDependency('library_fetch_init'); + } }, - _emscripten_fetch_xhr: function(fetch_attr, url) { - var xhr = new XMLHttpRequest(); - var requestMethod = Pointer_stringify(fetch_attr); + _emscripten_fetch_load_cached_data: function(db, fetch, onsuccess, onerror) { + if (!db) { +#if FETCH_DEBUG + console.error('fetch: IndexedDB not available!'); +#endif + onerror(fetch, 0, 'IndexedDB not available!'); + return; + } + + var fetch_attr = fetch + 108/*TODO:structs_info*/; + var path = {{{ makeGetValue('fetch_attr', 64/*TODO*/, 'i32') }}}; + if (!path) path = {{{ makeGetValue('fetch', 8/*TODO:structs_info.jsonify this*/, 'i32') }}}; + var pathStr = Pointer_stringify(path); + + try { + var transaction = db.transaction(['FILES'], 'readonly'); + var packages = transaction.objectStore('FILES'); + var getRequest = packages.get(pathStr); + getRequest.onsuccess = function(event) { + if (event.target.result) { + var value = event.target.result; + var len = value.byteLength || value.length; +#if FETCH_DEBUG + console.log('fetch: Loaded file ' + pathStr + ' from IndexedDB, length: ' + len); +#endif + + var ptr = _malloc(len); // TODO: make emscripten_fetch_close() free() this data. + HEAPU8.set(new Uint8Array(value), ptr); + {{{ makeSetValue('fetch', 12/*TODO:jsonify*/, 'ptr', 'i32')}}}; + {{{ makeSetValue('fetch', 16/*TODO:jsonify*/, 'len', 'i64')}}}; + {{{ makeSetValue('fetch', 24/*TODO:jsonify*/, '0', 'i64')}}}; + {{{ makeSetValue('fetch', 32/*TODO:jsonify*/, 'len', 'i64')}}}; + + onsuccess(fetch, 0, value); + } else { + // Succeeded to load, but the load came back with the value of undefined, treat that as an error since we never store undefined in db. +#if FETCH_DEBUG + console.error('fetch: Loaded file ' + pathStr + ' from IndexedDB, but it had 0 length!'); +#endif + onerror(fetch, 0, 'no data'); + } + }; + getRequest.onerror = function(error) { +#if FETCH_DEBUG + console.error('fetch: Failed to load file ' + pathStr + ' from IndexedDB!'); +#endif + onerror(fetch, 0, error); + }; + } catch(e) { +#if FETCH_DEBUG + console.error('fetch: Failed to load file ' + pathStr + ' from IndexedDB! Got exception ' + e); +#endif + onerror(fetch, 0, e); + } + }, + + _emscripten_fetch_cache_data: function(db, fetch, data, onsuccess, onerror) { + if (!db) { +#if FETCH_DEBUG + console.error('fetch: IndexedDB not available!'); +#endif + onerror(fetch, 0, 'IndexedDB not available!'); + return; + } + + var fetch_attr = fetch + 108/*TODO:structs_info*/; + var destinationPath = {{{ makeGetValue('fetch_attr', 64/*TODO*/, 'i32') }}}; + if (!destinationPath) destinationPath = {{{ makeGetValue('fetch', 8/*TODO*/, 'i32') }}}; + var destinationPathStr = Pointer_stringify(destinationPath); + + try { + var transaction = db.transaction(['FILES'], 'readwrite'); + var packages = transaction.objectStore('FILES'); + var putRequest = packages.put(data, destinationPathStr); + putRequest.onsuccess = function(event) { +#if FETCH_DEBUG + console.log('fetch: Stored file "' + destinationPathStr + '" to IndexedDB cache.'); +#endif + onsuccess(fetch, 0, destinationPathStr); + }; + putRequest.onerror = function(error) { +#if FETCH_DEBUG + console.error('fetch: Failed to store file "' + destinationPathStr + '" to IndexedDB cache!'); +#endif + onerror(fetch, 0, error); + }; + } catch(e) { +#if FETCH_DEBUG + console.error('fetch: Failed to store file "' + destinationPathStr + '" to IndexedDB cache! Exception: ' + e); +#endif + onerror(fetch, 0, e); + } + }, + + _emscripten_fetch_xhr: function(fetch, onsuccess, onerror, onprogress) { + var url = {{{ makeGetValue('fetch', 8/*TODO:structs_info.jsonify this*/, 'i32') }}}; + if (!url) { +#if FETCH_DEBUG + console.error('fetch: XHR failed, no URL specified!'); +#endif + onerror(fetch, 0, 'no url specified!'); + return; + } var url_ = Pointer_stringify(url); - xhr.responseType = 'arraybuffer'; - xhr.timeout = {{{ makeGetValue('fetch_attr', 52/*TODO*/, 'i32') }}}; - xhr.withCredentials = !!({{{ makeGetValue('fetch_attr', 56/*TODO*/, 'i32') }}}); + + var fetch_attr = fetch + 108/*TODO:structs_info*/; + var requestMethod = fetch_attr ? Pointer_stringify(fetch_attr) : 'GET'; + var userData = {{{ makeGetValue('fetch_attr', 32/*TODO:structs_info.jsonify this*/, 'i32') }}}; + var fetchAttributes = {{{ makeGetValue('fetch_attr', 48/*TODO*/, 'i32') }}}; + var timeoutMsecs = {{{ makeGetValue('fetch_attr', 52/*TODO*/, 'i32') }}}; + var withCredentials = !!({{{ makeGetValue('fetch_attr', 56/*TODO*/, 'i32') }}}); var synchronousRequest = !!({{{ makeGetValue('fetch_attr', 60/*TODO*/, 'i32') }}}); var destinationPath = {{{ makeGetValue('fetch_attr', 64/*TODO*/, 'i32') }}}; var userName = {{{ makeGetValue('fetch_attr', 68/*TODO*/, 'i32') }}}; - var userNameStr = userName ? Pointer_stringify(userName) : undefined; var password = {{{ makeGetValue('fetch_attr', 72/*TODO*/, 'i32') }}}; - var passwordStr = password ? Pointer_stringify(password) : undefined; var requestHeaders = {{{ makeGetValue('fetch_attr', 76/*TODO*/, 'i32') }}}; var overriddenMimeType = {{{ makeGetValue('fetch_attr', 80/*TODO*/, 'i32') }}}; + + var fetchAttrLoadToMemory = !!(fetchAttributes & 1/*EMSCRIPTEN_FETCH_LOAD_TO_MEMORY*/); + var fetchAttrStreamData = !!(fetchAttributes & 2/*EMSCRIPTEN_FETCH_STREAM_DATA*/); + var fetchAttrPersistFile = !!(fetchAttributes & 4/*EMSCRIPTEN_FETCH_PERSIST_FILE*/); + var fetchAttrAppend = !!(fetchAttributes & 8/*EMSCRIPTEN_FETCH_APPEND*/); + var fetchAttrReplace = !!(fetchAttributes & 16/*EMSCRIPTEN_FETCH_REPLACE*/); + var fetchAttrNoDownload = !!(fetchAttributes & 32/*EMSCRIPTEN_FETCH_NO_DOWNLOAD*/); + + var userNameStr = userName ? Pointer_stringify(userName) : undefined; + var passwordStr = password ? Pointer_stringify(password) : undefined; var overriddenMimeTypeStr = overriddenMimeType ? Pointer_stringify(overriddenMimeType) : undefined; + + var xhr = new XMLHttpRequest(); + xhr.responseType = fetchAttrStreamData ? 'moz-chunked-arraybuffer' : 'arraybuffer'; + xhr.timeout = timeoutMsecs; + xhr.withCredentials = withCredentials; #if FETCH_DEBUG console.log('fetch: xhr.timeout: ' + xhr.timeout + ', xhr.withCredentials: ' + xhr.withCredentials); console.log('fetch: xhr.open(requestMethod="' + requestMethod + '", url: "' + url_ +'", userName: ' + userNameStr + ', password: ' + passwordStr + ');'); @@ -47,63 +207,90 @@ var LibraryFetch = { } } Fetch.xhrs.push(xhr); - var fetch = _malloc(1000 /*TODO:structs_info.sizeof(fetch)*/); - _memset(fetch, 0, 1000/*TODO:structs_info.sizeof(fetch)*/); - var urlString = _malloc(url_.length /*TODO*/); - {{{ makeSetValue('fetch', 8/*TODO:jsonify*/, 'urlString', 'i32')}}}; - writeStringToMemory(url_, urlString); var id = Fetch.xhrs.length; {{{ makeSetValue('fetch', 0/*TODO:jsonify*/, 'id', 'i32')}}}; - var userData = {{{ makeGetValue('fetch_attr', 32/*TODO:structs_info.jsonify this*/, 'i32') }}}; - {{{ makeSetValue('fetch', 4/*TODO:jsonify*/, 'userData', 'i32')}}}; var data = null; // TODO: Support user to pass data to request. // TODO: Support specifying custom headers to the request. - var onsuccess = {{{ makeGetValue('fetch_attr', 36/*TODO:structs_info.jsonify this*/, 'i32') }}}; - var onerror = {{{ makeGetValue('fetch_attr', 40/*TODO:structs_info.jsonify this*/, 'i32') }}}; - var onprogress = {{{ makeGetValue('fetch_attr', 44/*TODO:structs_info.jsonify this*/, 'i32') }}}; - xhr.onload = function(e) { - var len = xhr.response.byteLength; - var ptr = _malloc(len); // TODO: make emscripten_fetch_close() free() this data. - HEAPU8.set(new Uint8Array(xhr.response), ptr); - {{{ makeSetValue('fetch', 12/*TODO:jsonify*/, 'ptr', 'i64')}}}; - {{{ makeSetValue('fetch', 16/*TODO:jsonify*/, 'len', 'i64')}}}; + var len = xhr.response ? xhr.response.byteLength : 0; + var ptr = 0; + var ptrLen = 0; + if (fetchAttrLoadToMemory && !fetchAttrStreamData) { + ptrLen = len; +#if FETCH_DEBUG + console.log('fetch: allocating ' + ptrLen + ' bytes in Emscripten heap for xhr data'); +#endif + ptr = _malloc(ptrLen); // TODO: make emscripten_fetch_close() free() this data. + HEAPU8.set(new Uint8Array(xhr.response), ptr); + } + {{{ makeSetValue('fetch', 12/*TODO:jsonify*/, 'ptr', 'i32')}}}; + {{{ makeSetValue('fetch', 16/*TODO:jsonify*/, 'ptrLen', 'i64')}}}; {{{ makeSetValue('fetch', 24/*TODO:jsonify*/, '0', 'i64')}}}; - {{{ makeSetValue('fetch', 32/*TODO:jsonify*/, 'len', 'i64')}}}; + if (len) { + {{{ makeSetValue('fetch', 32/*TODO:jsonify*/, 'len', 'i64')}}}; + } if (xhr.status == 200 || xhr.status == 0) { - if (onsuccess) Runtime.dynCall('vi', onsuccess, [fetch]); +#if FETCH_DEBUG + console.log('fetch: xhr succeeded with status 200'); +#endif + if (onsuccess) onsuccess(fetch, xhr, e); } else { - if (onerror) Runtime.dynCall('vi', onerror, [fetch]); // TODO: pass error reason +#if FETCH_DEBUG + console.error('fetch: xhr failed with status ' + xhr.status); +#endif + if (onerror) onerror(fetch, xhr, e); } } xhr.onerror = function(e) { - if (onerror) Runtime.dynCall('vi', onerror, [fetch]); // TODO: pass error reason +#if FETCH_DEBUG + console.error('fetch: xhr failed with error ' + e); +#endif + if (onerror) onerror(fetch, xhr, e); } xhr.ontimeout = function(e) { - if (onerror) Runtime.dynCall('vi', onerror, [fetch]); // TODO: pass error reason +#if FETCH_DEBUG + console.error('fetch: xhr timed out with error ' + e); +#endif + if (onerror) onerror(fetch, xhr, e); } xhr.onprogress = function(e) { - Module.print(xhr.status); - Module.print(xhr.readyState); - Module.print(xhr.statusText); - {{{ makeSetValue('fetch', 16/*TODO:jsonify*/, '0', 'i64')}}}; - {{{ makeSetValue('fetch', 24/*TODO:jsonify*/, 'e.loaded', 'i64')}}}; + var ptrLen = (fetchAttrLoadToMemory && fetchAttrStreamData && xhr.response) ? xhr.response.byteLength : 0; + var ptr = 0; + if (fetchAttrLoadToMemory && fetchAttrStreamData) { +#if FETCH_DEBUG + console.log('fetch: allocating ' + ptrLen + ' bytes in Emscripten heap for xhr data'); +#endif + ptr = _malloc(ptrLen); // TODO: make emscripten_fetch_close() free() this data. + HEAPU8.set(new Uint8Array(xhr.response), ptr); + } + {{{ makeSetValue('fetch', 12/*TODO:jsonify*/, 'ptr', 'i32')}}}; + {{{ makeSetValue('fetch', 16/*TODO:jsonify*/, 'ptrLen', 'i64')}}}; + {{{ makeSetValue('fetch', 24/*TODO:jsonify*/, 'e.loaded - ptrLen', 'i64')}}}; {{{ makeSetValue('fetch', 32/*TODO:jsonify*/, 'e.total', 'i64')}}}; - if (onprogress) Runtime.dynCall('vi', onprogress, [fetch]); + if (onprogress) onprogress(fetch, xhr, e); } - console.log('xhr.send(data=' + data + ')'); +#if FETCH_DEBUG + console.log('fetch: xhr.send(data=' + data + ')'); +#endif try { xhr.send(data); } catch(e) { - if (onerror) Runtime.dynCall('vi', onerror, [fetch]); // TODO: pass error reason +#if FETCH_DEBUG + console.error('fetch: xhr failed with exception: ' + e); +#endif + if (onerror) onerror(fetch, xhr, e); } }, - emscripten_fetch__deps: ['$Fetch', '_emscripten_fetch_xhr'], - emscripten_fetch: function(fetch_attr, url) { - Module['noExitRuntime'] = true; + emscripten_start_fetch__deps: ['$Fetch', '_emscripten_fetch_xhr', '_emscripten_fetch_cache_data', '_emscripten_fetch_load_cached_data'], + emscripten_start_fetch: function(fetch) { + Module['noExitRuntime'] = true; // TODO: assumes we are the main Emscripten runtime thread, in the future won't. + var fetch_attr = fetch + 108/*TODO:structs_info*/; + var onsuccess = {{{ makeGetValue('fetch_attr', 36/*TODO:structs_info.jsonify this*/, 'i32') }}}; + var onerror = {{{ makeGetValue('fetch_attr', 40/*TODO:structs_info.jsonify this*/, 'i32') }}}; + var onprogress = {{{ makeGetValue('fetch_attr', 44/*TODO:structs_info.jsonify this*/, 'i32') }}}; var fetchAttributes = {{{ makeGetValue('fetch_attr', 48/*TODO*/, 'i32') }}}; var fetchAttrLoadToMemory = !!(fetchAttributes & 1/*EMSCRIPTEN_FETCH_LOAD_TO_MEMORY*/); var fetchAttrStreamData = !!(fetchAttributes & 2/*EMSCRIPTEN_FETCH_STREAM_DATA*/); @@ -112,11 +299,86 @@ var LibraryFetch = { var fetchAttrReplace = !!(fetchAttributes & 16/*EMSCRIPTEN_FETCH_REPLACE*/); var fetchAttrNoDownload = !!(fetchAttributes & 32/*EMSCRIPTEN_FETCH_NO_DOWNLOAD*/); - if (!fetchAttrNoDownload) { - __emscripten_fetch_xhr(fetch_attr, url); + var reportSuccess = function(fetch, xhr, e) { +#if FETCH_DEBUG + console.log('fetch: operation success. e: ' + e); +#endif + if (onsuccess) Runtime.dynCall('vi', onsuccess, [fetch]); + }; + + var cacheResultAndReportSuccess = function(fetch, xhr, e) { +#if FETCH_DEBUG + console.log('fetch: operation success. Caching result.. e: ' + e); +#endif + var storeSuccess = function() { +#if FETCH_DEBUG + console.log('fetch: IndexedDB store succeeded.'); +#endif + }; + var storeError = function() { +#if FETCH_DEBUG + console.error('fetch: IndexedDB store failed.'); +#endif + }; + __emscripten_fetch_cache_data(Fetch.dbInstance, fetch, xhr.response, storeSuccess, storeError); + if (onsuccess) Runtime.dynCall('vi', onsuccess, [fetch]); + }; + + var reportProgress = function(fetch, xhr, e) { + if (onprogress) Runtime.dynCall('vi', onprogress, [fetch]); + }; + + var reportError = function(fetch, xhr, e) { +#if FETCH_DEBUG + console.error('fetch: operation failed: ' + e); +#endif + if (onerror) Runtime.dynCall('vi', onerror, [fetch]); + }; + + var performUncachedXhr = function(fetch, xhr, e) { +#if FETCH_DEBUG + console.error('fetch: starting (uncached) XHR: ' + e); +#endif + __emscripten_fetch_xhr(fetch, reportSuccess, reportError, reportProgress); + }; + + var performCachedXhr = function(fetch, xhr, e) { +#if FETCH_DEBUG + console.error('fetch: starting (cached) XHR: ' + e); +#endif + __emscripten_fetch_xhr(fetch, cacheResultAndReportSuccess, reportError, reportProgress); + }; + + // Should we try IndexedDB first? + if (!fetchAttrReplace) { + if (!Fetch.dbInstance) { +#if FETCH_DEBUG + console.error('fetch: failed to read IndexedDB! Database is not open.'); +#endif + reportError(fetch, 0, 'IndexedDB is not open'); + return 0; // todo: free + } + + if (fetchAttrNoDownload) { + __emscripten_fetch_load_cached_data(Fetch.dbInstance, fetch, reportSuccess, reportError); + } else if (fetchAttrPersistFile) { + __emscripten_fetch_load_cached_data(Fetch.dbInstance, fetch, reportSuccess, performCachedXhr); + } else { + __emscripten_fetch_load_cached_data(Fetch.dbInstance, fetch, reportSuccess, performUncachedXhr); + } + } else if (!fetchAttrNoDownload) { + if (fetchAttrPersistFile) { + __emscripten_fetch_xhr(fetch, cacheResultAndReportSuccess, reportError, reportProgress); + } else { + __emscripten_fetch_xhr(fetch, reportSuccess, reportError, reportProgress); + } } else { - throw 'not implemented!'; +#if FETCH_DEBUG + console.error('fetch: Invalid combination of flags passed.'); +#endif + return 0; // todo: free } + return fetch; } }; diff --git a/system/include/emscripten/fetch.h b/system/include/emscripten/fetch.h index 7403e06b5ecdf..6662b0719a024 100644 --- a/system/include/emscripten/fetch.h +++ b/system/include/emscripten/fetch.h @@ -9,54 +9,6 @@ extern "C" { #endif -struct emscripten_fetch_t -{ - // Unique identifier for this fetch in progress. - unsigned int id; - - // Custom data that can be tagged along the process. - void *userData; - - // The remote URL that is being downloaded. - const char *url; - - // In onsuccess() handler: - // - If the EMSCRIPTEN_FETCH_LOAD_TO_MEMORY attribute was specified for the transfer, this points to the - // body of the downloaded data. Otherwise this will be null. - // In onprogress() handler: - // - If the EMSCRIPTEN_FETCH_STREAM_DATA attribute was specified for the transfer, this points to a partial - // chunk of bytes related to the transfer. Otherwise this will be null. - const char *data; - - // Specifies the length of the above data block in bytes. When the download finishes, this field will be valid even if - // EMSCRIPTEN_FETCH_LOAD_TO_MEMORY was not specified. - uint64_t numBytes; - - // If EMSCRIPTEN_FETCH_STREAM_DATA is being performed, this indicates the byte offset from the start of the stream - // that the data block specifies. (for onprogress() streaming XHR transfer, the number of bytes downloaded so far before this chunk) - uint64_t dataOffset; - - // Specifies the total number of bytes that the response body will be. - uint64_t totalBytes; - - - - // Specifies the readyState of the XHR request: - // 0: UNSENT: request not sent yet - // 1: OPENED: emscripten_fetch has been called. - // 2: HEADERS_RECEIVED: emscripten_fetch has been called, and headers and status are available. - // 3: LOADING: download in progress. - // 4: DONE: download finished. - // See https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/readyState - unsigned short readyState; - - // Specifies the status code of the response. - unsigned short status; - - // Specifies a human-readable form of the status code. - char statusText[64]; -}; - // Emscripten fetch attributes: // If passed, the body of the request will be present in full in the onsuccess() handler. #define EMSCRIPTEN_FETCH_LOAD_TO_MEMORY 1 @@ -82,6 +34,8 @@ struct emscripten_fetch_t // EMSCRIPTEN_FETCH_APPEND, EMSCRIPTEN_FETCH_REPLACE and EMSCRIPTEN_FETCH_NO_DOWNLOAD are mutually exclusive. #define EMSCRIPTEN_FETCH_NO_DOWNLOAD 32 +struct emscripten_fetch_t; + // Specifies the parameters for a newly initiated fetch operation. struct emscripten_fetch_attr_t { @@ -134,6 +88,57 @@ struct emscripten_fetch_attr_t const char *overriddenMimeType; }; +struct emscripten_fetch_t +{ + // Unique identifier for this fetch in progress. + unsigned int id; + + // Custom data that can be tagged along the process. + void *userData; + + // The remote URL that is being downloaded. + const char *url; + + // In onsuccess() handler: + // - If the EMSCRIPTEN_FETCH_LOAD_TO_MEMORY attribute was specified for the transfer, this points to the + // body of the downloaded data. Otherwise this will be null. + // In onprogress() handler: + // - If the EMSCRIPTEN_FETCH_STREAM_DATA attribute was specified for the transfer, this points to a partial + // chunk of bytes related to the transfer. Otherwise this will be null. + const char *data; + + // Specifies the length of the above data block in bytes. When the download finishes, this field will be valid even if + // EMSCRIPTEN_FETCH_LOAD_TO_MEMORY was not specified. + uint64_t numBytes; + + // If EMSCRIPTEN_FETCH_STREAM_DATA is being performed, this indicates the byte offset from the start of the stream + // that the data block specifies. (for onprogress() streaming XHR transfer, the number of bytes downloaded so far before this chunk) + uint64_t dataOffset; + + // Specifies the total number of bytes that the response body will be. + uint64_t totalBytes; + + + + // Specifies the readyState of the XHR request: + // 0: UNSENT: request not sent yet + // 1: OPENED: emscripten_fetch has been called. + // 2: HEADERS_RECEIVED: emscripten_fetch has been called, and headers and status are available. + // 3: LOADING: download in progress. + // 4: DONE: download finished. + // See https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/readyState + unsigned short readyState; + + // Specifies the status code of the response. + unsigned short status; + + // Specifies a human-readable form of the status code. + char statusText[64]; + + // For internal use only. + emscripten_fetch_attr_t __attributes; +}; + // Clears the fields of an emscripten_fetch_attr_t structure to their default values in a future-compatible manner. void emscripten_fetch_attr_init(emscripten_fetch_attr_t *fetch_attr); diff --git a/system/lib/fetch/emscripten_fetch.cpp b/system/lib/fetch/emscripten_fetch.cpp index 0b072ad1cd508..100242b7fc9ec 100644 --- a/system/lib/fetch/emscripten_fetch.cpp +++ b/system/lib/fetch/emscripten_fetch.cpp @@ -1,12 +1,39 @@ #include +#include +#include #include #include +extern "C" { + void emscripten_start_fetch(emscripten_fetch_t *fetch); +} + void emscripten_fetch_attr_init(emscripten_fetch_attr_t *fetch_attr) { memset(fetch_attr, 0, sizeof(emscripten_fetch_attr_t)); } +static int globalFetchIdCounter = 1; +emscripten_fetch_t *emscripten_fetch(emscripten_fetch_attr_t *fetch_attr, const char *url) +{ + if (!fetch_attr) return 0; + if (!url) return 0; + emscripten_fetch_t *fetch = (emscripten_fetch_t *)malloc(sizeof(emscripten_fetch_t)); + memset(fetch, 0, sizeof(emscripten_fetch_t)); + fetch->id = globalFetchIdCounter++; // TODO: make this thread-safe! + fetch->userData = fetch_attr->userData; + fetch->url = strdup(url); // TODO: free + fetch->__attributes = *fetch_attr; + fetch->__attributes.destinationPath = fetch->__attributes.destinationPath ? strdup(fetch->__attributes.destinationPath) : 0; // TODO: free + fetch->__attributes.userName = fetch->__attributes.userName ? strdup(fetch->__attributes.userName) : 0; // TODO: free + fetch->__attributes.password = fetch->__attributes.password ? strdup(fetch->__attributes.password) : 0; // TODO: free + fetch->__attributes.requestHeaders = 0;// TODO:strdup(fetch->__attributes.requestHeaders); + fetch->__attributes.overriddenMimeType = fetch->__attributes.overriddenMimeType ? strdup(fetch->__attributes.overriddenMimeType) : 0; // TODO: free + + emscripten_start_fetch(fetch); + return fetch; +} + EMSCRIPTEN_RESULT emscripten_fetch_wait(emscripten_fetch_t *fetch, double timeoutMsecs) { return EMSCRIPTEN_RESULT_SUCCESS; @@ -14,5 +41,6 @@ EMSCRIPTEN_RESULT emscripten_fetch_wait(emscripten_fetch_t *fetch, double timeou EMSCRIPTEN_RESULT emscripten_fetch_close(emscripten_fetch_t *fetch) { + free(fetch); return EMSCRIPTEN_RESULT_SUCCESS; } diff --git a/tests/fetch/cached_xhr.cpp b/tests/fetch/cached_xhr.cpp index 6e3cdffd66aed..6750d1da27403 100644 --- a/tests/fetch/cached_xhr.cpp +++ b/tests/fetch/cached_xhr.cpp @@ -4,6 +4,8 @@ #include #include +int result = 0; + // Fetch file without XHRing. void fetchFromIndexedDB() { @@ -15,6 +17,12 @@ void fetchFromIndexedDB() assert(fetch->data != 0); printf("Finished downloading %llu bytes\n", fetch->numBytes); emscripten_fetch_close(fetch); + +#ifdef REPORT_RESULT + result = 1; + REPORT_RESULT(); +#endif + }; attr.onprogress = [](emscripten_fetch_t *fetch) { printf("Downloading.. %.2f%% complete.\n", (fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes); @@ -41,6 +49,8 @@ int main() attr.onprogress = [](emscripten_fetch_t *fetch) { printf("Downloading.. %.2f%% complete.\n", (fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes); }; - attr.attributes = EMSCRIPTEN_FETCH_APPEND | EMSCRIPTEN_FETCH_LOAD_TO_MEMORY | EMSCRIPTEN_FETCH_PERSIST_FILE; + attr.attributes = EMSCRIPTEN_FETCH_REPLACE | EMSCRIPTEN_FETCH_LOAD_TO_MEMORY | EMSCRIPTEN_FETCH_PERSIST_FILE; emscripten_fetch_t *fetch = emscripten_fetch(&attr, "gears.png"); + assert(fetch != 0); + memset(&attr, 0, sizeof(attr)); // emscripten_fetch() must be able to operate without referencing to this structure after the call. } diff --git a/tests/fetch/stream_file.cpp b/tests/fetch/stream_file.cpp index a1ba758d23955..b73213d7da995 100644 --- a/tests/fetch/stream_file.cpp +++ b/tests/fetch/stream_file.cpp @@ -4,24 +4,43 @@ #include #include +// Compute rudimentary checksum of data +uint32_t checksum = 0; + +int result = 0; + int main() { emscripten_fetch_attr_t attr; emscripten_fetch_attr_init(&attr); - strcpy(attr.requestType, "GET"); + strcpy(attr.requestMethod, "GET"); attr.onsuccess = [](emscripten_fetch_t *fetch) { assert(fetch->data == 0); // The data was streamed via onprogress, no bytes available here. assert(fetch->numBytes == 0); - assert(fetch->totalBytes > 0); + assert(fetch->totalBytes == 134217728); printf("Finished downloading %llu bytes\n", fetch->totalBytes); + printf("Data checksum: %08X\n", checksum); + assert(checksum == 0xA7F8E858U); emscripten_fetch_close(fetch); + +#ifdef REPORT_RESULT + result = 1; + REPORT_RESULT(); +#endif }; attr.onprogress = [](emscripten_fetch_t *fetch) { assert(fetch->data != 0); assert(fetch->numBytes > 0); - assert(fetch->numBytes <= fetch->totalBytes); - printf("Downloading.. %.2f%% complete.\n", (fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes); + assert(fetch->dataOffset + fetch->numBytes <= fetch->totalBytes); + assert(fetch->totalBytes <= 134217728); + printf("Downloading.. %.2f%% complete. Received chunk [%llu, %llu[\n", + (fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes, + fetch->dataOffset, + fetch->dataOffset + fetch->numBytes); + + for(int i = 0; i < fetch->numBytes; ++i) + checksum = ((checksum << 8) | (checksum >> 24)) * fetch->data[i] + fetch->data[i]; }; - attr.attributes = EMSCRIPTEN_FETCH_APPEND | EMSCRIPTEN_FETCH_STREAM_DATA; - emscripten_fetch_t *fetch = emscripten_fetch(&attr, "myfile.dat"); + attr.attributes = EMSCRIPTEN_FETCH_LOAD_TO_MEMORY | EMSCRIPTEN_FETCH_APPEND | EMSCRIPTEN_FETCH_STREAM_DATA; + emscripten_fetch_t *fetch = emscripten_fetch(&attr, "largefile.txt"); } diff --git a/tests/fetch/to_memory.cpp b/tests/fetch/to_memory.cpp index 9b6bc19a0e18c..7d8d29599089b 100644 --- a/tests/fetch/to_memory.cpp +++ b/tests/fetch/to_memory.cpp @@ -16,7 +16,7 @@ int main() emscripten_fetch_attr_init(&attr); strcpy(attr.requestMethod, "GET"); attr.userData = (void*)0x12345678; - attr.attributes = EMSCRIPTEN_FETCH_APPEND | EMSCRIPTEN_FETCH_LOAD_TO_MEMORY; + attr.attributes = EMSCRIPTEN_FETCH_REPLACE | EMSCRIPTEN_FETCH_LOAD_TO_MEMORY; attr.onsuccess = [](emscripten_fetch_t *fetch) { #if FILE_DOES_NOT_EXIST @@ -80,8 +80,9 @@ int main() #endif REPORT_RESULT(); #endif - }; emscripten_fetch_t *fetch = emscripten_fetch(&attr, "gears.png"); + assert(fetch != 0); + memset(&attr, 0, sizeof(attr)); // emscripten_fetch() must be able to operate without referencing to this structure after the call. } diff --git a/tests/test_browser.py b/tests/test_browser.py index a0fa8ccbe2514..7deadad7411eb 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -3293,19 +3293,35 @@ def test_webgl_offscreen_canvas_in_mainthread_after_pthread(self): def test_preallocated_heap(self): self.btest('test_preallocated_heap.cpp', expected='1', args=['-s', 'TOTAL_MEMORY='+str(16*1024*1024), '-s', 'ABORTING_MALLOC=0', '--shell-file', path_from_root('tests', 'test_preallocated_heap_shell.html')]) + # Tests emscripten_fetch() usage to XHR data directly to memory without persisting results to IndexedDB. def test_fetch_to_memory(self): temp_args = [path_from_root('system/lib/fetch/emscripten_fetch.cpp'), '--js-library', path_from_root('src/library_fetch.js')] # TODO: Emscripten system libs, pass these e.g. with -lfetch + + # Test error reporting in the negative case when the file URL doesn't exist. (http 404) self.btest('fetch/to_memory.cpp', expected='1', args=['--std=c++11', '-s', 'FETCH_DEBUG=1', '-DFILE_DOES_NOT_EXIST'] + temp_args) + # Test the positive case when the file URL exists. (http 200) shutil.copyfile(path_from_root('tests', 'gears.png'), os.path.join(self.get_dir(), 'gears.png')) self.btest('fetch/to_memory.cpp', expected='1', args=['--std=c++11', '-s', 'FETCH_DEBUG=1'] + temp_args) - def test_fetch_stream_file(self): - temp_args = [path_from_root('system/lib/fetch/emscripten_fetch.cpp'), '--js-library', path_from_root('src/library_fetch.js')] # TODO: Emscripten system libs, pass these e.g. with -lfetch - self.btest('fetch/stream_file.cpp', expected='0', args=['--std=c++11', '-s', 'FETCH_DEBUG=1'] + temp_args) - + # Tests emscripten_fetch() usage to persist an XHR into IndexedDB and subsequently load up from there. def test_fetch_cached_xhr(self): temp_args = [path_from_root('system/lib/fetch/emscripten_fetch.cpp'), '--js-library', path_from_root('src/library_fetch.js')] # TODO: Emscripten system libs, pass these e.g. with -lfetch shutil.copyfile(path_from_root('tests', 'gears.png'), os.path.join(self.get_dir(), 'gears.png')) - self.btest('fetch/cached_xhr.cpp', expected='0', args=['--std=c++11', '-s', 'FETCH_DEBUG=1'] + temp_args) + self.btest('fetch/cached_xhr.cpp', expected='1', args=['--std=c++11', '-s', 'FETCH_DEBUG=1'] + temp_args) + + # Test emscripten_fetch() usage to stream a XHR in to memory without storing the full file in memory + def test_fetch_stream_file(self): + temp_args = [path_from_root('system/lib/fetch/emscripten_fetch.cpp'), '--js-library', path_from_root('src/library_fetch.js')] # TODO: Emscripten system libs, pass these e.g. with -lfetch + + # Strategy: create a large 128MB file, and compile with a small 16MB Emscripten heap, so that the tested file + # won't fully fit in the heap. This verifies that streaming works properly. + f = open('largefile.txt', 'w') + s = '12345678' + for i in xrange(14): + s = s[::-1] + s # length of str will be 2^17=128KB + for i in xrange(1024): + f.write(s) + f.close() + self.btest('fetch/stream_file.cpp', expected='1', args=['--std=c++11', '-s', 'FETCH_DEBUG=1', '-s', 'TOTAL_MEMORY=536870912'] + temp_args) From 857e09bca3504a642ad5fa14a7d673d04b967b55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Wed, 8 Jun 2016 11:27:37 +0300 Subject: [PATCH 09/74] Add support and testing for synchronous emscripten_fetch() in --proxy-to-worker mode. --- src/library_fetch.js | 6 ++-- system/include/emscripten/fetch.h | 6 +++- tests/fetch/sync_xhr.cpp | 49 +++++++++++++++++++++++++++++++ tests/test_browser.py | 7 +++++ 4 files changed, 64 insertions(+), 4 deletions(-) create mode 100644 tests/fetch/sync_xhr.cpp diff --git a/src/library_fetch.js b/src/library_fetch.js index 166c1788600cf..6e8feac94b641 100644 --- a/src/library_fetch.js +++ b/src/library_fetch.js @@ -157,7 +157,6 @@ var LibraryFetch = { var fetchAttributes = {{{ makeGetValue('fetch_attr', 48/*TODO*/, 'i32') }}}; var timeoutMsecs = {{{ makeGetValue('fetch_attr', 52/*TODO*/, 'i32') }}}; var withCredentials = !!({{{ makeGetValue('fetch_attr', 56/*TODO*/, 'i32') }}}); - var synchronousRequest = !!({{{ makeGetValue('fetch_attr', 60/*TODO*/, 'i32') }}}); var destinationPath = {{{ makeGetValue('fetch_attr', 64/*TODO*/, 'i32') }}}; var userName = {{{ makeGetValue('fetch_attr', 68/*TODO*/, 'i32') }}}; var password = {{{ makeGetValue('fetch_attr', 72/*TODO*/, 'i32') }}}; @@ -170,20 +169,21 @@ var LibraryFetch = { var fetchAttrAppend = !!(fetchAttributes & 8/*EMSCRIPTEN_FETCH_APPEND*/); var fetchAttrReplace = !!(fetchAttributes & 16/*EMSCRIPTEN_FETCH_REPLACE*/); var fetchAttrNoDownload = !!(fetchAttributes & 32/*EMSCRIPTEN_FETCH_NO_DOWNLOAD*/); + var fetchAttrSynchronous = !!(fetchAttributes & 64/*EMSCRIPTEN_FETCH_SYNCHRONOUS*/); var userNameStr = userName ? Pointer_stringify(userName) : undefined; var passwordStr = password ? Pointer_stringify(password) : undefined; var overriddenMimeTypeStr = overriddenMimeType ? Pointer_stringify(overriddenMimeType) : undefined; var xhr = new XMLHttpRequest(); - xhr.responseType = fetchAttrStreamData ? 'moz-chunked-arraybuffer' : 'arraybuffer'; xhr.timeout = timeoutMsecs; xhr.withCredentials = withCredentials; #if FETCH_DEBUG console.log('fetch: xhr.timeout: ' + xhr.timeout + ', xhr.withCredentials: ' + xhr.withCredentials); console.log('fetch: xhr.open(requestMethod="' + requestMethod + '", url: "' + url_ +'", userName: ' + userNameStr + ', password: ' + passwordStr + ');'); #endif - xhr.open(requestMethod, url_, !synchronousRequest, userNameStr, passwordStr); + xhr.open(requestMethod, url_, !fetchAttrSynchronous, userNameStr, passwordStr); + xhr.responseType = fetchAttrStreamData ? 'moz-chunked-arraybuffer' : 'arraybuffer'; if (overriddenMimeType) { #if FETCH_DEBUG diff --git a/system/include/emscripten/fetch.h b/system/include/emscripten/fetch.h index 6662b0719a024..8316c9d0a657f 100644 --- a/system/include/emscripten/fetch.h +++ b/system/include/emscripten/fetch.h @@ -34,6 +34,10 @@ extern "C" { // EMSCRIPTEN_FETCH_APPEND, EMSCRIPTEN_FETCH_REPLACE and EMSCRIPTEN_FETCH_NO_DOWNLOAD are mutually exclusive. #define EMSCRIPTEN_FETCH_NO_DOWNLOAD 32 +// If specified, emscripten_fetch() will synchronously run to completion before returning. +// The callback handlers will be called from within emscripten_fetch() while the operation is in progress. +#define EMSCRIPTEN_FETCH_SYNCHRONOUS 64 + struct emscripten_fetch_t; // Specifies the parameters for a newly initiated fetch operation. @@ -61,7 +65,7 @@ struct emscripten_fetch_attr_t // If true, performs a synchronous blocking XHR. The emscripten_fetch() function call does not return until // the request has completed. Setting this to true in the main browser thread will fail with // EMSCRIPTEN_RESULT_NOT_SUPPORTED. - EM_BOOL synchronousRequest; + EM_BOOL synchronousRequestTODODELETEONCESTRUCTJSONIFIED; // Specifies the destination path in IndexedDB where to store the downloaded content body. If this is empty, the transfer // is not stored to IndexedDB at all. diff --git a/tests/fetch/sync_xhr.cpp b/tests/fetch/sync_xhr.cpp new file mode 100644 index 0000000000000..4a38ecb67fb5e --- /dev/null +++ b/tests/fetch/sync_xhr.cpp @@ -0,0 +1,49 @@ +#include +#include +#include +#include +#include + +int result = 0; + +int main() +{ + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.attributes = EMSCRIPTEN_FETCH_REPLACE | EMSCRIPTEN_FETCH_LOAD_TO_MEMORY | EMSCRIPTEN_FETCH_SYNCHRONOUS; + + attr.onsuccess = [](emscripten_fetch_t *fetch) { + printf("Finished downloading %llu bytes\n", fetch->numBytes); + // Compute rudimentary checksum of data + uint8_t checksum = 0; + for(int i = 0; i < fetch->numBytes; ++i) + checksum ^= fetch->data[i]; + printf("Data checksum: %02X\n", checksum); + assert(checksum == 0x08); + emscripten_fetch_close(fetch); + + if (result == 0) result = 1; +#ifdef REPORT_RESULT + REPORT_RESULT(); +#endif + }; + + attr.onprogress = [](emscripten_fetch_t *fetch) { + printf("Downloading.. %.2f%% complete.\n", (fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes); + }; + + attr.onerror = [](emscripten_fetch_t *fetch) { + printf("Download failed!\n"); + assert(false && "Shouldn't fail!"); + }; + + emscripten_fetch_t *fetch = emscripten_fetch(&attr, "gears.png"); + if (result == 0) { + result = 2; + printf("emscripten_fetch() failed to run synchronously!\n"); + } +#ifdef REPORT_RESULT + REPORT_RESULT(); +#endif +} diff --git a/tests/test_browser.py b/tests/test_browser.py index 7deadad7411eb..a06b5c382fa2a 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -3325,3 +3325,10 @@ def test_fetch_stream_file(self): f.write(s) f.close() self.btest('fetch/stream_file.cpp', expected='1', args=['--std=c++11', '-s', 'FETCH_DEBUG=1', '-s', 'TOTAL_MEMORY=536870912'] + temp_args) + + # Tests emscripten_fetch() usage in synchronous mode. + def test_fetch_sync_xhr(self): + temp_args = [path_from_root('system/lib/fetch/emscripten_fetch.cpp'), '--js-library', path_from_root('src/library_fetch.js')] # TODO: Emscripten system libs, pass these e.g. with -lfetch + + shutil.copyfile(path_from_root('tests', 'gears.png'), os.path.join(self.get_dir(), 'gears.png')) + self.btest('fetch/sync_xhr.cpp', expected='1', args=['--std=c++11', '-s', 'FETCH_DEBUG=1', '--proxy-to-worker'] + temp_args) From 437a815b92fa72299879e5d6de382b6554eea416 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Thu, 9 Jun 2016 00:09:34 +0300 Subject: [PATCH 10/74] Restructure library_fetch so it can be run from a worker. --- src/Fetch.js | 381 ++++++++++++++++++++++++++++++++++++++++++ src/library_fetch.js | 386 +------------------------------------------ 2 files changed, 388 insertions(+), 379 deletions(-) create mode 100644 src/Fetch.js diff --git a/src/Fetch.js b/src/Fetch.js new file mode 100644 index 0000000000000..dd21d72b1aa1e --- /dev/null +++ b/src/Fetch.js @@ -0,0 +1,381 @@ +var Fetch = { + xhrs: [], + // Specifies an instance to the IndexedDB database. The database is opened + // as a preload step before the Emscripten application starts. + dbInstance: undefined, + + openDatabase: function(dbname, dbversion, onsuccess, onerror) { + try { +#if FETCH_DEBUG + console.log('fetch: indexedDB.open(dbname="' + dbname + '", dbversion="' + dbversion + '");'); +#endif + var openRequest = indexedDB.open(dbname, dbversion); + } catch (e) { return onerror(e); } + + openRequest.onupgradeneeded = function(event) { +#if FETCH_DEBUG + console.log('fetch: IndexedDB upgrade needed. Clearing database.'); +#endif + var db = event.target.result; + if (db.objectStoreNames.contains('FILES')) { + db.deleteObjectStore('FILES'); + } + db.createObjectStore('FILES'); + }; + openRequest.onsuccess = function(event) { onsuccess(event.target.result); }; + openRequest.onerror = function(error) { onerror(error); }; + }, + + staticInit: function() { + var onsuccess = function(db) { +#if FETCH_DEBUG + console.log('fetch: IndexedDB successfully opened.'); +#endif + Fetch.dbInstance = db; + removeRunDependency('library_fetch_init'); + }; + var onerror = function() { +#if FETCH_DEBUG + console.error('fetch: IndexedDB open failed.'); +#endif + Fetch.dbInstance = false; + removeRunDependency('library_fetch_init'); + }; + Fetch.openDatabase('emscripten_filesystem', 1, onsuccess, onerror); + addRunDependency('library_fetch_init'); + } +} + +function _emscripten_fetch_load_cached_data(db, fetch, onsuccess, onerror) { + if (!db) { +#if FETCH_DEBUG + console.error('fetch: IndexedDB not available!'); +#endif + onerror(fetch, 0, 'IndexedDB not available!'); + return; + } + + var fetch_attr = fetch + 108/*TODO:structs_info*/; + var path = {{{ makeGetValue('fetch_attr', 64/*TODO*/, 'i32') }}}; + if (!path) path = {{{ makeGetValue('fetch', 8/*TODO:structs_info.jsonify this*/, 'i32') }}}; + var pathStr = Pointer_stringify(path); + + try { + var transaction = db.transaction(['FILES'], 'readonly'); + var packages = transaction.objectStore('FILES'); + var getRequest = packages.get(pathStr); + getRequest.onsuccess = function(event) { + if (event.target.result) { + var value = event.target.result; + var len = value.byteLength || value.length; +#if FETCH_DEBUG + console.log('fetch: Loaded file ' + pathStr + ' from IndexedDB, length: ' + len); +#endif + + var ptr = _malloc(len); // TODO: make emscripten_fetch_close() free() this data. + HEAPU8.set(new Uint8Array(value), ptr); + {{{ makeSetValue('fetch', 12/*TODO:jsonify*/, 'ptr', 'i32')}}}; + {{{ makeSetValue('fetch', 16/*TODO:jsonify*/, 'len', 'i64')}}}; + {{{ makeSetValue('fetch', 24/*TODO:jsonify*/, '0', 'i64')}}}; + {{{ makeSetValue('fetch', 32/*TODO:jsonify*/, 'len', 'i64')}}}; + + onsuccess(fetch, 0, value); + } else { + // Succeeded to load, but the load came back with the value of undefined, treat that as an error since we never store undefined in db. +#if FETCH_DEBUG + console.error('fetch: Loaded file ' + pathStr + ' from IndexedDB, but it had 0 length!'); +#endif + onerror(fetch, 0, 'no data'); + } + }; + getRequest.onerror = function(error) { +#if FETCH_DEBUG + console.error('fetch: Failed to load file ' + pathStr + ' from IndexedDB!'); +#endif + onerror(fetch, 0, error); + }; + } catch(e) { +#if FETCH_DEBUG + console.error('fetch: Failed to load file ' + pathStr + ' from IndexedDB! Got exception ' + e); +#endif + onerror(fetch, 0, e); + } +} + +function _emscripten_fetch_cache_data(db, fetch, data, onsuccess, onerror) { + if (!db) { +#if FETCH_DEBUG + console.error('fetch: IndexedDB not available!'); +#endif + onerror(fetch, 0, 'IndexedDB not available!'); + return; + } + + var fetch_attr = fetch + 108/*TODO:structs_info*/; + var destinationPath = {{{ makeGetValue('fetch_attr', 64/*TODO*/, 'i32') }}}; + if (!destinationPath) destinationPath = {{{ makeGetValue('fetch', 8/*TODO*/, 'i32') }}}; + var destinationPathStr = Pointer_stringify(destinationPath); + + try { + var transaction = db.transaction(['FILES'], 'readwrite'); + var packages = transaction.objectStore('FILES'); + var putRequest = packages.put(data, destinationPathStr); + putRequest.onsuccess = function(event) { +#if FETCH_DEBUG + console.log('fetch: Stored file "' + destinationPathStr + '" to IndexedDB cache.'); +#endif + onsuccess(fetch, 0, destinationPathStr); + }; + putRequest.onerror = function(error) { +#if FETCH_DEBUG + console.error('fetch: Failed to store file "' + destinationPathStr + '" to IndexedDB cache!'); +#endif + onerror(fetch, 0, error); + }; + } catch(e) { +#if FETCH_DEBUG + console.error('fetch: Failed to store file "' + destinationPathStr + '" to IndexedDB cache! Exception: ' + e); +#endif + onerror(fetch, 0, e); + } +} + +function _emscripten_fetch_xhr(fetch, onsuccess, onerror, onprogress) { + var url = {{{ makeGetValue('fetch', 8/*TODO:structs_info.jsonify this*/, 'i32') }}}; + if (!url) { +#if FETCH_DEBUG + console.error('fetch: XHR failed, no URL specified!'); +#endif + onerror(fetch, 0, 'no url specified!'); + return; + } + var url_ = Pointer_stringify(url); + + var fetch_attr = fetch + 108/*TODO:structs_info*/; + var requestMethod = fetch_attr ? Pointer_stringify(fetch_attr) : 'GET'; + var userData = {{{ makeGetValue('fetch_attr', 32/*TODO:structs_info.jsonify this*/, 'i32') }}}; + var fetchAttributes = {{{ makeGetValue('fetch_attr', 48/*TODO*/, 'i32') }}}; + var timeoutMsecs = {{{ makeGetValue('fetch_attr', 52/*TODO*/, 'i32') }}}; + var withCredentials = !!({{{ makeGetValue('fetch_attr', 56/*TODO*/, 'i32') }}}); + var destinationPath = {{{ makeGetValue('fetch_attr', 64/*TODO*/, 'i32') }}}; + var userName = {{{ makeGetValue('fetch_attr', 68/*TODO*/, 'i32') }}}; + var password = {{{ makeGetValue('fetch_attr', 72/*TODO*/, 'i32') }}}; + var requestHeaders = {{{ makeGetValue('fetch_attr', 76/*TODO*/, 'i32') }}}; + var overriddenMimeType = {{{ makeGetValue('fetch_attr', 80/*TODO*/, 'i32') }}}; + + var fetchAttrLoadToMemory = !!(fetchAttributes & 1/*EMSCRIPTEN_FETCH_LOAD_TO_MEMORY*/); + var fetchAttrStreamData = !!(fetchAttributes & 2/*EMSCRIPTEN_FETCH_STREAM_DATA*/); + var fetchAttrPersistFile = !!(fetchAttributes & 4/*EMSCRIPTEN_FETCH_PERSIST_FILE*/); + var fetchAttrAppend = !!(fetchAttributes & 8/*EMSCRIPTEN_FETCH_APPEND*/); + var fetchAttrReplace = !!(fetchAttributes & 16/*EMSCRIPTEN_FETCH_REPLACE*/); + var fetchAttrNoDownload = !!(fetchAttributes & 32/*EMSCRIPTEN_FETCH_NO_DOWNLOAD*/); + var fetchAttrSynchronous = !!(fetchAttributes & 64/*EMSCRIPTEN_FETCH_SYNCHRONOUS*/); + + var userNameStr = userName ? Pointer_stringify(userName) : undefined; + var passwordStr = password ? Pointer_stringify(password) : undefined; + var overriddenMimeTypeStr = overriddenMimeType ? Pointer_stringify(overriddenMimeType) : undefined; + + var xhr = new XMLHttpRequest(); + xhr.timeout = timeoutMsecs; + xhr.withCredentials = withCredentials; +#if FETCH_DEBUG + console.log('fetch: xhr.timeout: ' + xhr.timeout + ', xhr.withCredentials: ' + xhr.withCredentials); + console.log('fetch: xhr.open(requestMethod="' + requestMethod + '", url: "' + url_ +'", userName: ' + userNameStr + ', password: ' + passwordStr + ');'); +#endif + xhr.open(requestMethod, url_, !fetchAttrSynchronous, userNameStr, passwordStr); + xhr.responseType = fetchAttrStreamData ? 'moz-chunked-arraybuffer' : 'arraybuffer'; + + if (overriddenMimeType) { +#if FETCH_DEBUG + console.log('fetch: xhr.overrideMimeType("' + overriddenMimeTypeStr + '");'); +#endif + xhr.overrideMimeType(overriddenMimeTypeStr); + } + if (requestHeaders) { + for(;;) { + var key = {{{ makeGetValue('requestHeaders', 0, 'i32') }}}; + if (!key) break; + var value = {{{ makeGetValue('requestHeaders', 4, 'i32') }}}; + if (!value) break; + requestHeaders += 8; + var keyStr = Pointer_stringify(key); + var valueStr = Pointer_stringify(value); +#if FETCH_DEBUG + console.log('fetch: xhr.setRequestHeader("' + keyStr + '", "' + valueStr + '");'); +#endif + xhr.setRequestHeader(keyStr, valueStr); + } + } + Fetch.xhrs.push(xhr); + var id = Fetch.xhrs.length; + {{{ makeSetValue('fetch', 0/*TODO:jsonify*/, 'id', 'i32')}}}; + var data = null; // TODO: Support user to pass data to request. + // TODO: Support specifying custom headers to the request. + + xhr.onload = function(e) { + var len = xhr.response ? xhr.response.byteLength : 0; + var ptr = 0; + var ptrLen = 0; + if (fetchAttrLoadToMemory && !fetchAttrStreamData) { + ptrLen = len; +#if FETCH_DEBUG + console.log('fetch: allocating ' + ptrLen + ' bytes in Emscripten heap for xhr data'); +#endif + ptr = _malloc(ptrLen); // TODO: make emscripten_fetch_close() free() this data. + HEAPU8.set(new Uint8Array(xhr.response), ptr); + } + {{{ makeSetValue('fetch', 12/*TODO:jsonify*/, 'ptr', 'i32')}}}; + {{{ makeSetValue('fetch', 16/*TODO:jsonify*/, 'ptrLen', 'i64')}}}; + {{{ makeSetValue('fetch', 24/*TODO:jsonify*/, '0', 'i64')}}}; + if (len) { + {{{ makeSetValue('fetch', 32/*TODO:jsonify*/, 'len', 'i64')}}}; + } + if (xhr.status == 200 || xhr.status == 0) { +#if FETCH_DEBUG + console.log('fetch: xhr succeeded with status 200'); +#endif + if (onsuccess) onsuccess(fetch, xhr, e); + } else { +#if FETCH_DEBUG + console.error('fetch: xhr failed with status ' + xhr.status); +#endif + if (onerror) onerror(fetch, xhr, e); + } + } + xhr.onerror = function(e) { +#if FETCH_DEBUG + console.error('fetch: xhr failed with error ' + e); +#endif + if (onerror) onerror(fetch, xhr, e); + } + xhr.ontimeout = function(e) { +#if FETCH_DEBUG + console.error('fetch: xhr timed out with error ' + e); +#endif + if (onerror) onerror(fetch, xhr, e); + } + xhr.onprogress = function(e) { + var ptrLen = (fetchAttrLoadToMemory && fetchAttrStreamData && xhr.response) ? xhr.response.byteLength : 0; + var ptr = 0; + if (fetchAttrLoadToMemory && fetchAttrStreamData) { +#if FETCH_DEBUG + console.log('fetch: allocating ' + ptrLen + ' bytes in Emscripten heap for xhr data'); +#endif + ptr = _malloc(ptrLen); // TODO: make emscripten_fetch_close() free() this data. + HEAPU8.set(new Uint8Array(xhr.response), ptr); + } + {{{ makeSetValue('fetch', 12/*TODO:jsonify*/, 'ptr', 'i32')}}}; + {{{ makeSetValue('fetch', 16/*TODO:jsonify*/, 'ptrLen', 'i64')}}}; + {{{ makeSetValue('fetch', 24/*TODO:jsonify*/, 'e.loaded - ptrLen', 'i64')}}}; + {{{ makeSetValue('fetch', 32/*TODO:jsonify*/, 'e.total', 'i64')}}}; + if (onprogress) onprogress(fetch, xhr, e); + } +#if FETCH_DEBUG + console.log('fetch: xhr.send(data=' + data + ')'); +#endif + try { + xhr.send(data); + } catch(e) { +#if FETCH_DEBUG + console.error('fetch: xhr failed with exception: ' + e); +#endif + if (onerror) onerror(fetch, xhr, e); + } +} + +function emscripten_start_fetch(fetch) { + Module['noExitRuntime'] = true; // TODO: assumes we are the main Emscripten runtime thread, in the future won't. + + var fetch_attr = fetch + 108/*TODO:structs_info*/; + var onsuccess = {{{ makeGetValue('fetch_attr', 36/*TODO:structs_info.jsonify this*/, 'i32') }}}; + var onerror = {{{ makeGetValue('fetch_attr', 40/*TODO:structs_info.jsonify this*/, 'i32') }}}; + var onprogress = {{{ makeGetValue('fetch_attr', 44/*TODO:structs_info.jsonify this*/, 'i32') }}}; + var fetchAttributes = {{{ makeGetValue('fetch_attr', 48/*TODO*/, 'i32') }}}; + var fetchAttrLoadToMemory = !!(fetchAttributes & 1/*EMSCRIPTEN_FETCH_LOAD_TO_MEMORY*/); + var fetchAttrStreamData = !!(fetchAttributes & 2/*EMSCRIPTEN_FETCH_STREAM_DATA*/); + var fetchAttrPersistFile = !!(fetchAttributes & 4/*EMSCRIPTEN_FETCH_PERSIST_FILE*/); + var fetchAttrAppend = !!(fetchAttributes & 8/*EMSCRIPTEN_FETCH_APPEND*/); + var fetchAttrReplace = !!(fetchAttributes & 16/*EMSCRIPTEN_FETCH_REPLACE*/); + var fetchAttrNoDownload = !!(fetchAttributes & 32/*EMSCRIPTEN_FETCH_NO_DOWNLOAD*/); + + var reportSuccess = function(fetch, xhr, e) { +#if FETCH_DEBUG + console.log('fetch: operation success. e: ' + e); +#endif + if (onsuccess) Runtime.dynCall('vi', onsuccess, [fetch]); + }; + + var cacheResultAndReportSuccess = function(fetch, xhr, e) { +#if FETCH_DEBUG + console.log('fetch: operation success. Caching result.. e: ' + e); +#endif + var storeSuccess = function() { +#if FETCH_DEBUG + console.log('fetch: IndexedDB store succeeded.'); +#endif + }; + var storeError = function() { +#if FETCH_DEBUG + console.error('fetch: IndexedDB store failed.'); +#endif + }; + __emscripten_fetch_cache_data(Fetch.dbInstance, fetch, xhr.response, storeSuccess, storeError); + if (onsuccess) Runtime.dynCall('vi', onsuccess, [fetch]); + }; + + var reportProgress = function(fetch, xhr, e) { + if (onprogress) Runtime.dynCall('vi', onprogress, [fetch]); + }; + + var reportError = function(fetch, xhr, e) { +#if FETCH_DEBUG + console.error('fetch: operation failed: ' + e); +#endif + if (onerror) Runtime.dynCall('vi', onerror, [fetch]); + }; + + var performUncachedXhr = function(fetch, xhr, e) { +#if FETCH_DEBUG + console.error('fetch: starting (uncached) XHR: ' + e); +#endif + __emscripten_fetch_xhr(fetch, reportSuccess, reportError, reportProgress); + }; + + var performCachedXhr = function(fetch, xhr, e) { +#if FETCH_DEBUG + console.error('fetch: starting (cached) XHR: ' + e); +#endif + __emscripten_fetch_xhr(fetch, cacheResultAndReportSuccess, reportError, reportProgress); + }; + + // Should we try IndexedDB first? + if (!fetchAttrReplace) { + if (!Fetch.dbInstance) { +#if FETCH_DEBUG + console.error('fetch: failed to read IndexedDB! Database is not open.'); +#endif + reportError(fetch, 0, 'IndexedDB is not open'); + return 0; // todo: free + } + + if (fetchAttrNoDownload) { + __emscripten_fetch_load_cached_data(Fetch.dbInstance, fetch, reportSuccess, reportError); + } else if (fetchAttrPersistFile) { + __emscripten_fetch_load_cached_data(Fetch.dbInstance, fetch, reportSuccess, performCachedXhr); + } else { + __emscripten_fetch_load_cached_data(Fetch.dbInstance, fetch, reportSuccess, performUncachedXhr); + } + } else if (!fetchAttrNoDownload) { + if (fetchAttrPersistFile) { + __emscripten_fetch_xhr(fetch, cacheResultAndReportSuccess, reportError, reportProgress); + } else { + __emscripten_fetch_xhr(fetch, reportSuccess, reportError, reportProgress); + } + } else { +#if FETCH_DEBUG + console.error('fetch: Invalid combination of flags passed.'); +#endif + return 0; // todo: free + } + return fetch; +} diff --git a/src/library_fetch.js b/src/library_fetch.js index 6e8feac94b641..4e649d93be53d 100644 --- a/src/library_fetch.js +++ b/src/library_fetch.js @@ -1,385 +1,13 @@ +#include Fetch.js + var LibraryFetch = { $Fetch__postset: 'Fetch.staticInit();', - $Fetch: { - xhrs: [], - // Specifies an instance to the IndexedDB database. The database is opened - // as a preload step before the Emscripten application starts. - dbInstance: undefined, - - openDatabase: function(dbname, dbversion, onsuccess, onerror) { - try { -#if FETCH_DEBUG - console.log('fetch: indexedDB.open(dbname="' + dbname + '", dbversion="' + dbversion + '");') -#endif - var openRequest = indexedDB.open(dbname, dbversion); - } catch (e) { return onerror(e); } - - openRequest.onupgradeneeded = function(event) { -#if FETCH_DEBUG - console.log('fetch: IndexedDB upgrade needed. Clearing database.'); -#endif - var db = event.target.result; - if (db.objectStoreNames.contains('FILES')) db.deleteObjectStore('FILES'); - db.createObjectStore('FILES'); - }; - openRequest.onsuccess = function(event) { onsuccess(event.target.result); }; - openRequest.onerror = function(error) { onerror(error); }; - }, - - staticInit: function() { - var onsuccess = function(db) { -#if FETCH_DEBUG - console.log('fetch: IndexedDB successfully opened.'); -#endif - Fetch.dbInstance = db; - removeRunDependency('library_fetch_init'); - }; - var onerror = function() { -#if FETCH_DEBUG - console.error('fetch: IndexedDB open failed.'); -#endif - Fetch.dbInstance = false; - removeRunDependency('library_fetch_init'); - }; - Fetch.openDatabase('emscripten_filesystem', 1, onsuccess, onerror); - addRunDependency('library_fetch_init'); - } - }, - - _emscripten_fetch_load_cached_data: function(db, fetch, onsuccess, onerror) { - if (!db) { -#if FETCH_DEBUG - console.error('fetch: IndexedDB not available!'); -#endif - onerror(fetch, 0, 'IndexedDB not available!'); - return; - } - - var fetch_attr = fetch + 108/*TODO:structs_info*/; - var path = {{{ makeGetValue('fetch_attr', 64/*TODO*/, 'i32') }}}; - if (!path) path = {{{ makeGetValue('fetch', 8/*TODO:structs_info.jsonify this*/, 'i32') }}}; - var pathStr = Pointer_stringify(path); - - try { - var transaction = db.transaction(['FILES'], 'readonly'); - var packages = transaction.objectStore('FILES'); - var getRequest = packages.get(pathStr); - getRequest.onsuccess = function(event) { - if (event.target.result) { - var value = event.target.result; - var len = value.byteLength || value.length; -#if FETCH_DEBUG - console.log('fetch: Loaded file ' + pathStr + ' from IndexedDB, length: ' + len); -#endif - - var ptr = _malloc(len); // TODO: make emscripten_fetch_close() free() this data. - HEAPU8.set(new Uint8Array(value), ptr); - {{{ makeSetValue('fetch', 12/*TODO:jsonify*/, 'ptr', 'i32')}}}; - {{{ makeSetValue('fetch', 16/*TODO:jsonify*/, 'len', 'i64')}}}; - {{{ makeSetValue('fetch', 24/*TODO:jsonify*/, '0', 'i64')}}}; - {{{ makeSetValue('fetch', 32/*TODO:jsonify*/, 'len', 'i64')}}}; - - onsuccess(fetch, 0, value); - } else { - // Succeeded to load, but the load came back with the value of undefined, treat that as an error since we never store undefined in db. -#if FETCH_DEBUG - console.error('fetch: Loaded file ' + pathStr + ' from IndexedDB, but it had 0 length!'); -#endif - onerror(fetch, 0, 'no data'); - } - }; - getRequest.onerror = function(error) { -#if FETCH_DEBUG - console.error('fetch: Failed to load file ' + pathStr + ' from IndexedDB!'); -#endif - onerror(fetch, 0, error); - }; - } catch(e) { -#if FETCH_DEBUG - console.error('fetch: Failed to load file ' + pathStr + ' from IndexedDB! Got exception ' + e); -#endif - onerror(fetch, 0, e); - } - }, - - _emscripten_fetch_cache_data: function(db, fetch, data, onsuccess, onerror) { - if (!db) { -#if FETCH_DEBUG - console.error('fetch: IndexedDB not available!'); -#endif - onerror(fetch, 0, 'IndexedDB not available!'); - return; - } - - var fetch_attr = fetch + 108/*TODO:structs_info*/; - var destinationPath = {{{ makeGetValue('fetch_attr', 64/*TODO*/, 'i32') }}}; - if (!destinationPath) destinationPath = {{{ makeGetValue('fetch', 8/*TODO*/, 'i32') }}}; - var destinationPathStr = Pointer_stringify(destinationPath); - - try { - var transaction = db.transaction(['FILES'], 'readwrite'); - var packages = transaction.objectStore('FILES'); - var putRequest = packages.put(data, destinationPathStr); - putRequest.onsuccess = function(event) { -#if FETCH_DEBUG - console.log('fetch: Stored file "' + destinationPathStr + '" to IndexedDB cache.'); -#endif - onsuccess(fetch, 0, destinationPathStr); - }; - putRequest.onerror = function(error) { -#if FETCH_DEBUG - console.error('fetch: Failed to store file "' + destinationPathStr + '" to IndexedDB cache!'); -#endif - onerror(fetch, 0, error); - }; - } catch(e) { -#if FETCH_DEBUG - console.error('fetch: Failed to store file "' + destinationPathStr + '" to IndexedDB cache! Exception: ' + e); -#endif - onerror(fetch, 0, e); - } - }, - - _emscripten_fetch_xhr: function(fetch, onsuccess, onerror, onprogress) { - var url = {{{ makeGetValue('fetch', 8/*TODO:structs_info.jsonify this*/, 'i32') }}}; - if (!url) { -#if FETCH_DEBUG - console.error('fetch: XHR failed, no URL specified!'); -#endif - onerror(fetch, 0, 'no url specified!'); - return; - } - var url_ = Pointer_stringify(url); - - var fetch_attr = fetch + 108/*TODO:structs_info*/; - var requestMethod = fetch_attr ? Pointer_stringify(fetch_attr) : 'GET'; - var userData = {{{ makeGetValue('fetch_attr', 32/*TODO:structs_info.jsonify this*/, 'i32') }}}; - var fetchAttributes = {{{ makeGetValue('fetch_attr', 48/*TODO*/, 'i32') }}}; - var timeoutMsecs = {{{ makeGetValue('fetch_attr', 52/*TODO*/, 'i32') }}}; - var withCredentials = !!({{{ makeGetValue('fetch_attr', 56/*TODO*/, 'i32') }}}); - var destinationPath = {{{ makeGetValue('fetch_attr', 64/*TODO*/, 'i32') }}}; - var userName = {{{ makeGetValue('fetch_attr', 68/*TODO*/, 'i32') }}}; - var password = {{{ makeGetValue('fetch_attr', 72/*TODO*/, 'i32') }}}; - var requestHeaders = {{{ makeGetValue('fetch_attr', 76/*TODO*/, 'i32') }}}; - var overriddenMimeType = {{{ makeGetValue('fetch_attr', 80/*TODO*/, 'i32') }}}; - - var fetchAttrLoadToMemory = !!(fetchAttributes & 1/*EMSCRIPTEN_FETCH_LOAD_TO_MEMORY*/); - var fetchAttrStreamData = !!(fetchAttributes & 2/*EMSCRIPTEN_FETCH_STREAM_DATA*/); - var fetchAttrPersistFile = !!(fetchAttributes & 4/*EMSCRIPTEN_FETCH_PERSIST_FILE*/); - var fetchAttrAppend = !!(fetchAttributes & 8/*EMSCRIPTEN_FETCH_APPEND*/); - var fetchAttrReplace = !!(fetchAttributes & 16/*EMSCRIPTEN_FETCH_REPLACE*/); - var fetchAttrNoDownload = !!(fetchAttributes & 32/*EMSCRIPTEN_FETCH_NO_DOWNLOAD*/); - var fetchAttrSynchronous = !!(fetchAttributes & 64/*EMSCRIPTEN_FETCH_SYNCHRONOUS*/); - - var userNameStr = userName ? Pointer_stringify(userName) : undefined; - var passwordStr = password ? Pointer_stringify(password) : undefined; - var overriddenMimeTypeStr = overriddenMimeType ? Pointer_stringify(overriddenMimeType) : undefined; - - var xhr = new XMLHttpRequest(); - xhr.timeout = timeoutMsecs; - xhr.withCredentials = withCredentials; -#if FETCH_DEBUG - console.log('fetch: xhr.timeout: ' + xhr.timeout + ', xhr.withCredentials: ' + xhr.withCredentials); - console.log('fetch: xhr.open(requestMethod="' + requestMethod + '", url: "' + url_ +'", userName: ' + userNameStr + ', password: ' + passwordStr + ');'); -#endif - xhr.open(requestMethod, url_, !fetchAttrSynchronous, userNameStr, passwordStr); - xhr.responseType = fetchAttrStreamData ? 'moz-chunked-arraybuffer' : 'arraybuffer'; - - if (overriddenMimeType) { -#if FETCH_DEBUG - console.log('fetch: xhr.overrideMimeType("' + overriddenMimeTypeStr + '");'); -#endif - xhr.overrideMimeType(overriddenMimeTypeStr); - } - if (requestHeaders) { - for(;;) { - var key = {{{ makeGetValue('requestHeaders', 0, 'i32') }}}; - if (!key) break; - var value = {{{ makeGetValue('requestHeaders', 4, 'i32') }}}; - if (!value) break; - requestHeaders += 8; - var keyStr = Pointer_stringify(key); - var valueStr = Pointer_stringify(value); -#if FETCH_DEBUG - console.log('fetch: xhr.setRequestHeader("' + keyStr + '", "' + valueStr + '");'); -#endif - xhr.setRequestHeader(keyStr, valueStr); - } - } - Fetch.xhrs.push(xhr); - var id = Fetch.xhrs.length; - {{{ makeSetValue('fetch', 0/*TODO:jsonify*/, 'id', 'i32')}}}; - var data = null; // TODO: Support user to pass data to request. - // TODO: Support specifying custom headers to the request. - - xhr.onload = function(e) { - var len = xhr.response ? xhr.response.byteLength : 0; - var ptr = 0; - var ptrLen = 0; - if (fetchAttrLoadToMemory && !fetchAttrStreamData) { - ptrLen = len; -#if FETCH_DEBUG - console.log('fetch: allocating ' + ptrLen + ' bytes in Emscripten heap for xhr data'); -#endif - ptr = _malloc(ptrLen); // TODO: make emscripten_fetch_close() free() this data. - HEAPU8.set(new Uint8Array(xhr.response), ptr); - } - {{{ makeSetValue('fetch', 12/*TODO:jsonify*/, 'ptr', 'i32')}}}; - {{{ makeSetValue('fetch', 16/*TODO:jsonify*/, 'ptrLen', 'i64')}}}; - {{{ makeSetValue('fetch', 24/*TODO:jsonify*/, '0', 'i64')}}}; - if (len) { - {{{ makeSetValue('fetch', 32/*TODO:jsonify*/, 'len', 'i64')}}}; - } - if (xhr.status == 200 || xhr.status == 0) { -#if FETCH_DEBUG - console.log('fetch: xhr succeeded with status 200'); -#endif - if (onsuccess) onsuccess(fetch, xhr, e); - } else { -#if FETCH_DEBUG - console.error('fetch: xhr failed with status ' + xhr.status); -#endif - if (onerror) onerror(fetch, xhr, e); - } - } - xhr.onerror = function(e) { -#if FETCH_DEBUG - console.error('fetch: xhr failed with error ' + e); -#endif - if (onerror) onerror(fetch, xhr, e); - } - xhr.ontimeout = function(e) { -#if FETCH_DEBUG - console.error('fetch: xhr timed out with error ' + e); -#endif - if (onerror) onerror(fetch, xhr, e); - } - xhr.onprogress = function(e) { - var ptrLen = (fetchAttrLoadToMemory && fetchAttrStreamData && xhr.response) ? xhr.response.byteLength : 0; - var ptr = 0; - if (fetchAttrLoadToMemory && fetchAttrStreamData) { -#if FETCH_DEBUG - console.log('fetch: allocating ' + ptrLen + ' bytes in Emscripten heap for xhr data'); -#endif - ptr = _malloc(ptrLen); // TODO: make emscripten_fetch_close() free() this data. - HEAPU8.set(new Uint8Array(xhr.response), ptr); - } - {{{ makeSetValue('fetch', 12/*TODO:jsonify*/, 'ptr', 'i32')}}}; - {{{ makeSetValue('fetch', 16/*TODO:jsonify*/, 'ptrLen', 'i64')}}}; - {{{ makeSetValue('fetch', 24/*TODO:jsonify*/, 'e.loaded - ptrLen', 'i64')}}}; - {{{ makeSetValue('fetch', 32/*TODO:jsonify*/, 'e.total', 'i64')}}}; - if (onprogress) onprogress(fetch, xhr, e); - } -#if FETCH_DEBUG - console.log('fetch: xhr.send(data=' + data + ')'); -#endif - try { - xhr.send(data); - } catch(e) { -#if FETCH_DEBUG - console.error('fetch: xhr failed with exception: ' + e); -#endif - if (onerror) onerror(fetch, xhr, e); - } - }, - + $Fetch: Fetch, + _emscripten_fetch_load_cached_data: _emscripten_fetch_load_cached_data, + _emscripten_fetch_cache_data: _emscripten_fetch_cache_data, + _emscripten_fetch_xhr: _emscripten_fetch_xhr, emscripten_start_fetch__deps: ['$Fetch', '_emscripten_fetch_xhr', '_emscripten_fetch_cache_data', '_emscripten_fetch_load_cached_data'], - emscripten_start_fetch: function(fetch) { - Module['noExitRuntime'] = true; // TODO: assumes we are the main Emscripten runtime thread, in the future won't. - - var fetch_attr = fetch + 108/*TODO:structs_info*/; - var onsuccess = {{{ makeGetValue('fetch_attr', 36/*TODO:structs_info.jsonify this*/, 'i32') }}}; - var onerror = {{{ makeGetValue('fetch_attr', 40/*TODO:structs_info.jsonify this*/, 'i32') }}}; - var onprogress = {{{ makeGetValue('fetch_attr', 44/*TODO:structs_info.jsonify this*/, 'i32') }}}; - var fetchAttributes = {{{ makeGetValue('fetch_attr', 48/*TODO*/, 'i32') }}}; - var fetchAttrLoadToMemory = !!(fetchAttributes & 1/*EMSCRIPTEN_FETCH_LOAD_TO_MEMORY*/); - var fetchAttrStreamData = !!(fetchAttributes & 2/*EMSCRIPTEN_FETCH_STREAM_DATA*/); - var fetchAttrPersistFile = !!(fetchAttributes & 4/*EMSCRIPTEN_FETCH_PERSIST_FILE*/); - var fetchAttrAppend = !!(fetchAttributes & 8/*EMSCRIPTEN_FETCH_APPEND*/); - var fetchAttrReplace = !!(fetchAttributes & 16/*EMSCRIPTEN_FETCH_REPLACE*/); - var fetchAttrNoDownload = !!(fetchAttributes & 32/*EMSCRIPTEN_FETCH_NO_DOWNLOAD*/); - - var reportSuccess = function(fetch, xhr, e) { -#if FETCH_DEBUG - console.log('fetch: operation success. e: ' + e); -#endif - if (onsuccess) Runtime.dynCall('vi', onsuccess, [fetch]); - }; - - var cacheResultAndReportSuccess = function(fetch, xhr, e) { -#if FETCH_DEBUG - console.log('fetch: operation success. Caching result.. e: ' + e); -#endif - var storeSuccess = function() { -#if FETCH_DEBUG - console.log('fetch: IndexedDB store succeeded.'); -#endif - }; - var storeError = function() { -#if FETCH_DEBUG - console.error('fetch: IndexedDB store failed.'); -#endif - }; - __emscripten_fetch_cache_data(Fetch.dbInstance, fetch, xhr.response, storeSuccess, storeError); - if (onsuccess) Runtime.dynCall('vi', onsuccess, [fetch]); - }; - - var reportProgress = function(fetch, xhr, e) { - if (onprogress) Runtime.dynCall('vi', onprogress, [fetch]); - }; - - var reportError = function(fetch, xhr, e) { -#if FETCH_DEBUG - console.error('fetch: operation failed: ' + e); -#endif - if (onerror) Runtime.dynCall('vi', onerror, [fetch]); - }; - - var performUncachedXhr = function(fetch, xhr, e) { -#if FETCH_DEBUG - console.error('fetch: starting (uncached) XHR: ' + e); -#endif - __emscripten_fetch_xhr(fetch, reportSuccess, reportError, reportProgress); - }; - - var performCachedXhr = function(fetch, xhr, e) { -#if FETCH_DEBUG - console.error('fetch: starting (cached) XHR: ' + e); -#endif - __emscripten_fetch_xhr(fetch, cacheResultAndReportSuccess, reportError, reportProgress); - }; - - // Should we try IndexedDB first? - if (!fetchAttrReplace) { - if (!Fetch.dbInstance) { -#if FETCH_DEBUG - console.error('fetch: failed to read IndexedDB! Database is not open.'); -#endif - reportError(fetch, 0, 'IndexedDB is not open'); - return 0; // todo: free - } - - if (fetchAttrNoDownload) { - __emscripten_fetch_load_cached_data(Fetch.dbInstance, fetch, reportSuccess, reportError); - } else if (fetchAttrPersistFile) { - __emscripten_fetch_load_cached_data(Fetch.dbInstance, fetch, reportSuccess, performCachedXhr); - } else { - __emscripten_fetch_load_cached_data(Fetch.dbInstance, fetch, reportSuccess, performUncachedXhr); - } - } else if (!fetchAttrNoDownload) { - if (fetchAttrPersistFile) { - __emscripten_fetch_xhr(fetch, cacheResultAndReportSuccess, reportError, reportProgress); - } else { - __emscripten_fetch_xhr(fetch, reportSuccess, reportError, reportProgress); - } - } else { -#if FETCH_DEBUG - console.error('fetch: Invalid combination of flags passed.'); -#endif - return 0; // todo: free - } - return fetch; - } + emscripten_start_fetch: emscripten_start_fetch }; mergeInto(LibraryManager.library, LibraryFetch); From 2d847d93498683f43448a3c2b9fecd5e537675b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Thu, 9 Jun 2016 17:48:53 +0300 Subject: [PATCH 11/74] Add very hacky proof of concept of sync fetches on the main thread with SAB+proxy-to-worker. --- emcc.py | 21 ++++ src/Fetch.js | 139 +++++++++++++++++--------- src/library_fetch.js | 10 +- src/preamble.js | 4 +- system/include/emscripten/fetch.h | 6 ++ system/lib/fetch/emscripten_fetch.cpp | 62 +++++++++++- tools/shared.py | 4 + 7 files changed, 191 insertions(+), 55 deletions(-) diff --git a/emcc.py b/emcc.py index 3b113d063208b..1e1e6b84eb002 100755 --- a/emcc.py +++ b/emcc.py @@ -1754,6 +1754,27 @@ def repl(m): if shared.Settings.USE_PTHREADS: shutil.copyfile(shared.path_from_root('src', 'pthread-main.js'), os.path.join(os.path.dirname(os.path.abspath(target)), 'pthread-main.js')) + src = open(final, 'r').read() + funcs_to_import = ['alignMemoryPage', '_sbrk', '_pthread_mutex_lock', '_malloc', '_emscripten_sync_run_in_main_thread_1', '_emscripten_sync_run_in_main_thread', '_emscripten_is_main_runtime_thread', '_pthread_mutex_unlock', '_emscripten_set_current_thread_status', '_emscripten_futex_wait', 'writeStringToMemory', 'intArrayFromString', 'lengthBytesUTF8', 'stringToUTF8Array'] + function_prologue = '' + for func in funcs_to_import: + loc = src.find('function ' + func + '(') + if loc == -1: + logging.fatal('failed to find function ' + func + '!') + sys.exit(1) + end_loc = src.find('{', loc) + 1 + nesting_level = 1 + while nesting_level > 0: + if src[end_loc] == '{': nesting_level += 1 + if src[end_loc] == '}': nesting_level -= 1 + end_loc += 1 + + func_code = src[loc:end_loc] + function_prologue = function_prologue + '\n' + func_code + + fetch_worker_src = function_prologue + '\n' + shared.clang_preprocess(shared.path_from_root('src', 'fetch-worker.js')) + open('fetch-worker.js', 'w').write(fetch_worker_src) + if shared.Settings.BINARYEN: # Insert a call to integrate with wasm.js js = open(final).read() diff --git a/src/Fetch.js b/src/Fetch.js index dd21d72b1aa1e..43dbfb277d0cf 100644 --- a/src/Fetch.js +++ b/src/Fetch.js @@ -1,9 +1,16 @@ var Fetch = { xhrs: [], + // The web worker that runs proxied file I/O requests. + worker: undefined, // Specifies an instance to the IndexedDB database. The database is opened // as a preload step before the Emscripten application starts. dbInstance: undefined, + setu64: function(addr, val) { + HEAPU32[addr >> 2] = val; + HEAPU32[addr + 4 >> 2] = (val / 4294967296)|0; + }, + openDatabase: function(dbname, dbversion, onsuccess, onerror) { try { #if FETCH_DEBUG @@ -26,23 +33,46 @@ var Fetch = { openRequest.onerror = function(error) { onerror(error); }; }, + initFetchWorker: function() { + var stackSize = 128*1024; + var stack = allocate(stackSize>>2, "i32*", ALLOC_DYNAMIC); + Fetch.worker.postMessage({cmd: 'init', TOTAL_MEMORY: TOTAL_MEMORY, DYNAMICTOP: DYNAMICTOP + 1024*1024, STACKTOP: stack, STACK_MAX: stack + stackSize, queuePtr: _fetch_work_queue, buffer: HEAPU8.buffer}, [HEAPU8.buffer]); + }, + staticInit: function() { var onsuccess = function(db) { #if FETCH_DEBUG console.log('fetch: IndexedDB successfully opened.'); #endif Fetch.dbInstance = db; - removeRunDependency('library_fetch_init'); + if (typeof ENVIRONMENT_IS_FETCH_WORKER === 'undefined' && !ENVIRONMENT_IS_PTHREAD) { + Fetch.initFetchWorker(); + removeRunDependency('library_fetch_init'); + } }; var onerror = function() { #if FETCH_DEBUG console.error('fetch: IndexedDB open failed.'); #endif Fetch.dbInstance = false; - removeRunDependency('library_fetch_init'); + if (typeof ENVIRONMENT_IS_FETCH_WORKER === 'undefined' && !ENVIRONMENT_IS_PTHREAD) { + Fetch.initFetchWorker(); + removeRunDependency('library_fetch_init'); + } }; Fetch.openDatabase('emscripten_filesystem', 1, onsuccess, onerror); - addRunDependency('library_fetch_init'); + + if (typeof ENVIRONMENT_IS_FETCH_WORKER === 'undefined' && !ENVIRONMENT_IS_PTHREAD) { + addRunDependency('library_fetch_init'); + + var fetchJs = 'fetch-worker.js'; + // Allow HTML module to configure the location where the 'pthread-main.js' file will be loaded from, + // either via Module.locateFile() function, or via Module.pthreadMainPrefixURL string. If neither + // of these are passed, then the default URL 'pthread-main.js' relative to the main html file is loaded. + if (typeof Module['locateFile'] === 'function') fetchJs = Module['locateFile'](fetchJs); + else if (Module['pthreadMainPrefixURL']) fetchJs = Module['pthreadMainPrefixURL'] + fetchJs; + Fetch.worker = new Worker(fetchJs); + } } } @@ -55,9 +85,9 @@ function _emscripten_fetch_load_cached_data(db, fetch, onsuccess, onerror) { return; } - var fetch_attr = fetch + 108/*TODO:structs_info*/; - var path = {{{ makeGetValue('fetch_attr', 64/*TODO*/, 'i32') }}}; - if (!path) path = {{{ makeGetValue('fetch', 8/*TODO:structs_info.jsonify this*/, 'i32') }}}; + var fetch_attr = fetch + 112/*TODO:structs_info*/; + var path = HEAPU32[fetch_attr + 64 >> 2];//{{{ makeGetValue('fetch_attr', 64/*TODO*/, 'i32') }}}; + if (!path) path = HEAPU32[fetch + 8 >> 2];//{{{ makeGetValue('fetch', 8/*TODO:structs_info.jsonify this*/, 'i32') }}}; var pathStr = Pointer_stringify(path); try { @@ -74,10 +104,10 @@ function _emscripten_fetch_load_cached_data(db, fetch, onsuccess, onerror) { var ptr = _malloc(len); // TODO: make emscripten_fetch_close() free() this data. HEAPU8.set(new Uint8Array(value), ptr); - {{{ makeSetValue('fetch', 12/*TODO:jsonify*/, 'ptr', 'i32')}}}; - {{{ makeSetValue('fetch', 16/*TODO:jsonify*/, 'len', 'i64')}}}; - {{{ makeSetValue('fetch', 24/*TODO:jsonify*/, '0', 'i64')}}}; - {{{ makeSetValue('fetch', 32/*TODO:jsonify*/, 'len', 'i64')}}}; + HEAPU32[fetch + 12 >> 2] = ptr;//{{{ makeSetValue('fetch', 12/*TODO:jsonify*/, 'ptr', 'i32')}}}; + Fetch.setu64(fetch + 16, len);//{{{ makeSetValue('fetch', 16/*TODO:jsonify*/, 'len', 'i64')}}}; + Fetch.setu64(fetch + 24, 0);//{{{ makeSetValue('fetch', 24/*TODO:jsonify*/, '0', 'i64')}}}; + Fetch.setu64(fetch + 32, len);//{{{ makeSetValue('fetch', 32/*TODO:jsonify*/, 'len', 'i64')}}}; onsuccess(fetch, 0, value); } else { @@ -111,9 +141,9 @@ function _emscripten_fetch_cache_data(db, fetch, data, onsuccess, onerror) { return; } - var fetch_attr = fetch + 108/*TODO:structs_info*/; - var destinationPath = {{{ makeGetValue('fetch_attr', 64/*TODO*/, 'i32') }}}; - if (!destinationPath) destinationPath = {{{ makeGetValue('fetch', 8/*TODO*/, 'i32') }}}; + var fetch_attr = fetch + 112/*TODO:structs_info*/; + var destinationPath = HEAPU32[fetch_attr + 64 >> 2];//{{{ makeGetValue('fetch_attr', 64/*TODO*/, 'i32') }}}; + if (!destinationPath) destinationPath = HEAPU32[fetch + 8 >> 2];//{{{ makeGetValue('fetch', 8/*TODO*/, 'i32') }}}; var destinationPathStr = Pointer_stringify(destinationPath); try { @@ -141,7 +171,7 @@ function _emscripten_fetch_cache_data(db, fetch, data, onsuccess, onerror) { } function _emscripten_fetch_xhr(fetch, onsuccess, onerror, onprogress) { - var url = {{{ makeGetValue('fetch', 8/*TODO:structs_info.jsonify this*/, 'i32') }}}; + var url = HEAPU32[fetch + 8 >> 2];//{{{ makeGetValue('fetch', 8/*TODO:structs_info.jsonify this*/, 'i32') }}}; if (!url) { #if FETCH_DEBUG console.error('fetch: XHR failed, no URL specified!'); @@ -151,17 +181,18 @@ function _emscripten_fetch_xhr(fetch, onsuccess, onerror, onprogress) { } var url_ = Pointer_stringify(url); - var fetch_attr = fetch + 108/*TODO:structs_info*/; - var requestMethod = fetch_attr ? Pointer_stringify(fetch_attr) : 'GET'; - var userData = {{{ makeGetValue('fetch_attr', 32/*TODO:structs_info.jsonify this*/, 'i32') }}}; - var fetchAttributes = {{{ makeGetValue('fetch_attr', 48/*TODO*/, 'i32') }}}; - var timeoutMsecs = {{{ makeGetValue('fetch_attr', 52/*TODO*/, 'i32') }}}; - var withCredentials = !!({{{ makeGetValue('fetch_attr', 56/*TODO*/, 'i32') }}}); - var destinationPath = {{{ makeGetValue('fetch_attr', 64/*TODO*/, 'i32') }}}; - var userName = {{{ makeGetValue('fetch_attr', 68/*TODO*/, 'i32') }}}; - var password = {{{ makeGetValue('fetch_attr', 72/*TODO*/, 'i32') }}}; - var requestHeaders = {{{ makeGetValue('fetch_attr', 76/*TODO*/, 'i32') }}}; - var overriddenMimeType = {{{ makeGetValue('fetch_attr', 80/*TODO*/, 'i32') }}}; + var fetch_attr = fetch + 112/*TODO:structs_info*/; + var requestMethod = Pointer_stringify(fetch_attr); + if (!requestMethod) requestMethod = 'GET'; + var userData = HEAPU32[fetch_attr + 32 >> 2];//{{{ makeGetValue('fetch_attr', 32/*TODO:structs_info.jsonify this*/, 'i32') }}}; + var fetchAttributes = HEAPU32[fetch_attr + 48 >> 2];//{{{ makeGetValue('fetch_attr', 48/*TODO*/, 'i32') }}}; + var timeoutMsecs = HEAPU32[fetch_attr + 52 >> 2];//{{{ makeGetValue('fetch_attr', 52/*TODO*/, 'i32') }}}; + var withCredentials = !!HEAPU32[fetch_attr + 56 >> 2];//({{{ makeGetValue('fetch_attr', 56/*TODO*/, 'i32') }}}); + var destinationPath = HEAPU32[fetch_attr + 64 >> 2];//{{{ makeGetValue('fetch_attr', 64/*TODO*/, 'i32') }}}; + var userName = HEAPU32[fetch_attr + 68 >> 2];//{{{ makeGetValue('fetch_attr', 68/*TODO*/, 'i32') }}}; + var password = HEAPU32[fetch_attr + 72 >> 2];//{{{ makeGetValue('fetch_attr', 72/*TODO*/, 'i32') }}}; + var requestHeaders = HEAPU32[fetch_attr + 76 >> 2];//{{{ makeGetValue('fetch_attr', 76/*TODO*/, 'i32') }}}; + var overriddenMimeType = HEAPU32[fetch_attr + 80 >> 2];//{{{ makeGetValue('fetch_attr', 80/*TODO*/, 'i32') }}}; var fetchAttrLoadToMemory = !!(fetchAttributes & 1/*EMSCRIPTEN_FETCH_LOAD_TO_MEMORY*/); var fetchAttrStreamData = !!(fetchAttributes & 2/*EMSCRIPTEN_FETCH_STREAM_DATA*/); @@ -170,6 +201,7 @@ function _emscripten_fetch_xhr(fetch, onsuccess, onerror, onprogress) { var fetchAttrReplace = !!(fetchAttributes & 16/*EMSCRIPTEN_FETCH_REPLACE*/); var fetchAttrNoDownload = !!(fetchAttributes & 32/*EMSCRIPTEN_FETCH_NO_DOWNLOAD*/); var fetchAttrSynchronous = !!(fetchAttributes & 64/*EMSCRIPTEN_FETCH_SYNCHRONOUS*/); + var fetchAttrWaitable = !!(fetchAttributes & 128/*EMSCRIPTEN_FETCH_WAITABLE*/); var userNameStr = userName ? Pointer_stringify(userName) : undefined; var passwordStr = password ? Pointer_stringify(password) : undefined; @@ -193,9 +225,9 @@ function _emscripten_fetch_xhr(fetch, onsuccess, onerror, onprogress) { } if (requestHeaders) { for(;;) { - var key = {{{ makeGetValue('requestHeaders', 0, 'i32') }}}; + var key = HEAPU32[requestHeaders >> 2];//{{{ makeGetValue('requestHeaders', 0, 'i32') }}}; if (!key) break; - var value = {{{ makeGetValue('requestHeaders', 4, 'i32') }}}; + var value = HEAPU32[requestHeaders + 4 >> 2];//{{{ makeGetValue('requestHeaders', 4, 'i32') }}}; if (!value) break; requestHeaders += 8; var keyStr = Pointer_stringify(key); @@ -208,7 +240,7 @@ function _emscripten_fetch_xhr(fetch, onsuccess, onerror, onprogress) { } Fetch.xhrs.push(xhr); var id = Fetch.xhrs.length; - {{{ makeSetValue('fetch', 0/*TODO:jsonify*/, 'id', 'i32')}}}; + HEAPU32[fetch >> 2] = id;//{{{ makeSetValue('fetch', 0/*TODO:jsonify*/, 'id', 'i32')}}}; var data = null; // TODO: Support user to pass data to request. // TODO: Support specifying custom headers to the request. @@ -222,14 +254,17 @@ function _emscripten_fetch_xhr(fetch, onsuccess, onerror, onprogress) { console.log('fetch: allocating ' + ptrLen + ' bytes in Emscripten heap for xhr data'); #endif ptr = _malloc(ptrLen); // TODO: make emscripten_fetch_close() free() this data. - HEAPU8.set(new Uint8Array(xhr.response), ptr); + HEAPU8.set(new Uint8Array(xhr.response), ptr); // TODO: Since DYNAMICTOP is not coherent, this can corrupt } - {{{ makeSetValue('fetch', 12/*TODO:jsonify*/, 'ptr', 'i32')}}}; - {{{ makeSetValue('fetch', 16/*TODO:jsonify*/, 'ptrLen', 'i64')}}}; - {{{ makeSetValue('fetch', 24/*TODO:jsonify*/, '0', 'i64')}}}; + HEAPU32[fetch + 12 >> 2] = ptr;//{{{ makeSetValue('fetch', 12/*TODO:jsonify*/, 'ptr', 'i32')}}}; + Fetch.setu64(fetch + 16, ptrLen);//{{{ makeSetValue('fetch', 16/*TODO:jsonify*/, 'ptrLen', 'i64')}}}; + Fetch.setu64(fetch + 24, 0);//{{{ makeSetValue('fetch', 24/*TODO:jsonify*/, '0', 'i64')}}}; if (len) { - {{{ makeSetValue('fetch', 32/*TODO:jsonify*/, 'len', 'i64')}}}; + Fetch.setu64(fetch + 32, len);//{{{ makeSetValue('fetch', 32/*TODO:jsonify*/, 'len', 'i64')}}}; } + HEAPU16[fetch + 40 >> 1] = xhr.readyState; + HEAPU16[fetch + 42 >> 1] = xhr.status; +// if (xhr.statusText) writeStringToMemory(fetch + 44, xhr.statusText); if (xhr.status == 200 || xhr.status == 0) { #if FETCH_DEBUG console.log('fetch: xhr succeeded with status 200'); @@ -255,6 +290,7 @@ function _emscripten_fetch_xhr(fetch, onsuccess, onerror, onprogress) { if (onerror) onerror(fetch, xhr, e); } xhr.onprogress = function(e) { + console.log('fetch ptr ' + fetch + ', state ' + HEAPU32[fetch + 108 >> 2]); var ptrLen = (fetchAttrLoadToMemory && fetchAttrStreamData && xhr.response) ? xhr.response.byteLength : 0; var ptr = 0; if (fetchAttrLoadToMemory && fetchAttrStreamData) { @@ -264,10 +300,13 @@ function _emscripten_fetch_xhr(fetch, onsuccess, onerror, onprogress) { ptr = _malloc(ptrLen); // TODO: make emscripten_fetch_close() free() this data. HEAPU8.set(new Uint8Array(xhr.response), ptr); } - {{{ makeSetValue('fetch', 12/*TODO:jsonify*/, 'ptr', 'i32')}}}; - {{{ makeSetValue('fetch', 16/*TODO:jsonify*/, 'ptrLen', 'i64')}}}; - {{{ makeSetValue('fetch', 24/*TODO:jsonify*/, 'e.loaded - ptrLen', 'i64')}}}; - {{{ makeSetValue('fetch', 32/*TODO:jsonify*/, 'e.total', 'i64')}}}; + HEAPU32[fetch + 12 >> 2] = ptr;//{{{ makeSetValue('fetch', 12/*TODO:jsonify*/, 'ptr', 'i32')}}}; + Fetch.setu64(fetch + 16, ptrLen);//{{{ makeSetValue('fetch', 16/*TODO:jsonify*/, 'ptrLen', 'i64')}}}; + Fetch.setu64(fetch + 24, e.loaded - ptrLen);//{{{ makeSetValue('fetch', 24/*TODO:jsonify*/, 'e.loaded - ptrLen', 'i64')}}}; + Fetch.setu64(fetch + 32, e.total);//{{{ makeSetValue('fetch', 32/*TODO:jsonify*/, 'e.total', 'i64')}}}; + HEAPU16[fetch + 40 >> 1] = xhr.readyState; + HEAPU16[fetch + 42 >> 1] = xhr.status; + if (xhr.statusText) writeStringToMemory(fetch + 44, xhr.statusText); if (onprogress) onprogress(fetch, xhr, e); } #if FETCH_DEBUG @@ -283,14 +322,14 @@ function _emscripten_fetch_xhr(fetch, onsuccess, onerror, onprogress) { } } -function emscripten_start_fetch(fetch) { - Module['noExitRuntime'] = true; // TODO: assumes we are the main Emscripten runtime thread, in the future won't. +function emscripten_start_fetch(fetch, successcb, errorcb, progresscb) { + if (typeof Module !== 'undefined') Module['noExitRuntime'] = true; // If we are the main Emscripten runtime, we should not be closing down. - var fetch_attr = fetch + 108/*TODO:structs_info*/; - var onsuccess = {{{ makeGetValue('fetch_attr', 36/*TODO:structs_info.jsonify this*/, 'i32') }}}; - var onerror = {{{ makeGetValue('fetch_attr', 40/*TODO:structs_info.jsonify this*/, 'i32') }}}; - var onprogress = {{{ makeGetValue('fetch_attr', 44/*TODO:structs_info.jsonify this*/, 'i32') }}}; - var fetchAttributes = {{{ makeGetValue('fetch_attr', 48/*TODO*/, 'i32') }}}; + var fetch_attr = fetch + 112/*TODO:structs_info*/; + var onsuccess = HEAPU32[fetch_attr + 36 >> 2];//{{{ makeGetValue('fetch_attr', 36/*TODO:structs_info.jsonify this*/, 'i32') }}}; + var onerror = HEAPU32[fetch_attr + 40 >> 2];//{{{ makeGetValue('fetch_attr', 40/*TODO:structs_info.jsonify this*/, 'i32') }}}; + var onprogress = HEAPU32[fetch_attr + 44 >> 2];//{{{ makeGetValue('fetch_attr', 44/*TODO:structs_info.jsonify this*/, 'i32') }}}; + var fetchAttributes = HEAPU32[fetch_attr + 48 >> 2];//{{{ makeGetValue('fetch_attr', 48/*TODO*/, 'i32') }}}; var fetchAttrLoadToMemory = !!(fetchAttributes & 1/*EMSCRIPTEN_FETCH_LOAD_TO_MEMORY*/); var fetchAttrStreamData = !!(fetchAttributes & 2/*EMSCRIPTEN_FETCH_STREAM_DATA*/); var fetchAttrPersistFile = !!(fetchAttributes & 4/*EMSCRIPTEN_FETCH_PERSIST_FILE*/); @@ -302,7 +341,8 @@ function emscripten_start_fetch(fetch) { #if FETCH_DEBUG console.log('fetch: operation success. e: ' + e); #endif - if (onsuccess) Runtime.dynCall('vi', onsuccess, [fetch]); + if (onsuccess && Runtime.dynCall) Runtime.dynCall('vi', onsuccess, [fetch]); + else if (successcb) successcb(fetch); }; var cacheResultAndReportSuccess = function(fetch, xhr, e) { @@ -320,18 +360,21 @@ function emscripten_start_fetch(fetch) { #endif }; __emscripten_fetch_cache_data(Fetch.dbInstance, fetch, xhr.response, storeSuccess, storeError); - if (onsuccess) Runtime.dynCall('vi', onsuccess, [fetch]); + if (onsuccess && Runtime.dynCall) Runtime.dynCall('vi', onsuccess, [fetch]); + else if (successcb) successcb(fetch); }; var reportProgress = function(fetch, xhr, e) { - if (onprogress) Runtime.dynCall('vi', onprogress, [fetch]); + if (onprogress && Runtime.dynCall) Runtime.dynCall('vi', onprogress, [fetch]); + else if (progresscb) progresscb(fetch); }; var reportError = function(fetch, xhr, e) { #if FETCH_DEBUG console.error('fetch: operation failed: ' + e); #endif - if (onerror) Runtime.dynCall('vi', onerror, [fetch]); + if (onerror && Runtime.dynCall) Runtime.dynCall('vi', onerror, [fetch]); + else if (errorcb) errorcb(fetch); }; var performUncachedXhr = function(fetch, xhr, e) { diff --git a/src/library_fetch.js b/src/library_fetch.js index 4e649d93be53d..d1d193cdd6f2e 100644 --- a/src/library_fetch.js +++ b/src/library_fetch.js @@ -1,12 +1,18 @@ #include Fetch.js var LibraryFetch = { - $Fetch__postset: 'Fetch.staticInit();', + $Fetch__postset: 'if (!ENVIRONMENT_IS_PTHREAD) Fetch.staticInit();', $Fetch: Fetch, + fetch_work_queue: '; if (ENVIRONMENT_IS_PTHREAD) _fetch_work_queue = PthreadWorkerInit._fetch_work_queue; else PthreadWorkerInit._fetch_work_queue = _fetch_work_queue = allocate(12, "i32*", ALLOC_STATIC)', + _emscripten_get_fetch_work_queue__deps: ['fetch_work_queue'], + _emscripten_get_fetch_work_queue: function() { + return _fetch_work_queue; + }, + _emscripten_fetch_load_cached_data: _emscripten_fetch_load_cached_data, _emscripten_fetch_cache_data: _emscripten_fetch_cache_data, _emscripten_fetch_xhr: _emscripten_fetch_xhr, - emscripten_start_fetch__deps: ['$Fetch', '_emscripten_fetch_xhr', '_emscripten_fetch_cache_data', '_emscripten_fetch_load_cached_data'], + emscripten_start_fetch__deps: ['$Fetch', '_emscripten_fetch_xhr', '_emscripten_fetch_cache_data', '_emscripten_fetch_load_cached_data', '_emscripten_get_fetch_work_queue'], emscripten_start_fetch: emscripten_start_fetch }; diff --git a/src/preamble.js b/src/preamble.js index 4cfe89064e477..31c1eaa35d028 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -590,7 +590,7 @@ function UTF8ToString(ptr) { // Copies the given Javascript String object 'str' to the given byte array at address 'outIdx', // encoded in UTF8 form and null-terminated. The copy will require at most str.length*4+1 bytes of space in the HEAP. -// Use the function lengthBytesUTF8() to compute the exact number of bytes (excluding null terminator) that this function will write. +// Use the function lengthBytesUTF8 to compute the exact number of bytes (excluding null terminator) that this function will write. // Parameters: // str: the Javascript string to copy. // outU8Array: the array to copy to. Each index in this array is assumed to be one 8-byte element. @@ -655,7 +655,7 @@ function stringToUTF8Array(str, outU8Array, outIdx, maxBytesToWrite) { // Copies the given Javascript String object 'str' to the emscripten HEAP at address 'outPtr', // null-terminated and encoded in UTF8 form. The copy will require at most str.length*4+1 bytes of space in the HEAP. -// Use the function lengthBytesUTF8() to compute the exact number of bytes (excluding null terminator) that this function will write. +// Use the function lengthBytesUTF8 to compute the exact number of bytes (excluding null terminator) that this function will write. // Returns the number of bytes written, EXCLUDING the null terminator. function stringToUTF8(str, outPtr, maxBytesToWrite) { diff --git a/system/include/emscripten/fetch.h b/system/include/emscripten/fetch.h index 8316c9d0a657f..7c4594d96e2f4 100644 --- a/system/include/emscripten/fetch.h +++ b/system/include/emscripten/fetch.h @@ -38,6 +38,10 @@ extern "C" { // The callback handlers will be called from within emscripten_fetch() while the operation is in progress. #define EMSCRIPTEN_FETCH_SYNCHRONOUS 64 +// If specified, it will be possible to call emscripten_fetch_wait() on the fetch +// to test or wair for its completion. +#define EMSCRIPTEN_FETCH_WAITABLE 128 + struct emscripten_fetch_t; // Specifies the parameters for a newly initiated fetch operation. @@ -139,6 +143,8 @@ struct emscripten_fetch_t // Specifies a human-readable form of the status code. char statusText[64]; + uint32_t __proxyState; + // For internal use only. emscripten_fetch_attr_t __attributes; }; diff --git a/system/lib/fetch/emscripten_fetch.cpp b/system/lib/fetch/emscripten_fetch.cpp index 100242b7fc9ec..d25a4cc51b54d 100644 --- a/system/lib/fetch/emscripten_fetch.cpp +++ b/system/lib/fetch/emscripten_fetch.cpp @@ -1,11 +1,43 @@ #include #include +#include //todoremove #include #include #include +#include + +struct __emscripten_fetch_queue +{ + emscripten_fetch_t **queuedOperations; + int numQueuedItems; + int queueSize; +}; extern "C" { void emscripten_start_fetch(emscripten_fetch_t *fetch); + __emscripten_fetch_queue *_emscripten_get_fetch_work_queue(); + + __emscripten_fetch_queue *_emscripten_get_fetch_queue() + { + __emscripten_fetch_queue *queue = _emscripten_get_fetch_work_queue(); + if (!queue->queuedOperations) + { + queue->queueSize = 64; + queue->numQueuedItems = 0; + queue->queuedOperations = (emscripten_fetch_t**)malloc(sizeof(emscripten_fetch_t*) * queue->queueSize); + } + return queue; + } +} + +void emscripten_proxy_fetch(emscripten_fetch_t *fetch) +{ + // TODO: mutex lock + __emscripten_fetch_queue *queue = _emscripten_get_fetch_queue(); +// TODO handle case when queue->numQueuedItems >= queue->queueSize + queue->queuedOperations[queue->numQueuedItems++] = fetch; + printf("Queueing fetch\n"); + // TODO: mutex unlock } void emscripten_fetch_attr_init(emscripten_fetch_attr_t *fetch_attr) @@ -30,17 +62,41 @@ emscripten_fetch_t *emscripten_fetch(emscripten_fetch_attr_t *fetch_attr, const fetch->__attributes.requestHeaders = 0;// TODO:strdup(fetch->__attributes.requestHeaders); fetch->__attributes.overriddenMimeType = fetch->__attributes.overriddenMimeType ? strdup(fetch->__attributes.overriddenMimeType) : 0; // TODO: free - emscripten_start_fetch(fetch); + if ((fetch->__attributes.attributes & EMSCRIPTEN_FETCH_WAITABLE) != 0) + { + emscripten_atomic_store_u32(&fetch->__proxyState, 1); // sent to proxy worker. + emscripten_proxy_fetch(fetch); + } + else + emscripten_start_fetch(fetch); return fetch; } EMSCRIPTEN_RESULT emscripten_fetch_wait(emscripten_fetch_t *fetch, double timeoutMsecs) { - return EMSCRIPTEN_RESULT_SUCCESS; + if (!fetch) return EMSCRIPTEN_RESULT_INVALID_PARAM; + uint32_t proxyState = emscripten_atomic_load_u32(&fetch->__proxyState); + if (proxyState == 2) return EMSCRIPTEN_RESULT_SUCCESS; // already finished. + if (proxyState != 1) return EMSCRIPTEN_RESULT_INVALID_PARAM; // the fetch should be ongoing? +// #ifdef FETCH_DEBUG + printf("fetch: emscripten_fetch_wait..\n"); +// #endif + while(proxyState == 1/*sent to proxy worker*/) + { + emscripten_futex_wait(&fetch->__proxyState, proxyState, 100 /*TODO HACK:Sleep sometimes doesn't wake up?*/);//timeoutMsecs); + proxyState = emscripten_atomic_load_u32(&fetch->__proxyState); + } +// #ifdef FETCH_DEBUG + printf("fetch: emscripten_fetch_wait done..\n"); +// #endif + + if (proxyState == 2) return EMSCRIPTEN_RESULT_SUCCESS; + else return EMSCRIPTEN_RESULT_FAILED; } EMSCRIPTEN_RESULT emscripten_fetch_close(emscripten_fetch_t *fetch) { - free(fetch); + emscripten_atomic_store_u32(&fetch->__proxyState, 0); + free(fetch); // TODO: thread-safety before freeing (what if freeing an operation in progress? explicit emscripten_fetch_abort()?) return EMSCRIPTEN_RESULT_SUCCESS; } diff --git a/tools/shared.py b/tools/shared.py index 9dd73e4fb5a5d..b9f35903c7f71 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -2322,6 +2322,10 @@ def safe_copy(src, dst): if dst == '/dev/null': return shutil.copyfile(src, dst) +def clang_preprocess(filename): + # TODO: REMOVE HACK AND PASS PREPROCESSOR FLAGS TO CLANG. + return subprocess.check_output([CLANG_CC, '-DFETCH_DEBUG=1', '-E', '-P', '-C', '-x', 'c', filename]) + def read_and_preprocess(filename): f = open(filename, 'r').read() pos = 0 From da10d5de6dcd83f838575556d052fa24cc1d171e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Thu, 9 Jun 2016 17:49:08 +0300 Subject: [PATCH 12/74] Add two test files --- tests/fetch/sync_fetch_in_main_thread.cpp | 36 ++++++++++++++++++ tests/fetch/to_indexeddb.cpp | 46 +++++++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 tests/fetch/sync_fetch_in_main_thread.cpp create mode 100644 tests/fetch/to_indexeddb.cpp diff --git a/tests/fetch/sync_fetch_in_main_thread.cpp b/tests/fetch/sync_fetch_in_main_thread.cpp new file mode 100644 index 0000000000000..33689306582d9 --- /dev/null +++ b/tests/fetch/sync_fetch_in_main_thread.cpp @@ -0,0 +1,36 @@ +#include +#include +#include +#include +#include + +int result = 0; + +int main() +{ + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + attr.attributes = EMSCRIPTEN_FETCH_REPLACE | EMSCRIPTEN_FETCH_LOAD_TO_MEMORY | EMSCRIPTEN_FETCH_WAITABLE; + emscripten_fetch_t *fetch = emscripten_fetch(&attr, "gears.png"); + assert(fetch != 0); + memset(&attr, 0, sizeof(attr)); // emscripten_fetch() must be able to operate without referencing to this structure after the call. + printf("Main thread waiting for fetch to finish...\n"); + emscripten_fetch_wait(fetch, INFINITY); + printf("Main thread waiting for fetch to finish done...\n"); + assert(fetch->data != 0); + assert(fetch->numBytes > 0); + assert(fetch->totalBytes == fetch->numBytes); + assert(fetch->readyState == 4/*DONE*/); + assert(fetch->status == 200); + + uint8_t checksum = 0; + for(int i = 0; i < fetch->numBytes; ++i) + checksum ^= fetch->data[i]; + printf("Data checksum: %02X\n", checksum); + assert(checksum == 0x08); + emscripten_fetch_close(fetch); + +#ifdef REPORT_RESULT + REPORT_RESULT(); +#endif +} diff --git a/tests/fetch/to_indexeddb.cpp b/tests/fetch/to_indexeddb.cpp new file mode 100644 index 0000000000000..c61edd26f611b --- /dev/null +++ b/tests/fetch/to_indexeddb.cpp @@ -0,0 +1,46 @@ +#include +#include +#include +#include +#include + +int result = 0; + +int main() +{ + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.userData = (void*)0x12345678; + attr.attributes = EMSCRIPTEN_FETCH_REPLACE | EMSCRIPTEN_FETCH_PERSIST_FILE; + + attr.onsuccess = [](emscripten_fetch_t *fetch) { + assert(fetch); + printf("Finished downloading %llu bytes\n", fetch->totalBytes); + assert(fetch->url); + assert(!strcmp(fetch->url, "gears.png")); + assert(fetch->id != 0); + assert((uintptr_t)fetch->userData == 0x12345678); + assert(fetch->totalBytes == 6407); + emscripten_fetch_close(fetch); + +#ifdef REPORT_RESULT + result = 1; + REPORT_RESULT(); +#endif + }; + + attr.onprogress = [](emscripten_fetch_t *fetch) { + assert(fetch); + if (fetch->status != 200) return; + printf("Downloading.. %.2f%% complete.\n", (fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes); + }; + + attr.onerror = [](emscripten_fetch_t *fetch) { + printf("Download failed!\n"); + }; + + emscripten_fetch_t *fetch = emscripten_fetch(&attr, "gears.png"); + assert(fetch != 0); + memset(&attr, 0, sizeof(attr)); // emscripten_fetch() must be able to operate without referencing to this structure after the call. +} From 1808a3a34b964bc36dfb31ba2f289282c8c890e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Wed, 24 Aug 2016 01:54:20 +0300 Subject: [PATCH 13/74] Add fetch-worker.js --- src/fetch-worker.js | 139 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 src/fetch-worker.js diff --git a/src/fetch-worker.js b/src/fetch-worker.js new file mode 100644 index 0000000000000..d3c39fb564bb8 --- /dev/null +++ b/src/fetch-worker.js @@ -0,0 +1,139 @@ +#include "Fetch.js" + +var ENVIRONMENT_IS_FETCH_WORKER = true; +var ENVIRONMENT_IS_WORKER = true; +var ENVIRONMENT_IS_PTHREAD = true; +var __pthread_is_main_runtime_thread=0; +var Atomics_load = Atomics.load; +var Atomics_store = Atomics.store; +var Atomics_exchange = Atomics.exchange; +var Atomics_compareExchange = Atomics.compareExchange; +var Atomics_add = Atomics.add; +var Atomics_sub = Atomics.sub; +var Atomics_and = Atomics.and; +var Atomics_or = Atomics.or; +var Atomics_xor = Atomics.xor; +var DYNAMICTOP_PTR = 0; +var TOTAL_MEMORY = 0; +function enlargeMemory() { + abort('Cannot enlarge memory arrays, since compiling with pthreads support enabled (-s USE_PTHREADS=1).'); +} +var nan = NaN; +var inf = Infinity; + +__emscripten_fetch_xhr = _emscripten_fetch_xhr; +__emscripten_fetch_cache_data = _emscripten_fetch_cache_data; + +function _emscripten_asm_const_v() {} + +function assert(condition) { + if (!condition) console.error('assert failure!'); +} + +/// TODO: DO SOMETHING ABOUT ME. +function Pointer_stringify(ptr, /* optional */ length) { + if (length === 0 || !ptr) return ""; + // TODO: use TextDecoder + // Find the length, and check for UTF while doing so + var hasUtf = 0; + var t; + var i = 0; + while (1) { + t = HEAPU8[(((ptr)+(i))>>0)]; + hasUtf |= t; + if (t == 0 && !length) break; + i++; + if (length && i == length) break; + } + if (!length) length = i; + + var ret = ""; + + if (hasUtf < 128) { + var MAX_CHUNK = 1024; // split up into chunks, because .apply on a huge string can overflow the stack + var curr; + while (length > 0) { + curr = String.fromCharCode.apply(String, HEAPU8.subarray(ptr, ptr + Math.min(length, MAX_CHUNK))); + ret = ret ? ret + curr : curr; + ptr += MAX_CHUNK; + length -= MAX_CHUNK; + } + return ret; + } + return Module['UTF8ToString'](ptr); +} + + +console.log('fetch worker script loading.'); +Fetch.staticInit(); + +var queuePtr = 0; +var buffer = null; +var STACKTOP = 0; +var STACK_MAX = 0; +var HEAP8 = null; +var HEAPU8 = null; +var HEAP16 = null; +var HEAPU16 = null; +var HEAP32 = null; +var HEAPU32 = null; + +function processWorkQueue() { + if (!queuePtr) return; + + console.log('polling work to perform'); + var queuedOperations = HEAPU32[queuePtr >> 2]; + var numQueuedItems = HEAPU32[queuePtr + 4 >> 2]; + var queueSize = HEAPU32[queuePtr + 8 >> 2]; + for(var i = 0; i < numQueuedItems; ++i) { + var fetch = HEAPU32[(queuedOperations >> 2)+i]; + console.log('processWorkQueue: starting fetch'); + console.log('fetch ptr1 ' + fetch + ', state ' + HEAPU32[fetch + 108 >> 2]); + function successcb(fetch) { + console.log('fetch ptr ' + fetch + ', state ' + HEAPU32[fetch + 108 >> 2]); + console.log('FETCH-WORKER: fetch finished'); + var oldVal = Atomics.compareExchange(HEAPU32, fetch + 108 >> 2, 1, 2); + console.log('atomics wake ' + (fetch + 108)); + Atomics.wake(HEAP32, fetch + 108 >> 2, 1); + console.log('oldVal ' + oldVal); + } + function errorcb(fetch) { + console.log('FETCH-WORKER: fetch failed'); + } + function progresscb(fetch) { + console.log('FETCH-WORKER: fetch progress..'); + } + try { + emscripten_start_fetch(fetch, successcb, errorcb, progresscb); + } catch(e) { + console.error(e); + } + if (interval != undefined) { + clearInterval(interval); + interval = undefined; + } + } + HEAPU32[queuePtr + 4 >> 2] = 0; +} + +interval = 0; +this.onmessage = function(e) { + if (e.data.cmd == 'init') { + queuePtr = e.data.queuePtr; + buffer = e.data.buffer; + STACKTOP = e.data.STACKTOP; + STACK_MAX = e.data.STACK_MAX; + DYNAMICTOP_PTR = e.data.DYNAMICTOP_PTR; + TOTAL_MEMORY = e.data.TOTAL_MEMORY; + HEAP8 = new Int8Array(buffer); + HEAPU8 = new Uint8Array(buffer); + HEAP16 = new Int16Array(buffer); + HEAPU16 = new Uint16Array(buffer); + HEAP32 = new Int32Array(buffer); + HEAPU32 = new Uint32Array(buffer); + console.log('DYNAMICTOP_PTR: ' + DYNAMICTOP_PTR); + console.log('DYNAMICTOP: ' + HEAP32[DYNAMICTOP_PTR>>2]); + console.log('fetch worker init, queue ptr ' + queuePtr + ', heap length: ' + buffer.byteLength); + interval = setInterval(processWorkQueue, 1000); + } +} From 85ace77ee113b937b44f0d4d87a31cc8a3240b5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Wed, 24 Aug 2016 01:53:29 +0300 Subject: [PATCH 14/74] Use the new DYNAMICTOP_PTR variable. --- src/Fetch.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Fetch.js b/src/Fetch.js index 43dbfb277d0cf..95ffdce1b3773 100644 --- a/src/Fetch.js +++ b/src/Fetch.js @@ -36,7 +36,7 @@ var Fetch = { initFetchWorker: function() { var stackSize = 128*1024; var stack = allocate(stackSize>>2, "i32*", ALLOC_DYNAMIC); - Fetch.worker.postMessage({cmd: 'init', TOTAL_MEMORY: TOTAL_MEMORY, DYNAMICTOP: DYNAMICTOP + 1024*1024, STACKTOP: stack, STACK_MAX: stack + stackSize, queuePtr: _fetch_work_queue, buffer: HEAPU8.buffer}, [HEAPU8.buffer]); + Fetch.worker.postMessage({cmd: 'init', TOTAL_MEMORY: TOTAL_MEMORY, DYNAMICTOP_PTR: DYNAMICTOP_PTR, STACKTOP: stack, STACK_MAX: stack + stackSize, queuePtr: _fetch_work_queue, buffer: HEAPU8.buffer}, [HEAPU8.buffer]); }, staticInit: function() { From 669eee90623fa178037039ced06ed5365aa09673 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Wed, 24 Aug 2016 02:02:31 +0300 Subject: [PATCH 15/74] Make fetch work better in non-pthreads mode. --- src/Fetch.js | 12 +++++++++--- src/library_fetch.js | 7 ++++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/Fetch.js b/src/Fetch.js index 95ffdce1b3773..6cf0095a1caa0 100644 --- a/src/Fetch.js +++ b/src/Fetch.js @@ -40,12 +40,18 @@ var Fetch = { }, staticInit: function() { +#if USE_PTHREADS + var isMainThread = (typeof ENVIRONMENT_IS_FETCH_WORKER === 'undefined' && !ENVIRONMENT_IS_PTHREAD); +#else + var isMainThread = (typeof ENVIRONMENT_IS_FETCH_WORKER === 'undefined'); +#endif + var onsuccess = function(db) { #if FETCH_DEBUG console.log('fetch: IndexedDB successfully opened.'); #endif Fetch.dbInstance = db; - if (typeof ENVIRONMENT_IS_FETCH_WORKER === 'undefined' && !ENVIRONMENT_IS_PTHREAD) { + if (isMainThread) { Fetch.initFetchWorker(); removeRunDependency('library_fetch_init'); } @@ -55,14 +61,14 @@ var Fetch = { console.error('fetch: IndexedDB open failed.'); #endif Fetch.dbInstance = false; - if (typeof ENVIRONMENT_IS_FETCH_WORKER === 'undefined' && !ENVIRONMENT_IS_PTHREAD) { + if (isMainThread) { Fetch.initFetchWorker(); removeRunDependency('library_fetch_init'); } }; Fetch.openDatabase('emscripten_filesystem', 1, onsuccess, onerror); - if (typeof ENVIRONMENT_IS_FETCH_WORKER === 'undefined' && !ENVIRONMENT_IS_PTHREAD) { + if (isMainThread) { addRunDependency('library_fetch_init'); var fetchJs = 'fetch-worker.js'; diff --git a/src/library_fetch.js b/src/library_fetch.js index d1d193cdd6f2e..0d5f9c2afa10b 100644 --- a/src/library_fetch.js +++ b/src/library_fetch.js @@ -1,9 +1,14 @@ #include Fetch.js var LibraryFetch = { +#if USE_PTHREADS $Fetch__postset: 'if (!ENVIRONMENT_IS_PTHREAD) Fetch.staticInit();', - $Fetch: Fetch, fetch_work_queue: '; if (ENVIRONMENT_IS_PTHREAD) _fetch_work_queue = PthreadWorkerInit._fetch_work_queue; else PthreadWorkerInit._fetch_work_queue = _fetch_work_queue = allocate(12, "i32*", ALLOC_STATIC)', +#else + $Fetch__postset: 'Fetch.staticInit();', + fetch_work_queue: 'allocate(12, "i32*", ALLOC_STATIC)', +#endif + $Fetch: Fetch, _emscripten_get_fetch_work_queue__deps: ['fetch_work_queue'], _emscripten_get_fetch_work_queue: function() { return _fetch_work_queue; From c4a6ddcc9c83fd0115752432efdeaed264328d52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Thu, 25 Aug 2016 22:59:57 +0300 Subject: [PATCH 16/74] Start work on ASMFS. --- emcc.py | 13 ++- src/Fetch.js | 10 +-- src/fetch-worker.js | 3 - src/library_fetch.js | 8 +- src/settings.js | 4 + system/lib/fetch/asmfs.cpp | 171 +++++++++++++++++++++++++++++++++++++ 6 files changed, 196 insertions(+), 13 deletions(-) create mode 100644 system/lib/fetch/asmfs.cpp diff --git a/emcc.py b/emcc.py index 1e1e6b84eb002..df3d96e5553d5 100755 --- a/emcc.py +++ b/emcc.py @@ -1138,6 +1138,17 @@ def check(input_file): # stb_image 2.x need to have STB_IMAGE_IMPLEMENTATION defined to include the implementation when compiling newargs.append('-DSTB_IMAGE_IMPLEMENTATION') + if shared.Settings.ASMFS and final_suffix in JS_CONTAINING_SUFFIXES: + input_files.append((next_arg_index, shared.path_from_root('system', 'lib', 'fetch', 'asmfs.cpp'))) + next_arg_index += 1 + shared.Settings.NO_FILESYSTEM = 1 + shared.Settings.FETCH = 1 + + if shared.Settings.FETCH and final_suffix in JS_CONTAINING_SUFFIXES: + input_files.append((next_arg_index, shared.path_from_root('system', 'lib', 'fetch', 'emscripten_fetch.cpp'))) + next_arg_index += 1 + js_libraries.append(shared.path_from_root('src', 'library_fetch.js')) + forced_stdlibs = [] if shared.Settings.DEMANGLE_SUPPORT: shared.Settings.EXPORTED_FUNCTIONS += ['___cxa_demangle'] @@ -1755,7 +1766,7 @@ def repl(m): shutil.copyfile(shared.path_from_root('src', 'pthread-main.js'), os.path.join(os.path.dirname(os.path.abspath(target)), 'pthread-main.js')) src = open(final, 'r').read() - funcs_to_import = ['alignMemoryPage', '_sbrk', '_pthread_mutex_lock', '_malloc', '_emscripten_sync_run_in_main_thread_1', '_emscripten_sync_run_in_main_thread', '_emscripten_is_main_runtime_thread', '_pthread_mutex_unlock', '_emscripten_set_current_thread_status', '_emscripten_futex_wait', 'writeStringToMemory', 'intArrayFromString', 'lengthBytesUTF8', 'stringToUTF8Array'] + funcs_to_import = ['alignMemoryPage', '_sbrk', '_pthread_mutex_lock', '_malloc', '_emscripten_sync_run_in_main_thread_1', '_emscripten_sync_run_in_main_thread', '_emscripten_is_main_runtime_thread', '_pthread_mutex_unlock', '_emscripten_set_current_thread_status', '_emscripten_futex_wait', 'stringToUTF8', 'intArrayFromString', 'lengthBytesUTF8', 'stringToUTF8Array'] function_prologue = '' for func in funcs_to_import: loc = src.find('function ' + func + '(') diff --git a/src/Fetch.js b/src/Fetch.js index 6cf0095a1caa0..e3bc91f33bdaf 100644 --- a/src/Fetch.js +++ b/src/Fetch.js @@ -82,7 +82,7 @@ var Fetch = { } } -function _emscripten_fetch_load_cached_data(db, fetch, onsuccess, onerror) { +function __emscripten_fetch_load_cached_data(db, fetch, onsuccess, onerror) { if (!db) { #if FETCH_DEBUG console.error('fetch: IndexedDB not available!'); @@ -138,7 +138,7 @@ function _emscripten_fetch_load_cached_data(db, fetch, onsuccess, onerror) { } } -function _emscripten_fetch_cache_data(db, fetch, data, onsuccess, onerror) { +function __emscripten_fetch_cache_data(db, fetch, data, onsuccess, onerror) { if (!db) { #if FETCH_DEBUG console.error('fetch: IndexedDB not available!'); @@ -176,7 +176,7 @@ function _emscripten_fetch_cache_data(db, fetch, data, onsuccess, onerror) { } } -function _emscripten_fetch_xhr(fetch, onsuccess, onerror, onprogress) { +function __emscripten_fetch_xhr(fetch, onsuccess, onerror, onprogress) { var url = HEAPU32[fetch + 8 >> 2];//{{{ makeGetValue('fetch', 8/*TODO:structs_info.jsonify this*/, 'i32') }}}; if (!url) { #if FETCH_DEBUG @@ -270,7 +270,7 @@ function _emscripten_fetch_xhr(fetch, onsuccess, onerror, onprogress) { } HEAPU16[fetch + 40 >> 1] = xhr.readyState; HEAPU16[fetch + 42 >> 1] = xhr.status; -// if (xhr.statusText) writeStringToMemory(fetch + 44, xhr.statusText); +// if (xhr.statusText) stringToUTF8(fetch + 44, xhr.statusText, 64); if (xhr.status == 200 || xhr.status == 0) { #if FETCH_DEBUG console.log('fetch: xhr succeeded with status 200'); @@ -312,7 +312,7 @@ function _emscripten_fetch_xhr(fetch, onsuccess, onerror, onprogress) { Fetch.setu64(fetch + 32, e.total);//{{{ makeSetValue('fetch', 32/*TODO:jsonify*/, 'e.total', 'i64')}}}; HEAPU16[fetch + 40 >> 1] = xhr.readyState; HEAPU16[fetch + 42 >> 1] = xhr.status; - if (xhr.statusText) writeStringToMemory(fetch + 44, xhr.statusText); + if (xhr.statusText) stringToUTF8(fetch + 44, xhr.statusText, 64); if (onprogress) onprogress(fetch, xhr, e); } #if FETCH_DEBUG diff --git a/src/fetch-worker.js b/src/fetch-worker.js index d3c39fb564bb8..0652a58819ff9 100644 --- a/src/fetch-worker.js +++ b/src/fetch-worker.js @@ -21,9 +21,6 @@ function enlargeMemory() { var nan = NaN; var inf = Infinity; -__emscripten_fetch_xhr = _emscripten_fetch_xhr; -__emscripten_fetch_cache_data = _emscripten_fetch_cache_data; - function _emscripten_asm_const_v() {} function assert(condition) { diff --git a/src/library_fetch.js b/src/library_fetch.js index 0d5f9c2afa10b..7c37d3bb2c7b5 100644 --- a/src/library_fetch.js +++ b/src/library_fetch.js @@ -14,10 +14,10 @@ var LibraryFetch = { return _fetch_work_queue; }, - _emscripten_fetch_load_cached_data: _emscripten_fetch_load_cached_data, - _emscripten_fetch_cache_data: _emscripten_fetch_cache_data, - _emscripten_fetch_xhr: _emscripten_fetch_xhr, - emscripten_start_fetch__deps: ['$Fetch', '_emscripten_fetch_xhr', '_emscripten_fetch_cache_data', '_emscripten_fetch_load_cached_data', '_emscripten_get_fetch_work_queue'], + $__emscripten_fetch_load_cached_data: __emscripten_fetch_load_cached_data, + $__emscripten_fetch_cache_data: __emscripten_fetch_cache_data, + $__emscripten_fetch_xhr: __emscripten_fetch_xhr, + emscripten_start_fetch__deps: ['$Fetch', '$__emscripten_fetch_xhr', '$__emscripten_fetch_cache_data', '$__emscripten_fetch_load_cached_data', '_emscripten_get_fetch_work_queue'], emscripten_start_fetch: emscripten_start_fetch }; diff --git a/src/settings.js b/src/settings.js index c3995a7163b99..fa6506f93cb0c 100644 --- a/src/settings.js +++ b/src/settings.js @@ -802,4 +802,8 @@ var OFFSCREENCANVAS_SUPPORT = 0; // If set to 1, enables support for transferrin var FETCH_DEBUG = 0; // If nonzero, prints out debugging information in library_fetch.js +var FETCH = 0; // If nonzero, enables emscripten_fetch API. + +var ASMFS = 0; // If set to 1, uses the multithreaded filesystem that is implemented within the asm.js module, using emscripten_fetch. Implies -s FETCH=1. + // Reserved: variables containing POINTER_MASKING. diff --git a/system/lib/fetch/asmfs.cpp b/system/lib/fetch/asmfs.cpp new file mode 100644 index 0000000000000..ea862a3c2e03d --- /dev/null +++ b/system/lib/fetch/asmfs.cpp @@ -0,0 +1,171 @@ +#include +#include +#include +#define __NEED_struct_iovec +#include +#include +#include +#include +#include +#include + +extern "C" { + +#define EM_FILEDESCRIPTOR_MAGIC 0x64666d65U // 'emfd' +struct FileDescriptor +{ + uint32_t magic; + emscripten_fetch_t *fetch; + ssize_t file_pos; +}; + +// http://man7.org/linux/man-pages/man2/open.2.html +long __syscall5(int which, ...) // open +{ + va_list vl; + va_start(vl, which); + const char *pathname = va_arg(vl, const char*); + int flags = va_arg(vl, int); + int mode = va_arg(vl, int); + va_end(vl); + + EM_ASM_INT( { Module['printErr']('__syscall5 OPEN, which: ' + $0 + ', pathname ' + Pointer_stringify($1) + ', ' + $2 + ', ' + $3 + '.') }, + which, pathname, flags, mode); + + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.attributes = EMSCRIPTEN_FETCH_APPEND | EMSCRIPTEN_FETCH_LOAD_TO_MEMORY | EMSCRIPTEN_FETCH_WAITABLE | EMSCRIPTEN_FETCH_PERSIST_FILE; + + FileDescriptor *desc = (FileDescriptor*)malloc(sizeof(FileDescriptor)); + desc->magic = EM_FILEDESCRIPTOR_MAGIC; + desc->fetch = emscripten_fetch(&attr, pathname); + desc->file_pos = 0; + + return (long)desc; +} + +// http://man7.org/linux/man-pages/man2/close.2.html +long __syscall6(int which, ...) // close +{ + va_list vl; + va_start(vl, which); + int fd = va_arg(vl, int); + va_end(vl); + EM_ASM_INT( { Module['printErr']('__syscall6 CLOSE, which: ' + $0 + ', fd: ' + $1 + '.') }, which, fd); + + FileDescriptor *desc = (FileDescriptor*)fd; + if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) + { + fprintf(stderr, "Invalid or already closed file descriptor 0x%8X passed to close()!", (unsigned int)desc); + return -1; // TODO: set errno + } + if (desc->fetch) + { + emscripten_fetch_wait(desc->fetch, INFINITY); // TODO: This should not be necessary- test this out + emscripten_fetch_close(desc->fetch); + desc->fetch = 0; + } + desc->magic = 0; + free(desc); + return 0; +} + +// http://man7.org/linux/man-pages/man2/sysctl.2.html +long __syscall54(int which, ...) // sysctl +{ + EM_ASM_INT( { Module['printErr']('__syscall54 SYSCTL, which: ' + $0 + '.') }, which); + return 0; +} + +// http://man7.org/linux/man-pages/man2/llseek.2.html +long __syscall140(int which, ...) // llseek +{ + va_list vl; + va_start(vl, which); + unsigned int fd = va_arg(vl, unsigned int); + unsigned long offset_high = va_arg(vl, unsigned long); + unsigned long offset_low = va_arg(vl, unsigned long); + off_t *result = va_arg(vl, off_t *); + unsigned int whence = va_arg(vl, unsigned int); + va_end(vl); + EM_ASM_INT( { Module['printErr']('__syscall140 LLSEEK, which: ' + $0 + ', fd ' + $1 + ', offset high ' + $2 + ' offset low ' + $3 + ' result ' + $4 + ' whence ' + $5 + '.') }, + which, fd, offset_high, offset_low, result, whence); + + FileDescriptor *desc = (FileDescriptor*)fd; + if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) + { + fprintf(stderr, "Invalid or closed file descriptor 0x%8X passed to close()!", (unsigned int)desc); + return -1; // TODO: set errno + } + if (!desc->fetch) + { + fprintf(stderr, "Internal error: no file data available for fd 0x%8X!", (unsigned int)desc); + return -1; + } + + emscripten_fetch_wait(desc->fetch, INFINITY); + + uint64_t offset = ((uint64_t)offset_high << 32) | (uint64_t)offset_low; + switch(whence) + { + case SEEK_SET: desc->file_pos = offset; break; + case SEEK_CUR: desc->file_pos += offset; break; + case SEEK_END: desc->file_pos = desc->fetch->numBytes - offset; break; + default: return -1; + } + if (desc->file_pos < 0) desc->file_pos = 0; + if (desc->file_pos > desc->fetch->numBytes) desc->file_pos = desc->fetch->numBytes; + if (result) *result = desc->file_pos; + return 0; +} + +// http://man7.org/linux/man-pages/man2/readv.2.html +long __syscall145(int which, ...) // readv +{ + va_list vl; + va_start(vl, which); + int fd = va_arg(vl, int); + const iovec *iov = va_arg(vl, const iovec*); + int iovcnt = va_arg(vl, int); + va_end(vl); + EM_ASM_INT( { Module['printErr']('__syscall145 READV, which: ' + $0 + ', fd ' + $1 + ', iov: ' + $2 + ', iovcnt: ' + $3 + ' .') }, which, fd, iov, iovcnt); + + FileDescriptor *desc = (FileDescriptor*)fd; + if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) + { + fprintf(stderr, "Invalid or closed file descriptor 0x%8X passed to readv()!", (unsigned int)desc); + return -1; // TODO: set errno + } + if (!desc->fetch) + { + fprintf(stderr, "Internal error: no file data available for fd 0x%8X!", (unsigned int)desc); + return -1; + } + + emscripten_fetch_wait(desc->fetch, INFINITY); // TODO: Ensure that multiple waits are ok. + + size_t offset = desc->file_pos; + for(int i = 0; i < iovcnt; ++i) + { + ssize_t dataLeft = desc->fetch->numBytes - offset; + if (dataLeft <= 0) break; + size_t bytesToCopy = dataLeft < iov[i].iov_len ? dataLeft : iov[i].iov_len; + memcpy(iov[i].iov_base, &desc->fetch->data[offset], bytesToCopy); + offset += bytesToCopy; + } + ssize_t numRead = offset - desc->file_pos; + desc->file_pos = offset; + return numRead; +} + +/* +// http://man7.org/linux/man-pages/man2/writev.2.html +long __syscall146(int which, ...) // writev +{ + EM_ASM_INT( { Module['printErr']('__syscall146 WRITEV, which: ' + $0 + '.') }, which); + return 0; +} +*/ + +} // ~extern "C" From 89456a51cbbf9a950a0300e6a937f39756a4a73c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Thu, 25 Aug 2016 23:05:07 +0300 Subject: [PATCH 17/74] ASMFS requires USE_PTHREADS --- emcc.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/emcc.py b/emcc.py index df3d96e5553d5..ef7f8b3f558d9 100755 --- a/emcc.py +++ b/emcc.py @@ -1143,6 +1143,9 @@ def check(input_file): next_arg_index += 1 shared.Settings.NO_FILESYSTEM = 1 shared.Settings.FETCH = 1 + if not shared.Settings.USE_PTHREADS: + logging.error('-s ASMFS=1 requires either -s USE_PTHREADS=1 or -s USE_PTHREADS=2 to be set!') + sys.exit(1) if shared.Settings.FETCH and final_suffix in JS_CONTAINING_SUFFIXES: input_files.append((next_arg_index, shared.path_from_root('system', 'lib', 'fetch', 'emscripten_fetch.cpp'))) From 699f435d6de4c11cbd7db8e97cda11d11fa3731a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Fri, 26 Aug 2016 13:03:34 +0300 Subject: [PATCH 18/74] If HTTP gives result 0 and there is data, treat it as HTTP 200 (e.g. loading via file:// gets this). Mimic HTTP 200 code for downloads cached from IndexedDB --- src/Fetch.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Fetch.js b/src/Fetch.js index e3bc91f33bdaf..ab0b577bda525 100644 --- a/src/Fetch.js +++ b/src/Fetch.js @@ -114,6 +114,8 @@ function __emscripten_fetch_load_cached_data(db, fetch, onsuccess, onerror) { Fetch.setu64(fetch + 16, len);//{{{ makeSetValue('fetch', 16/*TODO:jsonify*/, 'len', 'i64')}}}; Fetch.setu64(fetch + 24, 0);//{{{ makeSetValue('fetch', 24/*TODO:jsonify*/, '0', 'i64')}}}; Fetch.setu64(fetch + 32, len);//{{{ makeSetValue('fetch', 32/*TODO:jsonify*/, 'len', 'i64')}}}; + HEAPU16[fetch + 40 >> 1] = 4; // Mimic XHR readyState 4 === 'DONE: The operation is complete' + HEAPU16[fetch + 42 >> 1] = 200; // Mimic XHR HTTP status code 200 "OK" onsuccess(fetch, 0, value); } else { @@ -269,9 +271,10 @@ function __emscripten_fetch_xhr(fetch, onsuccess, onerror, onprogress) { Fetch.setu64(fetch + 32, len);//{{{ makeSetValue('fetch', 32/*TODO:jsonify*/, 'len', 'i64')}}}; } HEAPU16[fetch + 40 >> 1] = xhr.readyState; + if (xhr.readyState === 4 && xhr.status === 0 && len > 0) xhr.status = 200; // If loading files from a source that does not give HTTP status code, assume success if we got data bytes HEAPU16[fetch + 42 >> 1] = xhr.status; // if (xhr.statusText) stringToUTF8(fetch + 44, xhr.statusText, 64); - if (xhr.status == 200 || xhr.status == 0) { + if (xhr.status == 200) { #if FETCH_DEBUG console.log('fetch: xhr succeeded with status 200'); #endif @@ -311,6 +314,7 @@ function __emscripten_fetch_xhr(fetch, onsuccess, onerror, onprogress) { Fetch.setu64(fetch + 24, e.loaded - ptrLen);//{{{ makeSetValue('fetch', 24/*TODO:jsonify*/, 'e.loaded - ptrLen', 'i64')}}}; Fetch.setu64(fetch + 32, e.total);//{{{ makeSetValue('fetch', 32/*TODO:jsonify*/, 'e.total', 'i64')}}}; HEAPU16[fetch + 40 >> 1] = xhr.readyState; + if (xhr.readyState >= 3 && xhr.status === 0 && e.loaded > 0) xhr.status = 200; // If loading files from a source that does not give HTTP status code, assume success if we get data bytes HEAPU16[fetch + 42 >> 1] = xhr.status; if (xhr.statusText) stringToUTF8(fetch + 44, xhr.statusText, 64); if (onprogress) onprogress(fetch, xhr, e); From 57804f5f6b69e8e8e39238910bd7ea90691df734 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Fri, 26 Aug 2016 13:03:54 +0300 Subject: [PATCH 19/74] Implement various failure modes for ASMFS open() --- system/lib/fetch/asmfs.cpp | 145 ++++++++++++++++++++++++++++++++++++- 1 file changed, 144 insertions(+), 1 deletion(-) diff --git a/system/lib/fetch/asmfs.cpp b/system/lib/fetch/asmfs.cpp index ea862a3c2e03d..53d599d411394 100644 --- a/system/lib/fetch/asmfs.cpp +++ b/system/lib/fetch/asmfs.cpp @@ -8,6 +8,8 @@ #include #include #include +#include +#include extern "C" { @@ -32,6 +34,122 @@ long __syscall5(int which, ...) // open EM_ASM_INT( { Module['printErr']('__syscall5 OPEN, which: ' + $0 + ', pathname ' + Pointer_stringify($1) + ', ' + $2 + ', ' + $3 + '.') }, which, pathname, flags, mode); + int accessMode = (flags & O_ACCMODE); + if (accessMode != O_RDONLY) // O_WRONLY or O_RDWR + { + EM_ASM(Module['printErr']('open() syscall failed! pathname refers to a file on a read-only filesystem and write access was requested.')); + errno = EROFS; // "pathname refers to a file on a read-only filesystem and write access was requested." + return -1; + } + + if ((flags & O_ASYNC)) + { + EM_ASM(Module['printErr']('open() syscall failed! Opening files with O_ASYNC flag is not supported in ASMFS (TODO?)')); + return -1; + } + + // The flags:O_CLOEXEC flag is ignored, doesn't have meaning for Emscripten + + if ((flags & O_CREAT)) + { + EM_ASM(Module['printErr']('open() syscall failed! Opening files with O_CREAT flag is not yet supported in ASMFS (TODO)')); + return -1; + } + + // TODO: the flags:O_DIRECT flag seems like a great way to let applications explicitly control XHR/IndexedDB read/write buffering behavior? + if ((flags & O_DIRECT)) + { + EM_ASM(Module['printErr']('open() syscall failed! The filesystem does not support the O_DIRECT flag.')); + errno = EINVAL; // "The filesystem does not support the O_DIRECT flag." + return -1; + } + + if ((flags & O_DIRECTORY)) + { + EM_ASM(Module['printErr']('open() syscall failed! Opening directories with O_DIRECTORY flag is not yet supported in ASMFS (TODO)')); + return -1; + } + + if ((flags & O_DSYNC)) + { + EM_ASM(Module['printErr']('open() syscall failed! Opening files with O_DSYNC flag is not yet supported in ASMFS (TODO)')); + return -1; + } + + if ((flags & O_EXCL)) + { + EM_ASM(Module['printErr']('open() syscall failed! Opening files with O_EXCL flag is not yet supported in ASMFS (TODO)')); + + // TODO: Check if the file exists and if so, return EEXIST error (the file should not exist, but with O_EXCL should always create a new one) + return -1; + } + + // The flags:O_LARGEFILE flag is ignored, we should always be largefile-compatible + + // TODO: The flags:O_NOATIME is ignored, file access times have not been implemented yet + // The flags O_NOCTTY, O_NOFOLLOW + + if ((flags & (O_NONBLOCK|O_NDELAY))) + { + EM_ASM(Module['printErr']('open() syscall failed! Opening files with O_NONBLOCK or O_NDELAY flags is not yet supported in ASMFS (TODO)')); + return -1; + } + + if ((flags & O_PATH)) + { + EM_ASM(Module['printErr']('open() syscall failed! Opening files with O_PATH flags is not yet supported in ASMFS (TODO)')); + return -1; + } + + if ((flags & O_SYNC)) + { + EM_ASM(Module['printErr']('open() syscall failed! Opening files with O_SYNC flags is not yet supported in ASMFS (TODO)')); + return -1; + } + + if ((flags & O_TMPFILE)) + { + if (accessMode != O_WRONLY && accessMode != O_RDWR) + { + EM_ASM(Module['printErr']('open() syscall failed! O_TMPFILE was specified in flags, but neither O_WRONLY nor O_RDWR was specified.')); + errno = EINVAL; // "O_TMPFILE was specified in flags, but neither O_WRONLY nor O_RDWR was specified." + return -1; + } + EM_ASM(Module['printErr']('open() syscall failed! "The filesystem containing pathname does not support O_TMPFILE.')); + errno = EOPNOTSUPP; // "The filesystem containing pathname does not support O_TMPFILE." + return -1; + } + + if ((flags & O_TRUNC)) + { + EM_ASM(Module['printErr']('open() syscall failed! Opening files with O_TRUNC flags is not yet supported in ASMFS (TODO)')); + return -1; + } + + /* TODO: + if (is_directory and (accessMode == O_WRONLY || accessMode == O_RDWR)) + { + errno = EISDIR; // "pathname refers to a directory and the access requested involved writing (that is, O_WRONLY or O_RDWR is set)." + return -1; + } + */ + + /* TODO: + if (too_many_files_open) + { + errno = EMFILE; // "The per-process limit on the number of open file descriptors has been reached, see getrlimit(RLIMIT_NOFILE)" + return -1; + } + */ + + // http://stackoverflow.com/questions/417142/what-is-the-maximum-length-of-a-url-in-different-browsers + if (strlen(pathname) > 2000) + { + errno = ENAMETOOLONG; + EM_ASM(Module['printErr']('open() syscall failed! The URL to open was more than 2000 characters long')); + return -1; + } + emscripten_fetch_attr_t attr; emscripten_fetch_attr_init(&attr); strcpy(attr.requestMethod, "GET"); @@ -40,8 +158,33 @@ long __syscall5(int which, ...) // open FileDescriptor *desc = (FileDescriptor*)malloc(sizeof(FileDescriptor)); desc->magic = EM_FILEDESCRIPTOR_MAGIC; desc->fetch = emscripten_fetch(&attr, pathname); - desc->file_pos = 0; + // switch(fopen_mode) + // { + // case synchronous_fopen: + emscripten_fetch_wait(desc->fetch, INFINITY); + + if (desc->fetch->status != 200 || desc->fetch->totalBytes == 0) + { + EM_ASM_INT( { Module['printErr']('__syscall5 OPEN failed! File ' + Pointer_stringify($0) + ' does not exist: XHR returned status code ' + $1 + ', and file length was ' + $2 + '.') }, + pathname, (int)desc->fetch->status, (int)desc->fetch->totalBytes); + emscripten_fetch_close(desc->fetch); + errno = ENOENT; + return -1; + } + + desc->file_pos = (flags & O_APPEND) ? desc->fetch->totalBytes : 0; + + // break; + // case asynchronous_fopen: + // break; + // } + + // TODO: The file descriptor needs to be a small number, man page: + // "a small, nonnegative integer for use in subsequent system calls + // (read(2), write(2), lseek(2), fcntl(2), etc.). The file descriptor + // returned by a successful call will be the lowest-numbered file + // descriptor not currently open for the process." return (long)desc; } From 4c02dc3559b75d7ac0173b0d6042d6013976216e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Fri, 26 Aug 2016 13:47:39 +0300 Subject: [PATCH 20/74] Implement more error detection on ASMFS and set errno values. --- system/lib/fetch/asmfs.cpp | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/system/lib/fetch/asmfs.cpp b/system/lib/fetch/asmfs.cpp index 53d599d411394..8ed886d6cab18 100644 --- a/system/lib/fetch/asmfs.cpp +++ b/system/lib/fetch/asmfs.cpp @@ -201,7 +201,8 @@ long __syscall6(int which, ...) // close if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) { fprintf(stderr, "Invalid or already closed file descriptor 0x%8X passed to close()!", (unsigned int)desc); - return -1; // TODO: set errno + errno = EBADF; // "fd isn't a valid open file descriptor." + return -1; } if (desc->fetch) { @@ -239,7 +240,8 @@ long __syscall140(int which, ...) // llseek if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) { fprintf(stderr, "Invalid or closed file descriptor 0x%8X passed to close()!", (unsigned int)desc); - return -1; // TODO: set errno + errno = EBADF; // "fd isn't a valid open file descriptor." + return -1; } if (!desc->fetch) { @@ -255,10 +257,14 @@ long __syscall140(int which, ...) // llseek case SEEK_SET: desc->file_pos = offset; break; case SEEK_CUR: desc->file_pos += offset; break; case SEEK_END: desc->file_pos = desc->fetch->numBytes - offset; break; - default: return -1; + default: + errno = EINVAL; // "whence is invalid." + return -1; } + // Clamp the seek to within the file size range. if (desc->file_pos < 0) desc->file_pos = 0; if (desc->file_pos > desc->fetch->numBytes) desc->file_pos = desc->fetch->numBytes; + if (result) *result = desc->file_pos; return 0; } @@ -278,7 +284,8 @@ long __syscall145(int which, ...) // readv if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) { fprintf(stderr, "Invalid or closed file descriptor 0x%8X passed to readv()!", (unsigned int)desc); - return -1; // TODO: set errno + errno = EBADF; // "fd is not a valid file descriptor or is not open for reading." + return -1; } if (!desc->fetch) { @@ -286,8 +293,29 @@ long __syscall145(int which, ...) // readv return -1; } + // TODO: Test and detect to return EISDIR. + + // TODO: Support nonblocking IO and check for EAGAIN/EWOULDBLOCK emscripten_fetch_wait(desc->fetch, INFINITY); // TODO: Ensure that multiple waits are ok. + if (iovcnt < 0) + { + errno = EINVAL; // "The vector count, iovcnt, is less than zero or greater than the permitted maximum." + return -1; + } + + ssize_t total_read_amount = 0; + for(int i = 0; i < iovcnt; ++i) + { + ssize_t n = total_read_amount + iov[i].iov_len; + if (n < total_read_amount || !iov[i].iov_base) + { + errno = EINVAL; // "The sum of the iov_len values overflows an ssize_t value." or "the address specified in buf is not valid" + return -1; + } + total_read_amount = n; + } + size_t offset = desc->file_pos; for(int i = 0; i < iovcnt; ++i) { From 6192d1b2cfe9bc1964bcfc090e2fddae90214401 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Fri, 26 Aug 2016 13:54:15 +0300 Subject: [PATCH 21/74] Add first test for ASMFS --- tests/asmfs/hello_file.cpp | 32 ++++++++++++++++++++++++++++++++ tests/asmfs/hello_file.txt | 1 + tests/test_browser.py | 4 ++++ 3 files changed, 37 insertions(+) create mode 100644 tests/asmfs/hello_file.cpp create mode 100644 tests/asmfs/hello_file.txt diff --git a/tests/asmfs/hello_file.cpp b/tests/asmfs/hello_file.cpp new file mode 100644 index 0000000000000..04ac15dce7450 --- /dev/null +++ b/tests/asmfs/hello_file.cpp @@ -0,0 +1,32 @@ +#include +#include +#include +#include +#include + +int main() +{ + FILE *file = fopen("hello_file.txt", "rb"); + assert(file); + fseek(file, 0, SEEK_END); + + long size = ftell(file); + rewind(file); + printf("size: %ld\n", size); + + char *buffer = (char*) malloc (sizeof(char)*(size+1)); + assert(buffer); + buffer[size] = '\0'; + + size_t read = fread(buffer, 1, size, file); + printf("File contents: %s\n", buffer); + assert(size == 6); + assert(!strcmp(buffer, "Hello!")); + + fclose(file); + +#ifdef REPORT_RESULT + int result = 0; + REPORT_RESULT(); +#endif +} diff --git a/tests/asmfs/hello_file.txt b/tests/asmfs/hello_file.txt new file mode 100644 index 0000000000000..05a682bd4e7c7 --- /dev/null +++ b/tests/asmfs/hello_file.txt @@ -0,0 +1 @@ +Hello! \ No newline at end of file diff --git a/tests/test_browser.py b/tests/test_browser.py index a06b5c382fa2a..b088a50fff804 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -3332,3 +3332,7 @@ def test_fetch_sync_xhr(self): shutil.copyfile(path_from_root('tests', 'gears.png'), os.path.join(self.get_dir(), 'gears.png')) self.btest('fetch/sync_xhr.cpp', expected='1', args=['--std=c++11', '-s', 'FETCH_DEBUG=1', '--proxy-to-worker'] + temp_args) + + def test_asmfs_hello_file(self): + shutil.copyfile(path_from_root('tests', 'asmfs', 'hello_file.txt'), os.path.join(self.get_dir(), 'hello_file.txt')) + self.btest('asmfs/hello_file.cpp', expected='0', args=['-s', 'ASMFS=1', '-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1', '--proxy-to-worker']) From 9725e543c78f336736ce7f643a5e490962e9e6e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Fri, 26 Aug 2016 18:58:36 +0300 Subject: [PATCH 22/74] Fix fetch worker import of asm.js _malloc function. Generate the fetch-worker.js script file to correct output directory. Remember to import getTotalMemory() in fetch worker. --- emcc.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/emcc.py b/emcc.py index ef7f8b3f558d9..3a04b4fefcc80 100755 --- a/emcc.py +++ b/emcc.py @@ -1768,11 +1768,18 @@ def repl(m): if shared.Settings.USE_PTHREADS: shutil.copyfile(shared.path_from_root('src', 'pthread-main.js'), os.path.join(os.path.dirname(os.path.abspath(target)), 'pthread-main.js')) + if shared.Settings.FETCH: src = open(final, 'r').read() - funcs_to_import = ['alignMemoryPage', '_sbrk', '_pthread_mutex_lock', '_malloc', '_emscripten_sync_run_in_main_thread_1', '_emscripten_sync_run_in_main_thread', '_emscripten_is_main_runtime_thread', '_pthread_mutex_unlock', '_emscripten_set_current_thread_status', '_emscripten_futex_wait', 'stringToUTF8', 'intArrayFromString', 'lengthBytesUTF8', 'stringToUTF8Array'] - function_prologue = '' - for func in funcs_to_import: - loc = src.find('function ' + func + '(') + funcs_to_import = ['alignMemoryPage', 'getTotalMemory', 'stringToUTF8', 'intArrayFromString', 'lengthBytesUTF8', 'stringToUTF8Array', '_emscripten_is_main_runtime_thread', '_emscripten_futex_wait'] + asm_funcs_to_import = ['_malloc', '_free', '_sbrk', '_pthread_mutex_lock', '_emscripten_sync_run_in_main_thread_1', '_emscripten_sync_run_in_main_thread', '_pthread_mutex_unlock', '_emscripten_set_current_thread_status'] + function_prologue = '''this.onerror = function(e) { + console.error(e); +} + +''' + asm_start = src.find('// EMSCRIPTEN_START_ASM') + for func in funcs_to_import + asm_funcs_to_import: + loc = src.find('function ' + func + '(', asm_start if func in asm_funcs_to_import else 0) if loc == -1: logging.fatal('failed to find function ' + func + '!') sys.exit(1) @@ -1787,7 +1794,7 @@ def repl(m): function_prologue = function_prologue + '\n' + func_code fetch_worker_src = function_prologue + '\n' + shared.clang_preprocess(shared.path_from_root('src', 'fetch-worker.js')) - open('fetch-worker.js', 'w').write(fetch_worker_src) + open(os.path.join(os.path.dirname(os.path.abspath(target)), 'fetch-worker.js'), 'w').write(fetch_worker_src) if shared.Settings.BINARYEN: # Insert a call to integrate with wasm.js From 0ca337856105050aee6b207b412f9960e087ee2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Fri, 26 Aug 2016 18:58:57 +0300 Subject: [PATCH 23/74] Print out error messages from fetch worker to console. --- src/Fetch.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Fetch.js b/src/Fetch.js index ab0b577bda525..446dcf17c2e65 100644 --- a/src/Fetch.js +++ b/src/Fetch.js @@ -78,6 +78,12 @@ var Fetch = { if (typeof Module['locateFile'] === 'function') fetchJs = Module['locateFile'](fetchJs); else if (Module['pthreadMainPrefixURL']) fetchJs = Module['pthreadMainPrefixURL'] + fetchJs; Fetch.worker = new Worker(fetchJs); + Fetch.worker.onmessage = function(e) { + Module['print']('fetch-worker sent a message: ' + e.filename + ':' + e.lineno + ': ' + e.message); + }; + Fetch.worker.onerror = function(e) { + Module['printErr']('fetch-worker sent an error! ' + e.filename + ':' + e.lineno + ': ' + e.message); + }; } } } From 71120e786f15ec4f2d7936df9f922ea343272c7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Fri, 26 Aug 2016 18:59:28 +0300 Subject: [PATCH 24/74] Use atomics in the processing of the fetch queue in fetch worker side. --- src/fetch-worker.js | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/fetch-worker.js b/src/fetch-worker.js index 0652a58819ff9..1cf74b36a4854 100644 --- a/src/fetch-worker.js +++ b/src/fetch-worker.js @@ -77,22 +77,22 @@ var HEAPU32 = null; function processWorkQueue() { if (!queuePtr) return; + var numQueuedItems = Atomics_load(HEAPU32, queuePtr + 4 >> 2); + if (numQueuedItems == 0) return; - console.log('polling work to perform'); - var queuedOperations = HEAPU32[queuePtr >> 2]; - var numQueuedItems = HEAPU32[queuePtr + 4 >> 2]; - var queueSize = HEAPU32[queuePtr + 8 >> 2]; + console.log('polling work to perform, there are ' + numQueuedItems + ' work items in the queue.'); + var queuedOperations = Atomics_load(HEAPU32, queuePtr >> 2); + var queueSize = Atomics_load(HEAPU32, queuePtr + 8 >> 2); for(var i = 0; i < numQueuedItems; ++i) { - var fetch = HEAPU32[(queuedOperations >> 2)+i]; + var fetch = Atomics_load(HEAPU32, (queuedOperations >> 2)+i); console.log('processWorkQueue: starting fetch'); - console.log('fetch ptr1 ' + fetch + ', state ' + HEAPU32[fetch + 108 >> 2]); + console.log('fetch ptr1 ' + fetch + ', state ' + Atomics_load(HEAPU32, fetch + 108 >> 2)); function successcb(fetch) { - console.log('fetch ptr ' + fetch + ', state ' + HEAPU32[fetch + 108 >> 2]); + console.log('fetch ptr ' + fetch + ', state ' + Atomics_load(HEAPU32, fetch + 108 >> 2)); console.log('FETCH-WORKER: fetch finished'); var oldVal = Atomics.compareExchange(HEAPU32, fetch + 108 >> 2, 1, 2); console.log('atomics wake ' + (fetch + 108)); Atomics.wake(HEAP32, fetch + 108 >> 2, 1); - console.log('oldVal ' + oldVal); } function errorcb(fetch) { console.log('FETCH-WORKER: fetch failed'); @@ -105,12 +105,14 @@ function processWorkQueue() { } catch(e) { console.error(e); } + /* if (interval != undefined) { clearInterval(interval); interval = undefined; } + */ } - HEAPU32[queuePtr + 4 >> 2] = 0; + Atomics_store(HEAPU32, queuePtr + 4 >> 2, 0); } interval = 0; @@ -131,6 +133,6 @@ this.onmessage = function(e) { console.log('DYNAMICTOP_PTR: ' + DYNAMICTOP_PTR); console.log('DYNAMICTOP: ' + HEAP32[DYNAMICTOP_PTR>>2]); console.log('fetch worker init, queue ptr ' + queuePtr + ', heap length: ' + buffer.byteLength); - interval = setInterval(processWorkQueue, 1000); + interval = setInterval(processWorkQueue, 100); } } From d8939c2d4add080f2d986c7ea5e7782a3d83908d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Fri, 26 Aug 2016 19:00:15 +0300 Subject: [PATCH 25/74] Improve pthread error message --- src/library_pthread.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/library_pthread.js b/src/library_pthread.js index 9cf5649e0071d..d9b1b3b2a8af5 100644 --- a/src/library_pthread.js +++ b/src/library_pthread.js @@ -308,7 +308,7 @@ var LibraryPThread = { }; worker.onerror = function(e) { - Module['printErr']('pthread sent an error! ' + e.message); + Module['printErr']('pthread sent an error! ' + e.filename + ':' + e.lineno + ': ' + e.message); }; // Allocate tempDoublePtr for the worker. This is done here on the worker's behalf, since we may need to do this statically From f21caccf3745d636af9ac576a5f3f0ed0a4a7c1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Fri, 26 Aug 2016 19:01:18 +0300 Subject: [PATCH 26/74] Clean up warnings in ASMFS file size comparison. --- system/lib/fetch/asmfs.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/system/lib/fetch/asmfs.cpp b/system/lib/fetch/asmfs.cpp index 8ed886d6cab18..4b259d262b5f7 100644 --- a/system/lib/fetch/asmfs.cpp +++ b/system/lib/fetch/asmfs.cpp @@ -263,7 +263,7 @@ long __syscall140(int which, ...) // llseek } // Clamp the seek to within the file size range. if (desc->file_pos < 0) desc->file_pos = 0; - if (desc->file_pos > desc->fetch->numBytes) desc->file_pos = desc->fetch->numBytes; + if ((size_t)desc->file_pos > desc->fetch->numBytes) desc->file_pos = desc->fetch->numBytes; if (result) *result = desc->file_pos; return 0; @@ -321,7 +321,7 @@ long __syscall145(int which, ...) // readv { ssize_t dataLeft = desc->fetch->numBytes - offset; if (dataLeft <= 0) break; - size_t bytesToCopy = dataLeft < iov[i].iov_len ? dataLeft : iov[i].iov_len; + size_t bytesToCopy = (size_t)dataLeft < iov[i].iov_len ? dataLeft : iov[i].iov_len; memcpy(iov[i].iov_base, &desc->fetch->data[offset], bytesToCopy); offset += bytesToCopy; } From c2e38c0a3eee55a86580e0262f53b99cf2023d39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Fri, 26 Aug 2016 19:02:38 +0300 Subject: [PATCH 27/74] Improve debug log print in emscripten_fetch wait api --- system/lib/fetch/emscripten_fetch.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system/lib/fetch/emscripten_fetch.cpp b/system/lib/fetch/emscripten_fetch.cpp index d25a4cc51b54d..7fec70d0103af 100644 --- a/system/lib/fetch/emscripten_fetch.cpp +++ b/system/lib/fetch/emscripten_fetch.cpp @@ -36,7 +36,7 @@ void emscripten_proxy_fetch(emscripten_fetch_t *fetch) __emscripten_fetch_queue *queue = _emscripten_get_fetch_queue(); // TODO handle case when queue->numQueuedItems >= queue->queueSize queue->queuedOperations[queue->numQueuedItems++] = fetch; - printf("Queueing fetch\n"); + printf("Queued fetch to fetch-worker to process. There are now %d operations in the queue\n", queue->numQueuedItems); // TODO: mutex unlock } From 4e72eb82fb73c39b763064cae7cdf27aa042c711 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Fri, 26 Aug 2016 23:48:27 +0300 Subject: [PATCH 28/74] Fix file seek behavior in llseek to work according to the semantics laid out in lseek man page --- system/lib/fetch/asmfs.cpp | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/system/lib/fetch/asmfs.cpp b/system/lib/fetch/asmfs.cpp index 4b259d262b5f7..da35320aab77c 100644 --- a/system/lib/fetch/asmfs.cpp +++ b/system/lib/fetch/asmfs.cpp @@ -223,6 +223,7 @@ long __syscall54(int which, ...) // sysctl } // http://man7.org/linux/man-pages/man2/llseek.2.html +// also useful: http://man7.org/linux/man-pages/man2/lseek.2.html long __syscall140(int which, ...) // llseek { va_list vl; @@ -251,19 +252,30 @@ long __syscall140(int which, ...) // llseek emscripten_fetch_wait(desc->fetch, INFINITY); - uint64_t offset = ((uint64_t)offset_high << 32) | (uint64_t)offset_low; + int64_t offset = (int64_t)(((uint64_t)offset_high << 32) | (uint64_t)offset_low); + int64_t newPos; switch(whence) { - case SEEK_SET: desc->file_pos = offset; break; - case SEEK_CUR: desc->file_pos += offset; break; - case SEEK_END: desc->file_pos = desc->fetch->numBytes - offset; break; + case SEEK_SET: newPos = offset; break; + case SEEK_CUR: newPos = desc->file_pos + offset; break; + case SEEK_END: newPos = desc->fetch->numBytes + offset; break; default: errno = EINVAL; // "whence is invalid." return -1; } - // Clamp the seek to within the file size range. - if (desc->file_pos < 0) desc->file_pos = 0; - if ((size_t)desc->file_pos > desc->fetch->numBytes) desc->file_pos = desc->fetch->numBytes; + if (newPos < 0) + { + errno = EINVAL; // "the resulting file offset would be negative" + return -1; + } + if (newPos > 0x7FFFFFFFLL) + { + errno = EOVERFLOW; // "The resulting file offset cannot be represented in an off_t." + EM_ASM_INT( { Module['printErr']('llseek EOVERFLOW error: fd ' + $0 + 'attempted to seek past unsupported 2^31-1 file size limit (TODO?).') }, + fd); + return -1; + } + desc->file_pos = newPos; if (result) *result = desc->file_pos; return 0; From b537772fab83cd3d7e10166b3ee6e457c02950b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Sat, 27 Aug 2016 21:18:39 +0300 Subject: [PATCH 29/74] Implement first version of writing files to memory in ASMFS. --- system/lib/fetch/asmfs.cpp | 527 ++++++++++++++++++++++++++++++++++--- tests/test_browser.py | 4 + 2 files changed, 491 insertions(+), 40 deletions(-) diff --git a/system/lib/fetch/asmfs.cpp b/system/lib/fetch/asmfs.cpp index da35320aab77c..62c721dfbb577 100644 --- a/system/lib/fetch/asmfs.cpp +++ b/system/lib/fetch/asmfs.cpp @@ -10,17 +10,297 @@ #include #include #include +#include extern "C" { +#define INODE_TYPE uint32_t +#define INODE_FILE 1 +#define INODE_DIR 2 + +struct inode +{ + char name[NAME_MAX+1]; // NAME_MAX actual bytes + one byte for null termination. + inode *parent; // ID of the parent node + inode *sibling; // ID of a sibling node (these form a singular linked list that specifies the content under a directory) + inode *child; // ID of the first child node in a chain of children (the root of a linked list of inodes) + uint32_t uid; // User ID of the owner + uint32_t gid; // Group ID of the owning group + uint32_t mode; // r/w/x modes + time_t ctime; // Time when the inode was last modified + time_t mtime; // Time when the content was last modified + time_t atime; // Time when the content was last accessed + size_t size; // Size of the file in bytes + size_t capacity; // Amount of bytes allocated to pointer data + uint8_t *data; // The actual file contents. + + INODE_TYPE type; + + emscripten_fetch_t *fetch; +}; + #define EM_FILEDESCRIPTOR_MAGIC 0x64666d65U // 'emfd' struct FileDescriptor { uint32_t magic; - emscripten_fetch_t *fetch; ssize_t file_pos; + uint32_t mode; + uint32_t flags; + + inode *node; }; +static inode *create_inode(INODE_TYPE type) +{ + inode *i = (inode*)malloc(sizeof(inode)); + memset(i, 0, sizeof(inode)); + i->ctime = i->mtime = i->atime = time(0); + i->type = type; + EM_ASM(Module['print']('create_inode allocated new inode object.')); + return i; +} + +// The current working directory of the application process. +static inode *cwd_inode = 0; + +static inode *filesystem_root() +{ + static inode *root_node = create_inode(INODE_DIR); + return root_node; +} + +static inode *get_cwd() +{ + if (!cwd_inode) cwd_inode = filesystem_root(); + return cwd_inode; +} + +static void set_cwd(inode *node) +{ + cwd_inode = node; +} + +static void delete_inode(inode *node) +{ + free(node); +} + +// Makes node the child of parent. +static void link_inode(inode *node, inode *parent) +{ + EM_ASM_INT( { Module['printErr']('link_inode: node ' + Pointer_stringify($0) + ' to parent ' + Pointer_stringify($1) + '.') }, + node->name, parent->name); + // When linking a node, it can't be part of the filesystem tree (but it can have children of its own) + assert(!node->parent); + assert(!node->sibling); + + // The inode pointed by 'node' is not yet part of the filesystem, so it's not shared memory and only this thread + // is accessing it. Therefore setting the node's parent here is not yet racy, do that operation first. + node->parent = parent; + + // This node is to become the first child of the parent, and the old first child of the parent should + // become the sibling of this node, i.e. + // 1) node->sibling = parent->child; + // 2) parent->child = node; + // However these two operations need to occur atomically in order to be coherent. To ensure that, run the two + // operations in a CAS loop, which is possible because the first operation is not racy until the node is 'published' + // to the filesystem tree by the compare_exchange operation. + do { __atomic_load(&parent->child, &node->sibling, __ATOMIC_SEQ_CST); // node->sibling <- parent->child + } while (!__atomic_compare_exchange(&parent->child, &node->sibling, &node, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)); // parent->child <- node if it had not raced to change value in between +} + +// Traverse back in sibling linked list, or 0 if no such node exist. +static inode *find_predecessor_sibling(inode *node, inode *parent) +{ + inode *child = parent->child; + if (child == node) + return 0; + while(child && child->sibling != node) + child = child->sibling; + if (!child->sibling) return 0; + return child; +} + +static void unlink_inode(inode *node) +{ + EM_ASM_INT( { Module['printErr']('unlink_inode: node ' + Pointer_stringify($0) + ' from its parent ' + Pointer_stringify($1) + '.') }, + node->name, node->parent->name); + inode *parent = node->parent; + if (!parent) return; + node->parent = 0; + + if (parent->child == node) + { + parent->child = node->sibling; + } + else + { + inode *predecessor = find_predecessor_sibling(node, parent); + if (predecessor) + predecessor->sibling = node->sibling; + } + node->parent = node->sibling = 0; +} + +// Compares two strings for equality until a '\0' or a '/' is hit. Returns 0 if the strings differ, +// or a pointer to the beginning of the next directory component name of s1 if the strings are equal. +static const char *path_cmp(const char *s1, const char *s2) +{ + while(*s1 == *s2) + { + if (*s1 == '/') return s1+1; + if (*s1 == '\0') return s1; + ++s1; + ++s2; + } + if (*s1 == '/' && *s2 == '\0') return s1+1; + if (*s1 == '\0' && *s2 == '/') return s1; + return 0; +} + +// Copies string 'path' to 'dst', but stops on the first forward slash '/' character. +// Returns number of bytes written, excluding null terminator +static int strcpy_inodename(char *dst, const char *path) +{ + char *d = dst; + while(*path && *path != '/') + *dst++ = *path++; + *dst = '\0'; + return dst - d; +} + +// Returns a pointer to the basename part of the string, i.e. the string after the last occurrence of a forward slash character +static const char *basename_part(const char *path) +{ + const char *s = path; + while(*path) + { + if (*path == '/') s = path+1; + ++path; + } + return s; +} + +static inode *create_directory_hierarchy_for_file(inode *root, const char *path_to_file) +{ + if (!root) return 0; + + inode *node = root->child; + while(node) + { + const char *child_path = path_cmp(path_to_file, node->name); + if (child_path) + { + // The directory name matches. + path_to_file = child_path; + if (path_to_file[0] == '\0') return node; + if (path_to_file[0] == '/' && path_to_file[1] == '\0' /* && node is a directory*/) return node; + root = node; + node = node->child; + } + else + { + node = node->sibling; + } + } + const char *basename_pos = basename_part(path_to_file); + while(*path_to_file && path_to_file < basename_pos) + { + node = create_inode(INODE_DIR); + path_to_file += strcpy_inodename(node->name, path_to_file) + 1; + link_inode(node, root); + EM_ASM_INT( { Module['print']('create_directory_hierarchy_for_file: created directory ' + Pointer_stringify($0) + ' under parent ' + Pointer_stringify($1) + '.') }, + node->name, node->parent->name); + root = node; + } + return root; +} + +// Given a root inode of the filesystem and a path relative to it, e.g. "some/directory/dir_or_file", +// returns the inode that corresponds to "dir_or_file", or 0 if it doesn't exist. +// If the parameter out_closest_parent is specified, the closest (grand)parent node will be returned. +static inode *find_inode(inode *root, const char *path, inode **out_closest_parent = 0) +{ + if (out_closest_parent) *out_closest_parent = root; + if (!root) + { + return 0; + } + + inode *node = root->child; + while(node) + { + const char *child_path = path_cmp(path, node->name); + if (child_path) + { + // The directory name matches. + path = child_path; + if (path[0] == '\0') return node; + if (path[0] == '/' && path[1] == '\0' /* && node is a directory*/) return node; + if (out_closest_parent) *out_closest_parent = node; + node = node->child; + } + else + { + node = node->sibling; + } + } + return 0; +} + +// Debug function that dumps out the filesystem tree to console. +void emscripten_dump_fs_tree(inode *root, char *path) +{ + printf("%s:\n", path); + // Print out: + // file mode | number of links | owner name | group name | file size in bytes | file last modified time | path name + // which aligns with "ls -AFTRl" on console + inode *child = root->child; + uint64_t totalSize = 0; + while(child) + { + printf("%c%c%c%c%c%c%c%c%c%c %d user%u group%u %u Jan 1 1970 %s%c\n", + child->type == INODE_DIR ? 'd' : '-', + (child->mode & S_IRUSR) ? 'r' : '-', + (child->mode & S_IWUSR) ? 'w' : '-', + (child->mode & S_IXUSR) ? 'x' : '-', + (child->mode & S_IRGRP) ? 'r' : '-', + (child->mode & S_IWGRP) ? 'w' : '-', + (child->mode & S_IXGRP) ? 'x' : '-', + (child->mode & S_IROTH) ? 'r' : '-', + (child->mode & S_IWOTH) ? 'w' : '-', + (child->mode & S_IXOTH) ? 'x' : '-', + 1, // number of links to this file + child->uid, + child->gid, + child->size, + child->name, + child->type == INODE_DIR ? '/' : ' '); + totalSize += child->size; + child = child->sibling; + } + printf("total %llu bytes\n\n", totalSize); + + child = root->child; + char *path_end = path + strlen(path); + while(child) + { + if (child->type == INODE_DIR) + { + strcpy(path_end, child->name); + strcat(path_end, "/"); + emscripten_dump_fs_tree(child, path); + } + child = child->sibling; + } +} + +void emscripten_dump_fs_root() +{ + char path[PATH_MAX] = "/"; + emscripten_dump_fs_tree(filesystem_root(), path); +} + // http://man7.org/linux/man-pages/man2/open.2.html long __syscall5(int which, ...) // open { @@ -35,12 +315,14 @@ long __syscall5(int which, ...) // open which, pathname, flags, mode); int accessMode = (flags & O_ACCMODE); + /* if (accessMode != O_RDONLY) // O_WRONLY or O_RDWR { EM_ASM(Module['printErr']('open() syscall failed! pathname refers to a file on a read-only filesystem and write access was requested.')); errno = EROFS; // "pathname refers to a file on a read-only filesystem and write access was requested." return -1; } + */ if ((flags & O_ASYNC)) { @@ -49,13 +331,13 @@ long __syscall5(int which, ...) // open } // The flags:O_CLOEXEC flag is ignored, doesn't have meaning for Emscripten - +/* if ((flags & O_CREAT)) { EM_ASM(Module['printErr']('open() syscall failed! Opening files with O_CREAT flag is not yet supported in ASMFS (TODO)')); return -1; } - +*/ // TODO: the flags:O_DIRECT flag seems like a great way to let applications explicitly control XHR/IndexedDB read/write buffering behavior? if ((flags & O_DIRECT)) { @@ -119,13 +401,13 @@ long __syscall5(int which, ...) // open errno = EOPNOTSUPP; // "The filesystem containing pathname does not support O_TMPFILE." return -1; } - +/* if ((flags & O_TRUNC)) { EM_ASM(Module['printErr']('open() syscall failed! Opening files with O_TRUNC flags is not yet supported in ASMFS (TODO)')); return -1; } - +*/ /* TODO: if (is_directory and (accessMode == O_WRONLY || accessMode == O_RDWR)) { @@ -150,36 +432,100 @@ long __syscall5(int which, ...) // open return -1; } - emscripten_fetch_attr_t attr; - emscripten_fetch_attr_init(&attr); - strcpy(attr.requestMethod, "GET"); - attr.attributes = EMSCRIPTEN_FETCH_APPEND | EMSCRIPTEN_FETCH_LOAD_TO_MEMORY | EMSCRIPTEN_FETCH_WAITABLE | EMSCRIPTEN_FETCH_PERSIST_FILE; + // Find if this file exists already in the filesystem? + inode *root; + if (pathname[0] == '/') + { + root = filesystem_root(); + ++pathname; + } + else + root = get_cwd(); - FileDescriptor *desc = (FileDescriptor*)malloc(sizeof(FileDescriptor)); - desc->magic = EM_FILEDESCRIPTOR_MAGIC; - desc->fetch = emscripten_fetch(&attr, pathname); + inode *grandparent = 0; + inode *node = find_inode(root, pathname, &grandparent); + if (node) + { + EM_ASM_INT( { Module['print']('__syscall5 OPEN: pathname ' + Pointer_stringify($0) + ', inode exists. data ptr: ' + $1 + ', data size: ' + $2 + ', fetch ptr: ' + $3) }, + pathname, node->data, node->size, node->fetch); + } + + if (node && node->fetch) emscripten_fetch_wait(node->fetch, INFINITY); + + if ((flags & O_CREAT) && (flags & O_TRUNC)) + { + // Create a new empty file or truncate existing one. + if (node) + { + if (node->fetch) + { + emscripten_fetch_close(node->fetch); + node->fetch = 0; + } + node->size = 0; + } + else + { + inode *directory = create_directory_hierarchy_for_file(root, pathname); + node = create_inode(INODE_FILE); + strcpy(node->name, basename_part(pathname)); + link_inode(node, directory); + printf("Created file %s in directory %s\n", node->name, directory->name); + } + + emscripten_dump_fs_root(); + } + else if (!node || (!node->fetch && !node->data)) + { + // If not, we'll need to fetch it. + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.attributes = EMSCRIPTEN_FETCH_APPEND | EMSCRIPTEN_FETCH_LOAD_TO_MEMORY | EMSCRIPTEN_FETCH_WAITABLE | EMSCRIPTEN_FETCH_PERSIST_FILE; + emscripten_fetch_t *fetch = emscripten_fetch(&attr, pathname); // switch(fopen_mode) // { // case synchronous_fopen: - emscripten_fetch_wait(desc->fetch, INFINITY); + emscripten_fetch_wait(fetch, INFINITY); - if (desc->fetch->status != 200 || desc->fetch->totalBytes == 0) + if (fetch->status != 200 || fetch->totalBytes == 0) { EM_ASM_INT( { Module['printErr']('__syscall5 OPEN failed! File ' + Pointer_stringify($0) + ' does not exist: XHR returned status code ' + $1 + ', and file length was ' + $2 + '.') }, - pathname, (int)desc->fetch->status, (int)desc->fetch->totalBytes); - emscripten_fetch_close(desc->fetch); + pathname, (int)fetch->status, (int)fetch->totalBytes); + emscripten_fetch_close(fetch); errno = ENOENT; return -1; } - - desc->file_pos = (flags & O_APPEND) ? desc->fetch->totalBytes : 0; - // break; // case asynchronous_fopen: // break; // } + if (node) + { + node->fetch = fetch; + } + else + { + inode *directory = create_directory_hierarchy_for_file(root, pathname); + node = create_inode(INODE_FILE); + strcpy(node->name, basename_part(pathname)); + node->fetch = fetch; + link_inode(node, directory); + printf("Created file %s in directory %s\n", node->name, directory->name); + } + node->size = node->fetch->totalBytes; + emscripten_dump_fs_root(); + } + + FileDescriptor *desc = (FileDescriptor*)malloc(sizeof(FileDescriptor)); + desc->magic = EM_FILEDESCRIPTOR_MAGIC; + desc->node = node; + desc->file_pos = (flags & O_APPEND) ? node->fetch->totalBytes : 0; + desc->mode = mode; + desc->flags = flags; + // TODO: The file descriptor needs to be a small number, man page: // "a small, nonnegative integer for use in subsequent system calls // (read(2), write(2), lseek(2), fcntl(2), etc.). The file descriptor @@ -204,11 +550,11 @@ long __syscall6(int which, ...) // close errno = EBADF; // "fd isn't a valid open file descriptor." return -1; } - if (desc->fetch) + if (desc->node && desc->node->fetch) { - emscripten_fetch_wait(desc->fetch, INFINITY); // TODO: This should not be necessary- test this out - emscripten_fetch_close(desc->fetch); - desc->fetch = 0; + emscripten_fetch_wait(desc->node->fetch, INFINITY); // TODO: This should not be necessary- test this out + emscripten_fetch_close(desc->node->fetch); + desc->node->fetch = 0; } desc->magic = 0; free(desc); @@ -244,13 +590,13 @@ long __syscall140(int which, ...) // llseek errno = EBADF; // "fd isn't a valid open file descriptor." return -1; } - if (!desc->fetch) + if (!desc->node->fetch) { fprintf(stderr, "Internal error: no file data available for fd 0x%8X!", (unsigned int)desc); return -1; } - emscripten_fetch_wait(desc->fetch, INFINITY); + emscripten_fetch_wait(desc->node->fetch, INFINITY); int64_t offset = (int64_t)(((uint64_t)offset_high << 32) | (uint64_t)offset_low); int64_t newPos; @@ -258,7 +604,7 @@ long __syscall140(int which, ...) // llseek { case SEEK_SET: newPos = offset; break; case SEEK_CUR: newPos = desc->file_pos + offset; break; - case SEEK_END: newPos = desc->fetch->numBytes + offset; break; + case SEEK_END: newPos = desc->node->fetch->numBytes + offset; break; default: errno = EINVAL; // "whence is invalid." return -1; @@ -295,20 +641,15 @@ long __syscall145(int which, ...) // readv FileDescriptor *desc = (FileDescriptor*)fd; if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) { - fprintf(stderr, "Invalid or closed file descriptor 0x%8X passed to readv()!", (unsigned int)desc); + EM_ASM_INT( { Module['printErr']('Invalid or closed file descriptor ' + $0 + ' passed to readv!') }, fd); errno = EBADF; // "fd is not a valid file descriptor or is not open for reading." return -1; } - if (!desc->fetch) - { - fprintf(stderr, "Internal error: no file data available for fd 0x%8X!", (unsigned int)desc); - return -1; - } // TODO: Test and detect to return EISDIR. // TODO: Support nonblocking IO and check for EAGAIN/EWOULDBLOCK - emscripten_fetch_wait(desc->fetch, INFINITY); // TODO: Ensure that multiple waits are ok. + if (desc->node->fetch) emscripten_fetch_wait(desc->node->fetch, INFINITY); if (iovcnt < 0) { @@ -320,7 +661,7 @@ long __syscall145(int which, ...) // readv for(int i = 0; i < iovcnt; ++i) { ssize_t n = total_read_amount + iov[i].iov_len; - if (n < total_read_amount || !iov[i].iov_base) + if (n < total_read_amount || (!iov[i].iov_base && iov[i].iov_len > 0)) { errno = EINVAL; // "The sum of the iov_len values overflows an ssize_t value." or "the address specified in buf is not valid" return -1; @@ -329,12 +670,14 @@ long __syscall145(int which, ...) // readv } size_t offset = desc->file_pos; + inode *node = desc->node; + uint8_t *data = node->data ? node->data : (uint8_t *)node->fetch->data; for(int i = 0; i < iovcnt; ++i) { - ssize_t dataLeft = desc->fetch->numBytes - offset; + ssize_t dataLeft = node->size - offset; if (dataLeft <= 0) break; size_t bytesToCopy = (size_t)dataLeft < iov[i].iov_len ? dataLeft : iov[i].iov_len; - memcpy(iov[i].iov_base, &desc->fetch->data[offset], bytesToCopy); + memcpy(iov[i].iov_base, &data[offset], bytesToCopy); offset += bytesToCopy; } ssize_t numRead = offset - desc->file_pos; @@ -342,13 +685,117 @@ long __syscall145(int which, ...) // readv return numRead; } -/* +static char stdout_buffer[4096] = {}; +static int stdout_buffer_end = 0; +static char stderr_buffer[4096] = {}; +static int stderr_buffer_end = 0; + +static void print_stream(void *bytes, int numBytes, bool stdout) +{ + char *buffer = stdout ? stdout_buffer : stderr_buffer; + int &buffer_end = stdout ? stdout_buffer_end : stderr_buffer_end; + + memcpy(buffer + buffer_end, bytes, numBytes); + buffer_end += numBytes; + int new_buffer_start = 0; + for(int i = 0; i < buffer_end; ++i) + { + if (buffer[i] == '\n') + { + buffer[i] = 0; + EM_ASM_INT( { Module['print'](Pointer_stringify($0)) }, buffer); + new_buffer_start = i+1; + } + } + size_t new_buffer_size = buffer_end - new_buffer_start; + memmove(buffer, buffer + new_buffer_start, new_buffer_size); + buffer_end = new_buffer_size; +} + // http://man7.org/linux/man-pages/man2/writev.2.html long __syscall146(int which, ...) // writev { - EM_ASM_INT( { Module['printErr']('__syscall146 WRITEV, which: ' + $0 + '.') }, which); - return 0; + va_list vl; + va_start(vl, which); + int fd = va_arg(vl, int); + const iovec *iov = va_arg(vl, const iovec*); + int iovcnt = va_arg(vl, int); + va_end(vl); + + FileDescriptor *desc = (FileDescriptor*)fd; + if (fd != 1/*stdout*/ && fd != 2/*stderr*/) + { +// EM_ASM_INT( { Module['printErr']('__syscall146 WRITEV, which: ' + $0 + ', fd ' + $1 + ', iov: ' + $2 + ', iovcnt: ' + $3 + ' .') }, which, fd, iov, iovcnt); + if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) + { + EM_ASM_INT( { Module['printErr']('Invalid or closed file descriptor ' + $0 + ' passed to writev!') }, fd); + errno = EBADF; // "fd is not a valid file descriptor or is not open for reading." + return -1; + } + } + + if (iovcnt < 0) + { + errno = EINVAL; // "The vector count, iovcnt, is less than zero or greater than the permitted maximum." + return -1; + } + + ssize_t total_write_amount = 0; + for(int i = 0; i < iovcnt; ++i) + { + ssize_t n = total_write_amount + iov[i].iov_len; + if (n < total_write_amount || (!iov[i].iov_base && iov[i].iov_len > 0)) + { + errno = EINVAL; // "The sum of the iov_len values overflows an ssize_t value." or "the address specified in buf is not valid" + return -1; + } + total_write_amount = n; + } +// EM_ASM_INT( { Module['printErr']('__syscall146 WRITEV, write amount to fd ' + $0 + ' is ' + $1 + '.') }, fd, total_write_amount); + + if (fd == 1/*stdout*/ || fd == 2/*stderr*/) + { +// EM_ASM_INT( { Module['printErr']('__syscall146 WRITEV, printing to stdout/stderr: ' + $0 + ' .') }, fd); + ssize_t bytesWritten = 0; + for(int i = 0; i < iovcnt; ++i) + { + print_stream(iov[i].iov_base, iov[i].iov_len, fd == 1); + bytesWritten += iov[i].iov_len; + } + return bytesWritten; + } + else + { +// EM_ASM_INT( { Module['printErr']('__syscall146 WRITEV, printing to file data ptr: ' + $0 + ' .') }, fd); + // Enlarge the file in memory to fit space for the new data + size_t newSize = desc->file_pos + total_write_amount; + inode *node = desc->node; + if (node->capacity < newSize) + { +// EM_ASM_INT( { Module['printErr']('__syscall146 WRITEV, enlarging data ptr: ' + $0 + ' -> ' + $1 + ' .') }, node->capacity, newSize); + size_t newCapacity = (newSize > (size_t)(node->capacity*1.25) ? newSize : (size_t)(node->capacity*1.25)); // Geometric increases in size for amortized O(1) behavior + uint8_t *newData = (uint8_t *)realloc(node->data, newCapacity); + if (!newData) + { + newData = (uint8_t *)malloc(newCapacity); + memcpy(newData, node->data, node->size); + // TODO: init gaps with zeroes. + free(node->data); + } + node->data = newData; + node->size = newSize; + node->capacity = newCapacity; + } + + for(int i = 0; i < iovcnt; ++i) + { + memcpy((uint8_t*)node->data + desc->file_pos, iov[i].iov_base, iov[i].iov_len); + desc->file_pos += iov[i].iov_len; + } +// EM_ASM_INT( { Module['print']('__syscall5 WRITE: wrote pathname ' + Pointer_stringify($0) + ', inode exists. data ptr: ' + $1 + ', data size: ' + $2 + ', fetch ptr: ' + $3) }, +// node->name, node->data, node->size, node->fetch); + } + return total_write_amount; } -*/ } // ~extern "C" diff --git a/tests/test_browser.py b/tests/test_browser.py index b088a50fff804..22317cae25e10 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -3336,3 +3336,7 @@ def test_fetch_sync_xhr(self): def test_asmfs_hello_file(self): shutil.copyfile(path_from_root('tests', 'asmfs', 'hello_file.txt'), os.path.join(self.get_dir(), 'hello_file.txt')) self.btest('asmfs/hello_file.cpp', expected='0', args=['-s', 'ASMFS=1', '-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1', '--proxy-to-worker']) + + def test_asmfs_read_file_twice(self): + shutil.copyfile(path_from_root('tests', 'asmfs', 'hello_file.txt'), os.path.join(self.get_dir(), 'hello_file.txt')) + self.btest('asmfs/read_file_twice.cpp', expected='0', args=['-s', 'ASMFS=1', '-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1', '--proxy-to-worker']) From f600fecce44dacc56fd474f5787fdcd3dcb8ec3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Sat, 27 Aug 2016 21:19:47 +0300 Subject: [PATCH 30/74] Add tests for file writing and reading in ASMFS. --- tests/asmfs/fopen_write.cpp | 35 +++++++++++++++++++++++++++++++ tests/asmfs/read_file_twice.cpp | 37 +++++++++++++++++++++++++++++++++ tests/test_browser.py | 3 +++ 3 files changed, 75 insertions(+) create mode 100644 tests/asmfs/fopen_write.cpp create mode 100644 tests/asmfs/read_file_twice.cpp diff --git a/tests/asmfs/fopen_write.cpp b/tests/asmfs/fopen_write.cpp new file mode 100644 index 0000000000000..0ec0c2e4d3f20 --- /dev/null +++ b/tests/asmfs/fopen_write.cpp @@ -0,0 +1,35 @@ +#include +#include +#include +#include +#include + +void create_file() +{ + FILE *file = fopen("hello_file.txt", "wb"); + assert(file); + const char *data = "Hello data!"; + fwrite(data, 1, strlen(data), file); + fclose(file); +} + +void read_file() +{ + FILE *file = fopen("hello_file.txt", "rb"); + char buffer[128] = {}; + size_t read = fread(buffer, 1, sizeof(buffer), file); + printf("read %u bytes. Result: %s\n", (unsigned int)read, buffer); + assert(read == strlen("Hello data!")); + assert(!strcmp(buffer, "Hello data!")); +} + +int main() +{ + create_file(); + read_file(); + +#ifdef REPORT_RESULT + int result = 0; + REPORT_RESULT(); +#endif +} diff --git a/tests/asmfs/read_file_twice.cpp b/tests/asmfs/read_file_twice.cpp new file mode 100644 index 0000000000000..94a4fa629bf3a --- /dev/null +++ b/tests/asmfs/read_file_twice.cpp @@ -0,0 +1,37 @@ +#include +#include +#include +#include +#include + +void read_file() +{ + FILE *file = fopen("hello_file.txt", "rb"); + assert(file); + fseek(file, 0, SEEK_END); + + long size = ftell(file); + rewind(file); + printf("size: %ld\n", size); + + char *buffer = (char*) malloc (sizeof(char)*(size+1)); + assert(buffer); + buffer[size] = '\0'; + + size_t read = fread(buffer, 1, size, file); + printf("File contents: %s\n", buffer); + assert(size == 6); + assert(!strcmp(buffer, "Hello!")); + + fclose(file); +} + +int main() +{ + read_file(); + read_file(); +#ifdef REPORT_RESULT + int result = 0; + REPORT_RESULT(); +#endif +} diff --git a/tests/test_browser.py b/tests/test_browser.py index 22317cae25e10..535a3f897e328 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -3340,3 +3340,6 @@ def test_asmfs_hello_file(self): def test_asmfs_read_file_twice(self): shutil.copyfile(path_from_root('tests', 'asmfs', 'hello_file.txt'), os.path.join(self.get_dir(), 'hello_file.txt')) self.btest('asmfs/read_file_twice.cpp', expected='0', args=['-s', 'ASMFS=1', '-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1', '--proxy-to-worker']) + + def test_asmfs_fopen_write(self): + self.btest('asmfs/fopen_write.cpp', expected='0', args=['-s', 'ASMFS=1', '-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1', '--proxy-to-worker']) From fb6141d52c13d43dde354a2d7057e2ed70b3d4fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Sat, 27 Aug 2016 22:34:20 +0300 Subject: [PATCH 31/74] Detect a special emscripten_fetch request type EM_IDB_DELETE which deletes the given file from IndexedDB. --- src/Fetch.js | 53 ++++++++++++++++++++++++++- src/fetch-worker.js | 4 +- system/lib/fetch/emscripten_fetch.cpp | 23 +++++++++++- 3 files changed, 74 insertions(+), 6 deletions(-) diff --git a/src/Fetch.js b/src/Fetch.js index 446dcf17c2e65..319fbe96d9919 100644 --- a/src/Fetch.js +++ b/src/Fetch.js @@ -88,6 +88,52 @@ var Fetch = { } } +function __emscripten_fetch_delete_cached_data(db, fetch, onsuccess, onerror) { + if (!db) { +#if FETCH_DEBUG + console.error('fetch: IndexedDB not available!'); +#endif + onerror(fetch, 0, 'IndexedDB not available!'); + return; + } + + var fetch_attr = fetch + 112/*TODO:structs_info*/; + var path = HEAPU32[fetch_attr + 64 >> 2];//{{{ makeGetValue('fetch_attr', 64/*TODO*/, 'i32') }}}; + if (!path) path = HEAPU32[fetch + 8 >> 2];//{{{ makeGetValue('fetch', 8/*TODO:structs_info.jsonify this*/, 'i32') }}}; + var pathStr = Pointer_stringify(path); + + try { + var transaction = db.transaction(['FILES'], 'readwrite'); + var packages = transaction.objectStore('FILES'); + var request = packages.delete(pathStr); + request.onsuccess = function(event) { + var value = event.target.result; +#if FETCH_DEBUG + console.log('fetch: Deleted file ' + pathStr + ' from IndexedDB'); +#endif + HEAPU32[fetch + 12 >> 2] = 0;//{{{ makeSetValue('fetch', 12/*TODO:jsonify*/, 'ptr', 'i32')}}}; + Fetch.setu64(fetch + 16, 0);//{{{ makeSetValue('fetch', 16/*TODO:jsonify*/, 'len', 'i64')}}}; + Fetch.setu64(fetch + 24, 0);//{{{ makeSetValue('fetch', 24/*TODO:jsonify*/, '0', 'i64')}}}; + Fetch.setu64(fetch + 32, 0);//{{{ makeSetValue('fetch', 32/*TODO:jsonify*/, 'len', 'i64')}}}; + HEAPU16[fetch + 40 >> 1] = 4; // Mimic XHR readyState 4 === 'DONE: The operation is complete' + HEAPU16[fetch + 42 >> 1] = 200; // Mimic XHR HTTP status code 200 "OK" + + onsuccess(fetch, 0, value); + }; + request.onerror = function(error) { +#if FETCH_DEBUG + console.error('fetch: Failed to delete file ' + pathStr + ' from IndexedDB! error: ' + error); +#endif + onerror(fetch, 0, error); + }; + } catch(e) { +#if FETCH_DEBUG + console.error('fetch: Failed to load file ' + pathStr + ' from IndexedDB! Got exception ' + e); +#endif + onerror(fetch, 0, e); + } +} + function __emscripten_fetch_load_cached_data(db, fetch, onsuccess, onerror) { if (!db) { #if FETCH_DEBUG @@ -342,6 +388,7 @@ function emscripten_start_fetch(fetch, successcb, errorcb, progresscb) { if (typeof Module !== 'undefined') Module['noExitRuntime'] = true; // If we are the main Emscripten runtime, we should not be closing down. var fetch_attr = fetch + 112/*TODO:structs_info*/; + var requestMethod = Pointer_stringify(fetch_attr); var onsuccess = HEAPU32[fetch_attr + 36 >> 2];//{{{ makeGetValue('fetch_attr', 36/*TODO:structs_info.jsonify this*/, 'i32') }}}; var onerror = HEAPU32[fetch_attr + 40 >> 2];//{{{ makeGetValue('fetch_attr', 40/*TODO:structs_info.jsonify this*/, 'i32') }}}; var onprogress = HEAPU32[fetch_attr + 44 >> 2];//{{{ makeGetValue('fetch_attr', 44/*TODO:structs_info.jsonify this*/, 'i32') }}}; @@ -408,7 +455,7 @@ function emscripten_start_fetch(fetch, successcb, errorcb, progresscb) { }; // Should we try IndexedDB first? - if (!fetchAttrReplace) { + if (!fetchAttrReplace /*|| requestMethod === 'EM_IDB_STORE'*/ || requestMethod === 'EM_IDB_DELETE') { if (!Fetch.dbInstance) { #if FETCH_DEBUG console.error('fetch: failed to read IndexedDB! Database is not open.'); @@ -417,7 +464,9 @@ function emscripten_start_fetch(fetch, successcb, errorcb, progresscb) { return 0; // todo: free } - if (fetchAttrNoDownload) { + if (requestMethod === 'EM_IDB_DELETE') { + __emscripten_fetch_delete_cached_data(Fetch.dbInstance, fetch, reportSuccess, reportError); + } else if (fetchAttrNoDownload) { __emscripten_fetch_load_cached_data(Fetch.dbInstance, fetch, reportSuccess, reportError); } else if (fetchAttrPersistFile) { __emscripten_fetch_load_cached_data(Fetch.dbInstance, fetch, reportSuccess, performCachedXhr); diff --git a/src/fetch-worker.js b/src/fetch-worker.js index 1cf74b36a4854..bfe2ec21774d0 100644 --- a/src/fetch-worker.js +++ b/src/fetch-worker.js @@ -130,9 +130,7 @@ this.onmessage = function(e) { HEAPU16 = new Uint16Array(buffer); HEAP32 = new Int32Array(buffer); HEAPU32 = new Uint32Array(buffer); - console.log('DYNAMICTOP_PTR: ' + DYNAMICTOP_PTR); - console.log('DYNAMICTOP: ' + HEAP32[DYNAMICTOP_PTR>>2]); - console.log('fetch worker init, queue ptr ' + queuePtr + ', heap length: ' + buffer.byteLength); + console.log('fetch: fetch Worker initialized. queue ptr ' + queuePtr + ', heap length: ' + buffer.byteLength); interval = setInterval(processWorkQueue, 100); } } diff --git a/system/lib/fetch/emscripten_fetch.cpp b/system/lib/fetch/emscripten_fetch.cpp index 7fec70d0103af..1e645763e2370 100644 --- a/system/lib/fetch/emscripten_fetch.cpp +++ b/system/lib/fetch/emscripten_fetch.cpp @@ -5,6 +5,7 @@ #include #include #include +#include struct __emscripten_fetch_queue { @@ -50,6 +51,19 @@ emscripten_fetch_t *emscripten_fetch(emscripten_fetch_attr_t *fetch_attr, const { if (!fetch_attr) return 0; if (!url) return 0; + + const bool waitable = (fetch_attr->attributes & EMSCRIPTEN_FETCH_WAITABLE) != 0; + const bool synchronous = (fetch_attr->attributes & EMSCRIPTEN_FETCH_SYNCHRONOUS) != 0; + const bool readFromIndexedDB = (fetch_attr->attributes & (EMSCRIPTEN_FETCH_APPEND | EMSCRIPTEN_FETCH_NO_DOWNLOAD)) != 0; + const bool writeToIndexedDB = (fetch_attr->attributes & EMSCRIPTEN_FETCH_PERSIST_FILE) != 0 || !strncmp(fetch_attr->requestMethod, "EM_IDB_", strlen("EM_IDB_")); + const bool performXhr = (fetch_attr->attributes & EMSCRIPTEN_FETCH_NO_DOWNLOAD) == 0; + const bool isMainBrowserThread = emscripten_is_main_browser_thread() != 0; + if (isMainBrowserThread && synchronous && (performXhr || readFromIndexedDB || writeToIndexedDB)) + { + printf("emscripten_fetch failed! Synchronous blocking XHRs and IndexedDB operations are not supported on the main browser thread. Try dropping the EMSCRIPTEN_FETCH_SYNCHRONOUS flag, or run with the linker flag --proxy-to-worker to decouple main C runtime thread from the main browser thread.\n"); + return 0; + } + emscripten_fetch_t *fetch = (emscripten_fetch_t *)malloc(sizeof(emscripten_fetch_t)); memset(fetch, 0, sizeof(emscripten_fetch_t)); fetch->id = globalFetchIdCounter++; // TODO: make this thread-safe! @@ -62,10 +76,17 @@ emscripten_fetch_t *emscripten_fetch(emscripten_fetch_attr_t *fetch_attr, const fetch->__attributes.requestHeaders = 0;// TODO:strdup(fetch->__attributes.requestHeaders); fetch->__attributes.overriddenMimeType = fetch->__attributes.overriddenMimeType ? strdup(fetch->__attributes.overriddenMimeType) : 0; // TODO: free - if ((fetch->__attributes.attributes & EMSCRIPTEN_FETCH_WAITABLE) != 0) + // Depending on the type of fetch, we can either perform it in the same Worker/thread than the caller, or we might need + // to run it in a separate Worker. There is a dedicated fetch worker that is available for the fetch, but in some scenarios + // it might be desirable to run in the same Worker as the caller, so deduce here whether to run the fetch in this thread, + // or if we need to use the fetch-worker instead. + if (waitable // Waitable fetches can be synchronously waited on, so must always be proxied + || (synchronous && (readFromIndexedDB || writeToIndexedDB))) // Synchronous IndexedDB access needs proxying { emscripten_atomic_store_u32(&fetch->__proxyState, 1); // sent to proxy worker. emscripten_proxy_fetch(fetch); + + if (synchronous) emscripten_fetch_wait(fetch, INFINITY); } else emscripten_start_fetch(fetch); From 5be2f7e14b69f8ff435df4779efcbf85e1bb852b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Sat, 27 Aug 2016 22:34:44 +0300 Subject: [PATCH 32/74] Add test for emscripten_fetch() with request type EM_IDB_DELETE --- tests/fetch/idb_delete.cpp | 52 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 tests/fetch/idb_delete.cpp diff --git a/tests/fetch/idb_delete.cpp b/tests/fetch/idb_delete.cpp new file mode 100644 index 0000000000000..03eecc4e1fe2d --- /dev/null +++ b/tests/fetch/idb_delete.cpp @@ -0,0 +1,52 @@ +#include +#include +#include +#include +#include + +int main() +{ + // 1. Populate file to IndexedDB by a GET. + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.attributes = EMSCRIPTEN_FETCH_REPLACE | EMSCRIPTEN_FETCH_SYNCHRONOUS | EMSCRIPTEN_FETCH_PERSIST_FILE; + emscripten_fetch_t *fetch = emscripten_fetch(&attr, "gears.png"); + assert(fetch->status == 200 && "Initial XHR GET of gears.png should have succeeded"); + emscripten_fetch_close(fetch); + + // 2. Make sure it is there. + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.attributes = EMSCRIPTEN_FETCH_NO_DOWNLOAD | EMSCRIPTEN_FETCH_SYNCHRONOUS | EMSCRIPTEN_FETCH_LOAD_TO_MEMORY; + fetch = emscripten_fetch(&attr, "gears.png"); + assert(fetch->status == 200 && "emscripten_fetch from IndexedDB should have found the file"); + assert(fetch->numBytes > 0); + assert(fetch->data != 0); + emscripten_fetch_close(fetch); + + // 3. Delete the file from IndexedDB by invoking an Emscripten-specific request "EM_IDB_DELETE". + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "EM_IDB_DELETE"); + attr.attributes = EMSCRIPTEN_FETCH_SYNCHRONOUS; + fetch = emscripten_fetch(&attr, "gears.png"); + assert(fetch->status == 200 && "Deleting the file from IndexedDB should have succeeded"); + assert(fetch->numBytes == 0); + assert(fetch->data == 0); + emscripten_fetch_close(fetch); + + // 4. Now try to get the file again from IndexedDB, and ensure it is no longer available. + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.attributes = EMSCRIPTEN_FETCH_NO_DOWNLOAD | EMSCRIPTEN_FETCH_SYNCHRONOUS | EMSCRIPTEN_FETCH_LOAD_TO_MEMORY; + fetch = emscripten_fetch(&attr, "gears.png"); + assert(fetch->status == 404 && "Attempting to GET the file from IndexedDB again should have failed after the file has been deleted"); + assert(fetch->numBytes == 0); + assert(fetch->data == 0); + emscripten_fetch_close(fetch); + +#ifdef REPORT_RESULT + int result = 0; + REPORT_RESULT(); +#endif +} From 6f33c81219983422cf15bfd5a6575da7717d3fdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Sat, 27 Aug 2016 23:11:38 +0300 Subject: [PATCH 33/74] Implement emscripten_fetch support for storing custom payloads as files on IndexedDB with EM_IDB_STORE verb. --- src/Fetch.js | 22 +++++++++- src/fetch-worker.js | 11 +++-- system/include/emscripten/fetch.h | 61 +++++++++++++++------------ system/lib/fetch/emscripten_fetch.cpp | 12 +++--- tests/fetch/idb_delete.cpp | 2 + 5 files changed, 67 insertions(+), 41 deletions(-) diff --git a/src/Fetch.js b/src/Fetch.js index 319fbe96d9919..78bbfc1279441 100644 --- a/src/Fetch.js +++ b/src/Fetch.js @@ -124,6 +124,8 @@ function __emscripten_fetch_delete_cached_data(db, fetch, onsuccess, onerror) { #if FETCH_DEBUG console.error('fetch: Failed to delete file ' + pathStr + ' from IndexedDB! error: ' + error); #endif + HEAPU16[fetch + 40 >> 1] = 4; // Mimic XHR readyState 4 === 'DONE: The operation is complete' + HEAPU16[fetch + 42 >> 1] = 404; // Mimic XHR HTTP status code 404 "Not Found" onerror(fetch, 0, error); }; } catch(e) { @@ -175,6 +177,8 @@ function __emscripten_fetch_load_cached_data(db, fetch, onsuccess, onerror) { #if FETCH_DEBUG console.error('fetch: Loaded file ' + pathStr + ' from IndexedDB, but it had 0 length!'); #endif + HEAPU16[fetch + 40 >> 1] = 4; // Mimic XHR readyState 4 === 'DONE: The operation is complete' + HEAPU16[fetch + 42 >> 1] = 404; // Mimic XHR HTTP status code 404 "Not Found" onerror(fetch, 0, 'no data'); } }; @@ -182,6 +186,8 @@ function __emscripten_fetch_load_cached_data(db, fetch, onsuccess, onerror) { #if FETCH_DEBUG console.error('fetch: Failed to load file ' + pathStr + ' from IndexedDB!'); #endif + HEAPU16[fetch + 40 >> 1] = 4; // Mimic XHR readyState 4 === 'DONE: The operation is complete' + HEAPU16[fetch + 42 >> 1] = 404; // Mimic XHR HTTP status code 404 "Not Found" onerror(fetch, 0, error); }; } catch(e) { @@ -214,12 +220,19 @@ function __emscripten_fetch_cache_data(db, fetch, data, onsuccess, onerror) { #if FETCH_DEBUG console.log('fetch: Stored file "' + destinationPathStr + '" to IndexedDB cache.'); #endif + HEAPU16[fetch + 40 >> 1] = 4; // Mimic XHR readyState 4 === 'DONE: The operation is complete' + HEAPU16[fetch + 42 >> 1] = 200; // Mimic XHR HTTP status code 200 "OK" onsuccess(fetch, 0, destinationPathStr); }; putRequest.onerror = function(error) { #if FETCH_DEBUG console.error('fetch: Failed to store file "' + destinationPathStr + '" to IndexedDB cache!'); #endif + // Most likely we got an error if IndexedDB is unwilling to store any more data for this page. + // TODO: Can we identify and break down different IndexedDB-provided errors and convert those + // to more HTTP status codes for more information? + HEAPU16[fetch + 40 >> 1] = 4; // Mimic XHR readyState 4 === 'DONE: The operation is complete' + HEAPU16[fetch + 42 >> 1] = 413; // Mimic XHR HTTP status code 413 "Payload Too Large" onerror(fetch, 0, error); }; } catch(e) { @@ -455,7 +468,7 @@ function emscripten_start_fetch(fetch, successcb, errorcb, progresscb) { }; // Should we try IndexedDB first? - if (!fetchAttrReplace /*|| requestMethod === 'EM_IDB_STORE'*/ || requestMethod === 'EM_IDB_DELETE') { + if (!fetchAttrReplace || requestMethod === 'EM_IDB_STORE' || requestMethod === 'EM_IDB_DELETE') { if (!Fetch.dbInstance) { #if FETCH_DEBUG console.error('fetch: failed to read IndexedDB! Database is not open.'); @@ -464,7 +477,12 @@ function emscripten_start_fetch(fetch, successcb, errorcb, progresscb) { return 0; // todo: free } - if (requestMethod === 'EM_IDB_DELETE') { + if (requestMethod === 'EM_IDB_STORE') { + var dataPtr = HEAPU32[fetch_attr + 84 >> 2]; + var dataLength = HEAPU32[fetch_attr + 88 >> 2]; + var data = HEAPU8.slice(dataPtr, dataPtr + dataLength); // TODO(?): Here we perform a clone of the data, because storing shared typed arrays to IndexedDB does not seem to be allowed. + __emscripten_fetch_cache_data(Fetch.dbInstance, fetch, data, reportSuccess, reportError); + } else if (requestMethod === 'EM_IDB_DELETE') { __emscripten_fetch_delete_cached_data(Fetch.dbInstance, fetch, reportSuccess, reportError); } else if (fetchAttrNoDownload) { __emscripten_fetch_load_cached_data(Fetch.dbInstance, fetch, reportSuccess, reportError); diff --git a/src/fetch-worker.js b/src/fetch-worker.js index bfe2ec21774d0..0005881ea19b4 100644 --- a/src/fetch-worker.js +++ b/src/fetch-worker.js @@ -86,16 +86,15 @@ function processWorkQueue() { for(var i = 0; i < numQueuedItems; ++i) { var fetch = Atomics_load(HEAPU32, (queuedOperations >> 2)+i); console.log('processWorkQueue: starting fetch'); - console.log('fetch ptr1 ' + fetch + ', state ' + Atomics_load(HEAPU32, fetch + 108 >> 2)); function successcb(fetch) { - console.log('fetch ptr ' + fetch + ', state ' + Atomics_load(HEAPU32, fetch + 108 >> 2)); - console.log('FETCH-WORKER: fetch finished'); - var oldVal = Atomics.compareExchange(HEAPU32, fetch + 108 >> 2, 1, 2); - console.log('atomics wake ' + (fetch + 108)); + console.log('FETCH-WORKER: fetch finished on success'); + Atomics.compareExchange(HEAPU32, fetch + 108 >> 2, 1, 2); Atomics.wake(HEAP32, fetch + 108 >> 2, 1); } function errorcb(fetch) { - console.log('FETCH-WORKER: fetch failed'); + console.log('FETCH-WORKER: fetch finished on failure'); + Atomics.compareExchange(HEAPU32, fetch + 108 >> 2, 1, 2); + Atomics.wake(HEAP32, fetch + 108 >> 2, 1); } function progresscb(fetch) { console.log('FETCH-WORKER: fetch progress..'); diff --git a/system/include/emscripten/fetch.h b/system/include/emscripten/fetch.h index 7c4594d96e2f4..35ee4a9ab8e10 100644 --- a/system/include/emscripten/fetch.h +++ b/system/include/emscripten/fetch.h @@ -48,64 +48,71 @@ struct emscripten_fetch_t; struct emscripten_fetch_attr_t { // 'POST', 'GET', etc. - char requestMethod[32]; + char requestMethod[32]; // XXX 0 // Custom data that can be tagged along the process. - void *userData; + void *userData; // XXX 32 - void (*onsuccess)(emscripten_fetch_t *fetch); - void (*onerror)(emscripten_fetch_t *fetch); - void (*onprogress)(emscripten_fetch_t *fetch); + void (*onsuccess)(emscripten_fetch_t *fetch); // XXX 36 + void (*onerror)(emscripten_fetch_t *fetch); // XXX 40 + void (*onprogress)(emscripten_fetch_t *fetch); // XXX 44 // EMSCRIPTEN_FETCH_* attributes - uint32_t attributes; + uint32_t attributes; // XXX 48 // Specifies the amount of time the request can take before failing due to a timeout. - unsigned long timeoutMSecs; + unsigned long timeoutMSecs; // XXX 52 // Indicates whether cross-site access control requests should be made using credentials. - EM_BOOL withCredentials; + EM_BOOL withCredentials; // XXX 56 // If true, performs a synchronous blocking XHR. The emscripten_fetch() function call does not return until // the request has completed. Setting this to true in the main browser thread will fail with // EMSCRIPTEN_RESULT_NOT_SUPPORTED. - EM_BOOL synchronousRequestTODODELETEONCESTRUCTJSONIFIED; + EM_BOOL synchronousRequestTODODELETEONCESTRUCTJSONIFIED; // XXX 60 // Specifies the destination path in IndexedDB where to store the downloaded content body. If this is empty, the transfer // is not stored to IndexedDB at all. // Note that this struct does not contain space to hold this string, it only carries a pointer. // Calling emscripten_fetch() will make an internal copy of this string. - const char *destinationPath; + const char *destinationPath; // XXX 64 // Specifies the authentication username to use for the request, if necessary. // Note that this struct does not contain space to hold this string, it only carries a pointer. // Calling emscripten_fetch() will make an internal copy of this string. - const char *userName; + const char *userName; // XXX 68 // Specifies the authentication username to use for the request, if necessary. // Note that this struct does not contain space to hold this string, it only carries a pointer. // Calling emscripten_fetch() will make an internal copy of this string. - const char *password; + const char *password; // XXX 72 // Points to an array of strings to pass custom headers to the request. This array takes the form // {"key1", "value1", "key2", "value2", "key3", "value3", ..., 0 }; Note especially that the array // needs to be terminated with a null pointer. - const char * const *requestHeaders; + const char * const *requestHeaders; // XXX 76 // Pass a custom MIME type here to force the browser to treat the received data with the given type. - const char *overriddenMimeType; + const char *overriddenMimeType; // XXX 80 + + // If non-zero, specified a pointer to the data that is to be passed as the body (payload) of the request + // that is being performed. Leave as zero if no request body needs to be sent. + const char *requestData; // XXX 84 + + // Specifies the length of the buffer pointed by 'requestData'. Leave as 0 if no request body needs to be sent. + size_t requestDataSize; // XXX 88 }; struct emscripten_fetch_t { // Unique identifier for this fetch in progress. - unsigned int id; + unsigned int id; // XXX 0 // Custom data that can be tagged along the process. - void *userData; + void *userData; // XXX 4 // The remote URL that is being downloaded. - const char *url; + const char *url; // XXX 8 // In onsuccess() handler: // - If the EMSCRIPTEN_FETCH_LOAD_TO_MEMORY attribute was specified for the transfer, this points to the @@ -113,20 +120,18 @@ struct emscripten_fetch_t // In onprogress() handler: // - If the EMSCRIPTEN_FETCH_STREAM_DATA attribute was specified for the transfer, this points to a partial // chunk of bytes related to the transfer. Otherwise this will be null. - const char *data; + const char *data; // XXX 12 // Specifies the length of the above data block in bytes. When the download finishes, this field will be valid even if // EMSCRIPTEN_FETCH_LOAD_TO_MEMORY was not specified. - uint64_t numBytes; + uint64_t numBytes; // XXX 16 // If EMSCRIPTEN_FETCH_STREAM_DATA is being performed, this indicates the byte offset from the start of the stream // that the data block specifies. (for onprogress() streaming XHR transfer, the number of bytes downloaded so far before this chunk) - uint64_t dataOffset; + uint64_t dataOffset; // XXX 24 // Specifies the total number of bytes that the response body will be. - uint64_t totalBytes; - - + uint64_t totalBytes; // XXX 32 // Specifies the readyState of the XHR request: // 0: UNSENT: request not sent yet @@ -135,18 +140,18 @@ struct emscripten_fetch_t // 3: LOADING: download in progress. // 4: DONE: download finished. // See https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/readyState - unsigned short readyState; + unsigned short readyState; // XXX 40 // Specifies the status code of the response. - unsigned short status; + unsigned short status; // XXX 42 // Specifies a human-readable form of the status code. - char statusText[64]; + char statusText[64]; // XXX 44 - uint32_t __proxyState; + uint32_t __proxyState; // XXX 108 // For internal use only. - emscripten_fetch_attr_t __attributes; + emscripten_fetch_attr_t __attributes; // XXX 112 }; // Clears the fields of an emscripten_fetch_attr_t structure to their default values in a future-compatible manner. diff --git a/system/lib/fetch/emscripten_fetch.cpp b/system/lib/fetch/emscripten_fetch.cpp index 1e645763e2370..fb74eddf35e4d 100644 --- a/system/lib/fetch/emscripten_fetch.cpp +++ b/system/lib/fetch/emscripten_fetch.cpp @@ -1,10 +1,10 @@ #include #include -#include //todoremove #include #include #include #include +#include #include struct __emscripten_fetch_queue @@ -37,7 +37,8 @@ void emscripten_proxy_fetch(emscripten_fetch_t *fetch) __emscripten_fetch_queue *queue = _emscripten_get_fetch_queue(); // TODO handle case when queue->numQueuedItems >= queue->queueSize queue->queuedOperations[queue->numQueuedItems++] = fetch; - printf("Queued fetch to fetch-worker to process. There are now %d operations in the queue\n", queue->numQueuedItems); + EM_ASM_INT( { console.log('Queued fetch to fetch-worker to process. There are now ' + $0 + ' operations in the queue.') }, + queue->numQueuedItems); // TODO: mutex unlock } @@ -60,7 +61,8 @@ emscripten_fetch_t *emscripten_fetch(emscripten_fetch_attr_t *fetch_attr, const const bool isMainBrowserThread = emscripten_is_main_browser_thread() != 0; if (isMainBrowserThread && synchronous && (performXhr || readFromIndexedDB || writeToIndexedDB)) { - printf("emscripten_fetch failed! Synchronous blocking XHRs and IndexedDB operations are not supported on the main browser thread. Try dropping the EMSCRIPTEN_FETCH_SYNCHRONOUS flag, or run with the linker flag --proxy-to-worker to decouple main C runtime thread from the main browser thread.\n"); + EM_ASM_INT( { Module['printErr']('emscripten_fetch("' + Pointer_stringify($0) + '") failed! Synchronous blocking XHRs and IndexedDB operations are not supported on the main browser thread. Try dropping the EMSCRIPTEN_FETCH_SYNCHRONOUS flag, or run with the linker flag --proxy-to-worker to decouple main C runtime thread from the main browser thread.') }, + url); return 0; } @@ -100,7 +102,7 @@ EMSCRIPTEN_RESULT emscripten_fetch_wait(emscripten_fetch_t *fetch, double timeou if (proxyState == 2) return EMSCRIPTEN_RESULT_SUCCESS; // already finished. if (proxyState != 1) return EMSCRIPTEN_RESULT_INVALID_PARAM; // the fetch should be ongoing? // #ifdef FETCH_DEBUG - printf("fetch: emscripten_fetch_wait..\n"); + EM_ASM({ console.log('fetch: emscripten_fetch_wait..') }); // #endif while(proxyState == 1/*sent to proxy worker*/) { @@ -108,7 +110,7 @@ EMSCRIPTEN_RESULT emscripten_fetch_wait(emscripten_fetch_t *fetch, double timeou proxyState = emscripten_atomic_load_u32(&fetch->__proxyState); } // #ifdef FETCH_DEBUG - printf("fetch: emscripten_fetch_wait done..\n"); + EM_ASM({ console.log('fetch: emscripten_fetch_wait done..') }); // #endif if (proxyState == 2) return EMSCRIPTEN_RESULT_SUCCESS; diff --git a/tests/fetch/idb_delete.cpp b/tests/fetch/idb_delete.cpp index 03eecc4e1fe2d..90c68d38c9a6b 100644 --- a/tests/fetch/idb_delete.cpp +++ b/tests/fetch/idb_delete.cpp @@ -45,6 +45,8 @@ int main() assert(fetch->data == 0); emscripten_fetch_close(fetch); + printf("Test succeeded!\n"); + #ifdef REPORT_RESULT int result = 0; REPORT_RESULT(); From 5f3a05b42cb34ab2f6e1ff9ebee189b09647b2cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Sat, 27 Aug 2016 23:11:56 +0300 Subject: [PATCH 34/74] Add test for storing custom payload data to IndexedDB. --- tests/fetch/idb_store.cpp | 67 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 tests/fetch/idb_store.cpp diff --git a/tests/fetch/idb_store.cpp b/tests/fetch/idb_store.cpp new file mode 100644 index 0000000000000..8d92de8668ebb --- /dev/null +++ b/tests/fetch/idb_store.cpp @@ -0,0 +1,67 @@ +#include +#include +#include +#include +#include +#include +#include + +#define TEST_SIZE 512000 + +int main() +{ + // 1. Populate an IndexedDB file entry with custom data bytes in memory. + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "EM_IDB_STORE"); + attr.attributes = EMSCRIPTEN_FETCH_REPLACE | EMSCRIPTEN_FETCH_SYNCHRONOUS | EMSCRIPTEN_FETCH_PERSIST_FILE; + uint8_t *data = (uint8_t*)malloc(TEST_SIZE); + srand(time(NULL)); + for(int i = 0; i < TEST_SIZE; ++i) + data[i] = (uint8_t)rand() | 0x40; + attr.requestData = (char *)data; + attr.requestDataSize = TEST_SIZE; + emscripten_fetch_t *fetch = emscripten_fetch(&attr, "myfile.dat"); + assert(fetch->status == 200 && "Initial IndexedDB store of myfile.dat should have succeeded"); + emscripten_fetch_close(fetch); + + // 2. Make sure it is there. + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.attributes = EMSCRIPTEN_FETCH_NO_DOWNLOAD | EMSCRIPTEN_FETCH_SYNCHRONOUS | EMSCRIPTEN_FETCH_LOAD_TO_MEMORY; + fetch = emscripten_fetch(&attr, "myfile.dat"); + assert(fetch->status == 200 && "emscripten_fetch from IndexedDB should have found the file"); + assert(fetch->numBytes == TEST_SIZE); + assert(fetch->totalBytes == TEST_SIZE); + assert(fetch->data != 0); + assert(fetch->data != (char *)data); // We should really have obtained a new copy of the memory. + assert(!memcmp((char *)data, fetch->data, TEST_SIZE)); + emscripten_fetch_close(fetch); + + // 3. Delete the file from IndexedDB by invoking an Emscripten-specific request "EM_IDB_DELETE". + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "EM_IDB_DELETE"); + attr.attributes = EMSCRIPTEN_FETCH_SYNCHRONOUS; + fetch = emscripten_fetch(&attr, "myfile.dat"); + assert(fetch->status == 200 && "Deleting the file from IndexedDB should have succeeded"); + assert(fetch->numBytes == 0); + assert(fetch->data == 0); + emscripten_fetch_close(fetch); + + // 4. Now try to get the file again from IndexedDB, and ensure it is no longer available. + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.attributes = EMSCRIPTEN_FETCH_NO_DOWNLOAD | EMSCRIPTEN_FETCH_SYNCHRONOUS | EMSCRIPTEN_FETCH_LOAD_TO_MEMORY; + fetch = emscripten_fetch(&attr, "myfile.dat"); + assert(fetch->status == 404 && "Attempting to GET the file from IndexedDB again should have failed after the file has been deleted"); + assert(fetch->numBytes == 0); + assert(fetch->data == 0); + emscripten_fetch_close(fetch); + + printf("Test succeeded!\n"); + +#ifdef REPORT_RESULT + int result = 0; + REPORT_RESULT(); +#endif +} From 371fda765fa1afe7c3ee60d52cc12ae40bde5614 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Sat, 27 Aug 2016 23:50:39 +0300 Subject: [PATCH 35/74] Improve emscripten_fetch() to be asynchronously usable on the main thread without pthreads enabled. --- emcc.py | 5 +++-- src/Fetch.js | 8 ++++++++ src/fetch-worker.js | 23 ++++++++++++++--------- src/library_fetch.js | 2 +- src/preamble.js | 1 + system/lib/fetch/emscripten_fetch.cpp | 9 +++++++++ tests/test_browser.py | 23 +++++++++++------------ 7 files changed, 47 insertions(+), 24 deletions(-) diff --git a/emcc.py b/emcc.py index 3a04b4fefcc80..e042f5bc7d14e 100755 --- a/emcc.py +++ b/emcc.py @@ -1768,10 +1768,11 @@ def repl(m): if shared.Settings.USE_PTHREADS: shutil.copyfile(shared.path_from_root('src', 'pthread-main.js'), os.path.join(os.path.dirname(os.path.abspath(target)), 'pthread-main.js')) - if shared.Settings.FETCH: + # Generate the fetch-worker.js script for multithreaded emscripten_fetch() support if targeting pthreads. + if shared.Settings.FETCH and shared.Settings.USE_PTHREADS: src = open(final, 'r').read() funcs_to_import = ['alignMemoryPage', 'getTotalMemory', 'stringToUTF8', 'intArrayFromString', 'lengthBytesUTF8', 'stringToUTF8Array', '_emscripten_is_main_runtime_thread', '_emscripten_futex_wait'] - asm_funcs_to_import = ['_malloc', '_free', '_sbrk', '_pthread_mutex_lock', '_emscripten_sync_run_in_main_thread_1', '_emscripten_sync_run_in_main_thread', '_pthread_mutex_unlock', '_emscripten_set_current_thread_status'] + asm_funcs_to_import = ['_malloc', '_free', '_sbrk', '_pthread_mutex_lock', '_pthread_mutex_unlock'] function_prologue = '''this.onerror = function(e) { console.error(e); } diff --git a/src/Fetch.js b/src/Fetch.js index 78bbfc1279441..0ca9c8381b1f7 100644 --- a/src/Fetch.js +++ b/src/Fetch.js @@ -51,23 +51,30 @@ var Fetch = { console.log('fetch: IndexedDB successfully opened.'); #endif Fetch.dbInstance = db; + +#if USE_PTHREADS if (isMainThread) { Fetch.initFetchWorker(); removeRunDependency('library_fetch_init'); } +#endif }; var onerror = function() { #if FETCH_DEBUG console.error('fetch: IndexedDB open failed.'); #endif Fetch.dbInstance = false; + +#if USE_PTHREADS if (isMainThread) { Fetch.initFetchWorker(); removeRunDependency('library_fetch_init'); } +#endif }; Fetch.openDatabase('emscripten_filesystem', 1, onsuccess, onerror); +#if USE_PTHREADS if (isMainThread) { addRunDependency('library_fetch_init'); @@ -85,6 +92,7 @@ var Fetch = { Module['printErr']('fetch-worker sent an error! ' + e.filename + ':' + e.lineno + ': ' + e.message); }; } +#endif } } diff --git a/src/fetch-worker.js b/src/fetch-worker.js index 0005881ea19b4..336b09f281a96 100644 --- a/src/fetch-worker.js +++ b/src/fetch-worker.js @@ -1,18 +1,23 @@ #include "Fetch.js" -var ENVIRONMENT_IS_FETCH_WORKER = true; -var ENVIRONMENT_IS_WORKER = true; -var ENVIRONMENT_IS_PTHREAD = true; -var __pthread_is_main_runtime_thread=0; -var Atomics_load = Atomics.load; -var Atomics_store = Atomics.store; -var Atomics_exchange = Atomics.exchange; -var Atomics_compareExchange = Atomics.compareExchange; var Atomics_add = Atomics.add; -var Atomics_sub = Atomics.sub; var Atomics_and = Atomics.and; +var Atomics_compareExchange = Atomics.compareExchange; +var Atomics_exchange = Atomics.exchange; +var Atomics_wait = Atomics.wait; +var Atomics_wake = Atomics.wake; +var Atomics_wakeOrRequeue = Atomics.wakeOrRequeue; +var Atomics_isLockFree = Atomics.isLockFree; +var Atomics_load = Atomics.load; var Atomics_or = Atomics.or; +var Atomics_store = Atomics.store; +var Atomics_sub = Atomics.sub; var Atomics_xor = Atomics.xor; + +var ENVIRONMENT_IS_FETCH_WORKER = true; +var ENVIRONMENT_IS_WORKER = true; +var ENVIRONMENT_IS_PTHREAD = true; +var __pthread_is_main_runtime_thread=0; var DYNAMICTOP_PTR = 0; var TOTAL_MEMORY = 0; function enlargeMemory() { diff --git a/src/library_fetch.js b/src/library_fetch.js index 7c37d3bb2c7b5..29411d845f253 100644 --- a/src/library_fetch.js +++ b/src/library_fetch.js @@ -17,7 +17,7 @@ var LibraryFetch = { $__emscripten_fetch_load_cached_data: __emscripten_fetch_load_cached_data, $__emscripten_fetch_cache_data: __emscripten_fetch_cache_data, $__emscripten_fetch_xhr: __emscripten_fetch_xhr, - emscripten_start_fetch__deps: ['$Fetch', '$__emscripten_fetch_xhr', '$__emscripten_fetch_cache_data', '$__emscripten_fetch_load_cached_data', '_emscripten_get_fetch_work_queue'], + emscripten_start_fetch__deps: ['$Fetch', '$__emscripten_fetch_xhr', '$__emscripten_fetch_cache_data', '$__emscripten_fetch_load_cached_data', '_emscripten_get_fetch_work_queue', 'emscripten_is_main_runtime_thread', 'pthread_mutex_lock', 'pthread_mutex_unlock'], emscripten_start_fetch: emscripten_start_fetch }; diff --git a/src/preamble.js b/src/preamble.js index 31c1eaa35d028..120b8a97f9346 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -1180,6 +1180,7 @@ if (typeof Atomics === 'undefined') { Atomics['add'] = function(t, i, v) { var w = t[i]; t[i] += v; return w; } Atomics['and'] = function(t, i, v) { var w = t[i]; t[i] &= v; return w; } Atomics['compareExchange'] = function(t, i, e, r) { var w = t[i]; if (w == e) t[i] = r; return w; } + Atomics['exchange'] = function(t, i, v) { var w = t[i]; t[i] = v; return w; } Atomics['wait'] = function(t, i, v, o) { if (t[i] != v) abort('Multithreading is not supported, cannot sleep to wait for futex!'); } Atomics['wake'] = function(t, i, c) { return 0; } Atomics['wakeOrRequeue'] = function(t, i1, c, i2, v) { return 0; } diff --git a/system/lib/fetch/emscripten_fetch.cpp b/system/lib/fetch/emscripten_fetch.cpp index fb74eddf35e4d..803d6e9cd601b 100644 --- a/system/lib/fetch/emscripten_fetch.cpp +++ b/system/lib/fetch/emscripten_fetch.cpp @@ -78,6 +78,7 @@ emscripten_fetch_t *emscripten_fetch(emscripten_fetch_attr_t *fetch_attr, const fetch->__attributes.requestHeaders = 0;// TODO:strdup(fetch->__attributes.requestHeaders); fetch->__attributes.overriddenMimeType = fetch->__attributes.overriddenMimeType ? strdup(fetch->__attributes.overriddenMimeType) : 0; // TODO: free +#if __EMSCRIPTEN_PTHREADS__ // Depending on the type of fetch, we can either perform it in the same Worker/thread than the caller, or we might need // to run it in a separate Worker. There is a dedicated fetch worker that is available for the fetch, but in some scenarios // it might be desirable to run in the same Worker as the caller, so deduce here whether to run the fetch in this thread, @@ -91,12 +92,14 @@ emscripten_fetch_t *emscripten_fetch(emscripten_fetch_attr_t *fetch_attr, const if (synchronous) emscripten_fetch_wait(fetch, INFINITY); } else +#endif emscripten_start_fetch(fetch); return fetch; } EMSCRIPTEN_RESULT emscripten_fetch_wait(emscripten_fetch_t *fetch, double timeoutMsecs) { +#if __EMSCRIPTEN_PTHREADS__ if (!fetch) return EMSCRIPTEN_RESULT_INVALID_PARAM; uint32_t proxyState = emscripten_atomic_load_u32(&fetch->__proxyState); if (proxyState == 2) return EMSCRIPTEN_RESULT_SUCCESS; // already finished. @@ -115,11 +118,17 @@ EMSCRIPTEN_RESULT emscripten_fetch_wait(emscripten_fetch_t *fetch, double timeou if (proxyState == 2) return EMSCRIPTEN_RESULT_SUCCESS; else return EMSCRIPTEN_RESULT_FAILED; +#else + EM_ASM({ console.error('fetch: emscripten_fetch_wait is not available when building without pthreads!') }); + return EMSCRIPTEN_RESULT_FAILED; +#endif } EMSCRIPTEN_RESULT emscripten_fetch_close(emscripten_fetch_t *fetch) { +#if __EMSCRIPTEN_PTHREADS__ emscripten_atomic_store_u32(&fetch->__proxyState, 0); +#endif free(fetch); // TODO: thread-safety before freeing (what if freeing an operation in progress? explicit emscripten_fetch_abort()?) return EMSCRIPTEN_RESULT_SUCCESS; } diff --git a/tests/test_browser.py b/tests/test_browser.py index 535a3f897e328..29db5493c0b71 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -3295,26 +3295,20 @@ def test_preallocated_heap(self): # Tests emscripten_fetch() usage to XHR data directly to memory without persisting results to IndexedDB. def test_fetch_to_memory(self): - temp_args = [path_from_root('system/lib/fetch/emscripten_fetch.cpp'), '--js-library', path_from_root('src/library_fetch.js')] # TODO: Emscripten system libs, pass these e.g. with -lfetch - # Test error reporting in the negative case when the file URL doesn't exist. (http 404) - self.btest('fetch/to_memory.cpp', expected='1', args=['--std=c++11', '-s', 'FETCH_DEBUG=1', '-DFILE_DOES_NOT_EXIST'] + temp_args) + self.btest('fetch/to_memory.cpp', expected='1', args=['--std=c++11', '-s', 'FETCH_DEBUG=1', '-s', 'FETCH=1', '-DFILE_DOES_NOT_EXIST']) # Test the positive case when the file URL exists. (http 200) shutil.copyfile(path_from_root('tests', 'gears.png'), os.path.join(self.get_dir(), 'gears.png')) - self.btest('fetch/to_memory.cpp', expected='1', args=['--std=c++11', '-s', 'FETCH_DEBUG=1'] + temp_args) + self.btest('fetch/to_memory.cpp', expected='1', args=['--std=c++11', '-s', 'FETCH_DEBUG=1', '-s', 'FETCH=1']) # Tests emscripten_fetch() usage to persist an XHR into IndexedDB and subsequently load up from there. def test_fetch_cached_xhr(self): - temp_args = [path_from_root('system/lib/fetch/emscripten_fetch.cpp'), '--js-library', path_from_root('src/library_fetch.js')] # TODO: Emscripten system libs, pass these e.g. with -lfetch - shutil.copyfile(path_from_root('tests', 'gears.png'), os.path.join(self.get_dir(), 'gears.png')) - self.btest('fetch/cached_xhr.cpp', expected='1', args=['--std=c++11', '-s', 'FETCH_DEBUG=1'] + temp_args) + self.btest('fetch/cached_xhr.cpp', expected='1', args=['--std=c++11', '-s', 'FETCH_DEBUG=1', '-s', 'FETCH=1']) # Test emscripten_fetch() usage to stream a XHR in to memory without storing the full file in memory def test_fetch_stream_file(self): - temp_args = [path_from_root('system/lib/fetch/emscripten_fetch.cpp'), '--js-library', path_from_root('src/library_fetch.js')] # TODO: Emscripten system libs, pass these e.g. with -lfetch - # Strategy: create a large 128MB file, and compile with a small 16MB Emscripten heap, so that the tested file # won't fully fit in the heap. This verifies that streaming works properly. f = open('largefile.txt', 'w') @@ -3324,14 +3318,19 @@ def test_fetch_stream_file(self): for i in xrange(1024): f.write(s) f.close() - self.btest('fetch/stream_file.cpp', expected='1', args=['--std=c++11', '-s', 'FETCH_DEBUG=1', '-s', 'TOTAL_MEMORY=536870912'] + temp_args) + self.btest('fetch/stream_file.cpp', expected='1', args=['--std=c++11', '-s', 'FETCH_DEBUG=1', '-s', 'FETCH=1', '-s', 'TOTAL_MEMORY=536870912']) # Tests emscripten_fetch() usage in synchronous mode. def test_fetch_sync_xhr(self): - temp_args = [path_from_root('system/lib/fetch/emscripten_fetch.cpp'), '--js-library', path_from_root('src/library_fetch.js')] # TODO: Emscripten system libs, pass these e.g. with -lfetch + shutil.copyfile(path_from_root('tests', 'gears.png'), os.path.join(self.get_dir(), 'gears.png')) + self.btest('fetch/sync_xhr.cpp', expected='1', args=['--std=c++11', '-s', 'FETCH_DEBUG=1', '-s', 'FETCH=1', '--proxy-to-worker', '-s', 'USE_PTHREADS=1']) + + def test_fetch_idb_store(self): + self.btest('fetch/idb_store.cpp', expected='0', args=['-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1', '-s', 'FETCH=1', '--proxy-to-worker']) + def test_fetch_idb_delete(self): shutil.copyfile(path_from_root('tests', 'gears.png'), os.path.join(self.get_dir(), 'gears.png')) - self.btest('fetch/sync_xhr.cpp', expected='1', args=['--std=c++11', '-s', 'FETCH_DEBUG=1', '--proxy-to-worker'] + temp_args) + self.btest('fetch/idb_delete.cpp', expected='0', args=['-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1', '-s', 'FETCH=1', '--proxy-to-worker']) def test_asmfs_hello_file(self): shutil.copyfile(path_from_root('tests', 'asmfs', 'hello_file.txt'), os.path.join(self.get_dir(), 'hello_file.txt')) From 46701591d4278f52ee6f5f5e24f96fe9783c2011 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Sun, 28 Aug 2016 00:43:13 +0300 Subject: [PATCH 36/74] Adds support for O_EXCL file open mode in ASMFS. --- system/lib/fetch/asmfs.cpp | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/system/lib/fetch/asmfs.cpp b/system/lib/fetch/asmfs.cpp index 62c721dfbb577..ca7eafe5d8db1 100644 --- a/system/lib/fetch/asmfs.cpp +++ b/system/lib/fetch/asmfs.cpp @@ -358,11 +358,9 @@ long __syscall5(int which, ...) // open return -1; } - if ((flags & O_EXCL)) + if ((flags & O_EXCL) && !(flags & O_CREAT)) { - EM_ASM(Module['printErr']('open() syscall failed! Opening files with O_EXCL flag is not yet supported in ASMFS (TODO)')); - - // TODO: Check if the file exists and if so, return EEXIST error (the file should not exist, but with O_EXCL should always create a new one) + EM_ASM(Module['printErr']('open() syscall failed! Opening files with O_EXCL flag needs to always be paired with O_CREAT')); return -1; } @@ -446,13 +444,20 @@ long __syscall5(int which, ...) // open inode *node = find_inode(root, pathname, &grandparent); if (node) { + if ((flags & O_CREAT) && (flags & O_EXCL)) + { + EM_ASM_INT( { Module['printErr']('__syscall5 OPEN: pathname ' + Pointer_stringify($0) + ', inode open failed because file exists and open flags had O_CREAT and O_EXCL') }, + pathname); + errno = EEXIST; + return -1; + } EM_ASM_INT( { Module['print']('__syscall5 OPEN: pathname ' + Pointer_stringify($0) + ', inode exists. data ptr: ' + $1 + ', data size: ' + $2 + ', fetch ptr: ' + $3) }, pathname, node->data, node->size, node->fetch); } if (node && node->fetch) emscripten_fetch_wait(node->fetch, INFINITY); - if ((flags & O_CREAT) && (flags & O_TRUNC)) + if ((flags & O_CREAT) || (flags & O_TRUNC) || (flags & O_EXCL)) { // Create a new empty file or truncate existing one. if (node) From 9fb36da76ddf27e480183d03b9efcab5373c008a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Sun, 28 Aug 2016 01:41:27 +0300 Subject: [PATCH 37/74] Implement syscalls mkdir, rmdir, write and unlink for ASMFS. Add test --- system/lib/fetch/asmfs.cpp | 138 ++++++++++++++++++++++++++++++++++- tests/cstdio/test_remove.cpp | 5 ++ tests/test_browser.py | 3 + 3 files changed, 145 insertions(+), 1 deletion(-) diff --git a/system/lib/fetch/asmfs.cpp b/system/lib/fetch/asmfs.cpp index ca7eafe5d8db1..a5ba31e00b7f0 100644 --- a/system/lib/fetch/asmfs.cpp +++ b/system/lib/fetch/asmfs.cpp @@ -183,12 +183,14 @@ static const char *basename_part(const char *path) static inode *create_directory_hierarchy_for_file(inode *root, const char *path_to_file) { + assert(root); if (!root) return 0; inode *node = root->child; while(node) { const char *child_path = path_cmp(path_to_file, node->name); + EM_ASM_INT( { Module['printErr']('path_cmp ' + Pointer_stringify($0) + ', ' + Pointer_stringify($1) + ', ' + Pointer_stringify($2) + ' .') }, path_to_file, node->name, child_path); if (child_path) { // The directory name matches. @@ -203,7 +205,9 @@ static inode *create_directory_hierarchy_for_file(inode *root, const char *path_ node = node->sibling; } } + EM_ASM_INT( { Module['printErr']('path_to_file ' + Pointer_stringify($0) + ' .') }, path_to_file); const char *basename_pos = basename_part(path_to_file); + EM_ASM_INT( { Module['printErr']('basename_pos ' + Pointer_stringify($0) + ' .') }, basename_pos); while(*path_to_file && path_to_file < basename_pos) { node = create_inode(INODE_DIR); @@ -216,6 +220,31 @@ static inode *create_directory_hierarchy_for_file(inode *root, const char *path_ return root; } +// Given a path to a file, finds the inode of the parent directory that contains the file, or 0 if the intermediate path doesn't exist. +static inode *find_parent_inode(inode *root, const char *path) +{ + if (!root) return 0; + const char *basename = basename_part(path); + inode *node = root->child; + while(node) + { + const char *child_path = path_cmp(path, node->name); + if (child_path) + { + // The directory name matches. + path = child_path; + if (path >= basename) return node; + if (!*path) return 0; + node = node->child; + } + else + { + node = node->sibling; + } + } + return 0; +} + // Given a root inode of the filesystem and a path relative to it, e.g. "some/directory/dir_or_file", // returns the inode that corresponds to "dir_or_file", or 0 if it doesn't exist. // If the parameter out_closest_parent is specified, the closest (grand)parent node will be returned. @@ -708,7 +737,7 @@ static void print_stream(void *bytes, int numBytes, bool stdout) if (buffer[i] == '\n') { buffer[i] = 0; - EM_ASM_INT( { Module['print'](Pointer_stringify($0)) }, buffer); + EM_ASM_INT( { Module['print'](Pointer_stringify($0)) }, buffer+new_buffer_start); new_buffer_start = i+1; } } @@ -803,4 +832,111 @@ long __syscall146(int which, ...) // writev return total_write_amount; } +// http://man7.org/linux/man-pages/man2/write.2.html +long __syscall4(int which, ...) // write +{ + va_list vl; + va_start(vl, which); + int fd = va_arg(vl, int); + void *buf = va_arg(vl, void *); + size_t count = va_arg(vl, size_t); + va_end(vl); + + iovec io = { buf, count }; + return __syscall146(146, fd, &io, 1); +} + +// http://man7.org/linux/man-pages/man2/mkdir.2.html +long __syscall39(int which, ...) // mkdir +{ + va_list vl; + va_start(vl, which); + const char *pathname = va_arg(vl, const char *); + mode_t mode = va_arg(vl, mode_t); + va_end(vl); + + EM_ASM_INT( { Module['printErr']('__syscall145 MKDIR, which: ' + $0 + ', pathname ' + Pointer_stringify($1) + ', mode: ' + $2 + ' .') }, which, pathname, mode); + + inode *root; + if (pathname[0] == '/') + { + root = filesystem_root(); + ++pathname; + } + else + root = get_cwd(); + inode *parent_dir = find_parent_inode(root, pathname); // TODO: This is wrong, shouldn't be recursive + inode *directory = create_inode(INODE_DIR); + strcpy(directory->name, basename_part(pathname)); + directory->mode = mode; + link_inode(directory, parent_dir); + emscripten_dump_fs_root(); + return 0; // TODO: error checking +} + +// http://man7.org/linux/man-pages/man2/rmdir.2.html +long __syscall40(int which, ...) // rmdir +{ + va_list vl; + va_start(vl, which); + const char *pathname = va_arg(vl, const char *); + va_end(vl); + + inode *root; + if (pathname[0] == '/') + { + root = filesystem_root(); + ++pathname; + } + else + root = get_cwd(); + inode *node = find_inode(root, pathname); + if (node && node->type == INODE_DIR && !node->child) + { + EM_ASM_INT( { Module['print']('__syscall145 RMDIR, pathname: ' + Pointer_stringify($0) + ' removed.') }, pathname); + unlink_inode(node); + emscripten_dump_fs_root(); + return 0; + } + else + { + EM_ASM_INT( { Module['printErr']('__syscall145 RMDIR, pathname: ' + Pointer_stringify($0) + ' not deleted.') }, pathname); + errno = ENOENT; + return -1; + } + +} + +// http://man7.org/linux/man-pages/man2/unlink.2.html +long __syscall10(int which, ...) // unlink +{ + va_list vl; + va_start(vl, which); + const char *pathname = va_arg(vl, const char *); + va_end(vl); + + inode *root; + if (pathname[0] == '/') + { + root = filesystem_root(); + ++pathname; + } + else + root = get_cwd(); + inode *node = find_inode(root, pathname); + if (node && !node->child) + { + EM_ASM_INT( { Module['print']('__syscall145 UNLINK, pathname: ' + Pointer_stringify($0) + ' removed.') }, pathname); + unlink_inode(node); + emscripten_dump_fs_root(); + return 0; + } + else + { + EM_ASM_INT( { Module['printErr']('__syscall145 UNLINK, pathname: ' + Pointer_stringify($0) + ' not deleted.') }, pathname); + errno = ENOENT; + return -1; + } +} + } // ~extern "C" diff --git a/tests/cstdio/test_remove.cpp b/tests/cstdio/test_remove.cpp index 6bc14bd2a1958..be310407ce4ff 100644 --- a/tests/cstdio/test_remove.cpp +++ b/tests/cstdio/test_remove.cpp @@ -59,5 +59,10 @@ int main() { atexit(cleanup); setup(); test(); + +#ifdef REPORT_RESULT + int result = 0; + REPORT_RESULT(); +#endif return EXIT_SUCCESS; } diff --git a/tests/test_browser.py b/tests/test_browser.py index 29db5493c0b71..2a45091945ed9 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -3342,3 +3342,6 @@ def test_asmfs_read_file_twice(self): def test_asmfs_fopen_write(self): self.btest('asmfs/fopen_write.cpp', expected='0', args=['-s', 'ASMFS=1', '-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1', '--proxy-to-worker']) + + def test_asmfs_mkdir_create_unlink_rmdir(self): + self.btest('cstdio/test_remove.cpp', expected='0', args=['-s', 'ASMFS=1', '-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1', '--proxy-to-worker']) From 56bf481a99457e00d710abaafa9e985218a827f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Mon, 29 Aug 2016 23:21:01 +0300 Subject: [PATCH 38/74] Implement better support for creating and opening directories in ASMFS. --- system/lib/fetch/asmfs.cpp | 138 +++++++++++++++++++++++++++---------- 1 file changed, 103 insertions(+), 35 deletions(-) diff --git a/system/lib/fetch/asmfs.cpp b/system/lib/fetch/asmfs.cpp index a5ba31e00b7f0..647edfa6d89f9 100644 --- a/system/lib/fetch/asmfs.cpp +++ b/system/lib/fetch/asmfs.cpp @@ -1,15 +1,16 @@ +#include +#include +#include #include #include #include #define __NEED_struct_iovec #include -#include #include #include #include #include #include -#include #include extern "C" { @@ -80,6 +81,41 @@ static void set_cwd(inode *node) cwd_inode = node; } +static void inode_abspath(inode *node, char *dst, int dstLen) +{ + if (!node) + { + assert(dstLen >= strlen("(null)")+1); + strcpy(dst, "(null)"); + return; + } + if (node == filesystem_root()) + { + assert(dstLen >= strlen("/")+1); + strcpy(dst, "/"); + return; + } +#define MAX_DIRECTORY_DEPTH 512 + inode *stack[MAX_DIRECTORY_DEPTH]; + int depth = 0; + while(node->parent && depth < MAX_DIRECTORY_DEPTH) + { + stack[depth++] = node; + node = node->parent; + } + char *dstEnd = dst + dstLen; + *dstEnd-- = '\0'; + while(depth > 0 && dst < dstEnd) + { + if (dst < dstEnd) *dst++ = '/'; + --depth; + int len = strlen(stack[depth]->name); + if (len > dstEnd - dst) len = dstEnd - dst; + strncpy(dst, stack[depth]->name, len); + dst += len; + } +} + static void delete_inode(inode *node) { free(node); @@ -88,8 +124,10 @@ static void delete_inode(inode *node) // Makes node the child of parent. static void link_inode(inode *node, inode *parent) { - EM_ASM_INT( { Module['printErr']('link_inode: node ' + Pointer_stringify($0) + ' to parent ' + Pointer_stringify($1) + '.') }, - node->name, parent->name); + char parentName[PATH_MAX]; + inode_abspath(parent, parentName, PATH_MAX); + EM_ASM_INT( { Module['printErr']('link_inode: node "' + Pointer_stringify($0) + '" to parent "' + Pointer_stringify($1) + '".') }, + node->name, parentName); // When linking a node, it can't be part of the filesystem tree (but it can have children of its own) assert(!node->parent); assert(!node->sibling); @@ -223,6 +261,8 @@ static inode *create_directory_hierarchy_for_file(inode *root, const char *path_ // Given a path to a file, finds the inode of the parent directory that contains the file, or 0 if the intermediate path doesn't exist. static inode *find_parent_inode(inode *root, const char *path) { + EM_ASM_INT( { Module['print']('find_parent_inode: inode: ' + Pointer_stringify($0) + ' path: ' + Pointer_stringify($1) + '.') }, + root ? root->name : "(null)", path); if (!root) return 0; const char *basename = basename_part(path); inode *node = root->child; @@ -242,7 +282,7 @@ static inode *find_parent_inode(inode *root, const char *path) node = node->sibling; } } - return 0; + return root; } // Given a root inode of the filesystem and a path relative to it, e.g. "some/directory/dir_or_file", @@ -374,13 +414,13 @@ long __syscall5(int which, ...) // open errno = EINVAL; // "The filesystem does not support the O_DIRECT flag." return -1; } - +/* if ((flags & O_DIRECTORY)) { EM_ASM(Module['printErr']('open() syscall failed! Opening directories with O_DIRECTORY flag is not yet supported in ASMFS (TODO)')); return -1; } - +*/ if ((flags & O_DSYNC)) { EM_ASM(Module['printErr']('open() syscall failed! Opening files with O_DSYNC flag is not yet supported in ASMFS (TODO)')); @@ -473,6 +513,22 @@ long __syscall5(int which, ...) // open inode *node = find_inode(root, pathname, &grandparent); if (node) { + if ((flags & O_DIRECTORY) && node->type != INODE_DIR) + { + EM_ASM_INT( { Module['printErr']('__syscall5 OPEN: asked to open directory with pathname ' + Pointer_stringify($0) + ', but the inode with that pathname is not a directory!') }, + pathname); + errno = ENOTDIR; + return -ENOTDIR; + } + + if (!(node->mode & 0444)) // Test if we have read permissions (todo: actually distinguish between user/group/other when that becomes interesting) + { + EM_ASM_INT( { Module['printErr']('__syscall5 OPEN: pathname ' + Pointer_stringify($0) + ' failed to open, no file permissions for read available') }, + pathname); + errno = EACCES; + return -EACCES; + } + if ((flags & O_CREAT) && (flags & O_EXCL)) { EM_ASM_INT( { Module['printErr']('__syscall5 OPEN: pathname ' + Pointer_stringify($0) + ', inode open failed because file exists and open flags had O_CREAT and O_EXCL') }, @@ -501,53 +557,65 @@ long __syscall5(int which, ...) // open else { inode *directory = create_directory_hierarchy_for_file(root, pathname); - node = create_inode(INODE_FILE); + node = create_inode((flags & O_DIRECTORY) ? INODE_DIR : INODE_FILE); strcpy(node->name, basename_part(pathname)); link_inode(node, directory); - printf("Created file %s in directory %s\n", node->name, directory->name); + printf("Created %s %s in directory %s\n", (flags & O_DIRECTORY) ? "directory" : "file", node->name, directory->name); } emscripten_dump_fs_root(); } else if (!node || (!node->fetch && !node->data)) { - // If not, we'll need to fetch it. - emscripten_fetch_attr_t attr; - emscripten_fetch_attr_init(&attr); - strcpy(attr.requestMethod, "GET"); - attr.attributes = EMSCRIPTEN_FETCH_APPEND | EMSCRIPTEN_FETCH_LOAD_TO_MEMORY | EMSCRIPTEN_FETCH_WAITABLE | EMSCRIPTEN_FETCH_PERSIST_FILE; - emscripten_fetch_t *fetch = emscripten_fetch(&attr, pathname); - - // switch(fopen_mode) - // { - // case synchronous_fopen: - emscripten_fetch_wait(fetch, INFINITY); - - if (fetch->status != 200 || fetch->totalBytes == 0) + emscripten_fetch_t *fetch = 0; + if (!(flags & O_DIRECTORY)) { - EM_ASM_INT( { Module['printErr']('__syscall5 OPEN failed! File ' + Pointer_stringify($0) + ' does not exist: XHR returned status code ' + $1 + ', and file length was ' + $2 + '.') }, - pathname, (int)fetch->status, (int)fetch->totalBytes); - emscripten_fetch_close(fetch); - errno = ENOENT; - return -1; + // If not, we'll need to fetch it. + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.attributes = EMSCRIPTEN_FETCH_APPEND | EMSCRIPTEN_FETCH_LOAD_TO_MEMORY | EMSCRIPTEN_FETCH_WAITABLE | EMSCRIPTEN_FETCH_PERSIST_FILE; + fetch = emscripten_fetch(&attr, pathname); + + // switch(fopen_mode) + // { + // case synchronous_fopen: + emscripten_fetch_wait(fetch, INFINITY); + + if (fetch->status != 200 || fetch->totalBytes == 0) + { + EM_ASM_INT( { Module['printErr']('__syscall5 OPEN failed! File ' + Pointer_stringify($0) + ' does not exist: XHR returned status code ' + $1 + ', and file length was ' + $2 + '.') }, + pathname, (int)fetch->status, (int)fetch->totalBytes); + emscripten_fetch_close(fetch); + errno = ENOENT; + return -1; + } + // break; + // case asynchronous_fopen: + // break; + // } } - // break; - // case asynchronous_fopen: - // break; - // } if (node) { node->fetch = fetch; } - else + else if ((flags & O_CREAT)) { inode *directory = create_directory_hierarchy_for_file(root, pathname); - node = create_inode(INODE_FILE); + node = create_inode((flags & O_DIRECTORY) ? INODE_DIR : INODE_FILE); strcpy(node->name, basename_part(pathname)); node->fetch = fetch; link_inode(node, directory); - printf("Created file %s in directory %s\n", node->name, directory->name); + printf("Created %s %s in directory %s\n", (flags & O_DIRECTORY) ? "directory" : "file", node->name, directory->name); + } + else + { + if (fetch) emscripten_fetch_close(fetch); + printf("Failed to open %s %s in directory, it does not exist and O_CREAT flag was not specified\n", + (flags & O_DIRECTORY) ? "directory" : "file", pathname); + errno = ENOENT; + return -ENOENT; // TODO: REVIEW THIS, NEED TO RETURN -errno here instead of -1? } node->size = node->fetch->totalBytes; emscripten_dump_fs_root(); @@ -855,7 +923,7 @@ long __syscall39(int which, ...) // mkdir mode_t mode = va_arg(vl, mode_t); va_end(vl); - EM_ASM_INT( { Module['printErr']('__syscall145 MKDIR, which: ' + $0 + ', pathname ' + Pointer_stringify($1) + ', mode: ' + $2 + ' .') }, which, pathname, mode); + EM_ASM_INT( { Module['printErr']('__syscall145 MKDIR, which: ' + $0 + ', pathname "' + Pointer_stringify($1) + '", mode: ' + $2 + ' .') }, which, pathname, mode); inode *root; if (pathname[0] == '/') From 957ad80113d414ce9d885fb32467e2e29665655a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Tue, 30 Aug 2016 00:31:34 +0300 Subject: [PATCH 39/74] Implement syscall getdents64 for ASMFS. --- system/lib/fetch/asmfs.cpp | 91 +++++++++++++++++++++++++++++++++++--- 1 file changed, 84 insertions(+), 7 deletions(-) diff --git a/system/lib/fetch/asmfs.cpp b/system/lib/fetch/asmfs.cpp index 647edfa6d89f9..d2b02dda137c4 100644 --- a/system/lib/fetch/asmfs.cpp +++ b/system/lib/fetch/asmfs.cpp @@ -624,7 +624,7 @@ long __syscall5(int which, ...) // open FileDescriptor *desc = (FileDescriptor*)malloc(sizeof(FileDescriptor)); desc->magic = EM_FILEDESCRIPTOR_MAGIC; desc->node = node; - desc->file_pos = (flags & O_APPEND) ? node->fetch->totalBytes : 0; + desc->file_pos = ((flags & O_APPEND) && node->fetch) ? node->fetch->totalBytes : 0; desc->mode = mode; desc->flags = flags; @@ -692,21 +692,19 @@ long __syscall140(int which, ...) // llseek errno = EBADF; // "fd isn't a valid open file descriptor." return -1; } - if (!desc->node->fetch) + + if (desc->node->fetch) { - fprintf(stderr, "Internal error: no file data available for fd 0x%8X!", (unsigned int)desc); - return -1; + emscripten_fetch_wait(desc->node->fetch, INFINITY); } - emscripten_fetch_wait(desc->node->fetch, INFINITY); - int64_t offset = (int64_t)(((uint64_t)offset_high << 32) | (uint64_t)offset_low); int64_t newPos; switch(whence) { case SEEK_SET: newPos = offset; break; case SEEK_CUR: newPos = desc->file_pos + offset; break; - case SEEK_END: newPos = desc->node->fetch->numBytes + offset; break; + case SEEK_END: newPos = (desc->node->fetch ? desc->node->fetch->numBytes : desc->node->size) + offset; break; default: errno = EINVAL; // "whence is invalid." return -1; @@ -1007,4 +1005,83 @@ long __syscall10(int which, ...) // unlink } } +// http://man7.org/linux/man-pages/man2/getdents.2.html +long __syscall220(int which, ...) // getdents64 (get directory entries 64-bit) +{ + va_list vl; + va_start(vl, which); + unsigned int fd = va_arg(vl, unsigned int); + dirent *de = va_arg(vl, dirent*); + unsigned int count = va_arg(vl, unsigned int); + unsigned int dirents_size = count / sizeof(de); // The number of dirent structures that can fit into the provided buffer. + dirent *de_end = de + dirents_size; + va_end(vl); + + FileDescriptor *desc = (FileDescriptor*)fd; + if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) + { + fprintf(stderr, "Invalid or closed file descriptor 0x%8X passed to close()!", (unsigned int)desc); + errno = EBADF; // "fd isn't a valid open file descriptor." + return -1; + } + + inode *node = desc->node; + if (!node) + { + assert(false); // TODO: Internal error handling? + return -1; + } + + inode *dotdot = node->parent ? node->parent : node; // In "/", the directory ".." refers to itself. + + ssize_t orig_file_pos = desc->file_pos; + ssize_t file_pos = 0; + // There are always two hardcoded directories "." and ".." + if (de >= de_end) return desc->file_pos - orig_file_pos; + if (desc->file_pos <= file_pos) + { + de->d_ino = (ino_t)node; // TODO: Create inode numbers instead of using pointers + de->d_off = file_pos; + de->d_reclen = sizeof(dirent); + de->d_type = DT_DIR; + strcpy(de->d_name, "."); + ++de; + desc->file_pos += sizeof(dirent); + } + file_pos += sizeof(dirent); + + if (de >= de_end) return desc->file_pos - orig_file_pos; + if (desc->file_pos <= file_pos) + { + de->d_ino = (ino_t)dotdot; // TODO: Create inode numbers instead of using pointers + de->d_off = file_pos; + de->d_reclen = sizeof(dirent); + de->d_type = DT_DIR; + strcpy(de->d_name, ".."); + ++de; + desc->file_pos += sizeof(dirent); + } + file_pos += sizeof(dirent); + + node = node->child; + while(node && de < de_end) + { + if (desc->file_pos <= file_pos) + { + de->d_ino = (ino_t)node; // TODO: Create inode numbers instead of using pointers + de->d_off = file_pos; + de->d_reclen = sizeof(dirent); + de->d_type = (node->type == INODE_DIR) ? DT_DIR : DT_REG /*Regular file*/; + de->d_name[255] = 0; + strncpy(de->d_name, node->name, 255); + ++de; + desc->file_pos += sizeof(dirent); + } + node = node->sibling; + file_pos += sizeof(dirent); + } + + return desc->file_pos - orig_file_pos; +} + } // ~extern "C" From 89eb305adb9eaf70cd0ce6e04c94e02b34cab1ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Tue, 30 Aug 2016 00:33:39 +0300 Subject: [PATCH 40/74] Add test_readdir.c test for ASMFS. --- tests/dirent/test_readdir.c | 5 +++++ tests/test_browser.py | 3 +++ 2 files changed, 8 insertions(+) diff --git a/tests/dirent/test_readdir.c b/tests/dirent/test_readdir.c index e5bb9d7581e2e..85920a78c4112 100644 --- a/tests/dirent/test_readdir.c +++ b/tests/dirent/test_readdir.c @@ -175,5 +175,10 @@ int main() { setup(); test(); test_scandir(); + +#ifdef REPORT_RESULT + int result = 0; + REPORT_RESULT(); +#endif return EXIT_SUCCESS; } diff --git a/tests/test_browser.py b/tests/test_browser.py index 2a45091945ed9..4cd97ecafcc2b 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -3345,3 +3345,6 @@ def test_asmfs_fopen_write(self): def test_asmfs_mkdir_create_unlink_rmdir(self): self.btest('cstdio/test_remove.cpp', expected='0', args=['-s', 'ASMFS=1', '-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1', '--proxy-to-worker']) + + def test_asmfs_dirent_test_readdir(self): + self.btest('dirent/test_readdir.c', expected='0', args=['-s', 'ASMFS=1', '-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1', '--proxy-to-worker']) From fa9703562717e52994db9da1b5a892d29b972ef7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Tue, 30 Aug 2016 00:48:01 +0300 Subject: [PATCH 41/74] Fix file creation modes in ASMFS and file open when opening creates a new inode to existing data obtained with emscripten_fetch(). --- system/lib/fetch/asmfs.cpp | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/system/lib/fetch/asmfs.cpp b/system/lib/fetch/asmfs.cpp index d2b02dda137c4..563701d9a4f18 100644 --- a/system/lib/fetch/asmfs.cpp +++ b/system/lib/fetch/asmfs.cpp @@ -67,6 +67,7 @@ static inode *cwd_inode = 0; static inode *filesystem_root() { static inode *root_node = create_inode(INODE_DIR); + root_node->mode = 0777; return root_node; } @@ -219,7 +220,7 @@ static const char *basename_part(const char *path) return s; } -static inode *create_directory_hierarchy_for_file(inode *root, const char *path_to_file) +static inode *create_directory_hierarchy_for_file(inode *root, const char *path_to_file, unsigned int mode) { assert(root); if (!root) return 0; @@ -249,6 +250,7 @@ static inode *create_directory_hierarchy_for_file(inode *root, const char *path_ while(*path_to_file && path_to_file < basename_pos) { node = create_inode(INODE_DIR); + node->mode = mode; path_to_file += strcpy_inodename(node->name, path_to_file) + 1; link_inode(node, root); EM_ASM_INT( { Module['print']('create_directory_hierarchy_for_file: created directory ' + Pointer_stringify($0) + ' under parent ' + Pointer_stringify($1) + '.') }, @@ -556,8 +558,9 @@ long __syscall5(int which, ...) // open } else { - inode *directory = create_directory_hierarchy_for_file(root, pathname); + inode *directory = create_directory_hierarchy_for_file(root, pathname, mode); node = create_inode((flags & O_DIRECTORY) ? INODE_DIR : INODE_FILE); + node->mode = mode; strcpy(node->name, basename_part(pathname)); link_inode(node, directory); printf("Created %s %s in directory %s\n", (flags & O_DIRECTORY) ? "directory" : "file", node->name, directory->name); @@ -598,12 +601,19 @@ long __syscall5(int which, ...) // open if (node) { - node->fetch = fetch; + if (node->type == INODE_FILE) + { + // If we had an existing inode entry, just associate the entry with the newly fetched data. + node->fetch = fetch; + } } - else if ((flags & O_CREAT)) + else if ((flags & O_CREAT) // If the filesystem entry did not exist, but we have a create flag, ... + || (!node && fetch)) // ... or if it did not exist in our fs, but it could be found via fetch(), ... { - inode *directory = create_directory_hierarchy_for_file(root, pathname); + // ... add it as a new entry to the fs. + inode *directory = create_directory_hierarchy_for_file(root, pathname, mode); node = create_inode((flags & O_DIRECTORY) ? INODE_DIR : INODE_FILE); + node->mode = mode; strcpy(node->name, basename_part(pathname)); node->fetch = fetch; link_inode(node, directory); From c4e820e2cbaf6385bb079bb575fdadc2c120da56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Tue, 30 Aug 2016 02:11:36 +0300 Subject: [PATCH 42/74] Add browser.test_asmfs_dirent_test_readdir_empty --- tests/dirent/test_readdir_empty.c | 5 +++++ tests/test_browser.py | 3 +++ 2 files changed, 8 insertions(+) diff --git a/tests/dirent/test_readdir_empty.c b/tests/dirent/test_readdir_empty.c index 00102733102ff..66ba79cf38d73 100644 --- a/tests/dirent/test_readdir_empty.c +++ b/tests/dirent/test_readdir_empty.c @@ -42,6 +42,11 @@ int main(int argc, char** argv) { closedir(dir); printf("success\n"); + +#ifdef REPORT_RESULT + int result = 0; + REPORT_RESULT(); +#endif return 0; } diff --git a/tests/test_browser.py b/tests/test_browser.py index 4cd97ecafcc2b..048db6149a6cc 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -3348,3 +3348,6 @@ def test_asmfs_mkdir_create_unlink_rmdir(self): def test_asmfs_dirent_test_readdir(self): self.btest('dirent/test_readdir.c', expected='0', args=['-s', 'ASMFS=1', '-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1', '--proxy-to-worker']) + + def test_asmfs_dirent_test_readdir_empty(self): + self.btest('dirent/test_readdir_empty.c', expected='0', args=['-s', 'ASMFS=1', '-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1', '--proxy-to-worker']) From 079fc43501cfd98e2a36c0a4b1a47eac019badac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Tue, 30 Aug 2016 02:24:44 +0300 Subject: [PATCH 43/74] Implement a stub of fsync() for test_unistd_close to pass --- system/lib/fetch/asmfs.cpp | 55 +++++++++++++++++++++++++++++++++++++- tests/test_browser.py | 3 +++ tests/unistd/close.c | 25 ++++++++++++++--- 3 files changed, 78 insertions(+), 5 deletions(-) diff --git a/system/lib/fetch/asmfs.cpp b/system/lib/fetch/asmfs.cpp index 563701d9a4f18..2ecad3e1486d7 100644 --- a/system/lib/fetch/asmfs.cpp +++ b/system/lib/fetch/asmfs.cpp @@ -660,7 +660,7 @@ long __syscall6(int which, ...) // close { fprintf(stderr, "Invalid or already closed file descriptor 0x%8X passed to close()!", (unsigned int)desc); errno = EBADF; // "fd isn't a valid open file descriptor." - return -1; + return -EBADF; } if (desc->node && desc->node->fetch) { @@ -1094,4 +1094,57 @@ long __syscall220(int which, ...) // getdents64 (get directory entries 64-bit) return desc->file_pos - orig_file_pos; } +// http://man7.org/linux/man-pages/man2/fsync.2.html +long __syscall118(int which, ...) // fsync +{ + va_list vl; + va_start(vl, which); + unsigned int fd = va_arg(vl, unsigned int); + va_end(vl); + + FileDescriptor *desc = (FileDescriptor*)fd; + if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) + { + fprintf(stderr, "Invalid or closed file descriptor 0x%8X passed to close()!", (unsigned int)desc); + errno = EBADF; // "fd isn't a valid open file descriptor." + return -EBADF; + } + + inode *node = desc->node; + if (!node) + { + assert(false); // TODO: Internal error handling? + return -1; + } + + return 0; +} + +// http://man7.org/linux/man-pages/man2/dup.2.html +long __syscall41(int which, ...) // dup +{ + va_list vl; + va_start(vl, which); + unsigned int fd = va_arg(vl, unsigned int); + va_end(vl); + + FileDescriptor *desc = (FileDescriptor*)fd; + if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) + { + fprintf(stderr, "Invalid or closed file descriptor 0x%8X passed to close()!", (unsigned int)desc); + errno = EBADF; // "fd isn't a valid open file descriptor." + return -EBADF; + } + + inode *node = desc->node; + if (!node) + { + assert(false); // TODO: Internal error handling? + return -1; + } + + // TODO: Implementing dup() requires separating out file descriptors and file descriptions + return 0; +} + } // ~extern "C" diff --git a/tests/test_browser.py b/tests/test_browser.py index 048db6149a6cc..25ddb1767461e 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -3351,3 +3351,6 @@ def test_asmfs_dirent_test_readdir(self): def test_asmfs_dirent_test_readdir_empty(self): self.btest('dirent/test_readdir_empty.c', expected='0', args=['-s', 'ASMFS=1', '-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1', '--proxy-to-worker']) + + def test_asmfs_unistd_close(self): + self.btest('unistd/close.c', expected='0', args=['-s', 'ASMFS=1', '-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1', '--proxy-to-worker']) diff --git a/tests/unistd/close.c b/tests/unistd/close.c index 7110d18a7f04f..212c63374ec2e 100644 --- a/tests/unistd/close.c +++ b/tests/unistd/close.c @@ -2,25 +2,42 @@ #include #include #include +#include int main() { int f = open("/", O_RDONLY); - printf("fsync(opened): %d\n", fsync(f)); + int ret = fsync(f); + printf("fsync(opened): %d\n", ret); printf("errno: %d\n", errno); + assert(ret == 0); + assert(errno == 0); errno = 0; - printf("close(opened): %d\n", close(f)); + ret = close(f); + printf("close(opened): %d\n", ret); printf("errno: %d\n", errno); + assert(ret == 0); + assert(errno == 0); errno = 0; - printf("fsync(closed): %d\n", fsync(f)); + ret = fsync(f); + printf("fsync(closed): %d\n", ret); printf("errno: %d\n", errno); + assert(ret == -1); + assert(errno == EBADF); errno = 0; - printf("close(closed): %d\n", close(f)); + ret = close(f); + printf("close(closed): %d\n", ret); printf("errno: %d\n", errno); + assert(ret == -1); + assert(errno == EBADF); errno = 0; +#ifdef REPORT_RESULT + int result = 0; + REPORT_RESULT(); +#endif return 0; } From 63831fb61d484e007ba090ff2551e95f1c357dbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Tue, 30 Aug 2016 16:54:05 +0300 Subject: [PATCH 44/74] Implement ASMFS to pass unlink.c test. Don't set errno in ASMFS syscalls, musl does that. Implement ASMFS syscalls chdir, chmod, access and getcwd. --- emcc.py | 1 + system/lib/fetch/asmfs.cpp | 264 +++++++++++++++++++++++++++---------- tests/test_browser.py | 4 + tests/unistd/unlink.c | 10 ++ 4 files changed, 212 insertions(+), 67 deletions(-) diff --git a/emcc.py b/emcc.py index e042f5bc7d14e..edbe3f34fbf08 100755 --- a/emcc.py +++ b/emcc.py @@ -1140,6 +1140,7 @@ def check(input_file): if shared.Settings.ASMFS and final_suffix in JS_CONTAINING_SUFFIXES: input_files.append((next_arg_index, shared.path_from_root('system', 'lib', 'fetch', 'asmfs.cpp'))) + newargs.append('-D__EMSCRIPTEN_ASMFS__=1') next_arg_index += 1 shared.Settings.NO_FILESYSTEM = 1 shared.Settings.FETCH = 1 diff --git a/system/lib/fetch/asmfs.cpp b/system/lib/fetch/asmfs.cpp index 2ecad3e1486d7..4f009f3821af5 100644 --- a/system/lib/fetch/asmfs.cpp +++ b/system/lib/fetch/asmfs.cpp @@ -298,6 +298,10 @@ static inode *find_inode(inode *root, const char *path, inode **out_closest_pare return 0; } + if (path[0] == 0 + || (path[0] == '/' && path[1] == '\0')) + return root; // special-case finding empty string path "" or "/" returns the root searched in. + inode *node = root->child; while(node) { @@ -390,8 +394,7 @@ long __syscall5(int which, ...) // open if (accessMode != O_RDONLY) // O_WRONLY or O_RDWR { EM_ASM(Module['printErr']('open() syscall failed! pathname refers to a file on a read-only filesystem and write access was requested.')); - errno = EROFS; // "pathname refers to a file on a read-only filesystem and write access was requested." - return -1; + return -EROFS; // "pathname refers to a file on a read-only filesystem and write access was requested." } */ @@ -413,8 +416,7 @@ long __syscall5(int which, ...) // open if ((flags & O_DIRECT)) { EM_ASM(Module['printErr']('open() syscall failed! The filesystem does not support the O_DIRECT flag.')); - errno = EINVAL; // "The filesystem does not support the O_DIRECT flag." - return -1; + return -EINVAL; // "The filesystem does not support the O_DIRECT flag." } /* if ((flags & O_DIRECTORY)) @@ -463,12 +465,10 @@ long __syscall5(int which, ...) // open if (accessMode != O_WRONLY && accessMode != O_RDWR) { EM_ASM(Module['printErr']('open() syscall failed! O_TMPFILE was specified in flags, but neither O_WRONLY nor O_RDWR was specified.')); - errno = EINVAL; // "O_TMPFILE was specified in flags, but neither O_WRONLY nor O_RDWR was specified." - return -1; + return -EINVAL; // "O_TMPFILE was specified in flags, but neither O_WRONLY nor O_RDWR was specified." } EM_ASM(Module['printErr']('open() syscall failed! "The filesystem containing pathname does not support O_TMPFILE.')); - errno = EOPNOTSUPP; // "The filesystem containing pathname does not support O_TMPFILE." - return -1; + return -EOPNOTSUPP; // "The filesystem containing pathname does not support O_TMPFILE." } /* if ((flags & O_TRUNC)) @@ -480,25 +480,22 @@ long __syscall5(int which, ...) // open /* TODO: if (is_directory and (accessMode == O_WRONLY || accessMode == O_RDWR)) { - errno = EISDIR; // "pathname refers to a directory and the access requested involved writing (that is, O_WRONLY or O_RDWR is set)." - return -1; + return -EISDIR; // "pathname refers to a directory and the access requested involved writing (that is, O_WRONLY or O_RDWR is set)." } */ /* TODO: if (too_many_files_open) { - errno = EMFILE; // "The per-process limit on the number of open file descriptors has been reached, see getrlimit(RLIMIT_NOFILE)" - return -1; + return -EMFILE; // "The per-process limit on the number of open file descriptors has been reached, see getrlimit(RLIMIT_NOFILE)" } */ // http://stackoverflow.com/questions/417142/what-is-the-maximum-length-of-a-url-in-different-browsers if (strlen(pathname) > 2000) { - errno = ENAMETOOLONG; EM_ASM(Module['printErr']('open() syscall failed! The URL to open was more than 2000 characters long')); - return -1; + return -ENAMETOOLONG; } // Find if this file exists already in the filesystem? @@ -519,7 +516,6 @@ long __syscall5(int which, ...) // open { EM_ASM_INT( { Module['printErr']('__syscall5 OPEN: asked to open directory with pathname ' + Pointer_stringify($0) + ', but the inode with that pathname is not a directory!') }, pathname); - errno = ENOTDIR; return -ENOTDIR; } @@ -527,7 +523,6 @@ long __syscall5(int which, ...) // open { EM_ASM_INT( { Module['printErr']('__syscall5 OPEN: pathname ' + Pointer_stringify($0) + ' failed to open, no file permissions for read available') }, pathname); - errno = EACCES; return -EACCES; } @@ -535,8 +530,7 @@ long __syscall5(int which, ...) // open { EM_ASM_INT( { Module['printErr']('__syscall5 OPEN: pathname ' + Pointer_stringify($0) + ', inode open failed because file exists and open flags had O_CREAT and O_EXCL') }, pathname); - errno = EEXIST; - return -1; + return -EEXIST; } EM_ASM_INT( { Module['print']('__syscall5 OPEN: pathname ' + Pointer_stringify($0) + ', inode exists. data ptr: ' + $1 + ', data size: ' + $2 + ', fetch ptr: ' + $3) }, pathname, node->data, node->size, node->fetch); @@ -590,8 +584,7 @@ long __syscall5(int which, ...) // open EM_ASM_INT( { Module['printErr']('__syscall5 OPEN failed! File ' + Pointer_stringify($0) + ' does not exist: XHR returned status code ' + $1 + ', and file length was ' + $2 + '.') }, pathname, (int)fetch->status, (int)fetch->totalBytes); emscripten_fetch_close(fetch); - errno = ENOENT; - return -1; + return -ENOENT; } // break; // case asynchronous_fopen: @@ -624,8 +617,7 @@ long __syscall5(int which, ...) // open if (fetch) emscripten_fetch_close(fetch); printf("Failed to open %s %s in directory, it does not exist and O_CREAT flag was not specified\n", (flags & O_DIRECTORY) ? "directory" : "file", pathname); - errno = ENOENT; - return -ENOENT; // TODO: REVIEW THIS, NEED TO RETURN -errno here instead of -1? + return -ENOENT; } node->size = node->fetch->totalBytes; emscripten_dump_fs_root(); @@ -659,8 +651,7 @@ long __syscall6(int which, ...) // close if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) { fprintf(stderr, "Invalid or already closed file descriptor 0x%8X passed to close()!", (unsigned int)desc); - errno = EBADF; // "fd isn't a valid open file descriptor." - return -EBADF; + return -EBADF; // "fd isn't a valid open file descriptor." } if (desc->node && desc->node->fetch) { @@ -699,8 +690,7 @@ long __syscall140(int which, ...) // llseek if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) { fprintf(stderr, "Invalid or closed file descriptor 0x%8X passed to close()!", (unsigned int)desc); - errno = EBADF; // "fd isn't a valid open file descriptor." - return -1; + return -EBADF; // "fd isn't a valid open file descriptor." } if (desc->node->fetch) @@ -716,20 +706,17 @@ long __syscall140(int which, ...) // llseek case SEEK_CUR: newPos = desc->file_pos + offset; break; case SEEK_END: newPos = (desc->node->fetch ? desc->node->fetch->numBytes : desc->node->size) + offset; break; default: - errno = EINVAL; // "whence is invalid." - return -1; + return -EINVAL; // "whence is invalid." } if (newPos < 0) { - errno = EINVAL; // "the resulting file offset would be negative" - return -1; + return -EINVAL; // "the resulting file offset would be negative" } if (newPos > 0x7FFFFFFFLL) { - errno = EOVERFLOW; // "The resulting file offset cannot be represented in an off_t." EM_ASM_INT( { Module['printErr']('llseek EOVERFLOW error: fd ' + $0 + 'attempted to seek past unsupported 2^31-1 file size limit (TODO?).') }, fd); - return -1; + return -EOVERFLOW; // "The resulting file offset cannot be represented in an off_t." } desc->file_pos = newPos; @@ -752,8 +739,7 @@ long __syscall145(int which, ...) // readv if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) { EM_ASM_INT( { Module['printErr']('Invalid or closed file descriptor ' + $0 + ' passed to readv!') }, fd); - errno = EBADF; // "fd is not a valid file descriptor or is not open for reading." - return -1; + return -EBADF; // "fd is not a valid file descriptor or is not open for reading." } // TODO: Test and detect to return EISDIR. @@ -763,8 +749,7 @@ long __syscall145(int which, ...) // readv if (iovcnt < 0) { - errno = EINVAL; // "The vector count, iovcnt, is less than zero or greater than the permitted maximum." - return -1; + return -EINVAL; // "The vector count, iovcnt, is less than zero or greater than the permitted maximum." } ssize_t total_read_amount = 0; @@ -773,8 +758,7 @@ long __syscall145(int which, ...) // readv ssize_t n = total_read_amount + iov[i].iov_len; if (n < total_read_amount || (!iov[i].iov_base && iov[i].iov_len > 0)) { - errno = EINVAL; // "The sum of the iov_len values overflows an ssize_t value." or "the address specified in buf is not valid" - return -1; + return -EINVAL; // "The sum of the iov_len values overflows an ssize_t value." or "the address specified in buf is not valid" } total_read_amount = n; } @@ -839,15 +823,13 @@ long __syscall146(int which, ...) // writev if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) { EM_ASM_INT( { Module['printErr']('Invalid or closed file descriptor ' + $0 + ' passed to writev!') }, fd); - errno = EBADF; // "fd is not a valid file descriptor or is not open for reading." - return -1; + return -EBADF; // "fd is not a valid file descriptor or is not open for reading." } } if (iovcnt < 0) { - errno = EINVAL; // "The vector count, iovcnt, is less than zero or greater than the permitted maximum." - return -1; + return -EINVAL; // "The vector count, iovcnt, is less than zero or greater than the permitted maximum." } ssize_t total_write_amount = 0; @@ -856,8 +838,7 @@ long __syscall146(int which, ...) // writev ssize_t n = total_write_amount + iov[i].iov_len; if (n < total_write_amount || (!iov[i].iov_base && iov[i].iov_len > 0)) { - errno = EINVAL; // "The sum of the iov_len values overflows an ssize_t value." or "the address specified in buf is not valid" - return -1; + return -EINVAL; // "The sum of the iov_len values overflows an ssize_t value." or "the address specified in buf is not valid" } total_write_amount = n; } @@ -922,6 +903,75 @@ long __syscall4(int which, ...) // write return __syscall146(146, fd, &io, 1); } +// http://man7.org/linux/man-pages/man2/chdir.2.html +long __syscall12(int which, ...) // chdir +{ + va_list vl; + va_start(vl, which); + const char *pathname = va_arg(vl, const char *); + va_end(vl); + + EM_ASM_INT( { Module['printErr']('__syscall145 CHDIR, which: ' + $0 + ', pathname "' + Pointer_stringify($1) + ' .') }, which, pathname); + + inode *root; + if (pathname[0] == '/') + { + root = filesystem_root(); + ++pathname; + } + else + root = get_cwd(); + char old_cwd[PATH_MAX]; + inode_abspath(root, old_cwd, PATH_MAX); + + inode *cwd = find_inode(root, pathname); + if (!cwd) + { + EM_ASM_INT( { Module['printErr']('CHDIR("' + Pointer_stringify($0) + '") FAILED: old working directory: "' + Pointer_stringify($1) + '".') }, + pathname, old_cwd); + return -1; + } + + char new_cwd[PATH_MAX]; + inode_abspath(cwd, new_cwd, PATH_MAX); + EM_ASM_INT( { Module['printErr']('CHDIR("' + Pointer_stringify($0) + '"): old working directory: "' + Pointer_stringify($1) + '", new working directory: "' + Pointer_stringify($2) + '".') }, + pathname, old_cwd, new_cwd); + + set_cwd(cwd); + return 0; +} + +// http://man7.org/linux/man-pages/man2/chmod.2.html +long __syscall15(int which, ...) // chmod +{ + va_list vl; + va_start(vl, which); + const char *pathname = va_arg(vl, const char *); + int mode = va_arg(vl, int); + va_end(vl); + + inode *root; + if (pathname[0] == '/') + { + root = filesystem_root(); + ++pathname; + } + else + root = get_cwd(); + + inode *node = find_inode(root, pathname); + if (!node) + { + assert(false); // TODO: Internal error handling? + return -1; + } + + // TODO: Access control + node->mode = mode; + return 0; +} + + // http://man7.org/linux/man-pages/man2/mkdir.2.html long __syscall39(int which, ...) // mkdir { @@ -967,20 +1017,31 @@ long __syscall40(int which, ...) // rmdir else root = get_cwd(); inode *node = find_inode(root, pathname); - if (node && node->type == INODE_DIR && !node->child) + if (!node) { - EM_ASM_INT( { Module['print']('__syscall145 RMDIR, pathname: ' + Pointer_stringify($0) + ' removed.') }, pathname); - unlink_inode(node); - emscripten_dump_fs_root(); - return 0; + EM_ASM_INT( { Module['printErr']('__syscall145 RMDIR, pathname: ' + Pointer_stringify($0) + ' not deleted.') }, pathname); + return -ENOENT; } - else + + if (node == filesystem_root() || node == get_cwd()) + return -EBUSY; + + if (node->parent && !(node->parent->mode & 0222)) // Need to have write access to the the parent directory { - EM_ASM_INT( { Module['printErr']('__syscall145 RMDIR, pathname: ' + Pointer_stringify($0) + ' not deleted.') }, pathname); - errno = ENOENT; - return -1; + EM_ASM_INT( { Module['print']('__syscall145 UNLINK failed: no write access to parent directory of "' + Pointer_stringify($0) + '".') }, pathname); + return -EACCES; } + if (node->type != INODE_DIR) + return -ENOTDIR; + + if (node->child) + return -ENOTEMPTY; + + EM_ASM_INT( { Module['print']('__syscall145 RMDIR, pathname: ' + Pointer_stringify($0) + ' removed.') }, pathname); + unlink_inode(node); + emscripten_dump_fs_root(); + return 0; } // http://man7.org/linux/man-pages/man2/unlink.2.html @@ -1000,19 +1061,71 @@ long __syscall10(int which, ...) // unlink else root = get_cwd(); inode *node = find_inode(root, pathname); - if (node && !node->child) + if (!node) { - EM_ASM_INT( { Module['print']('__syscall145 UNLINK, pathname: ' + Pointer_stringify($0) + ' removed.') }, pathname); - unlink_inode(node); - emscripten_dump_fs_root(); - return 0; + EM_ASM_INT( { Module['printErr']('__syscall145 UNLINK, ENOENT: ' + Pointer_stringify($0) + ' does not exist.') }, pathname); + return -ENOENT; + } + + if (!(node->mode & 0222) // Need to have write access to the file/directory to delete it ... + || node->child) // ... and if it's a directory, it can't be deleted if it's not empty + { + EM_ASM_INT( { Module['print']('__syscall145 UNLINK failed: no write access to "' + Pointer_stringify($0) + '".') }, pathname); + if (node->type == INODE_DIR) return -EISDIR; // Linux quirk: Return EISDIR error for not having permission to delete a directory. + else return -EPERM; // but return EPERM error for no permission to delete a file. + } + + if (node->parent && !(node->parent->mode & 0222)) // Need to have write access to the the parent directory + { + EM_ASM_INT( { Module['print']('__syscall145 UNLINK failed: no write access to parent directory of "' + Pointer_stringify($0) + '".') }, pathname); + return -EACCES; + } + + EM_ASM_INT( { Module['print']('__syscall145 UNLINK, pathname: ' + Pointer_stringify($0) + ' removed.') }, pathname); + unlink_inode(node); + emscripten_dump_fs_root(); + return 0; +} + +// http://man7.org/linux/man-pages/man2/faccessat.2.html +long __syscall33(int which, ...) // access +{ + va_list vl; + va_start(vl, which); + const char *pathname = va_arg(vl, const char *); + int mode = va_arg(vl, int); + va_end(vl); + + if ((mode & F_OK) && (mode & (R_OK | W_OK | X_OK))) + return -EINVAL; // mode was incorrectly specified. + + inode *root; + if (pathname[0] == '/') + { + root = filesystem_root(); + ++pathname; } else + root = get_cwd(); + inode *node = find_inode(root, pathname); + if (!node) { - EM_ASM_INT( { Module['printErr']('__syscall145 UNLINK, pathname: ' + Pointer_stringify($0) + ' not deleted.') }, pathname); - errno = ENOENT; - return -1; + EM_ASM_INT( { Module['printErr']('__syscall33 ACCESS, ENOENT: ' + Pointer_stringify($0) + ' does not exist.') }, pathname); + return -ENOENT; + } + + if ((mode & F_OK)) // Just test if file exists + { + return 0; } + + if (((mode & R_OK) && !(node->mode & 0444)) + || ((mode & W_OK) && !(node->mode & 0222)) + || ((mode & X_OK) && !(node->mode & 0111))) + { + return -EACCES; + } + return 0; } // http://man7.org/linux/man-pages/man2/getdents.2.html @@ -1031,8 +1144,7 @@ long __syscall220(int which, ...) // getdents64 (get directory entries 64-bit) if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) { fprintf(stderr, "Invalid or closed file descriptor 0x%8X passed to close()!", (unsigned int)desc); - errno = EBADF; // "fd isn't a valid open file descriptor." - return -1; + return -EBADF; // "fd isn't a valid open file descriptor." } inode *node = desc->node; @@ -1106,8 +1218,7 @@ long __syscall118(int which, ...) // fsync if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) { fprintf(stderr, "Invalid or closed file descriptor 0x%8X passed to close()!", (unsigned int)desc); - errno = EBADF; // "fd isn't a valid open file descriptor." - return -EBADF; + return -EBADF; // "fd isn't a valid open file descriptor." } inode *node = desc->node; @@ -1132,8 +1243,7 @@ long __syscall41(int which, ...) // dup if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) { fprintf(stderr, "Invalid or closed file descriptor 0x%8X passed to close()!", (unsigned int)desc); - errno = EBADF; // "fd isn't a valid open file descriptor." - return -EBADF; + return -EBADF; // "fd isn't a valid open file descriptor." } inode *node = desc->node; @@ -1147,4 +1257,24 @@ long __syscall41(int which, ...) // dup return 0; } +// http://man7.org/linux/man-pages/man2/getcwd.2.html +long __syscall183(int which, ...) // getcwd +{ + va_list vl; + va_start(vl, which); + char *buf = va_arg(vl, char *); + size_t size = va_arg(vl, size_t); + va_end(vl); + + inode *cwd = get_cwd(); + assert(cwd); + inode_abspath(cwd, buf, size); + + EM_ASM_INT( { Module['printErr']('getcwd: node "' + Pointer_stringify($0) + '" has abspath "' + Pointer_stringify($1) + '".') }, + cwd->name, buf); + + return 0; +} + + } // ~extern "C" diff --git a/tests/test_browser.py b/tests/test_browser.py index 25ddb1767461e..f9364607c1274 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -3354,3 +3354,7 @@ def test_asmfs_dirent_test_readdir_empty(self): def test_asmfs_unistd_close(self): self.btest('unistd/close.c', expected='0', args=['-s', 'ASMFS=1', '-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1', '--proxy-to-worker']) + + def test_asmfs_unistd_unlink(self): + # TODO: Once symlinks are supported, remove -DNO_SYMLINK=1 + self.btest('unistd/unlink.c', expected='0', args=['-s', 'ASMFS=1', '-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1', '--proxy-to-worker', '-DNO_SYMLINK=1']) diff --git a/tests/unistd/unlink.c b/tests/unistd/unlink.c index 68051c1370649..1d2803b1442fc 100644 --- a/tests/unistd/unlink.c +++ b/tests/unistd/unlink.c @@ -24,11 +24,16 @@ static void create_file(const char *path, const char *buffer, int mode) { void setup() { mkdir("working", 0777); #ifdef __EMSCRIPTEN__ + +#ifdef __EMSCRIPTEN_ASMFS__ + mkdir("working", 0777); +#else EM_ASM( #if NODEFS FS.mount(NODEFS, { root: '.' }, 'working'); #endif ); +#endif #endif chdir("working"); create_file("file", "test", 0777); @@ -163,5 +168,10 @@ int main() { signal(SIGABRT, cleanup); setup(); test(); + +#ifdef REPORT_RESULT + int result = 0; + REPORT_RESULT(); +#endif return EXIT_SUCCESS; } From e08680568f223b882f8590e514dab84763dbaa88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Tue, 30 Aug 2016 18:46:36 +0300 Subject: [PATCH 45/74] Test unistd/access.c for ASMFS. --- system/lib/fetch/asmfs.cpp | 2 +- tests/test_browser.py | 3 +++ tests/unistd/access.c | 20 +++++++++++++++++++- 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/system/lib/fetch/asmfs.cpp b/system/lib/fetch/asmfs.cpp index 4f009f3821af5..8752040d44154 100644 --- a/system/lib/fetch/asmfs.cpp +++ b/system/lib/fetch/asmfs.cpp @@ -565,7 +565,7 @@ long __syscall5(int which, ...) // open else if (!node || (!node->fetch && !node->data)) { emscripten_fetch_t *fetch = 0; - if (!(flags & O_DIRECTORY)) + if (!(flags & O_DIRECTORY) && accessMode != O_WRONLY) { // If not, we'll need to fetch it. emscripten_fetch_attr_t attr; diff --git a/tests/test_browser.py b/tests/test_browser.py index f9364607c1274..d95dbf7d96701 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -3355,6 +3355,9 @@ def test_asmfs_dirent_test_readdir_empty(self): def test_asmfs_unistd_close(self): self.btest('unistd/close.c', expected='0', args=['-s', 'ASMFS=1', '-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1', '--proxy-to-worker']) + def test_asmfs_unistd_access(self): + self.btest('unistd/access.c', expected='0', args=['-s', 'ASMFS=1', '-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1', '--proxy-to-worker']) + def test_asmfs_unistd_unlink(self): # TODO: Once symlinks are supported, remove -DNO_SYMLINK=1 self.btest('unistd/unlink.c', expected='0', args=['-s', 'ASMFS=1', '-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1', '--proxy-to-worker', '-DNO_SYMLINK=1']) diff --git a/tests/unistd/access.c b/tests/unistd/access.c index 92b52e5ef1c78..d90d7938b5482 100644 --- a/tests/unistd/access.c +++ b/tests/unistd/access.c @@ -2,8 +2,18 @@ #include #include #include +#include +#include int main() { +#ifdef __EMSCRIPTEN_ASMFS__ + mkdir("working", 0777); + chdir("working"); + close(open("forbidden", O_WRONLY|O_CREAT|O_TRUNC|O_EXCL, 0000)); + close(open("readable", O_WRONLY|O_CREAT|O_TRUNC|O_EXCL, 0444)); + close(open("writeable", O_WRONLY|O_CREAT|O_TRUNC|O_EXCL, 0222)); + close(open("allaccess", O_WRONLY|O_CREAT|O_TRUNC|O_EXCL, 0777)); +#else EM_ASM( FS.mkdir('working'); #if NODEFS @@ -15,7 +25,7 @@ int main() { FS.writeFile('writeable', ''); FS.chmod('writeable', 0222); FS.writeFile('allaccess', ''); FS.chmod('allaccess', 0777); ); - +#endif char* files[] = {"readable", "writeable", "allaccess", "forbidden", "nonexistent"}; for (int i = 0; i < sizeof files / sizeof files[0]; i++) { @@ -34,6 +44,7 @@ int main() { printf("\n"); } +#ifndef __EMSCRIPTEN_ASMFS__ // Restore full permissions on all created files so that python test runner rmtree // won't have problems on deleting the files. On Windows, calling shutil.rmtree() // will fail if any of the files are read-only. @@ -43,5 +54,12 @@ int main() { FS.chmod('writeable', 0777); FS.chmod('allaccess', 0777); ); +#endif + +#ifdef REPORT_RESULT + int result = 0; + REPORT_RESULT(); +#endif + return 0; } From d0e21d19cc9ed191227bd0f4307c5d578e8135c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Tue, 30 Aug 2016 19:29:30 +0300 Subject: [PATCH 46/74] Add RETURN_ERRNO macro to clean up error handling. --- system/lib/fetch/asmfs.cpp | 142 ++++++++----------------------------- 1 file changed, 31 insertions(+), 111 deletions(-) diff --git a/system/lib/fetch/asmfs.cpp b/system/lib/fetch/asmfs.cpp index 8752040d44154..3d5f7c0dd26df 100644 --- a/system/lib/fetch/asmfs.cpp +++ b/system/lib/fetch/asmfs.cpp @@ -376,6 +376,11 @@ void emscripten_dump_fs_root() emscripten_dump_fs_tree(filesystem_root(), path); } +#define RETURN_ERRNO(errno, error_reason) do { \ + EM_ASM({ Module['printErr'](__FUNCTION__ + '() returned errno ' + #errno + ': ' + error_reason + '!')}); \ + return -errno; \ + } while(0) + // http://man7.org/linux/man-pages/man2/open.2.html long __syscall5(int which, ...) // open { @@ -386,56 +391,24 @@ long __syscall5(int which, ...) // open int mode = va_arg(vl, int); va_end(vl); - EM_ASM_INT( { Module['printErr']('__syscall5 OPEN, which: ' + $0 + ', pathname ' + Pointer_stringify($1) + ', ' + $2 + ', ' + $3 + '.') }, - which, pathname, flags, mode); + EM_ASM_INT( { Module['printErr']('open(pathname="' + Pointer_stringify($0) + '", flags=0x' + ($1).toString(16) + ', mode=0' + ($2).toString(8) + ')') }, pathname, flags, mode); int accessMode = (flags & O_ACCMODE); - /* - if (accessMode != O_RDONLY) // O_WRONLY or O_RDWR - { - EM_ASM(Module['printErr']('open() syscall failed! pathname refers to a file on a read-only filesystem and write access was requested.')); - return -EROFS; // "pathname refers to a file on a read-only filesystem and write access was requested." - } - */ if ((flags & O_ASYNC)) - { - EM_ASM(Module['printErr']('open() syscall failed! Opening files with O_ASYNC flag is not supported in ASMFS (TODO?)')); - return -1; - } + RETURN_ERRNO(ENOTSUP, "TODO: Opening files with O_ASYNC flag is not supported in ASMFS"); // The flags:O_CLOEXEC flag is ignored, doesn't have meaning for Emscripten -/* - if ((flags & O_CREAT)) - { - EM_ASM(Module['printErr']('open() syscall failed! Opening files with O_CREAT flag is not yet supported in ASMFS (TODO)')); - return -1; - } -*/ + // TODO: the flags:O_DIRECT flag seems like a great way to let applications explicitly control XHR/IndexedDB read/write buffering behavior? if ((flags & O_DIRECT)) - { - EM_ASM(Module['printErr']('open() syscall failed! The filesystem does not support the O_DIRECT flag.')); - return -EINVAL; // "The filesystem does not support the O_DIRECT flag." - } -/* - if ((flags & O_DIRECTORY)) - { - EM_ASM(Module['printErr']('open() syscall failed! Opening directories with O_DIRECTORY flag is not yet supported in ASMFS (TODO)')); - return -1; - } -*/ + RETURN_ERRNO(ENOTSUP, "TODO: O_DIRECT flag is not supported in ASMFS"); + if ((flags & O_DSYNC)) - { - EM_ASM(Module['printErr']('open() syscall failed! Opening files with O_DSYNC flag is not yet supported in ASMFS (TODO)')); - return -1; - } + RETURN_ERRNO(ENOTSUP, "TODO: O_DSYNC flag is not supported in ASMFS"); if ((flags & O_EXCL) && !(flags & O_CREAT)) - { - EM_ASM(Module['printErr']('open() syscall failed! Opening files with O_EXCL flag needs to always be paired with O_CREAT')); - return -1; - } + RETURN_ERRNO(EINVAL, "open() with O_EXCL flag needs to always be paired with O_CREAT"); // Spec says the behavior is undefined, we can just enforce it // The flags:O_LARGEFILE flag is ignored, we should always be largefile-compatible @@ -443,46 +416,21 @@ long __syscall5(int which, ...) // open // The flags O_NOCTTY, O_NOFOLLOW if ((flags & (O_NONBLOCK|O_NDELAY))) - { - EM_ASM(Module['printErr']('open() syscall failed! Opening files with O_NONBLOCK or O_NDELAY flags is not yet supported in ASMFS (TODO)')); - return -1; - } + RETURN_ERRNO(ENOTSUP, "TODO: Opening files with O_NONBLOCK or O_NDELAY flags is not supported in ASMFS"); if ((flags & O_PATH)) - { - EM_ASM(Module['printErr']('open() syscall failed! Opening files with O_PATH flags is not yet supported in ASMFS (TODO)')); - return -1; - } + RETURN_ERRNO(ENOTSUP, "TODO: Opening files with O_PATH flag is not supported in ASMFS"); if ((flags & O_SYNC)) - { - EM_ASM(Module['printErr']('open() syscall failed! Opening files with O_SYNC flags is not yet supported in ASMFS (TODO)')); - return -1; - } + RETURN_ERRNO(ENOTSUP, "TODO: Opening files with O_SYNC flag is not supported in ASMFS"); if ((flags & O_TMPFILE)) { if (accessMode != O_WRONLY && accessMode != O_RDWR) - { - EM_ASM(Module['printErr']('open() syscall failed! O_TMPFILE was specified in flags, but neither O_WRONLY nor O_RDWR was specified.')); - return -EINVAL; // "O_TMPFILE was specified in flags, but neither O_WRONLY nor O_RDWR was specified." - } - EM_ASM(Module['printErr']('open() syscall failed! "The filesystem containing pathname does not support O_TMPFILE.')); - return -EOPNOTSUPP; // "The filesystem containing pathname does not support O_TMPFILE." - } -/* - if ((flags & O_TRUNC)) - { - EM_ASM(Module['printErr']('open() syscall failed! Opening files with O_TRUNC flags is not yet supported in ASMFS (TODO)')); - return -1; - } -*/ - /* TODO: - if (is_directory and (accessMode == O_WRONLY || accessMode == O_RDWR)) - { - return -EISDIR; // "pathname refers to a directory and the access requested involved writing (that is, O_WRONLY or O_RDWR is set)." + RETURN_ERRNO(EINVAL, "O_TMPFILE was specified in flags, but neither O_WRONLY nor O_RDWR was specified"); + + RETURN_ERRNO(EOPNOTSUPP, "TODO: The filesystem containing pathname does not support O_TMPFILE"); } - */ /* TODO: if (too_many_files_open) @@ -493,10 +441,7 @@ long __syscall5(int which, ...) // open // http://stackoverflow.com/questions/417142/what-is-the-maximum-length-of-a-url-in-different-browsers if (strlen(pathname) > 2000) - { - EM_ASM(Module['printErr']('open() syscall failed! The URL to open was more than 2000 characters long')); - return -ENAMETOOLONG; - } + RETURN_ERRNO(ENAMETOOLONG, "pathname was too long"); // Find if this file exists already in the filesystem? inode *root; @@ -513,27 +458,16 @@ long __syscall5(int which, ...) // open if (node) { if ((flags & O_DIRECTORY) && node->type != INODE_DIR) - { - EM_ASM_INT( { Module['printErr']('__syscall5 OPEN: asked to open directory with pathname ' + Pointer_stringify($0) + ', but the inode with that pathname is not a directory!') }, - pathname); - return -ENOTDIR; - } + RETURN_ERRNO(ENOTDIR, "O_DIRECTORY was specified and pathname was not a directory"); if (!(node->mode & 0444)) // Test if we have read permissions (todo: actually distinguish between user/group/other when that becomes interesting) - { - EM_ASM_INT( { Module['printErr']('__syscall5 OPEN: pathname ' + Pointer_stringify($0) + ' failed to open, no file permissions for read available') }, - pathname); - return -EACCES; - } + RETURN_ERRNO(EACCES, "The requested access to the file is not allowed"); if ((flags & O_CREAT) && (flags & O_EXCL)) - { - EM_ASM_INT( { Module['printErr']('__syscall5 OPEN: pathname ' + Pointer_stringify($0) + ', inode open failed because file exists and open flags had O_CREAT and O_EXCL') }, - pathname); - return -EEXIST; - } - EM_ASM_INT( { Module['print']('__syscall5 OPEN: pathname ' + Pointer_stringify($0) + ', inode exists. data ptr: ' + $1 + ', data size: ' + $2 + ', fetch ptr: ' + $3) }, - pathname, node->data, node->size, node->fetch); + RETURN_ERRNO(EEXIST, "pathname already exists and O_CREAT and O_EXCL were used"); + + if (node->type == INODE_DIR && accessMode != O_RDONLY) + RETURN_ERRNO(EISDIR, "pathname refers to a directory and the access requested involved writing (that is, O_WRONLY or O_RDWR is set)"); } if (node && node->fetch) emscripten_fetch_wait(node->fetch, INFINITY); @@ -543,11 +477,8 @@ long __syscall5(int which, ...) // open // Create a new empty file or truncate existing one. if (node) { - if (node->fetch) - { - emscripten_fetch_close(node->fetch); - node->fetch = 0; - } + if (node->fetch) emscripten_fetch_close(node->fetch); + node->fetch = 0; node->size = 0; } else @@ -557,10 +488,7 @@ long __syscall5(int which, ...) // open node->mode = mode; strcpy(node->name, basename_part(pathname)); link_inode(node, directory); - printf("Created %s %s in directory %s\n", (flags & O_DIRECTORY) ? "directory" : "file", node->name, directory->name); } - - emscripten_dump_fs_root(); } else if (!node || (!node->fetch && !node->data)) { @@ -581,10 +509,8 @@ long __syscall5(int which, ...) // open if (fetch->status != 200 || fetch->totalBytes == 0) { - EM_ASM_INT( { Module['printErr']('__syscall5 OPEN failed! File ' + Pointer_stringify($0) + ' does not exist: XHR returned status code ' + $1 + ', and file length was ' + $2 + '.') }, - pathname, (int)fetch->status, (int)fetch->totalBytes); emscripten_fetch_close(fetch); - return -ENOENT; + RETURN_ERRNO(ENOENT, "O_CREAT is not set and the named file does not exist (attempted emscripten_fetch() XHR to download)"); } // break; // case asynchronous_fopen: @@ -594,11 +520,8 @@ long __syscall5(int which, ...) // open if (node) { - if (node->type == INODE_FILE) - { - // If we had an existing inode entry, just associate the entry with the newly fetched data. - node->fetch = fetch; - } + // If we had an existing inode entry, just associate the entry with the newly fetched data. + if (node->type == INODE_FILE) node->fetch = fetch; } else if ((flags & O_CREAT) // If the filesystem entry did not exist, but we have a create flag, ... || (!node && fetch)) // ... or if it did not exist in our fs, but it could be found via fetch(), ... @@ -610,14 +533,11 @@ long __syscall5(int which, ...) // open strcpy(node->name, basename_part(pathname)); node->fetch = fetch; link_inode(node, directory); - printf("Created %s %s in directory %s\n", (flags & O_DIRECTORY) ? "directory" : "file", node->name, directory->name); } else { if (fetch) emscripten_fetch_close(fetch); - printf("Failed to open %s %s in directory, it does not exist and O_CREAT flag was not specified\n", - (flags & O_DIRECTORY) ? "directory" : "file", pathname); - return -ENOENT; + RETURN_ERRNO(ENOENT, "O_CREAT is not set and the named file does not exist"); } node->size = node->fetch->totalBytes; emscripten_dump_fs_root(); From 8f7137686eb3128c6bec10b532745046dd8c3000 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Tue, 30 Aug 2016 19:48:38 +0300 Subject: [PATCH 47/74] ASMFS code cleanup --- system/lib/fetch/asmfs.cpp | 102 ++++++++++++------------------------- 1 file changed, 33 insertions(+), 69 deletions(-) diff --git a/system/lib/fetch/asmfs.cpp b/system/lib/fetch/asmfs.cpp index 3d5f7c0dd26df..a9dd1067a5157 100644 --- a/system/lib/fetch/asmfs.cpp +++ b/system/lib/fetch/asmfs.cpp @@ -259,6 +259,14 @@ static inode *create_directory_hierarchy_for_file(inode *root, const char *path_ } return root; } +// Same as above, but the root node is deduced from 'path'. (either absolute if path starts with "/", or relative) +static inode *create_directory_hierarchy_for_file(const char *path, unsigned int mode) +{ + inode *root; + if (path[0] == '/') root = filesystem_root(), ++path; + else root = get_cwd(); + return create_directory_hierarchy_for_file(root, path, mode); +} // Given a path to a file, finds the inode of the parent directory that contains the file, or 0 if the intermediate path doesn't exist. static inode *find_parent_inode(inode *root, const char *path) @@ -323,6 +331,15 @@ static inode *find_inode(inode *root, const char *path, inode **out_closest_pare return 0; } +// Same as above, but the root node is deduced from 'path'. (either absolute if path starts with "/", or relative) +static inode *find_inode(const char *path, inode **out_closest_parent = 0) +{ + inode *root; + if (path[0] == '/') root = filesystem_root(), ++path; + else root = get_cwd(); + return find_inode(root, path, out_closest_parent); +} + // Debug function that dumps out the filesystem tree to console. void emscripten_dump_fs_tree(inode *root, char *path) { @@ -377,7 +394,7 @@ void emscripten_dump_fs_root() } #define RETURN_ERRNO(errno, error_reason) do { \ - EM_ASM({ Module['printErr'](__FUNCTION__ + '() returned errno ' + #errno + ': ' + error_reason + '!')}); \ + EM_ASM_INT({ Module['printErr'](Pointer_stringify($0) + '() returned errno ' + #errno + ': ' + error_reason + '!')}, __FUNCTION__); \ return -errno; \ } while(0) @@ -444,17 +461,10 @@ long __syscall5(int which, ...) // open RETURN_ERRNO(ENAMETOOLONG, "pathname was too long"); // Find if this file exists already in the filesystem? - inode *root; - if (pathname[0] == '/') - { - root = filesystem_root(); - ++pathname; - } - else - root = get_cwd(); + inode *root = (pathname[0] == '/') ? filesystem_root() : get_cwd(); + const char *relpath = (pathname[0] == '/') ? pathname+1 : pathname; - inode *grandparent = 0; - inode *node = find_inode(root, pathname, &grandparent); + inode *node = find_inode(root, relpath); if (node) { if ((flags & O_DIRECTORY) && node->type != INODE_DIR) @@ -483,7 +493,7 @@ long __syscall5(int which, ...) // open } else { - inode *directory = create_directory_hierarchy_for_file(root, pathname, mode); + inode *directory = create_directory_hierarchy_for_file(root, relpath, mode); node = create_inode((flags & O_DIRECTORY) ? INODE_DIR : INODE_FILE); node->mode = mode; strcpy(node->name, basename_part(pathname)); @@ -527,7 +537,7 @@ long __syscall5(int which, ...) // open || (!node && fetch)) // ... or if it did not exist in our fs, but it could be found via fetch(), ... { // ... add it as a new entry to the fs. - inode *directory = create_directory_hierarchy_for_file(root, pathname, mode); + inode *directory = create_directory_hierarchy_for_file(root, relpath, mode); node = create_inode((flags & O_DIRECTORY) ? INODE_DIR : INODE_FILE); node->mode = mode; strcpy(node->name, basename_part(pathname)); @@ -833,18 +843,10 @@ long __syscall12(int which, ...) // chdir EM_ASM_INT( { Module['printErr']('__syscall145 CHDIR, which: ' + $0 + ', pathname "' + Pointer_stringify($1) + ' .') }, which, pathname); - inode *root; - if (pathname[0] == '/') - { - root = filesystem_root(); - ++pathname; - } - else - root = get_cwd(); char old_cwd[PATH_MAX]; - inode_abspath(root, old_cwd, PATH_MAX); + inode_abspath(get_cwd(), old_cwd, PATH_MAX); - inode *cwd = find_inode(root, pathname); + inode *cwd = find_inode(pathname); if (!cwd) { EM_ASM_INT( { Module['printErr']('CHDIR("' + Pointer_stringify($0) + '") FAILED: old working directory: "' + Pointer_stringify($1) + '".') }, @@ -870,16 +872,7 @@ long __syscall15(int which, ...) // chmod int mode = va_arg(vl, int); va_end(vl); - inode *root; - if (pathname[0] == '/') - { - root = filesystem_root(); - ++pathname; - } - else - root = get_cwd(); - - inode *node = find_inode(root, pathname); + inode *node = find_inode(pathname); if (!node) { assert(false); // TODO: Internal error handling? @@ -903,15 +896,10 @@ long __syscall39(int which, ...) // mkdir EM_ASM_INT( { Module['printErr']('__syscall145 MKDIR, which: ' + $0 + ', pathname "' + Pointer_stringify($1) + '", mode: ' + $2 + ' .') }, which, pathname, mode); - inode *root; - if (pathname[0] == '/') - { - root = filesystem_root(); - ++pathname; - } - else - root = get_cwd(); - inode *parent_dir = find_parent_inode(root, pathname); // TODO: This is wrong, shouldn't be recursive + inode *root = (pathname[0] == '/') ? filesystem_root() : get_cwd(); + const char *relpath = (pathname[0] == '/') ? pathname+1 : pathname; + + inode *parent_dir = find_parent_inode(root, relpath); inode *directory = create_inode(INODE_DIR); strcpy(directory->name, basename_part(pathname)); directory->mode = mode; @@ -928,15 +916,7 @@ long __syscall40(int which, ...) // rmdir const char *pathname = va_arg(vl, const char *); va_end(vl); - inode *root; - if (pathname[0] == '/') - { - root = filesystem_root(); - ++pathname; - } - else - root = get_cwd(); - inode *node = find_inode(root, pathname); + inode *node = find_inode(pathname); if (!node) { EM_ASM_INT( { Module['printErr']('__syscall145 RMDIR, pathname: ' + Pointer_stringify($0) + ' not deleted.') }, pathname); @@ -972,15 +952,7 @@ long __syscall10(int which, ...) // unlink const char *pathname = va_arg(vl, const char *); va_end(vl); - inode *root; - if (pathname[0] == '/') - { - root = filesystem_root(); - ++pathname; - } - else - root = get_cwd(); - inode *node = find_inode(root, pathname); + inode *node = find_inode(pathname); if (!node) { EM_ASM_INT( { Module['printErr']('__syscall145 UNLINK, ENOENT: ' + Pointer_stringify($0) + ' does not exist.') }, pathname); @@ -1019,15 +991,7 @@ long __syscall33(int which, ...) // access if ((mode & F_OK) && (mode & (R_OK | W_OK | X_OK))) return -EINVAL; // mode was incorrectly specified. - inode *root; - if (pathname[0] == '/') - { - root = filesystem_root(); - ++pathname; - } - else - root = get_cwd(); - inode *node = find_inode(root, pathname); + inode *node = find_inode(pathname); if (!node) { EM_ASM_INT( { Module['printErr']('__syscall33 ACCESS, ENOENT: ' + Pointer_stringify($0) + ' does not exist.') }, pathname); From 23d992de3a9c8e7ab378b63a1ac606b3d29e9339 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Tue, 30 Aug 2016 22:17:24 +0300 Subject: [PATCH 48/74] Cleanup error handling in ASMFS. --- system/lib/fetch/asmfs.cpp | 416 +++++++++++++++++-------------------- 1 file changed, 190 insertions(+), 226 deletions(-) diff --git a/system/lib/fetch/asmfs.cpp b/system/lib/fetch/asmfs.cpp index a9dd1067a5157..ee2cea0635c93 100644 --- a/system/lib/fetch/asmfs.cpp +++ b/system/lib/fetch/asmfs.cpp @@ -15,6 +15,9 @@ extern "C" { +// http://stackoverflow.com/questions/417142/what-is-the-maximum-length-of-a-url-in-different-browsers +#define MAX_PATHNAME_LENGTH 2000 + #define INODE_TYPE uint32_t #define INODE_FILE 1 #define INODE_DIR 2 @@ -407,58 +410,39 @@ long __syscall5(int which, ...) // open int flags = va_arg(vl, int); int mode = va_arg(vl, int); va_end(vl); - - EM_ASM_INT( { Module['printErr']('open(pathname="' + Pointer_stringify($0) + '", flags=0x' + ($1).toString(16) + ', mode=0' + ($2).toString(8) + ')') }, pathname, flags, mode); + EM_ASM_INT({ Module['printErr']('open(pathname="' + Pointer_stringify($0) + '", flags=0x' + ($1).toString(16) + ', mode=0' + ($2).toString(8) + ')') }, + pathname, flags, mode); int accessMode = (flags & O_ACCMODE); - if ((flags & O_ASYNC)) - RETURN_ERRNO(ENOTSUP, "TODO: Opening files with O_ASYNC flag is not supported in ASMFS"); + if ((flags & O_ASYNC)) RETURN_ERRNO(ENOTSUP, "TODO: Opening files with O_ASYNC flag is not supported in ASMFS"); + if ((flags & O_DIRECT)) RETURN_ERRNO(ENOTSUP, "TODO: O_DIRECT flag is not supported in ASMFS"); + if ((flags & O_DSYNC)) RETURN_ERRNO(ENOTSUP, "TODO: O_DSYNC flag is not supported in ASMFS"); + if ((flags & O_EXCL) && !(flags & O_CREAT)) RETURN_ERRNO(EINVAL, "open() with O_EXCL flag needs to always be paired with O_CREAT"); // Spec says the behavior is undefined, we can just enforce it + if ((flags & (O_NONBLOCK|O_NDELAY))) RETURN_ERRNO(ENOTSUP, "TODO: Opening files with O_NONBLOCK or O_NDELAY flags is not supported in ASMFS"); + if ((flags & O_PATH)) RETURN_ERRNO(ENOTSUP, "TODO: Opening files with O_PATH flag is not supported in ASMFS"); + if ((flags & O_SYNC)) RETURN_ERRNO(ENOTSUP, "TODO: Opening files with O_SYNC flag is not supported in ASMFS"); // The flags:O_CLOEXEC flag is ignored, doesn't have meaning for Emscripten // TODO: the flags:O_DIRECT flag seems like a great way to let applications explicitly control XHR/IndexedDB read/write buffering behavior? - if ((flags & O_DIRECT)) - RETURN_ERRNO(ENOTSUP, "TODO: O_DIRECT flag is not supported in ASMFS"); - - if ((flags & O_DSYNC)) - RETURN_ERRNO(ENOTSUP, "TODO: O_DSYNC flag is not supported in ASMFS"); - - if ((flags & O_EXCL) && !(flags & O_CREAT)) - RETURN_ERRNO(EINVAL, "open() with O_EXCL flag needs to always be paired with O_CREAT"); // Spec says the behavior is undefined, we can just enforce it // The flags:O_LARGEFILE flag is ignored, we should always be largefile-compatible // TODO: The flags:O_NOATIME is ignored, file access times have not been implemented yet // The flags O_NOCTTY, O_NOFOLLOW - if ((flags & (O_NONBLOCK|O_NDELAY))) - RETURN_ERRNO(ENOTSUP, "TODO: Opening files with O_NONBLOCK or O_NDELAY flags is not supported in ASMFS"); - - if ((flags & O_PATH)) - RETURN_ERRNO(ENOTSUP, "TODO: Opening files with O_PATH flag is not supported in ASMFS"); - - if ((flags & O_SYNC)) - RETURN_ERRNO(ENOTSUP, "TODO: Opening files with O_SYNC flag is not supported in ASMFS"); - if ((flags & O_TMPFILE)) { - if (accessMode != O_WRONLY && accessMode != O_RDWR) - RETURN_ERRNO(EINVAL, "O_TMPFILE was specified in flags, but neither O_WRONLY nor O_RDWR was specified"); - - RETURN_ERRNO(EOPNOTSUPP, "TODO: The filesystem containing pathname does not support O_TMPFILE"); + if (accessMode != O_WRONLY && accessMode != O_RDWR) RETURN_ERRNO(EINVAL, "O_TMPFILE was specified in flags, but neither O_WRONLY nor O_RDWR was specified"); + else RETURN_ERRNO(EOPNOTSUPP, "TODO: The filesystem containing pathname does not support O_TMPFILE"); } - /* TODO: - if (too_many_files_open) - { - return -EMFILE; // "The per-process limit on the number of open file descriptors has been reached, see getrlimit(RLIMIT_NOFILE)" - } - */ + // TODO: if (too_many_files_open) RETURN_ERRNO(EMFILE, "The per-process limit on the number of open file descriptors has been reached, see getrlimit(RLIMIT_NOFILE)"); - // http://stackoverflow.com/questions/417142/what-is-the-maximum-length-of-a-url-in-different-browsers - if (strlen(pathname) > 2000) - RETURN_ERRNO(ENAMETOOLONG, "pathname was too long"); + int len = strlen(pathname); + if (len > MAX_PATHNAME_LENGTH) RETURN_ERRNO(ENAMETOOLONG, "pathname was too long"); + if (len == 0) RETURN_ERRNO(ENOENT, "pathname is empty"); // Find if this file exists already in the filesystem? inode *root = (pathname[0] == '/') ? filesystem_root() : get_cwd(); @@ -467,17 +451,10 @@ long __syscall5(int which, ...) // open inode *node = find_inode(root, relpath); if (node) { - if ((flags & O_DIRECTORY) && node->type != INODE_DIR) - RETURN_ERRNO(ENOTDIR, "O_DIRECTORY was specified and pathname was not a directory"); - - if (!(node->mode & 0444)) // Test if we have read permissions (todo: actually distinguish between user/group/other when that becomes interesting) - RETURN_ERRNO(EACCES, "The requested access to the file is not allowed"); - - if ((flags & O_CREAT) && (flags & O_EXCL)) - RETURN_ERRNO(EEXIST, "pathname already exists and O_CREAT and O_EXCL were used"); - - if (node->type == INODE_DIR && accessMode != O_RDONLY) - RETURN_ERRNO(EISDIR, "pathname refers to a directory and the access requested involved writing (that is, O_WRONLY or O_RDWR is set)"); + if ((flags & O_DIRECTORY) && node->type != INODE_DIR) RETURN_ERRNO(ENOTDIR, "O_DIRECTORY was specified and pathname was not a directory"); + if (!(node->mode & 0444)) RETURN_ERRNO(EACCES, "The requested access to the file is not allowed"); + if ((flags & O_CREAT) && (flags & O_EXCL)) RETURN_ERRNO(EEXIST, "pathname already exists and O_CREAT and O_EXCL were used"); + if (node->type == INODE_DIR && accessMode != O_RDONLY) RETURN_ERRNO(EISDIR, "pathname refers to a directory and the access requested involved writing (that is, O_WRONLY or O_RDWR is set)"); } if (node && node->fetch) emscripten_fetch_wait(node->fetch, INFINITY); @@ -575,14 +552,11 @@ long __syscall6(int which, ...) // close va_start(vl, which); int fd = va_arg(vl, int); va_end(vl); - EM_ASM_INT( { Module['printErr']('__syscall6 CLOSE, which: ' + $0 + ', fd: ' + $1 + '.') }, which, fd); + EM_ASM_INT({ Module['printErr']('close(fd=' + $0 + ')') }, fd); FileDescriptor *desc = (FileDescriptor*)fd; - if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) - { - fprintf(stderr, "Invalid or already closed file descriptor 0x%8X passed to close()!", (unsigned int)desc); - return -EBADF; // "fd isn't a valid open file descriptor." - } + if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) RETURN_ERRNO(EBADF, "fd isn't a valid open file descriptor"); + if (desc->node && desc->node->fetch) { emscripten_fetch_wait(desc->node->fetch, INFINITY); // TODO: This should not be necessary- test this out @@ -597,7 +571,7 @@ long __syscall6(int which, ...) // close // http://man7.org/linux/man-pages/man2/sysctl.2.html long __syscall54(int which, ...) // sysctl { - EM_ASM_INT( { Module['printErr']('__syscall54 SYSCTL, which: ' + $0 + '.') }, which); + EM_ASM( { Module['printErr']('sysctl() is ignored') }); return 0; } @@ -613,20 +587,13 @@ long __syscall140(int which, ...) // llseek off_t *result = va_arg(vl, off_t *); unsigned int whence = va_arg(vl, unsigned int); va_end(vl); - EM_ASM_INT( { Module['printErr']('__syscall140 LLSEEK, which: ' + $0 + ', fd ' + $1 + ', offset high ' + $2 + ' offset low ' + $3 + ' result ' + $4 + ' whence ' + $5 + '.') }, - which, fd, offset_high, offset_low, result, whence); + EM_ASM_INT({ Module['printErr']('llseek(fd=' + $0 + ', offset=0x' + (($1<<32)|$2) + ', result=0x' + ($3).toString(16) + ', whence=' + $4 + ')') }, + fd, offset_high, offset_low, result, whence); FileDescriptor *desc = (FileDescriptor*)fd; - if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) - { - fprintf(stderr, "Invalid or closed file descriptor 0x%8X passed to close()!", (unsigned int)desc); - return -EBADF; // "fd isn't a valid open file descriptor." - } + if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) RETURN_ERRNO(EBADF, "fd isn't a valid open file descriptor"); - if (desc->node->fetch) - { - emscripten_fetch_wait(desc->node->fetch, INFINITY); - } + if (desc->node->fetch) emscripten_fetch_wait(desc->node->fetch, INFINITY); int64_t offset = (int64_t)(((uint64_t)offset_high << 32) | (uint64_t)offset_low); int64_t newPos; @@ -635,19 +602,13 @@ long __syscall140(int which, ...) // llseek case SEEK_SET: newPos = offset; break; case SEEK_CUR: newPos = desc->file_pos + offset; break; case SEEK_END: newPos = (desc->node->fetch ? desc->node->fetch->numBytes : desc->node->size) + offset; break; - default: - return -EINVAL; // "whence is invalid." - } - if (newPos < 0) - { - return -EINVAL; // "the resulting file offset would be negative" - } - if (newPos > 0x7FFFFFFFLL) - { - EM_ASM_INT( { Module['printErr']('llseek EOVERFLOW error: fd ' + $0 + 'attempted to seek past unsupported 2^31-1 file size limit (TODO?).') }, - fd); - return -EOVERFLOW; // "The resulting file offset cannot be represented in an off_t." + case 3/*SEEK_DATA*/: RETURN_ERRNO(EINVAL, "whence is invalid (sparse files, whence=SEEK_DATA, is not supported"); + case 4/*SEEK_HOLE*/: RETURN_ERRNO(EINVAL, "whence is invalid (sparse files, whence=SEEK_HOLE, is not supported"); + default: RETURN_ERRNO(EINVAL, "whence is invalid"); } + if (newPos < 0) RETURN_ERRNO(EINVAL, "The resulting file offset would be negative"); + if (newPos > 0x7FFFFFFFLL) RETURN_ERRNO(EOVERFLOW, "The resulting file offset cannot be represented in an off_t"); + desc->file_pos = newPos; if (result) *result = desc->file_pos; @@ -663,39 +624,35 @@ long __syscall145(int which, ...) // readv const iovec *iov = va_arg(vl, const iovec*); int iovcnt = va_arg(vl, int); va_end(vl); - EM_ASM_INT( { Module['printErr']('__syscall145 READV, which: ' + $0 + ', fd ' + $1 + ', iov: ' + $2 + ', iovcnt: ' + $3 + ' .') }, which, fd, iov, iovcnt); + EM_ASM_INT({ Module['printErr']('readv(fd=' + $0 + ', iov=0x' + ($1).toString(16) + ', iovcnt=' + $2 + ')') }, fd, iov, iovcnt); FileDescriptor *desc = (FileDescriptor*)fd; - if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) - { - EM_ASM_INT( { Module['printErr']('Invalid or closed file descriptor ' + $0 + ' passed to readv!') }, fd); - return -EBADF; // "fd is not a valid file descriptor or is not open for reading." - } + if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) RETURN_ERRNO(EBADF, "fd isn't a valid open file descriptor"); - // TODO: Test and detect to return EISDIR. + inode *node = desc->node; + if (!node) RETURN_ERRNO(-1, "ASMFS internal error: file descriptor points to a non-file"); + if (node->type == INODE_DIR) RETURN_ERRNO(EISDIR, "fd refers to a directory"); + if (node->type != INODE_FILE /* TODO: && node->type != socket */) RETURN_ERRNO(EINVAL, "fd is attached to an object which is unsuitable for reading"); - // TODO: Support nonblocking IO and check for EAGAIN/EWOULDBLOCK - if (desc->node->fetch) emscripten_fetch_wait(desc->node->fetch, INFINITY); + // TODO: if (node->type == INODE_FILE && desc has O_NONBLOCK && read would block) RETURN_ERRNO(EAGAIN, "The file descriptor fd refers to a file other than a socket and has been marked nonblocking (O_NONBLOCK), and the read would block"); + // TODO: if (node->type == socket && desc has O_NONBLOCK && read would block) RETURN_ERRNO(EWOULDBLOCK, "The file descriptor fd refers to a socket and has been marked nonblocking (O_NONBLOCK), and the read would block"); - if (iovcnt < 0) - { - return -EINVAL; // "The vector count, iovcnt, is less than zero or greater than the permitted maximum." - } + if (node->fetch) emscripten_fetch_wait(node->fetch, INFINITY); + + if (node->size > 0 && !node->data && (!node->fetch || !node->fetch->data)) RETURN_ERRNO(-1, "ASMFS internal error: no file data available"); + if (iovcnt < 0) RETURN_ERRNO(EINVAL, "The vector count, iovcnt, is less than zero"); ssize_t total_read_amount = 0; for(int i = 0; i < iovcnt; ++i) { ssize_t n = total_read_amount + iov[i].iov_len; - if (n < total_read_amount || (!iov[i].iov_base && iov[i].iov_len > 0)) - { - return -EINVAL; // "The sum of the iov_len values overflows an ssize_t value." or "the address specified in buf is not valid" - } + if (n < total_read_amount) RETURN_ERRNO(EINVAL, "The sum of the iov_len values overflows an ssize_t value"); + if (!iov[i].iov_base && iov[i].iov_len > 0) RETURN_ERRNO(EINVAL, "iov_len specifies a positive length buffer but iov_base is a null pointer"); total_read_amount = n; } size_t offset = desc->file_pos; - inode *node = desc->node; - uint8_t *data = node->data ? node->data : (uint8_t *)node->fetch->data; + uint8_t *data = node->data ? node->data : (node->fetch ? (uint8_t *)node->fetch->data : 0); for(int i = 0; i < iovcnt; ++i) { ssize_t dataLeft = node->size - offset; @@ -745,38 +702,27 @@ long __syscall146(int which, ...) // writev const iovec *iov = va_arg(vl, const iovec*); int iovcnt = va_arg(vl, int); va_end(vl); + EM_ASM_INT({ Module['printErr']('writev(fd=' + $0 + ', iov=0x' + ($1).toString(16) + ', iovcnt=' + $2 + ')') }, fd, iov, iovcnt); FileDescriptor *desc = (FileDescriptor*)fd; - if (fd != 1/*stdout*/ && fd != 2/*stderr*/) + if (fd != 1/*stdout*/ && fd != 2/*stderr*/) // TODO: Resolve the hardcoding of stdin,stdout & stderr { -// EM_ASM_INT( { Module['printErr']('__syscall146 WRITEV, which: ' + $0 + ', fd ' + $1 + ', iov: ' + $2 + ', iovcnt: ' + $3 + ' .') }, which, fd, iov, iovcnt); - if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) - { - EM_ASM_INT( { Module['printErr']('Invalid or closed file descriptor ' + $0 + ' passed to writev!') }, fd); - return -EBADF; // "fd is not a valid file descriptor or is not open for reading." - } + if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) RETURN_ERRNO(EBADF, "fd isn't a valid open file descriptor"); } - if (iovcnt < 0) - { - return -EINVAL; // "The vector count, iovcnt, is less than zero or greater than the permitted maximum." - } + if (iovcnt < 0) RETURN_ERRNO(EINVAL, "The vector count, iovcnt, is less than zero"); ssize_t total_write_amount = 0; for(int i = 0; i < iovcnt; ++i) { ssize_t n = total_write_amount + iov[i].iov_len; - if (n < total_write_amount || (!iov[i].iov_base && iov[i].iov_len > 0)) - { - return -EINVAL; // "The sum of the iov_len values overflows an ssize_t value." or "the address specified in buf is not valid" - } + if (n < total_write_amount) RETURN_ERRNO(EINVAL, "The sum of the iov_len values overflows an ssize_t value"); + if (!iov[i].iov_base && iov[i].iov_len > 0) RETURN_ERRNO(EINVAL, "iov_len specifies a positive length buffer but iov_base is a null pointer"); total_write_amount = n; } -// EM_ASM_INT( { Module['printErr']('__syscall146 WRITEV, write amount to fd ' + $0 + ' is ' + $1 + '.') }, fd, total_write_amount); if (fd == 1/*stdout*/ || fd == 2/*stderr*/) { -// EM_ASM_INT( { Module['printErr']('__syscall146 WRITEV, printing to stdout/stderr: ' + $0 + ' .') }, fd); ssize_t bytesWritten = 0; for(int i = 0; i < iovcnt; ++i) { @@ -787,13 +733,11 @@ long __syscall146(int which, ...) // writev } else { -// EM_ASM_INT( { Module['printErr']('__syscall146 WRITEV, printing to file data ptr: ' + $0 + ' .') }, fd); // Enlarge the file in memory to fit space for the new data size_t newSize = desc->file_pos + total_write_amount; inode *node = desc->node; if (node->capacity < newSize) { -// EM_ASM_INT( { Module['printErr']('__syscall146 WRITEV, enlarging data ptr: ' + $0 + ' -> ' + $1 + ' .') }, node->capacity, newSize); size_t newCapacity = (newSize > (size_t)(node->capacity*1.25) ? newSize : (size_t)(node->capacity*1.25)); // Geometric increases in size for amortized O(1) behavior uint8_t *newData = (uint8_t *)realloc(node->data, newCapacity); if (!newData) @@ -813,8 +757,6 @@ long __syscall146(int which, ...) // writev memcpy((uint8_t*)node->data + desc->file_pos, iov[i].iov_base, iov[i].iov_len); desc->file_pos += iov[i].iov_len; } -// EM_ASM_INT( { Module['print']('__syscall5 WRITE: wrote pathname ' + Pointer_stringify($0) + ', inode exists. data ptr: ' + $1 + ', data size: ' + $2 + ', fetch ptr: ' + $3) }, -// node->name, node->data, node->size, node->fetch); } return total_write_amount; } @@ -828,6 +770,7 @@ long __syscall4(int which, ...) // write void *buf = va_arg(vl, void *); size_t count = va_arg(vl, size_t); va_end(vl); + EM_ASM_INT({ Module['printErr']('write(fd=' + $0 + ', buf=0x' + ($1).toString(16) + ', count=' + $2 + ')') }, fd, buf, count); iovec io = { buf, count }; return __syscall146(146, fd, &io, 1); @@ -840,26 +783,24 @@ long __syscall12(int which, ...) // chdir va_start(vl, which); const char *pathname = va_arg(vl, const char *); va_end(vl); + EM_ASM_INT({ Module['printErr']('chdir(pathname="' + Pointer_stringify($0) + '")') }, pathname); - EM_ASM_INT( { Module['printErr']('__syscall145 CHDIR, which: ' + $0 + ', pathname "' + Pointer_stringify($1) + ' .') }, which, pathname); + int len = strlen(pathname); + if (len > MAX_PATHNAME_LENGTH) RETURN_ERRNO(ENAMETOOLONG, "pathname was too long"); + if (len == 0) RETURN_ERRNO(ENOENT, "pathname is empty"); - char old_cwd[PATH_MAX]; - inode_abspath(get_cwd(), old_cwd, PATH_MAX); + inode *node = find_inode(pathname); - inode *cwd = find_inode(pathname); - if (!cwd) - { - EM_ASM_INT( { Module['printErr']('CHDIR("' + Pointer_stringify($0) + '") FAILED: old working directory: "' + Pointer_stringify($1) + '".') }, - pathname, old_cwd); - return -1; - } + // TODO: if (no permissions to navigate the tree to the path) RETURN_ERRNO(EACCES, "Search permission is denied for one of the components of path"); + // TODO: if (too many symlinks) RETURN_ERRNO(ELOOP, "Too many symbolic links were encountered in resolving path"); + + // TODO: Ensure that this is checked for all components of the path + if (!node) RETURN_ERRNO(ENOENT, "The directory specified in path does not exist"); - char new_cwd[PATH_MAX]; - inode_abspath(cwd, new_cwd, PATH_MAX); - EM_ASM_INT( { Module['printErr']('CHDIR("' + Pointer_stringify($0) + '"): old working directory: "' + Pointer_stringify($1) + '", new working directory: "' + Pointer_stringify($2) + '".') }, - pathname, old_cwd, new_cwd); + // TODO: Ensure that this is checked for all components of the path + if (node->type != INODE_DIR) RETURN_ERRNO(ENOTDIR, "A component of path is not a directory"); - set_cwd(cwd); + set_cwd(node); return 0; } @@ -871,20 +812,30 @@ long __syscall15(int which, ...) // chmod const char *pathname = va_arg(vl, const char *); int mode = va_arg(vl, int); va_end(vl); + EM_ASM_INT({ Module['printErr']('chmod(pathname="' + Pointer_stringify($0) + '", mode=0' + ($1).toString(8) + ')') }, pathname, mode); + + int len = strlen(pathname); + if (len > MAX_PATHNAME_LENGTH) RETURN_ERRNO(ENAMETOOLONG, "pathname was too long"); + if (len == 0) RETURN_ERRNO(ENOENT, "pathname is empty"); inode *node = find_inode(pathname); - if (!node) - { - assert(false); // TODO: Internal error handling? - return -1; - } - // TODO: Access control + // TODO: if (no permissions to navigate the tree to the path) RETURN_ERRNO(EACCES, "Search permission is denied on a component of the path prefix"); + // TODO: if (too many symlinks) RETURN_ERRNO(ELOOP, "Too many symbolic links were encountered in resolving pathname"); + + // TODO: Ensure that this is checked for all components of the path + if (!node) RETURN_ERRNO(ENOENT, "The file does not exist"); + + // TODO: Ensure that this is checked for all components of the path + if (node->type != INODE_DIR) RETURN_ERRNO(ENOTDIR, "A component of the path prefix is not a directory"); + + // TODO: if (not allowed) RETURN_ERRNO(EPERM, "The effective UID does not match the owner of the file"); + // TODO: read-only filesystems: if (fs is read-only) RETURN_ERRNO(EROFS, "The named file resides on a read-only filesystem"); + node->mode = mode; return 0; } - // http://man7.org/linux/man-pages/man2/mkdir.2.html long __syscall39(int which, ...) // mkdir { @@ -893,19 +844,33 @@ long __syscall39(int which, ...) // mkdir const char *pathname = va_arg(vl, const char *); mode_t mode = va_arg(vl, mode_t); va_end(vl); + EM_ASM_INT({ Module['printErr']('mkdir(pathname="' + Pointer_stringify($0) + '", mode=0' + ($1).toString(8) + ')') }, pathname, mode); - EM_ASM_INT( { Module['printErr']('__syscall145 MKDIR, which: ' + $0 + ', pathname "' + Pointer_stringify($1) + '", mode: ' + $2 + ' .') }, which, pathname, mode); + int len = strlen(pathname); + if (len > MAX_PATHNAME_LENGTH) RETURN_ERRNO(ENAMETOOLONG, "pathname was too long"); + if (len == 0) RETURN_ERRNO(ENOENT, "pathname is empty"); inode *root = (pathname[0] == '/') ? filesystem_root() : get_cwd(); const char *relpath = (pathname[0] == '/') ? pathname+1 : pathname; - inode *parent_dir = find_parent_inode(root, relpath); + + if (!parent_dir) RETURN_ERRNO(ENOENT, "A directory component in pathname does not exist or is a dangling symbolic link"); + + // TODO: if (component of path wasn't actually a directory) RETURN_ERRNO(ENOTDIR, "A component used as a directory in pathname is not, in fact, a directory"); + + inode *existing = find_inode(parent_dir, basename_part(pathname)); + if (existing) RETURN_ERRNO(EEXIST, "pathname already exists (not necessarily as a directory)"); + if (!(parent_dir->mode & 0222)) RETURN_ERRNO(EACCES, "The parent directory does not allow write permission to the process"); + + // TODO: if (too many symlinks when traversing path) RETURN_ERRNO(ELOOP, "Too many symbolic links were encountered in resolving pathname"); + // TODO: if (any parent dir in path doesn't have search permissions) RETURN_ERRNO(EACCES, "One of the directories in pathname did not allow search permission"); + // TODO: read-only filesystems: if (fs is read-only) RETURN_ERRNO(EROFS, "Pathname refers to a file on a read-only filesystem"); + inode *directory = create_inode(INODE_DIR); strcpy(directory->name, basename_part(pathname)); directory->mode = mode; link_inode(directory, parent_dir); - emscripten_dump_fs_root(); - return 0; // TODO: error checking + return 0; } // http://man7.org/linux/man-pages/man2/rmdir.2.html @@ -915,32 +880,32 @@ long __syscall40(int which, ...) // rmdir va_start(vl, which); const char *pathname = va_arg(vl, const char *); va_end(vl); + EM_ASM_INT({ Module['printErr']('rmdir(pathname="' + Pointer_stringify($0) + '")') }, pathname); - inode *node = find_inode(pathname); - if (!node) - { - EM_ASM_INT( { Module['printErr']('__syscall145 RMDIR, pathname: ' + Pointer_stringify($0) + ' not deleted.') }, pathname); - return -ENOENT; - } + int len = strlen(pathname); + if (len > MAX_PATHNAME_LENGTH) RETURN_ERRNO(ENAMETOOLONG, "pathname was too long"); + if (len == 0) RETURN_ERRNO(ENOENT, "pathname is empty"); - if (node == filesystem_root() || node == get_cwd()) - return -EBUSY; + if (!strcmp(pathname, ".") || (len >= 2 && !strcmp(pathname+len-2, "/."))) RETURN_ERRNO(EINVAL, "pathname has . as last component"); + if (!strcmp(pathname, "..") || (len >= 3 && !strcmp(pathname+len-3, "/.."))) RETURN_ERRNO(ENOTEMPTY, "pathname has .. as its final component"); - if (node->parent && !(node->parent->mode & 0222)) // Need to have write access to the the parent directory - { - EM_ASM_INT( { Module['print']('__syscall145 UNLINK failed: no write access to parent directory of "' + Pointer_stringify($0) + '".') }, pathname); - return -EACCES; - } + inode *node = find_inode(pathname); + if (!node) RETURN_ERRNO(ENOENT, "directory does not exist"); - if (node->type != INODE_DIR) - return -ENOTDIR; + // TODO: RETURN_ERRNO(ENOENT, "A directory component in pathname does not exist or is a dangling symbolic link"); + // TODO: RETURN_ERRNO(ELOOP, "Too many symbolic links were encountered in resolving pathname"); + // TODO: RETURN_ERRNO(EACCES, "one of the directories in the path prefix of pathname did not allow search permission"); - if (node->child) - return -ENOTEMPTY; + if (node == filesystem_root() || node == get_cwd()) RETURN_ERRNO(EBUSY, "pathname is currently in use by the system or some process that prevents its removal (pathname is currently used as a mount point or is the root directory of the calling process)"); + if (node->parent && !(node->parent->mode & 0222)) RETURN_ERRNO(EACCES, "Write access to the directory containing pathname was not allowed"); + if (node->type != INODE_DIR) RETURN_ERRNO(ENOTDIR, "pathname is not a directory"); + if (node->child) RETURN_ERRNO(ENOTEMPTY, "pathname contains entries other than . and .."); + + // TODO: RETURN_ERRNO(EPERM, "The directory containing pathname has the sticky bit (S_ISVTX) set and the process's effective user ID is neither the user ID of the file to be deleted nor that of the directory containing it, and the process is not privileged"); + // TODO: RETURN_ERRNO(EROFS, "pathname refers to a directory on a read-only filesystem"); - EM_ASM_INT( { Module['print']('__syscall145 RMDIR, pathname: ' + Pointer_stringify($0) + ' removed.') }, pathname); unlink_inode(node); - emscripten_dump_fs_root(); + return 0; } @@ -951,31 +916,38 @@ long __syscall10(int which, ...) // unlink va_start(vl, which); const char *pathname = va_arg(vl, const char *); va_end(vl); + EM_ASM_INT({ Module['printErr']('unlink(pathname="' + Pointer_stringify($0) + '")') }, pathname); + + int len = strlen(pathname); + if (len > MAX_PATHNAME_LENGTH) RETURN_ERRNO(ENAMETOOLONG, "pathname was too long"); + if (len == 0) RETURN_ERRNO(ENOENT, "pathname is empty"); inode *node = find_inode(pathname); - if (!node) - { - EM_ASM_INT( { Module['printErr']('__syscall145 UNLINK, ENOENT: ' + Pointer_stringify($0) + ' does not exist.') }, pathname); - return -ENOENT; - } + if (!node) RETURN_ERRNO(ENOENT, "file does not exist"); - if (!(node->mode & 0222) // Need to have write access to the file/directory to delete it ... - || node->child) // ... and if it's a directory, it can't be deleted if it's not empty - { - EM_ASM_INT( { Module['print']('__syscall145 UNLINK failed: no write access to "' + Pointer_stringify($0) + '".') }, pathname); - if (node->type == INODE_DIR) return -EISDIR; // Linux quirk: Return EISDIR error for not having permission to delete a directory. - else return -EPERM; // but return EPERM error for no permission to delete a file. - } + inode *parent = node->parent; + + // TODO: RETURN_ERRNO(ENOENT, "A component in pathname does not exist or is a dangling symbolic link"); + // TODO: RETURN_ERRNO(ELOOP, "Too many symbolic links were encountered in translating pathname"); + // TODO: RETURN_ERRNO(EACCES, "one of the directories in the path prefix of pathname did not allow search permission"); + + if (parent && !(parent->mode & 0222)) + RETURN_ERRNO(EACCES, "Write access to the directory containing pathname is not allowed for the process's effective UID"); - if (node->parent && !(node->parent->mode & 0222)) // Need to have write access to the the parent directory + // TODO: RETURN_ERRNO(ENOTDIR, "A component used as a directory in pathname is not, in fact, a directory"); + // TODO: RETURN_ERRNO(EPERM, "The directory containing pathname has the sticky bit (S_ISVTX) set and the process's effective user ID is neither the user ID of the file to be deleted nor that of the directory containing it, and the process is not privileged"); + // TODO: RETURN_ERRNO(EROFS, "pathname refers to a file on a read-only filesystem"); + + if (!(node->mode & 0222)) { - EM_ASM_INT( { Module['print']('__syscall145 UNLINK failed: no write access to parent directory of "' + Pointer_stringify($0) + '".') }, pathname); - return -EACCES; + if (node->type == INODE_DIR) RETURN_ERRNO(EISDIR, "directory deletion not permitted"); // Linux quirk: Return EISDIR error for not having permission to delete a directory. + else RETURN_ERRNO(EPERM, "file deletion not permitted"); // but return EPERM error for no permission to delete a file. } - EM_ASM_INT( { Module['print']('__syscall145 UNLINK, pathname: ' + Pointer_stringify($0) + ' removed.') }, pathname); + if (node->child) RETURN_ERRNO(EISDIR, "directory is not empty"); // Linux quirk: Return EISDIR error if not being able to delete a nonempty directory. + unlink_inode(node); - emscripten_dump_fs_root(); + return 0; } @@ -987,28 +959,31 @@ long __syscall33(int which, ...) // access const char *pathname = va_arg(vl, const char *); int mode = va_arg(vl, int); va_end(vl); + EM_ASM_INT({ Module['printErr']('access(pathname="' + Pointer_stringify($0) + '", mode=0' + ($1).toString(8) + ')') }, pathname, mode); + + int len = strlen(pathname); + if (len > MAX_PATHNAME_LENGTH) RETURN_ERRNO(ENAMETOOLONG, "pathname was too long"); + if (len == 0) RETURN_ERRNO(ENOENT, "pathname is empty"); - if ((mode & F_OK) && (mode & (R_OK | W_OK | X_OK))) - return -EINVAL; // mode was incorrectly specified. + if ((mode & F_OK) && (mode & (R_OK | W_OK | X_OK))) RETURN_ERRNO(EINVAL, "mode was incorrectly specified"); inode *node = find_inode(pathname); - if (!node) - { - EM_ASM_INT( { Module['printErr']('__syscall33 ACCESS, ENOENT: ' + Pointer_stringify($0) + ' does not exist.') }, pathname); - return -ENOENT; - } + if (!node) RETURN_ERRNO(ENOENT, "A component of pathname does not exist or is a dangling symbolic link"); - if ((mode & F_OK)) // Just test if file exists - { - return 0; - } + // TODO: RETURN_ERRNO(ENOENT, "A component of pathname does not exist or is a dangling symbolic link"); + + // Just testing if a file exists? + if ((mode & F_OK)) return 0; + + // TODO: RETURN_ERRNO(ELOOP, "Too many symbolic links were encountered in resolving pathname"); + // TODO: RETURN_ERRNO(EACCES, "search permission is denied for one of the directories in the path prefix of pathname"); + // TODO: RETURN_ERRNO(ENOTDIR, "A component used as a directory in pathname is not, in fact, a directory"); + // TODO: RETURN_ERRNO(EROFS, "Write permission was requested for a file on a read-only filesystem"); + + if ((mode & R_OK) && !(node->mode & 0444)) RETURN_ERRNO(EACCES, "Read access would be denied to the file"); + if ((mode & W_OK) && !(node->mode & 0222)) RETURN_ERRNO(EACCES, "Write access would be denied to the file"); + if ((mode & X_OK) && !(node->mode & 0111)) RETURN_ERRNO(EACCES, "Execute access would be denied to the file"); - if (((mode & R_OK) && !(node->mode & 0444)) - || ((mode & W_OK) && !(node->mode & 0222)) - || ((mode & X_OK) && !(node->mode & 0111))) - { - return -EACCES; - } return 0; } @@ -1020,23 +995,18 @@ long __syscall220(int which, ...) // getdents64 (get directory entries 64-bit) unsigned int fd = va_arg(vl, unsigned int); dirent *de = va_arg(vl, dirent*); unsigned int count = va_arg(vl, unsigned int); + va_end(vl); unsigned int dirents_size = count / sizeof(de); // The number of dirent structures that can fit into the provided buffer. dirent *de_end = de + dirents_size; - va_end(vl); + EM_ASM_INT({ Module['printErr']('getdents64(fd=' + $0 + ', de=0x' + ($1).toString(16) + ', count=' + $2 + ')') }, fd, de, count); FileDescriptor *desc = (FileDescriptor*)fd; - if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) - { - fprintf(stderr, "Invalid or closed file descriptor 0x%8X passed to close()!", (unsigned int)desc); - return -EBADF; // "fd isn't a valid open file descriptor." - } + if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) RETURN_ERRNO(EBADF, "Invalid file descriptor fd"); inode *node = desc->node; - if (!node) - { - assert(false); // TODO: Internal error handling? - return -1; - } + if (!node) RETURN_ERRNO(ENOENT, "No such directory"); + if (dirents_size == 0) RETURN_ERRNO(EINVAL, "Result buffer is too small"); + if (node->type != INODE_DIR) RETURN_ERRNO(ENOTDIR, "File descriptor does not refer to a directory"); inode *dotdot = node->parent ? node->parent : node; // In "/", the directory ".." refers to itself. @@ -1099,11 +1069,7 @@ long __syscall118(int which, ...) // fsync va_end(vl); FileDescriptor *desc = (FileDescriptor*)fd; - if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) - { - fprintf(stderr, "Invalid or closed file descriptor 0x%8X passed to close()!", (unsigned int)desc); - return -EBADF; // "fd isn't a valid open file descriptor." - } + if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) RETURN_ERRNO(EBADF, "fd isn't a valid open file descriptor"); inode *node = desc->node; if (!node) @@ -1122,22 +1088,17 @@ long __syscall41(int which, ...) // dup va_start(vl, which); unsigned int fd = va_arg(vl, unsigned int); va_end(vl); + EM_ASM_INT({ Module['printErr']('dup(fd=' + $0 + ')') }, fd); FileDescriptor *desc = (FileDescriptor*)fd; - if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) - { - fprintf(stderr, "Invalid or closed file descriptor 0x%8X passed to close()!", (unsigned int)desc); - return -EBADF; // "fd isn't a valid open file descriptor." - } + if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) RETURN_ERRNO(EBADF, "fd isn't a valid open file descriptor"); inode *node = desc->node; - if (!node) - { - assert(false); // TODO: Internal error handling? - return -1; - } + if (!node) RETURN_ERRNO(-1, "ASMFS internal error: file descriptor points to a nonexisting file"); + + // TODO: RETURN_ERRNO(EMFILE, "The per-process limit on the number of open file descriptors has been reached (see RLIMIT_NOFILE)"); - // TODO: Implementing dup() requires separating out file descriptors and file descriptions + EM_ASM({ Module['printErr']('TODO: dup() is a stub and not yet implemented') }); return 0; } @@ -1149,16 +1110,19 @@ long __syscall183(int which, ...) // getcwd char *buf = va_arg(vl, char *); size_t size = va_arg(vl, size_t); va_end(vl); + EM_ASM_INT({ Module['printErr']('getcwd(buf=0x' + $0 + ', size= ' + $1 + ')') }, buf, size); + + if (!buf && size > 0) RETURN_ERRNO(EFAULT, "buf points to a bad address"); + if (buf && size == 0) RETURN_ERRNO(EINVAL, "The size argument is zero and buf is not a null pointer"); inode *cwd = get_cwd(); - assert(cwd); + if (!cwd) RETURN_ERRNO(-1, "ASMFS internal error: no current working directory?!"); + // TODO: RETURN_ERRNO(ENOENT, "The current working directory has been unlinked"); + // TODO: RETURN_ERRNO(EACCES, "Permission to read or search a component of the filename was denied"); inode_abspath(cwd, buf, size); - - EM_ASM_INT( { Module['printErr']('getcwd: node "' + Pointer_stringify($0) + '" has abspath "' + Pointer_stringify($1) + '".') }, - cwd->name, buf); + if (strlen(buf) >= size-1) RETURN_ERRNO(ERANGE, "The size argument is less than the length of the absolute pathname of the working directory, including the terminating null byte. You need to allocate a bigger array and try again"); return 0; } - } // ~extern "C" From adee2959df1bb789beee94abf0f2817d428a2432 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Tue, 30 Aug 2016 22:21:18 +0300 Subject: [PATCH 49/74] Cleanup ASMFS fsync error handling. --- system/lib/fetch/asmfs.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/system/lib/fetch/asmfs.cpp b/system/lib/fetch/asmfs.cpp index ee2cea0635c93..c2328cb4d1891 100644 --- a/system/lib/fetch/asmfs.cpp +++ b/system/lib/fetch/asmfs.cpp @@ -1072,11 +1072,7 @@ long __syscall118(int which, ...) // fsync if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) RETURN_ERRNO(EBADF, "fd isn't a valid open file descriptor"); inode *node = desc->node; - if (!node) - { - assert(false); // TODO: Internal error handling? - return -1; - } + if (!node) RETURN_ERRNO(-1, "ASMFS internal error: file descriptor points to a non-file"); return 0; } From d37ddad65eabedfbc38a7062f4c6d05483cc5465 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Wed, 31 Aug 2016 01:26:35 +0300 Subject: [PATCH 50/74] Sort ASMFS syscalls in ascending order. --- system/lib/fetch/asmfs.cpp | 593 +++++++++--------- .../libc/musl/arch/emscripten/syscall_arch.h | 7 + 2 files changed, 304 insertions(+), 296 deletions(-) diff --git a/system/lib/fetch/asmfs.cpp b/system/lib/fetch/asmfs.cpp index c2328cb4d1891..16868e6522ad1 100644 --- a/system/lib/fetch/asmfs.cpp +++ b/system/lib/fetch/asmfs.cpp @@ -12,6 +12,7 @@ #include #include #include +#include "syscall_arch.h" extern "C" { @@ -401,6 +402,48 @@ void emscripten_dump_fs_root() return -errno; \ } while(0) +static char stdout_buffer[4096] = {}; +static int stdout_buffer_end = 0; +static char stderr_buffer[4096] = {}; +static int stderr_buffer_end = 0; + +static void print_stream(void *bytes, int numBytes, bool stdout) +{ + char *buffer = stdout ? stdout_buffer : stderr_buffer; + int &buffer_end = stdout ? stdout_buffer_end : stderr_buffer_end; + + memcpy(buffer + buffer_end, bytes, numBytes); + buffer_end += numBytes; + int new_buffer_start = 0; + for(int i = 0; i < buffer_end; ++i) + { + if (buffer[i] == '\n') + { + buffer[i] = 0; + EM_ASM_INT( { Module['print'](Pointer_stringify($0)) }, buffer+new_buffer_start); + new_buffer_start = i+1; + } + } + size_t new_buffer_size = buffer_end - new_buffer_start; + memmove(buffer, buffer + new_buffer_start, new_buffer_size); + buffer_end = new_buffer_size; +} + +// http://man7.org/linux/man-pages/man2/write.2.html +long __syscall4(int which, ...) // write +{ + va_list vl; + va_start(vl, which); + int fd = va_arg(vl, int); + void *buf = va_arg(vl, void *); + size_t count = va_arg(vl, size_t); + va_end(vl); + EM_ASM_INT({ Module['printErr']('write(fd=' + $0 + ', buf=0x' + ($1).toString(16) + ', count=' + $2 + ')') }, fd, buf, count); + + iovec io = { buf, count }; + return __syscall146(146, fd, &io, 1); +} + // http://man7.org/linux/man-pages/man2/open.2.html long __syscall5(int which, ...) // open { @@ -568,212 +611,46 @@ long __syscall6(int which, ...) // close return 0; } -// http://man7.org/linux/man-pages/man2/sysctl.2.html -long __syscall54(int which, ...) // sysctl -{ - EM_ASM( { Module['printErr']('sysctl() is ignored') }); - return 0; -} - -// http://man7.org/linux/man-pages/man2/llseek.2.html -// also useful: http://man7.org/linux/man-pages/man2/lseek.2.html -long __syscall140(int which, ...) // llseek -{ - va_list vl; - va_start(vl, which); - unsigned int fd = va_arg(vl, unsigned int); - unsigned long offset_high = va_arg(vl, unsigned long); - unsigned long offset_low = va_arg(vl, unsigned long); - off_t *result = va_arg(vl, off_t *); - unsigned int whence = va_arg(vl, unsigned int); - va_end(vl); - EM_ASM_INT({ Module['printErr']('llseek(fd=' + $0 + ', offset=0x' + (($1<<32)|$2) + ', result=0x' + ($3).toString(16) + ', whence=' + $4 + ')') }, - fd, offset_high, offset_low, result, whence); - - FileDescriptor *desc = (FileDescriptor*)fd; - if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) RETURN_ERRNO(EBADF, "fd isn't a valid open file descriptor"); - - if (desc->node->fetch) emscripten_fetch_wait(desc->node->fetch, INFINITY); - - int64_t offset = (int64_t)(((uint64_t)offset_high << 32) | (uint64_t)offset_low); - int64_t newPos; - switch(whence) - { - case SEEK_SET: newPos = offset; break; - case SEEK_CUR: newPos = desc->file_pos + offset; break; - case SEEK_END: newPos = (desc->node->fetch ? desc->node->fetch->numBytes : desc->node->size) + offset; break; - case 3/*SEEK_DATA*/: RETURN_ERRNO(EINVAL, "whence is invalid (sparse files, whence=SEEK_DATA, is not supported"); - case 4/*SEEK_HOLE*/: RETURN_ERRNO(EINVAL, "whence is invalid (sparse files, whence=SEEK_HOLE, is not supported"); - default: RETURN_ERRNO(EINVAL, "whence is invalid"); - } - if (newPos < 0) RETURN_ERRNO(EINVAL, "The resulting file offset would be negative"); - if (newPos > 0x7FFFFFFFLL) RETURN_ERRNO(EOVERFLOW, "The resulting file offset cannot be represented in an off_t"); - - desc->file_pos = newPos; - - if (result) *result = desc->file_pos; - return 0; -} - -// http://man7.org/linux/man-pages/man2/readv.2.html -long __syscall145(int which, ...) // readv +// http://man7.org/linux/man-pages/man2/unlink.2.html +long __syscall10(int which, ...) // unlink { va_list vl; va_start(vl, which); - int fd = va_arg(vl, int); - const iovec *iov = va_arg(vl, const iovec*); - int iovcnt = va_arg(vl, int); + const char *pathname = va_arg(vl, const char *); va_end(vl); - EM_ASM_INT({ Module['printErr']('readv(fd=' + $0 + ', iov=0x' + ($1).toString(16) + ', iovcnt=' + $2 + ')') }, fd, iov, iovcnt); - - FileDescriptor *desc = (FileDescriptor*)fd; - if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) RETURN_ERRNO(EBADF, "fd isn't a valid open file descriptor"); - - inode *node = desc->node; - if (!node) RETURN_ERRNO(-1, "ASMFS internal error: file descriptor points to a non-file"); - if (node->type == INODE_DIR) RETURN_ERRNO(EISDIR, "fd refers to a directory"); - if (node->type != INODE_FILE /* TODO: && node->type != socket */) RETURN_ERRNO(EINVAL, "fd is attached to an object which is unsuitable for reading"); - - // TODO: if (node->type == INODE_FILE && desc has O_NONBLOCK && read would block) RETURN_ERRNO(EAGAIN, "The file descriptor fd refers to a file other than a socket and has been marked nonblocking (O_NONBLOCK), and the read would block"); - // TODO: if (node->type == socket && desc has O_NONBLOCK && read would block) RETURN_ERRNO(EWOULDBLOCK, "The file descriptor fd refers to a socket and has been marked nonblocking (O_NONBLOCK), and the read would block"); - - if (node->fetch) emscripten_fetch_wait(node->fetch, INFINITY); - - if (node->size > 0 && !node->data && (!node->fetch || !node->fetch->data)) RETURN_ERRNO(-1, "ASMFS internal error: no file data available"); - if (iovcnt < 0) RETURN_ERRNO(EINVAL, "The vector count, iovcnt, is less than zero"); - - ssize_t total_read_amount = 0; - for(int i = 0; i < iovcnt; ++i) - { - ssize_t n = total_read_amount + iov[i].iov_len; - if (n < total_read_amount) RETURN_ERRNO(EINVAL, "The sum of the iov_len values overflows an ssize_t value"); - if (!iov[i].iov_base && iov[i].iov_len > 0) RETURN_ERRNO(EINVAL, "iov_len specifies a positive length buffer but iov_base is a null pointer"); - total_read_amount = n; - } - - size_t offset = desc->file_pos; - uint8_t *data = node->data ? node->data : (node->fetch ? (uint8_t *)node->fetch->data : 0); - for(int i = 0; i < iovcnt; ++i) - { - ssize_t dataLeft = node->size - offset; - if (dataLeft <= 0) break; - size_t bytesToCopy = (size_t)dataLeft < iov[i].iov_len ? dataLeft : iov[i].iov_len; - memcpy(iov[i].iov_base, &data[offset], bytesToCopy); - offset += bytesToCopy; - } - ssize_t numRead = offset - desc->file_pos; - desc->file_pos = offset; - return numRead; -} - -static char stdout_buffer[4096] = {}; -static int stdout_buffer_end = 0; -static char stderr_buffer[4096] = {}; -static int stderr_buffer_end = 0; + EM_ASM_INT({ Module['printErr']('unlink(pathname="' + Pointer_stringify($0) + '")') }, pathname); -static void print_stream(void *bytes, int numBytes, bool stdout) -{ - char *buffer = stdout ? stdout_buffer : stderr_buffer; - int &buffer_end = stdout ? stdout_buffer_end : stderr_buffer_end; + int len = strlen(pathname); + if (len > MAX_PATHNAME_LENGTH) RETURN_ERRNO(ENAMETOOLONG, "pathname was too long"); + if (len == 0) RETURN_ERRNO(ENOENT, "pathname is empty"); - memcpy(buffer + buffer_end, bytes, numBytes); - buffer_end += numBytes; - int new_buffer_start = 0; - for(int i = 0; i < buffer_end; ++i) - { - if (buffer[i] == '\n') - { - buffer[i] = 0; - EM_ASM_INT( { Module['print'](Pointer_stringify($0)) }, buffer+new_buffer_start); - new_buffer_start = i+1; - } - } - size_t new_buffer_size = buffer_end - new_buffer_start; - memmove(buffer, buffer + new_buffer_start, new_buffer_size); - buffer_end = new_buffer_size; -} + inode *node = find_inode(pathname); + if (!node) RETURN_ERRNO(ENOENT, "file does not exist"); -// http://man7.org/linux/man-pages/man2/writev.2.html -long __syscall146(int which, ...) // writev -{ - va_list vl; - va_start(vl, which); - int fd = va_arg(vl, int); - const iovec *iov = va_arg(vl, const iovec*); - int iovcnt = va_arg(vl, int); - va_end(vl); - EM_ASM_INT({ Module['printErr']('writev(fd=' + $0 + ', iov=0x' + ($1).toString(16) + ', iovcnt=' + $2 + ')') }, fd, iov, iovcnt); + inode *parent = node->parent; - FileDescriptor *desc = (FileDescriptor*)fd; - if (fd != 1/*stdout*/ && fd != 2/*stderr*/) // TODO: Resolve the hardcoding of stdin,stdout & stderr - { - if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) RETURN_ERRNO(EBADF, "fd isn't a valid open file descriptor"); - } + // TODO: RETURN_ERRNO(ENOENT, "A component in pathname does not exist or is a dangling symbolic link"); + // TODO: RETURN_ERRNO(ELOOP, "Too many symbolic links were encountered in translating pathname"); + // TODO: RETURN_ERRNO(EACCES, "one of the directories in the path prefix of pathname did not allow search permission"); - if (iovcnt < 0) RETURN_ERRNO(EINVAL, "The vector count, iovcnt, is less than zero"); + if (parent && !(parent->mode & 0222)) + RETURN_ERRNO(EACCES, "Write access to the directory containing pathname is not allowed for the process's effective UID"); - ssize_t total_write_amount = 0; - for(int i = 0; i < iovcnt; ++i) - { - ssize_t n = total_write_amount + iov[i].iov_len; - if (n < total_write_amount) RETURN_ERRNO(EINVAL, "The sum of the iov_len values overflows an ssize_t value"); - if (!iov[i].iov_base && iov[i].iov_len > 0) RETURN_ERRNO(EINVAL, "iov_len specifies a positive length buffer but iov_base is a null pointer"); - total_write_amount = n; - } + // TODO: RETURN_ERRNO(ENOTDIR, "A component used as a directory in pathname is not, in fact, a directory"); + // TODO: RETURN_ERRNO(EPERM, "The directory containing pathname has the sticky bit (S_ISVTX) set and the process's effective user ID is neither the user ID of the file to be deleted nor that of the directory containing it, and the process is not privileged"); + // TODO: RETURN_ERRNO(EROFS, "pathname refers to a file on a read-only filesystem"); - if (fd == 1/*stdout*/ || fd == 2/*stderr*/) + if (!(node->mode & 0222)) { - ssize_t bytesWritten = 0; - for(int i = 0; i < iovcnt; ++i) - { - print_stream(iov[i].iov_base, iov[i].iov_len, fd == 1); - bytesWritten += iov[i].iov_len; - } - return bytesWritten; + if (node->type == INODE_DIR) RETURN_ERRNO(EISDIR, "directory deletion not permitted"); // Linux quirk: Return EISDIR error for not having permission to delete a directory. + else RETURN_ERRNO(EPERM, "file deletion not permitted"); // but return EPERM error for no permission to delete a file. } - else - { - // Enlarge the file in memory to fit space for the new data - size_t newSize = desc->file_pos + total_write_amount; - inode *node = desc->node; - if (node->capacity < newSize) - { - size_t newCapacity = (newSize > (size_t)(node->capacity*1.25) ? newSize : (size_t)(node->capacity*1.25)); // Geometric increases in size for amortized O(1) behavior - uint8_t *newData = (uint8_t *)realloc(node->data, newCapacity); - if (!newData) - { - newData = (uint8_t *)malloc(newCapacity); - memcpy(newData, node->data, node->size); - // TODO: init gaps with zeroes. - free(node->data); - } - node->data = newData; - node->size = newSize; - node->capacity = newCapacity; - } - for(int i = 0; i < iovcnt; ++i) - { - memcpy((uint8_t*)node->data + desc->file_pos, iov[i].iov_base, iov[i].iov_len); - desc->file_pos += iov[i].iov_len; - } - } - return total_write_amount; -} + if (node->child) RETURN_ERRNO(EISDIR, "directory is not empty"); // Linux quirk: Return EISDIR error if not being able to delete a nonempty directory. -// http://man7.org/linux/man-pages/man2/write.2.html -long __syscall4(int which, ...) // write -{ - va_list vl; - va_start(vl, which); - int fd = va_arg(vl, int); - void *buf = va_arg(vl, void *); - size_t count = va_arg(vl, size_t); - va_end(vl); - EM_ASM_INT({ Module['printErr']('write(fd=' + $0 + ', buf=0x' + ($1).toString(16) + ', count=' + $2 + ')') }, fd, buf, count); + unlink_inode(node); - iovec io = { buf, count }; - return __syscall146(146, fd, &io, 1); + return 0; } // http://man7.org/linux/man-pages/man2/chdir.2.html @@ -836,6 +713,42 @@ long __syscall15(int which, ...) // chmod return 0; } +// http://man7.org/linux/man-pages/man2/faccessat.2.html +long __syscall33(int which, ...) // access +{ + va_list vl; + va_start(vl, which); + const char *pathname = va_arg(vl, const char *); + int mode = va_arg(vl, int); + va_end(vl); + EM_ASM_INT({ Module['printErr']('access(pathname="' + Pointer_stringify($0) + '", mode=0' + ($1).toString(8) + ')') }, pathname, mode); + + int len = strlen(pathname); + if (len > MAX_PATHNAME_LENGTH) RETURN_ERRNO(ENAMETOOLONG, "pathname was too long"); + if (len == 0) RETURN_ERRNO(ENOENT, "pathname is empty"); + + if ((mode & F_OK) && (mode & (R_OK | W_OK | X_OK))) RETURN_ERRNO(EINVAL, "mode was incorrectly specified"); + + inode *node = find_inode(pathname); + if (!node) RETURN_ERRNO(ENOENT, "A component of pathname does not exist or is a dangling symbolic link"); + + // TODO: RETURN_ERRNO(ENOENT, "A component of pathname does not exist or is a dangling symbolic link"); + + // Just testing if a file exists? + if ((mode & F_OK)) return 0; + + // TODO: RETURN_ERRNO(ELOOP, "Too many symbolic links were encountered in resolving pathname"); + // TODO: RETURN_ERRNO(EACCES, "search permission is denied for one of the directories in the path prefix of pathname"); + // TODO: RETURN_ERRNO(ENOTDIR, "A component used as a directory in pathname is not, in fact, a directory"); + // TODO: RETURN_ERRNO(EROFS, "Write permission was requested for a file on a read-only filesystem"); + + if ((mode & R_OK) && !(node->mode & 0444)) RETURN_ERRNO(EACCES, "Read access would be denied to the file"); + if ((mode & W_OK) && !(node->mode & 0222)) RETURN_ERRNO(EACCES, "Write access would be denied to the file"); + if ((mode & X_OK) && !(node->mode & 0111)) RETURN_ERRNO(EACCES, "Execute access would be denied to the file"); + + return 0; +} + // http://man7.org/linux/man-pages/man2/mkdir.2.html long __syscall39(int which, ...) // mkdir { @@ -909,80 +822,229 @@ long __syscall40(int which, ...) // rmdir return 0; } -// http://man7.org/linux/man-pages/man2/unlink.2.html -long __syscall10(int which, ...) // unlink +// http://man7.org/linux/man-pages/man2/dup.2.html +long __syscall41(int which, ...) // dup { va_list vl; va_start(vl, which); - const char *pathname = va_arg(vl, const char *); + unsigned int fd = va_arg(vl, unsigned int); va_end(vl); - EM_ASM_INT({ Module['printErr']('unlink(pathname="' + Pointer_stringify($0) + '")') }, pathname); + EM_ASM_INT({ Module['printErr']('dup(fd=' + $0 + ')') }, fd); - int len = strlen(pathname); - if (len > MAX_PATHNAME_LENGTH) RETURN_ERRNO(ENAMETOOLONG, "pathname was too long"); - if (len == 0) RETURN_ERRNO(ENOENT, "pathname is empty"); + FileDescriptor *desc = (FileDescriptor*)fd; + if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) RETURN_ERRNO(EBADF, "fd isn't a valid open file descriptor"); - inode *node = find_inode(pathname); - if (!node) RETURN_ERRNO(ENOENT, "file does not exist"); + inode *node = desc->node; + if (!node) RETURN_ERRNO(-1, "ASMFS internal error: file descriptor points to a nonexisting file"); - inode *parent = node->parent; + // TODO: RETURN_ERRNO(EMFILE, "The per-process limit on the number of open file descriptors has been reached (see RLIMIT_NOFILE)"); - // TODO: RETURN_ERRNO(ENOENT, "A component in pathname does not exist or is a dangling symbolic link"); - // TODO: RETURN_ERRNO(ELOOP, "Too many symbolic links were encountered in translating pathname"); - // TODO: RETURN_ERRNO(EACCES, "one of the directories in the path prefix of pathname did not allow search permission"); + EM_ASM({ Module['printErr']('TODO: dup() is a stub and not yet implemented') }); + return 0; +} - if (parent && !(parent->mode & 0222)) - RETURN_ERRNO(EACCES, "Write access to the directory containing pathname is not allowed for the process's effective UID"); +// http://man7.org/linux/man-pages/man2/sysctl.2.html +long __syscall54(int which, ...) // sysctl +{ + EM_ASM( { Module['printErr']('sysctl() is ignored') }); + return 0; +} - // TODO: RETURN_ERRNO(ENOTDIR, "A component used as a directory in pathname is not, in fact, a directory"); - // TODO: RETURN_ERRNO(EPERM, "The directory containing pathname has the sticky bit (S_ISVTX) set and the process's effective user ID is neither the user ID of the file to be deleted nor that of the directory containing it, and the process is not privileged"); - // TODO: RETURN_ERRNO(EROFS, "pathname refers to a file on a read-only filesystem"); +// http://man7.org/linux/man-pages/man2/fsync.2.html +long __syscall118(int which, ...) // fsync +{ + va_list vl; + va_start(vl, which); + unsigned int fd = va_arg(vl, unsigned int); + va_end(vl); - if (!(node->mode & 0222)) + FileDescriptor *desc = (FileDescriptor*)fd; + if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) RETURN_ERRNO(EBADF, "fd isn't a valid open file descriptor"); + + inode *node = desc->node; + if (!node) RETURN_ERRNO(-1, "ASMFS internal error: file descriptor points to a non-file"); + + return 0; +} + +// http://man7.org/linux/man-pages/man2/llseek.2.html +// also useful: http://man7.org/linux/man-pages/man2/lseek.2.html +long __syscall140(int which, ...) // llseek +{ + va_list vl; + va_start(vl, which); + unsigned int fd = va_arg(vl, unsigned int); + unsigned long offset_high = va_arg(vl, unsigned long); + unsigned long offset_low = va_arg(vl, unsigned long); + off_t *result = va_arg(vl, off_t *); + unsigned int whence = va_arg(vl, unsigned int); + va_end(vl); + EM_ASM_INT({ Module['printErr']('llseek(fd=' + $0 + ', offset=0x' + (($1<<32)|$2) + ', result=0x' + ($3).toString(16) + ', whence=' + $4 + ')') }, + fd, offset_high, offset_low, result, whence); + + FileDescriptor *desc = (FileDescriptor*)fd; + if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) RETURN_ERRNO(EBADF, "fd isn't a valid open file descriptor"); + + if (desc->node->fetch) emscripten_fetch_wait(desc->node->fetch, INFINITY); + + int64_t offset = (int64_t)(((uint64_t)offset_high << 32) | (uint64_t)offset_low); + int64_t newPos; + switch(whence) { - if (node->type == INODE_DIR) RETURN_ERRNO(EISDIR, "directory deletion not permitted"); // Linux quirk: Return EISDIR error for not having permission to delete a directory. - else RETURN_ERRNO(EPERM, "file deletion not permitted"); // but return EPERM error for no permission to delete a file. + case SEEK_SET: newPos = offset; break; + case SEEK_CUR: newPos = desc->file_pos + offset; break; + case SEEK_END: newPos = (desc->node->fetch ? desc->node->fetch->numBytes : desc->node->size) + offset; break; + case 3/*SEEK_DATA*/: RETURN_ERRNO(EINVAL, "whence is invalid (sparse files, whence=SEEK_DATA, is not supported"); + case 4/*SEEK_HOLE*/: RETURN_ERRNO(EINVAL, "whence is invalid (sparse files, whence=SEEK_HOLE, is not supported"); + default: RETURN_ERRNO(EINVAL, "whence is invalid"); } + if (newPos < 0) RETURN_ERRNO(EINVAL, "The resulting file offset would be negative"); + if (newPos > 0x7FFFFFFFLL) RETURN_ERRNO(EOVERFLOW, "The resulting file offset cannot be represented in an off_t"); - if (node->child) RETURN_ERRNO(EISDIR, "directory is not empty"); // Linux quirk: Return EISDIR error if not being able to delete a nonempty directory. - - unlink_inode(node); + desc->file_pos = newPos; + if (result) *result = desc->file_pos; return 0; } -// http://man7.org/linux/man-pages/man2/faccessat.2.html -long __syscall33(int which, ...) // access +// http://man7.org/linux/man-pages/man2/readv.2.html +long __syscall145(int which, ...) // readv { va_list vl; va_start(vl, which); - const char *pathname = va_arg(vl, const char *); - int mode = va_arg(vl, int); + int fd = va_arg(vl, int); + const iovec *iov = va_arg(vl, const iovec*); + int iovcnt = va_arg(vl, int); va_end(vl); - EM_ASM_INT({ Module['printErr']('access(pathname="' + Pointer_stringify($0) + '", mode=0' + ($1).toString(8) + ')') }, pathname, mode); + EM_ASM_INT({ Module['printErr']('readv(fd=' + $0 + ', iov=0x' + ($1).toString(16) + ', iovcnt=' + $2 + ')') }, fd, iov, iovcnt); - int len = strlen(pathname); - if (len > MAX_PATHNAME_LENGTH) RETURN_ERRNO(ENAMETOOLONG, "pathname was too long"); - if (len == 0) RETURN_ERRNO(ENOENT, "pathname is empty"); + FileDescriptor *desc = (FileDescriptor*)fd; + if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) RETURN_ERRNO(EBADF, "fd isn't a valid open file descriptor"); - if ((mode & F_OK) && (mode & (R_OK | W_OK | X_OK))) RETURN_ERRNO(EINVAL, "mode was incorrectly specified"); + inode *node = desc->node; + if (!node) RETURN_ERRNO(-1, "ASMFS internal error: file descriptor points to a non-file"); + if (node->type == INODE_DIR) RETURN_ERRNO(EISDIR, "fd refers to a directory"); + if (node->type != INODE_FILE /* TODO: && node->type != socket */) RETURN_ERRNO(EINVAL, "fd is attached to an object which is unsuitable for reading"); - inode *node = find_inode(pathname); - if (!node) RETURN_ERRNO(ENOENT, "A component of pathname does not exist or is a dangling symbolic link"); + // TODO: if (node->type == INODE_FILE && desc has O_NONBLOCK && read would block) RETURN_ERRNO(EAGAIN, "The file descriptor fd refers to a file other than a socket and has been marked nonblocking (O_NONBLOCK), and the read would block"); + // TODO: if (node->type == socket && desc has O_NONBLOCK && read would block) RETURN_ERRNO(EWOULDBLOCK, "The file descriptor fd refers to a socket and has been marked nonblocking (O_NONBLOCK), and the read would block"); - // TODO: RETURN_ERRNO(ENOENT, "A component of pathname does not exist or is a dangling symbolic link"); + if (node->fetch) emscripten_fetch_wait(node->fetch, INFINITY); - // Just testing if a file exists? - if ((mode & F_OK)) return 0; + if (node->size > 0 && !node->data && (!node->fetch || !node->fetch->data)) RETURN_ERRNO(-1, "ASMFS internal error: no file data available"); + if (iovcnt < 0) RETURN_ERRNO(EINVAL, "The vector count, iovcnt, is less than zero"); - // TODO: RETURN_ERRNO(ELOOP, "Too many symbolic links were encountered in resolving pathname"); - // TODO: RETURN_ERRNO(EACCES, "search permission is denied for one of the directories in the path prefix of pathname"); - // TODO: RETURN_ERRNO(ENOTDIR, "A component used as a directory in pathname is not, in fact, a directory"); - // TODO: RETURN_ERRNO(EROFS, "Write permission was requested for a file on a read-only filesystem"); + ssize_t total_read_amount = 0; + for(int i = 0; i < iovcnt; ++i) + { + ssize_t n = total_read_amount + iov[i].iov_len; + if (n < total_read_amount) RETURN_ERRNO(EINVAL, "The sum of the iov_len values overflows an ssize_t value"); + if (!iov[i].iov_base && iov[i].iov_len > 0) RETURN_ERRNO(EINVAL, "iov_len specifies a positive length buffer but iov_base is a null pointer"); + total_read_amount = n; + } - if ((mode & R_OK) && !(node->mode & 0444)) RETURN_ERRNO(EACCES, "Read access would be denied to the file"); - if ((mode & W_OK) && !(node->mode & 0222)) RETURN_ERRNO(EACCES, "Write access would be denied to the file"); - if ((mode & X_OK) && !(node->mode & 0111)) RETURN_ERRNO(EACCES, "Execute access would be denied to the file"); + size_t offset = desc->file_pos; + uint8_t *data = node->data ? node->data : (node->fetch ? (uint8_t *)node->fetch->data : 0); + for(int i = 0; i < iovcnt; ++i) + { + ssize_t dataLeft = node->size - offset; + if (dataLeft <= 0) break; + size_t bytesToCopy = (size_t)dataLeft < iov[i].iov_len ? dataLeft : iov[i].iov_len; + memcpy(iov[i].iov_base, &data[offset], bytesToCopy); + offset += bytesToCopy; + } + ssize_t numRead = offset - desc->file_pos; + desc->file_pos = offset; + return numRead; +} + +// http://man7.org/linux/man-pages/man2/writev.2.html +long __syscall146(int which, ...) // writev +{ + va_list vl; + va_start(vl, which); + int fd = va_arg(vl, int); + const iovec *iov = va_arg(vl, const iovec*); + int iovcnt = va_arg(vl, int); + va_end(vl); + EM_ASM_INT({ Module['printErr']('writev(fd=' + $0 + ', iov=0x' + ($1).toString(16) + ', iovcnt=' + $2 + ')') }, fd, iov, iovcnt); + + FileDescriptor *desc = (FileDescriptor*)fd; + if (fd != 1/*stdout*/ && fd != 2/*stderr*/) // TODO: Resolve the hardcoding of stdin,stdout & stderr + { + if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) RETURN_ERRNO(EBADF, "fd isn't a valid open file descriptor"); + } + + if (iovcnt < 0) RETURN_ERRNO(EINVAL, "The vector count, iovcnt, is less than zero"); + + ssize_t total_write_amount = 0; + for(int i = 0; i < iovcnt; ++i) + { + ssize_t n = total_write_amount + iov[i].iov_len; + if (n < total_write_amount) RETURN_ERRNO(EINVAL, "The sum of the iov_len values overflows an ssize_t value"); + if (!iov[i].iov_base && iov[i].iov_len > 0) RETURN_ERRNO(EINVAL, "iov_len specifies a positive length buffer but iov_base is a null pointer"); + total_write_amount = n; + } + + if (fd == 1/*stdout*/ || fd == 2/*stderr*/) + { + ssize_t bytesWritten = 0; + for(int i = 0; i < iovcnt; ++i) + { + print_stream(iov[i].iov_base, iov[i].iov_len, fd == 1); + bytesWritten += iov[i].iov_len; + } + return bytesWritten; + } + else + { + // Enlarge the file in memory to fit space for the new data + size_t newSize = desc->file_pos + total_write_amount; + inode *node = desc->node; + if (node->capacity < newSize) + { + size_t newCapacity = (newSize > (size_t)(node->capacity*1.25) ? newSize : (size_t)(node->capacity*1.25)); // Geometric increases in size for amortized O(1) behavior + uint8_t *newData = (uint8_t *)realloc(node->data, newCapacity); + if (!newData) + { + newData = (uint8_t *)malloc(newCapacity); + memcpy(newData, node->data, node->size); + // TODO: init gaps with zeroes. + free(node->data); + } + node->data = newData; + node->size = newSize; + node->capacity = newCapacity; + } + + for(int i = 0; i < iovcnt; ++i) + { + memcpy((uint8_t*)node->data + desc->file_pos, iov[i].iov_base, iov[i].iov_len); + desc->file_pos += iov[i].iov_len; + } + } + return total_write_amount; +} + +// http://man7.org/linux/man-pages/man2/getcwd.2.html +long __syscall183(int which, ...) // getcwd +{ + va_list vl; + va_start(vl, which); + char *buf = va_arg(vl, char *); + size_t size = va_arg(vl, size_t); + va_end(vl); + EM_ASM_INT({ Module['printErr']('getcwd(buf=0x' + $0 + ', size= ' + $1 + ')') }, buf, size); + + if (!buf && size > 0) RETURN_ERRNO(EFAULT, "buf points to a bad address"); + if (buf && size == 0) RETURN_ERRNO(EINVAL, "The size argument is zero and buf is not a null pointer"); + + inode *cwd = get_cwd(); + if (!cwd) RETURN_ERRNO(-1, "ASMFS internal error: no current working directory?!"); + // TODO: RETURN_ERRNO(ENOENT, "The current working directory has been unlinked"); + // TODO: RETURN_ERRNO(EACCES, "Permission to read or search a component of the filename was denied"); + inode_abspath(cwd, buf, size); + if (strlen(buf) >= size-1) RETURN_ERRNO(ERANGE, "The size argument is less than the length of the absolute pathname of the working directory, including the terminating null byte. You need to allocate a bigger array and try again"); return 0; } @@ -1060,65 +1122,4 @@ long __syscall220(int which, ...) // getdents64 (get directory entries 64-bit) return desc->file_pos - orig_file_pos; } -// http://man7.org/linux/man-pages/man2/fsync.2.html -long __syscall118(int which, ...) // fsync -{ - va_list vl; - va_start(vl, which); - unsigned int fd = va_arg(vl, unsigned int); - va_end(vl); - - FileDescriptor *desc = (FileDescriptor*)fd; - if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) RETURN_ERRNO(EBADF, "fd isn't a valid open file descriptor"); - - inode *node = desc->node; - if (!node) RETURN_ERRNO(-1, "ASMFS internal error: file descriptor points to a non-file"); - - return 0; -} - -// http://man7.org/linux/man-pages/man2/dup.2.html -long __syscall41(int which, ...) // dup -{ - va_list vl; - va_start(vl, which); - unsigned int fd = va_arg(vl, unsigned int); - va_end(vl); - EM_ASM_INT({ Module['printErr']('dup(fd=' + $0 + ')') }, fd); - - FileDescriptor *desc = (FileDescriptor*)fd; - if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) RETURN_ERRNO(EBADF, "fd isn't a valid open file descriptor"); - - inode *node = desc->node; - if (!node) RETURN_ERRNO(-1, "ASMFS internal error: file descriptor points to a nonexisting file"); - - // TODO: RETURN_ERRNO(EMFILE, "The per-process limit on the number of open file descriptors has been reached (see RLIMIT_NOFILE)"); - - EM_ASM({ Module['printErr']('TODO: dup() is a stub and not yet implemented') }); - return 0; -} - -// http://man7.org/linux/man-pages/man2/getcwd.2.html -long __syscall183(int which, ...) // getcwd -{ - va_list vl; - va_start(vl, which); - char *buf = va_arg(vl, char *); - size_t size = va_arg(vl, size_t); - va_end(vl); - EM_ASM_INT({ Module['printErr']('getcwd(buf=0x' + $0 + ', size= ' + $1 + ')') }, buf, size); - - if (!buf && size > 0) RETURN_ERRNO(EFAULT, "buf points to a bad address"); - if (buf && size == 0) RETURN_ERRNO(EINVAL, "The size argument is zero and buf is not a null pointer"); - - inode *cwd = get_cwd(); - if (!cwd) RETURN_ERRNO(-1, "ASMFS internal error: no current working directory?!"); - // TODO: RETURN_ERRNO(ENOENT, "The current working directory has been unlinked"); - // TODO: RETURN_ERRNO(EACCES, "Permission to read or search a component of the filename was denied"); - inode_abspath(cwd, buf, size); - if (strlen(buf) >= size-1) RETURN_ERRNO(ERANGE, "The size argument is less than the length of the absolute pathname of the working directory, including the terminating null byte. You need to allocate a bigger array and try again"); - - return 0; -} - } // ~extern "C" diff --git a/system/lib/libc/musl/arch/emscripten/syscall_arch.h b/system/lib/libc/musl/arch/emscripten/syscall_arch.h index b4d5fcfdc32b6..931599112aac7 100644 --- a/system/lib/libc/musl/arch/emscripten/syscall_arch.h +++ b/system/lib/libc/musl/arch/emscripten/syscall_arch.h @@ -26,6 +26,10 @@ // static syscalls. we must have one non-variadic argument before the rest due to ISO C. +#ifdef __cplusplus +extern "C" { +#endif + long __syscall1(int which, ...); long __syscall3(int which, ...); long __syscall4(int which, ...); @@ -138,3 +142,6 @@ long __syscall333(int which, ...); long __syscall334(int which, ...); long __syscall340(int which, ...); +#ifdef __cplusplus +} +#endif From 03c3c6e2e5a1863d89d05a1b4bdad8880f1cbab8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Wed, 31 Aug 2016 02:01:35 +0300 Subject: [PATCH 51/74] Add a few ASMFS stubs and mark down TODOs for more. --- system/lib/fetch/asmfs.cpp | 137 ++++++++++++++++++++++++++++++------- 1 file changed, 113 insertions(+), 24 deletions(-) diff --git a/system/lib/fetch/asmfs.cpp b/system/lib/fetch/asmfs.cpp index 16868e6522ad1..4d9a57f38938b 100644 --- a/system/lib/fetch/asmfs.cpp +++ b/system/lib/fetch/asmfs.cpp @@ -429,7 +429,20 @@ static void print_stream(void *bytes, int numBytes, bool stdout) buffer_end = new_buffer_size; } -// http://man7.org/linux/man-pages/man2/write.2.html +long __syscall3(int which, ...) // read +{ + va_list vl; + va_start(vl, which); + int fd = va_arg(vl, int); + void *buf = va_arg(vl, void *); + size_t count = va_arg(vl, size_t); + va_end(vl); + EM_ASM_INT({ Module['printErr']('read(fd=' + $0 + ', buf=0x' + ($1).toString(16) + ', count=' + $2 + ')') }, fd, buf, count); + + iovec io = { buf, count }; + return __syscall145(145/*readv*/, fd, &io, 1); +} + long __syscall4(int which, ...) // write { va_list vl; @@ -441,10 +454,9 @@ long __syscall4(int which, ...) // write EM_ASM_INT({ Module['printErr']('write(fd=' + $0 + ', buf=0x' + ($1).toString(16) + ', count=' + $2 + ')') }, fd, buf, count); iovec io = { buf, count }; - return __syscall146(146, fd, &io, 1); + return __syscall146(146/*writev*/, fd, &io, 1); } -// http://man7.org/linux/man-pages/man2/open.2.html long __syscall5(int which, ...) // open { va_list vl; @@ -588,7 +600,6 @@ long __syscall5(int which, ...) // open return (long)desc; } -// http://man7.org/linux/man-pages/man2/close.2.html long __syscall6(int which, ...) // close { va_list vl; @@ -611,7 +622,18 @@ long __syscall6(int which, ...) // close return 0; } -// http://man7.org/linux/man-pages/man2/unlink.2.html +long __syscall9(int which, ...) // link +{ + va_list vl; + va_start(vl, which); + const char *oldpath = va_arg(vl, const char *); + const char *newpath = va_arg(vl, const char *); + va_end(vl); + EM_ASM_INT({ Module['printErr']('link(oldpath="' + Pointer_stringify($0) + '", newpath="' + Pointer_stringify($1) + '")') }, oldpath, newpath); + + RETURN_ERRNO(ENOTSUP, "TODO: link() is a stub and not yet implemented in ASMFS"); +} + long __syscall10(int which, ...) // unlink { va_list vl; @@ -653,7 +675,6 @@ long __syscall10(int which, ...) // unlink return 0; } -// http://man7.org/linux/man-pages/man2/chdir.2.html long __syscall12(int which, ...) // chdir { va_list vl; @@ -681,7 +702,19 @@ long __syscall12(int which, ...) // chdir return 0; } -// http://man7.org/linux/man-pages/man2/chmod.2.html +long __syscall14(int which, ...) // mknod +{ + va_list vl; + va_start(vl, which); + const char *pathname = va_arg(vl, const char *); + mode_t mode = va_arg(vl, mode_t); + dev_t dev = va_arg(vl, dev_t); + va_end(vl); + EM_ASM_INT({ Module['printErr']('mknod(pathname="' + Pointer_stringify($0) + '", mode=0' + ($1).toString(8) + ', dev=' + $2 + ')') }, pathname, mode, dev); + + RETURN_ERRNO(ENOTSUP, "TODO: mknod() is a stub and not yet implemented in ASMFS"); +} + long __syscall15(int which, ...) // chmod { va_list vl; @@ -713,7 +746,6 @@ long __syscall15(int which, ...) // chmod return 0; } -// http://man7.org/linux/man-pages/man2/faccessat.2.html long __syscall33(int which, ...) // access { va_list vl; @@ -749,7 +781,16 @@ long __syscall33(int which, ...) // access return 0; } -// http://man7.org/linux/man-pages/man2/mkdir.2.html +long __syscall36(int which, ...) // sync +{ + EM_ASM({ Module['printErr']('sync()') }); + + // Spec mandates that "sync() is always successful". + return 0; +} + +// TODO: syscall38, int rename(const char *oldpath, const char *newpath); + long __syscall39(int which, ...) // mkdir { va_list vl; @@ -786,7 +827,6 @@ long __syscall39(int which, ...) // mkdir return 0; } -// http://man7.org/linux/man-pages/man2/rmdir.2.html long __syscall40(int which, ...) // rmdir { va_list vl; @@ -822,7 +862,6 @@ long __syscall40(int which, ...) // rmdir return 0; } -// http://man7.org/linux/man-pages/man2/dup.2.html long __syscall41(int which, ...) // dup { va_list vl; @@ -839,18 +878,32 @@ long __syscall41(int which, ...) // dup // TODO: RETURN_ERRNO(EMFILE, "The per-process limit on the number of open file descriptors has been reached (see RLIMIT_NOFILE)"); - EM_ASM({ Module['printErr']('TODO: dup() is a stub and not yet implemented') }); - return 0; + RETURN_ERRNO(ENOTSUP, "TODO: dup() is a stub and not yet implemented in ASMFS"); } -// http://man7.org/linux/man-pages/man2/sysctl.2.html -long __syscall54(int which, ...) // sysctl +// TODO: syscall42: int pipe(int pipefd[2]); + +long __syscall54(int which, ...) // ioctl/sysctl { - EM_ASM( { Module['printErr']('sysctl() is ignored') }); - return 0; + va_list vl; + va_start(vl, which); + int fd = va_arg(vl, int); + int request = va_arg(vl, int); + char *argp = va_arg(vl, char *); + va_end(vl); + EM_ASM_INT({ Module['printErr']('ioctl(fd=' + $0 + ', request=' + $1 + ', argp=0x' + $2 + ')') }, fd, request, argp); + RETURN_ERRNO(ENOTSUP, "TODO: ioctl() is a stub and not yet implemented in ASMFS"); } -// http://man7.org/linux/man-pages/man2/fsync.2.html +// TODO: syscall60: mode_t umask(mode_t mask); +// TODO: syscall63: dup2 +// TODO: syscall83: symlink +// TODO: syscall85: readlink +// TODO: syscall91: munmap +// TODO: syscall94: fchmod +// TODO: syscall102: socketcall + + long __syscall118(int which, ...) // fsync { va_list vl; @@ -867,8 +920,8 @@ long __syscall118(int which, ...) // fsync return 0; } -// http://man7.org/linux/man-pages/man2/llseek.2.html -// also useful: http://man7.org/linux/man-pages/man2/lseek.2.html +// TODO: syscall133: fchdir + long __syscall140(int which, ...) // llseek { va_list vl; @@ -907,7 +960,8 @@ long __syscall140(int which, ...) // llseek return 0; } -// http://man7.org/linux/man-pages/man2/readv.2.html +// TODO: syscall144 msync + long __syscall145(int which, ...) // readv { va_list vl; @@ -958,7 +1012,6 @@ long __syscall145(int which, ...) // readv return numRead; } -// http://man7.org/linux/man-pages/man2/writev.2.html long __syscall146(int which, ...) // writev { va_list vl; @@ -1026,7 +1079,12 @@ long __syscall146(int which, ...) // writev return total_write_amount; } -// http://man7.org/linux/man-pages/man2/getcwd.2.html +// TODO: syscall148: fdatasync +// TODO: syscall168: poll + +// TODO: syscall180: pread64 +// TODO: syscall181: pwrite64 + long __syscall183(int which, ...) // getcwd { va_list vl; @@ -1049,7 +1107,16 @@ long __syscall183(int which, ...) // getcwd return 0; } -// http://man7.org/linux/man-pages/man2/getdents.2.html +// TODO: syscall192: mmap2 +// TODO: syscall193: truncate64 +// TODO: syscall194: ftruncate64 +// TODO: syscall195: SYS_stat64 +// TODO: syscall196: SYS_lstat64 +// TODO: syscall197: SYS_fstat64 +// TODO: syscall198: lchown +// TODO: syscall207: fchown32 +// TODO: syscall212: chown32 + long __syscall220(int which, ...) // getdents64 (get directory entries 64-bit) { va_list vl; @@ -1122,4 +1189,26 @@ long __syscall220(int which, ...) // getdents64 (get directory entries 64-bit) return desc->file_pos - orig_file_pos; } +// TODO: syscall221: fcntl64 +// TODO: syscall268: statfs64 +// TODO: syscall269: fstatfs64 +// TODO: syscall295: openat +// TODO: syscall296: mkdirat +// TODO: syscall297: mknodat +// TODO: syscall298: fchownat +// TODO: syscall300: fstatat64 +// TODO: syscall301: unlinkat +// TODO: syscall302: renameat +// TODO: syscall303: linkat +// TODO: syscall304: symlinkat +// TODO: syscall305: readlinkat +// TODO: syscall306: fchmodat +// TODO: syscall307: faccessat +// TODO: syscall320: utimensat +// TODO: syscall324: fallocate +// TODO: syscall330: dup3 +// TODO: syscall331: pipe2 +// TODO: syscall333: preadv +// TODO: syscall334: pwritev + } // ~extern "C" From d76d047fb93e189fc93b1f43d42fa66bb4bc55ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Wed, 31 Aug 2016 02:04:22 +0300 Subject: [PATCH 52/74] Whitespace --- system/lib/fetch/asmfs.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/system/lib/fetch/asmfs.cpp b/system/lib/fetch/asmfs.cpp index 4d9a57f38938b..f5d991454926f 100644 --- a/system/lib/fetch/asmfs.cpp +++ b/system/lib/fetch/asmfs.cpp @@ -655,8 +655,7 @@ long __syscall10(int which, ...) // unlink // TODO: RETURN_ERRNO(ELOOP, "Too many symbolic links were encountered in translating pathname"); // TODO: RETURN_ERRNO(EACCES, "one of the directories in the path prefix of pathname did not allow search permission"); - if (parent && !(parent->mode & 0222)) - RETURN_ERRNO(EACCES, "Write access to the directory containing pathname is not allowed for the process's effective UID"); + if (parent && !(parent->mode & 0222)) RETURN_ERRNO(EACCES, "Write access to the directory containing pathname is not allowed for the process's effective UID"); // TODO: RETURN_ERRNO(ENOTDIR, "A component used as a directory in pathname is not, in fact, a directory"); // TODO: RETURN_ERRNO(EPERM, "The directory containing pathname has the sticky bit (S_ISVTX) set and the process's effective user ID is neither the user ID of the file to be deleted nor that of the directory containing it, and the process is not privileged"); From de434c207e95fc8ebed977f0f538d685c1ef3b43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Wed, 31 Aug 2016 17:06:28 +0300 Subject: [PATCH 53/74] Clean up find_inode() and make it report errors. --- system/lib/fetch/asmfs.cpp | 142 ++++++++++++++++++++++--------------- 1 file changed, 83 insertions(+), 59 deletions(-) diff --git a/system/lib/fetch/asmfs.cpp b/system/lib/fetch/asmfs.cpp index f5d991454926f..9a1c47aab619c 100644 --- a/system/lib/fetch/asmfs.cpp +++ b/system/lib/fetch/asmfs.cpp @@ -302,17 +302,28 @@ static inode *find_parent_inode(inode *root, const char *path) // Given a root inode of the filesystem and a path relative to it, e.g. "some/directory/dir_or_file", // returns the inode that corresponds to "dir_or_file", or 0 if it doesn't exist. // If the parameter out_closest_parent is specified, the closest (grand)parent node will be returned. -static inode *find_inode(inode *root, const char *path, inode **out_closest_parent = 0) +static inode *find_inode(inode *root, const char *path, int *out_errno) { - if (out_closest_parent) *out_closest_parent = root; - if (!root) + char rootName[PATH_MAX]; + inode_abspath(root, rootName, PATH_MAX); + EM_ASM_INT({ Module['printErr']('find_inode(root="' + Pointer_stringify($0) + '", path="' + Pointer_stringify($1) + '")') }, rootName, path); + +#define RETURN_NODE_AND_ERRNO(node, errno) do { *out_errno = (errno); return (node); } while(0) + assert(out_errno); // Passing in field to error is not optional. + + if (!root) RETURN_NODE_AND_ERRNO(0, ENOENT); + + // TODO: RETURN_ERRNO(ELOOP, "Too many symbolic links were encountered in translating pathname"); + // TODO: RETURN_ERRNO(EACCES, "one of the directories in the path prefix of pathname did not allow search permission"); + + // special-case finding empty string path "" or "/" returns the root searched in. + if (!path || path[0] == '\0') RETURN_NODE_AND_ERRNO(root, 0); + if (path[0] == '/' && path[1] == '\0') { - return 0; + if (root->type == INODE_DIR) RETURN_NODE_AND_ERRNO(root, 0); + else RETURN_NODE_AND_ERRNO(0, ENOTDIR); // "A component used as a directory in pathname is not, in fact, a directory" } - - if (path[0] == 0 - || (path[0] == '/' && path[1] == '\0')) - return root; // special-case finding empty string path "" or "/" returns the root searched in. + if (root->type != INODE_DIR) RETURN_NODE_AND_ERRNO(0, ENOTDIR); // "A component used as a directory in pathname is not, in fact, a directory" inode *node = root->child; while(node) @@ -322,9 +333,14 @@ static inode *find_inode(inode *root, const char *path, inode **out_closest_pare { // The directory name matches. path = child_path; - if (path[0] == '\0') return node; - if (path[0] == '/' && path[1] == '\0' /* && node is a directory*/) return node; - if (out_closest_parent) *out_closest_parent = node; + + // If we arrived to the end of the search, this is the node we were looking for. + if (path[0] == '\0') RETURN_NODE_AND_ERRNO(node, 0); + if (path[0] == '/' && path[1] == '\0') + { + if (node->type == INODE_DIR) RETURN_NODE_AND_ERRNO(node, 0); + else RETURN_NODE_AND_ERRNO(0, ENOTDIR); // "A component used as a directory in pathname is not, in fact, a directory" + } node = node->child; } else @@ -332,16 +348,16 @@ static inode *find_inode(inode *root, const char *path, inode **out_closest_pare node = node->sibling; } } - return 0; + RETURN_NODE_AND_ERRNO(0, ENOENT); } // Same as above, but the root node is deduced from 'path'. (either absolute if path starts with "/", or relative) -static inode *find_inode(const char *path, inode **out_closest_parent = 0) +static inode *find_inode(const char *path, int *out_errno) { inode *root; if (path[0] == '/') root = filesystem_root(), ++path; else root = get_cwd(); - return find_inode(root, path, out_closest_parent); + return find_inode(root, path, out_errno); } // Debug function that dumps out the filesystem tree to console. @@ -398,7 +414,7 @@ void emscripten_dump_fs_root() } #define RETURN_ERRNO(errno, error_reason) do { \ - EM_ASM_INT({ Module['printErr'](Pointer_stringify($0) + '() returned errno ' + #errno + ': ' + error_reason + '!')}, __FUNCTION__); \ + EM_ASM_INT({ Module['printErr'](Pointer_stringify($0) + '() returned errno ' + #errno + '(' + $1 + '): ' + error_reason + '!')}, __FUNCTION__, errno); \ return -errno; \ } while(0) @@ -503,17 +519,21 @@ long __syscall5(int which, ...) // open inode *root = (pathname[0] == '/') ? filesystem_root() : get_cwd(); const char *relpath = (pathname[0] == '/') ? pathname+1 : pathname; - inode *node = find_inode(root, relpath); + int err; + inode *node = find_inode(root, relpath, &err); + if (err == ENOTDIR) RETURN_ERRNO(ENOTDIR, "A component used as a directory in pathname is not, in fact, a directory"); + if (err == ELOOP) RETURN_ERRNO(ELOOP, "Too many symbolic links were encountered in resolving pathname"); + if (err == EACCES) RETURN_ERRNO(EACCES, "Search permission is denied for one of the directories in the path prefix of pathname"); + if (err && err != ENOENT) RETURN_ERRNO(err, "find_inode() error"); if (node) { if ((flags & O_DIRECTORY) && node->type != INODE_DIR) RETURN_ERRNO(ENOTDIR, "O_DIRECTORY was specified and pathname was not a directory"); if (!(node->mode & 0444)) RETURN_ERRNO(EACCES, "The requested access to the file is not allowed"); if ((flags & O_CREAT) && (flags & O_EXCL)) RETURN_ERRNO(EEXIST, "pathname already exists and O_CREAT and O_EXCL were used"); if (node->type == INODE_DIR && accessMode != O_RDONLY) RETURN_ERRNO(EISDIR, "pathname refers to a directory and the access requested involved writing (that is, O_WRONLY or O_RDWR is set)"); + if (node->fetch) emscripten_fetch_wait(node->fetch, INFINITY); } - if (node && node->fetch) emscripten_fetch_wait(node->fetch, INFINITY); - if ((flags & O_CREAT) || (flags & O_TRUNC) || (flags & O_EXCL)) { // Create a new empty file or truncate existing one. @@ -646,18 +666,20 @@ long __syscall10(int which, ...) // unlink if (len > MAX_PATHNAME_LENGTH) RETURN_ERRNO(ENAMETOOLONG, "pathname was too long"); if (len == 0) RETURN_ERRNO(ENOENT, "pathname is empty"); - inode *node = find_inode(pathname); + int err; + inode *node = find_inode(pathname, &err); + if (err == ENOTDIR) RETURN_ERRNO(ENOTDIR, "A component used as a directory in pathname is not, in fact, a directory"); + if (err == ELOOP) RETURN_ERRNO(ELOOP, "Too many symbolic links were encountered in translating pathname"); + if (err == EACCES) RETURN_ERRNO(EACCES, "One of the directories in the path prefix of pathname did not allow search permission"); + if (err == ENOENT) RETURN_ERRNO(ENOENT, "A component in pathname does not exist or is a dangling symbolic link"); + if (err) RETURN_ERRNO(err, "find_inode() error"); + if (!node) RETURN_ERRNO(ENOENT, "file does not exist"); inode *parent = node->parent; - // TODO: RETURN_ERRNO(ENOENT, "A component in pathname does not exist or is a dangling symbolic link"); - // TODO: RETURN_ERRNO(ELOOP, "Too many symbolic links were encountered in translating pathname"); - // TODO: RETURN_ERRNO(EACCES, "one of the directories in the path prefix of pathname did not allow search permission"); - if (parent && !(parent->mode & 0222)) RETURN_ERRNO(EACCES, "Write access to the directory containing pathname is not allowed for the process's effective UID"); - // TODO: RETURN_ERRNO(ENOTDIR, "A component used as a directory in pathname is not, in fact, a directory"); // TODO: RETURN_ERRNO(EPERM, "The directory containing pathname has the sticky bit (S_ISVTX) set and the process's effective user ID is neither the user ID of the file to be deleted nor that of the directory containing it, and the process is not privileged"); // TODO: RETURN_ERRNO(EROFS, "pathname refers to a file on a read-only filesystem"); @@ -686,16 +708,15 @@ long __syscall12(int which, ...) // chdir if (len > MAX_PATHNAME_LENGTH) RETURN_ERRNO(ENAMETOOLONG, "pathname was too long"); if (len == 0) RETURN_ERRNO(ENOENT, "pathname is empty"); - inode *node = find_inode(pathname); - - // TODO: if (no permissions to navigate the tree to the path) RETURN_ERRNO(EACCES, "Search permission is denied for one of the components of path"); - // TODO: if (too many symlinks) RETURN_ERRNO(ELOOP, "Too many symbolic links were encountered in resolving path"); - - // TODO: Ensure that this is checked for all components of the path + int err; + inode *node = find_inode(pathname, &err); + if (err == ENOTDIR) RETURN_ERRNO(ENOTDIR, "A component used as a directory in pathname is not, in fact, a directory"); + if (err == ELOOP) RETURN_ERRNO(ELOOP, "Too many symbolic links were encountered in resolving path"); + if (err == EACCES) RETURN_ERRNO(EACCES, "Search permission is denied for one of the components of path"); + if (err == ENOENT) RETURN_ERRNO(ENOENT, "Directory component in pathname does not exist or is a dangling symbolic link"); + if (err) RETURN_ERRNO(err, "find_inode() error"); if (!node) RETURN_ERRNO(ENOENT, "The directory specified in path does not exist"); - - // TODO: Ensure that this is checked for all components of the path - if (node->type != INODE_DIR) RETURN_ERRNO(ENOTDIR, "A component of path is not a directory"); + if (node->type != INODE_DIR) RETURN_ERRNO(ENOTDIR, "Path is not a directory"); set_cwd(node); return 0; @@ -707,7 +728,7 @@ long __syscall14(int which, ...) // mknod va_start(vl, which); const char *pathname = va_arg(vl, const char *); mode_t mode = va_arg(vl, mode_t); - dev_t dev = va_arg(vl, dev_t); + int dev = va_arg(vl, int); va_end(vl); EM_ASM_INT({ Module['printErr']('mknod(pathname="' + Pointer_stringify($0) + '", mode=0' + ($1).toString(8) + ', dev=' + $2 + ')') }, pathname, mode, dev); @@ -727,17 +748,15 @@ long __syscall15(int which, ...) // chmod if (len > MAX_PATHNAME_LENGTH) RETURN_ERRNO(ENAMETOOLONG, "pathname was too long"); if (len == 0) RETURN_ERRNO(ENOENT, "pathname is empty"); - inode *node = find_inode(pathname); - - // TODO: if (no permissions to navigate the tree to the path) RETURN_ERRNO(EACCES, "Search permission is denied on a component of the path prefix"); - // TODO: if (too many symlinks) RETURN_ERRNO(ELOOP, "Too many symbolic links were encountered in resolving pathname"); - - // TODO: Ensure that this is checked for all components of the path + int err; + inode *node = find_inode(pathname, &err); + if (err == ENOTDIR) RETURN_ERRNO(ENOTDIR, "A component used as a directory in pathname is not, in fact, a directory"); + if (err == ELOOP) RETURN_ERRNO(ELOOP, "Too many symbolic links were encountered in resolving pathname"); + if (err == EACCES) RETURN_ERRNO(EACCES, "Search permission is denied on a component of the path prefix"); + if (err == ENOENT) RETURN_ERRNO(ENOENT, "Directory component in pathname does not exist or is a dangling symbolic link"); + if (err) RETURN_ERRNO(err, "find_inode() error"); if (!node) RETURN_ERRNO(ENOENT, "The file does not exist"); - // TODO: Ensure that this is checked for all components of the path - if (node->type != INODE_DIR) RETURN_ERRNO(ENOTDIR, "A component of the path prefix is not a directory"); - // TODO: if (not allowed) RETURN_ERRNO(EPERM, "The effective UID does not match the owner of the file"); // TODO: read-only filesystems: if (fs is read-only) RETURN_ERRNO(EROFS, "The named file resides on a read-only filesystem"); @@ -760,17 +779,18 @@ long __syscall33(int which, ...) // access if ((mode & F_OK) && (mode & (R_OK | W_OK | X_OK))) RETURN_ERRNO(EINVAL, "mode was incorrectly specified"); - inode *node = find_inode(pathname); - if (!node) RETURN_ERRNO(ENOENT, "A component of pathname does not exist or is a dangling symbolic link"); - - // TODO: RETURN_ERRNO(ENOENT, "A component of pathname does not exist or is a dangling symbolic link"); + int err; + inode *node = find_inode(pathname, &err); + if (err == ENOTDIR) RETURN_ERRNO(ENOTDIR, "A component used as a directory in pathname is not, in fact, a directory"); + if (err == ELOOP) RETURN_ERRNO(ELOOP, "Too many symbolic links were encountered in resolving pathname"); + if (err == EACCES) RETURN_ERRNO(EACCES, "Search permission is denied for one of the directories in the path prefix of pathname"); + if (err == ENOENT) RETURN_ERRNO(ENOENT, "A component of pathname does not exist or is a dangling symbolic link"); + if (err) RETURN_ERRNO(err, "find_inode() error"); + if (!node) RETURN_ERRNO(ENOENT, "Pathname does not exist"); // Just testing if a file exists? if ((mode & F_OK)) return 0; - // TODO: RETURN_ERRNO(ELOOP, "Too many symbolic links were encountered in resolving pathname"); - // TODO: RETURN_ERRNO(EACCES, "search permission is denied for one of the directories in the path prefix of pathname"); - // TODO: RETURN_ERRNO(ENOTDIR, "A component used as a directory in pathname is not, in fact, a directory"); // TODO: RETURN_ERRNO(EROFS, "Write permission was requested for a file on a read-only filesystem"); if ((mode & R_OK) && !(node->mode & 0444)) RETURN_ERRNO(EACCES, "Read access would be denied to the file"); @@ -811,12 +831,15 @@ long __syscall39(int which, ...) // mkdir // TODO: if (component of path wasn't actually a directory) RETURN_ERRNO(ENOTDIR, "A component used as a directory in pathname is not, in fact, a directory"); - inode *existing = find_inode(parent_dir, basename_part(pathname)); + int err; + inode *existing = find_inode(parent_dir, basename_part(pathname), &err); + if (err == ENOTDIR) RETURN_ERRNO(ENOTDIR, "A component used as a directory in pathname is not, in fact, a directory"); + if (err == ELOOP) RETURN_ERRNO(ELOOP, "Too many symbolic links were encountered in resolving pathname"); + if (err == EACCES) RETURN_ERRNO(EACCES, "One of the directories in pathname did not allow search permission"); + if (err && err != ENOENT) RETURN_ERRNO(err, "find_inode() error"); if (existing) RETURN_ERRNO(EEXIST, "pathname already exists (not necessarily as a directory)"); if (!(parent_dir->mode & 0222)) RETURN_ERRNO(EACCES, "The parent directory does not allow write permission to the process"); - // TODO: if (too many symlinks when traversing path) RETURN_ERRNO(ELOOP, "Too many symbolic links were encountered in resolving pathname"); - // TODO: if (any parent dir in path doesn't have search permissions) RETURN_ERRNO(EACCES, "One of the directories in pathname did not allow search permission"); // TODO: read-only filesystems: if (fs is read-only) RETURN_ERRNO(EROFS, "Pathname refers to a file on a read-only filesystem"); inode *directory = create_inode(INODE_DIR); @@ -841,13 +864,14 @@ long __syscall40(int which, ...) // rmdir if (!strcmp(pathname, ".") || (len >= 2 && !strcmp(pathname+len-2, "/."))) RETURN_ERRNO(EINVAL, "pathname has . as last component"); if (!strcmp(pathname, "..") || (len >= 3 && !strcmp(pathname+len-3, "/.."))) RETURN_ERRNO(ENOTEMPTY, "pathname has .. as its final component"); - inode *node = find_inode(pathname); + int err; + inode *node = find_inode(pathname, &err); + if (err == ENOTDIR) RETURN_ERRNO(ENOTDIR, "A component used as a directory in pathname is not, in fact, a directory"); + if (err == ELOOP) RETURN_ERRNO(ELOOP, "Too many symbolic links were encountered in resolving pathname"); + if (err == EACCES) RETURN_ERRNO(EACCES, "one of the directories in the path prefix of pathname did not allow search permission"); + if (err == ENOENT) RETURN_ERRNO(ENOENT, "A directory component in pathname does not exist or is a dangling symbolic link"); + if (err) RETURN_ERRNO(err, "find_inode() error"); if (!node) RETURN_ERRNO(ENOENT, "directory does not exist"); - - // TODO: RETURN_ERRNO(ENOENT, "A directory component in pathname does not exist or is a dangling symbolic link"); - // TODO: RETURN_ERRNO(ELOOP, "Too many symbolic links were encountered in resolving pathname"); - // TODO: RETURN_ERRNO(EACCES, "one of the directories in the path prefix of pathname did not allow search permission"); - if (node == filesystem_root() || node == get_cwd()) RETURN_ERRNO(EBUSY, "pathname is currently in use by the system or some process that prevents its removal (pathname is currently used as a mount point or is the root directory of the calling process)"); if (node->parent && !(node->parent->mode & 0222)) RETURN_ERRNO(EACCES, "Write access to the directory containing pathname was not allowed"); if (node->type != INODE_DIR) RETURN_ERRNO(ENOTDIR, "pathname is not a directory"); From 6a70ce9af52707aa98f5342806f63b603c11d8e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Wed, 31 Aug 2016 17:12:35 +0300 Subject: [PATCH 54/74] Make create_inode carry the file mode. --- system/lib/fetch/asmfs.cpp | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/system/lib/fetch/asmfs.cpp b/system/lib/fetch/asmfs.cpp index 9a1c47aab619c..be1bbb1e557b4 100644 --- a/system/lib/fetch/asmfs.cpp +++ b/system/lib/fetch/asmfs.cpp @@ -55,13 +55,13 @@ struct FileDescriptor inode *node; }; -static inode *create_inode(INODE_TYPE type) +static inode *create_inode(INODE_TYPE type, int mode) { inode *i = (inode*)malloc(sizeof(inode)); memset(i, 0, sizeof(inode)); i->ctime = i->mtime = i->atime = time(0); i->type = type; - EM_ASM(Module['print']('create_inode allocated new inode object.')); + i->mode = mode; return i; } @@ -70,8 +70,7 @@ static inode *cwd_inode = 0; static inode *filesystem_root() { - static inode *root_node = create_inode(INODE_DIR); - root_node->mode = 0777; + static inode *root_node = create_inode(INODE_DIR, 0777); return root_node; } @@ -253,8 +252,7 @@ static inode *create_directory_hierarchy_for_file(inode *root, const char *path_ EM_ASM_INT( { Module['printErr']('basename_pos ' + Pointer_stringify($0) + ' .') }, basename_pos); while(*path_to_file && path_to_file < basename_pos) { - node = create_inode(INODE_DIR); - node->mode = mode; + node = create_inode(INODE_DIR, mode); path_to_file += strcpy_inodename(node->name, path_to_file) + 1; link_inode(node, root); EM_ASM_INT( { Module['print']('create_directory_hierarchy_for_file: created directory ' + Pointer_stringify($0) + ' under parent ' + Pointer_stringify($1) + '.') }, @@ -546,8 +544,7 @@ long __syscall5(int which, ...) // open else { inode *directory = create_directory_hierarchy_for_file(root, relpath, mode); - node = create_inode((flags & O_DIRECTORY) ? INODE_DIR : INODE_FILE); - node->mode = mode; + node = create_inode((flags & O_DIRECTORY) ? INODE_DIR : INODE_FILE, mode); strcpy(node->name, basename_part(pathname)); link_inode(node, directory); } @@ -590,8 +587,7 @@ long __syscall5(int which, ...) // open { // ... add it as a new entry to the fs. inode *directory = create_directory_hierarchy_for_file(root, relpath, mode); - node = create_inode((flags & O_DIRECTORY) ? INODE_DIR : INODE_FILE); - node->mode = mode; + node = create_inode((flags & O_DIRECTORY) ? INODE_DIR : INODE_FILE, mode); strcpy(node->name, basename_part(pathname)); node->fetch = fetch; link_inode(node, directory); @@ -842,9 +838,8 @@ long __syscall39(int which, ...) // mkdir // TODO: read-only filesystems: if (fs is read-only) RETURN_ERRNO(EROFS, "Pathname refers to a file on a read-only filesystem"); - inode *directory = create_inode(INODE_DIR); + inode *directory = create_inode(INODE_DIR, mode); strcpy(directory->name, basename_part(pathname)); - directory->mode = mode; link_inode(directory, parent_dir); return 0; } From 8070d06b4729fe5251baeebe4dd91f6fcf4c40a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Wed, 31 Aug 2016 17:27:07 +0300 Subject: [PATCH 55/74] Clean up find_parent_inode() --- system/lib/fetch/asmfs.cpp | 56 ++++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 21 deletions(-) diff --git a/system/lib/fetch/asmfs.cpp b/system/lib/fetch/asmfs.cpp index be1bbb1e557b4..ff6658bd0193c 100644 --- a/system/lib/fetch/asmfs.cpp +++ b/system/lib/fetch/asmfs.cpp @@ -155,10 +155,8 @@ static void link_inode(inode *node, inode *parent) static inode *find_predecessor_sibling(inode *node, inode *parent) { inode *child = parent->child; - if (child == node) - return 0; - while(child && child->sibling != node) - child = child->sibling; + if (child == node) return 0; + while(child && child->sibling != node) child = child->sibling; if (!child->sibling) return 0; return child; } @@ -178,8 +176,7 @@ static void unlink_inode(inode *node) else { inode *predecessor = find_predecessor_sibling(node, parent); - if (predecessor) - predecessor->sibling = node->sibling; + if (predecessor) predecessor->sibling = node->sibling; } node->parent = node->sibling = 0; } @@ -205,8 +202,7 @@ static const char *path_cmp(const char *s1, const char *s2) static int strcpy_inodename(char *dst, const char *path) { char *d = dst; - while(*path && *path != '/') - *dst++ = *path++; + while(*path && *path != '/') *dst++ = *path++; *dst = '\0'; return dst - d; } @@ -270,13 +266,28 @@ static inode *create_directory_hierarchy_for_file(const char *path, unsigned int return create_directory_hierarchy_for_file(root, path, mode); } -// Given a path to a file, finds the inode of the parent directory that contains the file, or 0 if the intermediate path doesn't exist. -static inode *find_parent_inode(inode *root, const char *path) +#define RETURN_NODE_AND_ERRNO(node, errno) do { *out_errno = (errno); return (node); } while(0) + +// Given a pathname to a file/directory, finds the inode of the directory that would contain the file/directory, or 0 if the intermediate path doesn't exist. +// Note that the file/directory pointed to by path does not need to exist, only its parent does. +static inode *find_parent_inode(inode *root, const char *path, int *out_errno) { - EM_ASM_INT( { Module['print']('find_parent_inode: inode: ' + Pointer_stringify($0) + ' path: ' + Pointer_stringify($1) + '.') }, - root ? root->name : "(null)", path); - if (!root) return 0; + char rootName[PATH_MAX]; + inode_abspath(root, rootName, PATH_MAX); + EM_ASM_INT({ Module['printErr']('find_parent_inode(root="' + Pointer_stringify($0) + '", path="' + Pointer_stringify($1) + '")') }, rootName, path); + + assert(out_errno); // Passing in error is mandatory. + + if (!root) RETURN_NODE_AND_ERRNO(0, ENOENT); + if (!path || path[0] == '\0') RETURN_NODE_AND_ERRNO(0, ENOENT); + if (path[0] == '/' && path[1] == '\0') RETURN_NODE_AND_ERRNO(0, ENOENT); + if (root->type != INODE_DIR) RETURN_NODE_AND_ERRNO(0, ENOTDIR); // "A component used as a directory in pathname is not, in fact, a directory" + + // TODO: RETURN_ERRNO(ELOOP, "Too many symbolic links were encountered in translating pathname"); + // TODO: RETURN_ERRNO(EACCES, "one of the directories in the path prefix of pathname did not allow search permission"); + const char *basename = basename_part(path); + if (path == basename) RETURN_NODE_AND_ERRNO(root, 0); inode *node = root->child; while(node) { @@ -285,16 +296,17 @@ static inode *find_parent_inode(inode *root, const char *path) { // The directory name matches. path = child_path; - if (path >= basename) return node; - if (!*path) return 0; + if (path >= basename) RETURN_NODE_AND_ERRNO(node, 0); + if (!*path) RETURN_NODE_AND_ERRNO(0, ENOENT); node = node->child; + if (node->type != INODE_DIR) RETURN_NODE_AND_ERRNO(0, ENOTDIR); // "A component used as a directory in pathname is not, in fact, a directory" } else { node = node->sibling; } } - return root; + RETURN_NODE_AND_ERRNO(0, ENOTDIR); // "A component used as a directory in pathname is not, in fact, a directory" } // Given a root inode of the filesystem and a path relative to it, e.g. "some/directory/dir_or_file", @@ -306,8 +318,7 @@ static inode *find_inode(inode *root, const char *path, int *out_errno) inode_abspath(root, rootName, PATH_MAX); EM_ASM_INT({ Module['printErr']('find_inode(root="' + Pointer_stringify($0) + '", path="' + Pointer_stringify($1) + '")') }, rootName, path); -#define RETURN_NODE_AND_ERRNO(node, errno) do { *out_errno = (errno); return (node); } while(0) - assert(out_errno); // Passing in field to error is not optional. + assert(out_errno); // Passing in error is mandatory. if (!root) RETURN_NODE_AND_ERRNO(0, ENOENT); @@ -821,13 +832,16 @@ long __syscall39(int which, ...) // mkdir inode *root = (pathname[0] == '/') ? filesystem_root() : get_cwd(); const char *relpath = (pathname[0] == '/') ? pathname+1 : pathname; - inode *parent_dir = find_parent_inode(root, relpath); - + int err; + inode *parent_dir = find_parent_inode(root, relpath, &err); + if (err == ENOTDIR) RETURN_ERRNO(ENOTDIR, "A component used as a directory in pathname is not, in fact, a directory"); + if (err == ELOOP) RETURN_ERRNO(ELOOP, "Too many symbolic links were encountered in resolving pathname"); + if (err == EACCES) RETURN_ERRNO(EACCES, "One of the directories in pathname did not allow search permission"); + if (err) RETURN_ERRNO(err, "find_inode() error"); if (!parent_dir) RETURN_ERRNO(ENOENT, "A directory component in pathname does not exist or is a dangling symbolic link"); // TODO: if (component of path wasn't actually a directory) RETURN_ERRNO(ENOTDIR, "A component used as a directory in pathname is not, in fact, a directory"); - int err; inode *existing = find_inode(parent_dir, basename_part(pathname), &err); if (err == ENOTDIR) RETURN_ERRNO(ENOTDIR, "A component used as a directory in pathname is not, in fact, a directory"); if (err == ELOOP) RETURN_ERRNO(ELOOP, "Too many symbolic links were encountered in resolving pathname"); From d88599d3767a4efede125ba29fc77c9b877ef92d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Mon, 5 Sep 2016 20:57:57 +0300 Subject: [PATCH 56/74] Improve fetch API handling of failure cases. --- src/Fetch.js | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/Fetch.js b/src/Fetch.js index 0ca9c8381b1f7..7e2b831a5a215 100644 --- a/src/Fetch.js +++ b/src/Fetch.js @@ -57,6 +57,8 @@ var Fetch = { Fetch.initFetchWorker(); removeRunDependency('library_fetch_init'); } +#else + removeRunDependency('library_fetch_init'); #endif }; var onerror = function() { @@ -92,6 +94,8 @@ var Fetch = { Module['printErr']('fetch-worker sent an error! ' + e.filename + ':' + e.lineno + ': ' + e.message); }; } +#else + addRunDependency('library_fetch_init'); #endif } } @@ -183,7 +187,7 @@ function __emscripten_fetch_load_cached_data(db, fetch, onsuccess, onerror) { } else { // Succeeded to load, but the load came back with the value of undefined, treat that as an error since we never store undefined in db. #if FETCH_DEBUG - console.error('fetch: Loaded file ' + pathStr + ' from IndexedDB, but it had 0 length!'); + console.error('fetch: File ' + pathStr + ' not found in IndexedDB'); #endif HEAPU16[fetch + 40 >> 1] = 4; // Mimic XHR readyState 4 === 'DONE: The operation is complete' HEAPU16[fetch + 42 >> 1] = 404; // Mimic XHR HTTP status code 404 "Not Found" @@ -344,7 +348,10 @@ function __emscripten_fetch_xhr(fetch, onsuccess, onerror, onprogress) { Fetch.setu64(fetch + 32, len);//{{{ makeSetValue('fetch', 32/*TODO:jsonify*/, 'len', 'i64')}}}; } HEAPU16[fetch + 40 >> 1] = xhr.readyState; - if (xhr.readyState === 4 && xhr.status === 0 && len > 0) xhr.status = 200; // If loading files from a source that does not give HTTP status code, assume success if we got data bytes + if (xhr.readyState === 4 && xhr.status === 0) { + if (len > 0) xhr.status = 200; // If loading files from a source that does not give HTTP status code, assume success if we got data bytes. + else xhr.status = 404; // Conversely, no data bytes is 404. + } HEAPU16[fetch + 42 >> 1] = xhr.status; // if (xhr.statusText) stringToUTF8(fetch + 44, xhr.statusText, 64); if (xhr.status == 200) { @@ -360,19 +367,26 @@ function __emscripten_fetch_xhr(fetch, onsuccess, onerror, onprogress) { } } xhr.onerror = function(e) { + var status = xhr.status; // XXX TODO: Overwriting xhr.status doesn't work here, so don't override anywhere else either. + if (xhr.readyState == 4 && status == 0) status = 404; // If no error recorded, pretend it was 404 Not Found. #if FETCH_DEBUG - console.error('fetch: xhr failed with error ' + e); + console.error('fetch: xhr error with readyState ' + xhr.readyState + ' and status ' + status); #endif + HEAPU32[fetch + 12 >> 2] = 0; + Fetch.setu64(fetch + 16, 0); + Fetch.setu64(fetch + 24, 0); + Fetch.setu64(fetch + 32, 0); + HEAPU16[fetch + 40 >> 1] = xhr.readyState; + HEAPU16[fetch + 42 >> 1] = status; if (onerror) onerror(fetch, xhr, e); } xhr.ontimeout = function(e) { #if FETCH_DEBUG - console.error('fetch: xhr timed out with error ' + e); + console.error('fetch: xhr timed out with readyState ' + xhr.readyState + ' and status ' + xhr.status); #endif if (onerror) onerror(fetch, xhr, e); } xhr.onprogress = function(e) { - console.log('fetch ptr ' + fetch + ', state ' + HEAPU32[fetch + 108 >> 2]); var ptrLen = (fetchAttrLoadToMemory && fetchAttrStreamData && xhr.response) ? xhr.response.byteLength : 0; var ptr = 0; if (fetchAttrLoadToMemory && fetchAttrStreamData) { From 83833bf55f908da229fd32f60e3ca62964e2b4ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Fri, 9 Sep 2016 15:30:01 +0300 Subject: [PATCH 57/74] Avoid using printf() on ASMFS implementation, stdout not yet properly implemented. --- site/source/docs/api_reference/index.rst | 4 ++++ src/Fetch.js | 4 ++-- system/lib/fetch/asmfs.cpp | 14 +++++++++++--- system/lib/fetch/emscripten_fetch.cpp | 1 + 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/site/source/docs/api_reference/index.rst b/site/source/docs/api_reference/index.rst index d94a8a23275e3..833040ef77129 100644 --- a/site/source/docs/api_reference/index.rst +++ b/site/source/docs/api_reference/index.rst @@ -18,6 +18,9 @@ This section lists Emscripten's public API, organised by header file. At a very - :ref:`Filesystem-API` (**library_fs.js**): APIs for managing file systems and synchronous file operations. +- :ref:`Fetch API`: + API for managing accesses to network XHR and IndexedDB. + - :ref:`Module`: Global JavaScript object that can be used to control code execution and access exported methods. @@ -41,6 +44,7 @@ This section lists Emscripten's public API, organised by header file. At a very html5.h preamble.js Filesystem-API + fetch module val.h bind.h diff --git a/src/Fetch.js b/src/Fetch.js index 7e2b831a5a215..5af76d60a021d 100644 --- a/src/Fetch.js +++ b/src/Fetch.js @@ -58,7 +58,7 @@ var Fetch = { removeRunDependency('library_fetch_init'); } #else - removeRunDependency('library_fetch_init'); + if (typeof ENVIRONMENT_IS_FETCH_WORKER === 'undefined' || !ENVIRONMENT_IS_FETCH_WORKER) removeRunDependency('library_fetch_init'); #endif }; var onerror = function() { @@ -95,7 +95,7 @@ var Fetch = { }; } #else - addRunDependency('library_fetch_init'); + if (typeof ENVIRONMENT_IS_FETCH_WORKER === 'undefined' || !ENVIRONMENT_IS_FETCH_WORKER) addRunDependency('library_fetch_init'); #endif } } diff --git a/system/lib/fetch/asmfs.cpp b/system/lib/fetch/asmfs.cpp index ff6658bd0193c..31df3b8b19524 100644 --- a/system/lib/fetch/asmfs.cpp +++ b/system/lib/fetch/asmfs.cpp @@ -372,7 +372,10 @@ static inode *find_inode(const char *path, int *out_errno) // Debug function that dumps out the filesystem tree to console. void emscripten_dump_fs_tree(inode *root, char *path) { - printf("%s:\n", path); + char str[256]; + sprintf(str,"%s:", path); + EM_ASM_INT( { Module['print'](Pointer_stringify($0)) }, str); + // Print out: // file mode | number of links | owner name | group name | file size in bytes | file last modified time | path name // which aligns with "ls -AFTRl" on console @@ -380,7 +383,7 @@ void emscripten_dump_fs_tree(inode *root, char *path) uint64_t totalSize = 0; while(child) { - printf("%c%c%c%c%c%c%c%c%c%c %d user%u group%u %u Jan 1 1970 %s%c\n", + sprintf(str,"%c%c%c%c%c%c%c%c%c%c %d user%u group%u %u Jan 1 1970 %s%c", child->type == INODE_DIR ? 'd' : '-', (child->mode & S_IRUSR) ? 'r' : '-', (child->mode & S_IWUSR) ? 'w' : '-', @@ -397,10 +400,14 @@ void emscripten_dump_fs_tree(inode *root, char *path) child->size, child->name, child->type == INODE_DIR ? '/' : ' '); + EM_ASM_INT( { Module['print'](Pointer_stringify($0)) }, str); + totalSize += child->size; child = child->sibling; } - printf("total %llu bytes\n\n", totalSize); + + sprintf(str, "total %llu bytes\n", totalSize); + EM_ASM_INT( { Module['print'](Pointer_stringify($0)) }, str); child = root->child; char *path_end = path + strlen(path); @@ -418,6 +425,7 @@ void emscripten_dump_fs_tree(inode *root, char *path) void emscripten_dump_fs_root() { + EM_ASM({ Module['printErr']('emscripten_dump_fs_root()') }); char path[PATH_MAX] = "/"; emscripten_dump_fs_tree(filesystem_root(), path); } diff --git a/system/lib/fetch/emscripten_fetch.cpp b/system/lib/fetch/emscripten_fetch.cpp index 803d6e9cd601b..ed5eed81cb140 100644 --- a/system/lib/fetch/emscripten_fetch.cpp +++ b/system/lib/fetch/emscripten_fetch.cpp @@ -107,6 +107,7 @@ EMSCRIPTEN_RESULT emscripten_fetch_wait(emscripten_fetch_t *fetch, double timeou // #ifdef FETCH_DEBUG EM_ASM({ console.log('fetch: emscripten_fetch_wait..') }); // #endif + // TODO: timeoutMsecs is currently ignored. Return EMSCRIPTEN_RESULT_TIMED_OUT on timeout. while(proxyState == 1/*sent to proxy worker*/) { emscripten_futex_wait(&fetch->__proxyState, proxyState, 100 /*TODO HACK:Sleep sometimes doesn't wake up?*/);//timeoutMsecs); From 6bee9aa7dcabf19afee96f7fb8b7287250532779 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Fri, 9 Sep 2016 15:30:35 +0300 Subject: [PATCH 58/74] Add initial Fetch API reference documentation and examples --- site/source/docs/api_reference/fetch.rst | 293 ++++++++++++++++++ tests/fetch/example_async_xhr_to_memory.cpp | 27 ++ ...mple_async_xhr_to_memory_via_indexeddb.cpp | 27 ++ tests/fetch/example_idb_delete.cpp | 22 ++ tests/fetch/example_idb_store.cpp | 40 +++ tests/fetch/example_stream_async_xhr.cpp | 43 +++ tests/fetch/example_sync_xhr_to_memory.cpp | 20 ++ tests/fetch/example_synchronous_fetch.cpp | 21 ++ .../fetch/example_waitable_xhr_to_memory.cpp | 29 ++ tests/fetch/example_xhr_progress.cpp | 39 +++ 10 files changed, 561 insertions(+) create mode 100644 site/source/docs/api_reference/fetch.rst create mode 100644 tests/fetch/example_async_xhr_to_memory.cpp create mode 100644 tests/fetch/example_async_xhr_to_memory_via_indexeddb.cpp create mode 100644 tests/fetch/example_idb_delete.cpp create mode 100644 tests/fetch/example_idb_store.cpp create mode 100644 tests/fetch/example_stream_async_xhr.cpp create mode 100644 tests/fetch/example_sync_xhr_to_memory.cpp create mode 100644 tests/fetch/example_synchronous_fetch.cpp create mode 100644 tests/fetch/example_waitable_xhr_to_memory.cpp create mode 100644 tests/fetch/example_xhr_progress.cpp diff --git a/site/source/docs/api_reference/fetch.rst b/site/source/docs/api_reference/fetch.rst new file mode 100644 index 0000000000000..8233569e9b532 --- /dev/null +++ b/site/source/docs/api_reference/fetch.rst @@ -0,0 +1,293 @@ +.. _fetch-api: + +========= +Fetch API +========= + +The Emscripten Fetch API allows native code to transfer files via XHR (HTTP GET, PUT, POST) from remote servers, and to persist the downloaded files locally in browser's IndexedDB storage, so that they can be reaccessed locally on subsequent page visits. The Fetch API is callable from multiple threads, and the network requests can be run either synchronously or asynchronously as desired. + +Introduction +============ + +The use of the Fetch API is quick to illustrate via an example. The following application downloads a file from a web server asynchronously to memory inside the application heap. + + .. code-block:: cpp + + #include + #include + #include + + void downloadSucceeded(emscripten_fetch_t *fetch) { + printf("Finished downloading %llu bytes from URL %s.\n", fetch->numBytes, fetch->url); + // The data is now available at fetch->data[0] through fetch->data[fetch->numBytes-1]; + emscripten_fetch_close(fetch); // Free data associated with the fetch. + } + + void downloadFailed(emscripten_fetch_t *fetch) { + printf("Downloading %s failed, HTTP failure status code: %d.\n", fetch->url, fetch->status); + emscripten_fetch_close(fetch); // Also free data on failure. + } + + int main() { + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.attributes = EMSCRIPTEN_FETCH_LOAD_TO_MEMORY; + attr.onsuccess = downloadSucceeded; + attr.onerror = downloadFailed; + emscripten_fetch(&attr, "myfile.dat"); + } + +If a relative pathname is specified to a call to emscripten_fetch, like in the above example, the XHR is performed relative to the href (URL) of the current page. Passing a fully qualified absolute URL allows downloading files across domains, however these are subject to `HTTP access control (CORS) rules `_. + +By default the Fetch API runs asynchronously, which means that the emscripten_fetch() function call returns immediately and the operation will continue to occur on the background. When the operation finishes, either the success or the failure callback will be invoked. + +Persisting data +=============== + +The XHR requests issued by the Fetch API are subject to the usual browser caching behavior. These caches are transient (temporary) so there is no guarantee that the data will persist in the cache for a given period of time. Additionally, if the files are somewhat large (multiple megabytes), browsers typically don't cache the downloads at all. + +To enable a more explicit control for persisting the downloaded files, the Fetch API interacts with the browser's IndexedDB API, which can load and store large data files that are available on subsequent visits to the page. To enable IndexedDB storage, pass the EMSCRIPTEN_FETCH_PERSIST_FILE flag in the fetch attributes: + + .. code-block:: cpp + + int main() { + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + ... + attr.attributes = EMSCRIPTEN_FETCH_LOAD_TO_MEMORY | EMSCRIPTEN_FETCH_PERSIST_FILE; + ... + emscripten_fetch(&attr, "myfile.dat"); + } + +For a full example, see the file tests/fetch/example_async_xhr_to_memory_via_indexeddb.cpp. + +Persisting data bytes from memory +--------------------------------- + +Sometimes it is useful to persist a range of bytes from application memory to IndexedDB (without having to perform any XHRs). This is possible with the Emscripten Fetch API by passing the special HTTP action verb "EM_IDB_STORE" to the Emscripten Fetch operation. + + .. code-block:: cpp + + void success(emscripten_fetch_t *fetch) { + printf("IDB store succeeded.\n"); + emscripten_fetch_close(fetch); + } + + void failure(emscripten_fetch_t *fetch) { + printf("IDB store failed.\n"); + emscripten_fetch_close(fetch); + } + + void persistFileToIndexedDB(const char *outputFilename, uint8_t *data, size_t numBytes) { + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "EM_IDB_STORE"); + attr.attributes = EMSCRIPTEN_FETCH_REPLACE | EMSCRIPTEN_FETCH_PERSIST_FILE; + attr.requestData = (char *)data; + attr.requestDataSize = numBytes; + attr.onsuccess = success; + attr.onerror = failure; + emscripten_fetch(&attr, outputFilename); + } + + int main() { + // Create data + uint8_t *data = (uint8_t*)malloc(10240); + srand(time(NULL)); + for(int i = 0; i < 10240; ++i) data[i] = (uint8_t)rand(); + + persistFileToIndexedDB("outputfile.dat", data, 10240); + } + +Deleting a file from IndexedDB +------------------------------ + +Files can be cleaned up from IndexedDB by using the HTTP action verb "EM_IDB_DELETE": + + .. code-block:: cpp + + void success(emscripten_fetch_t *fetch) { + printf("Deleting file from IDB succeeded.\n"); + emscripten_fetch_close(fetch); + } + + void failure(emscripten_fetch_t *fetch) { + printf("Deleting file from IDB failed.\n"); + emscripten_fetch_close(fetch); + } + + int main() { + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "EM_IDB_DELETE"); + emscripten_fetch(&attr, "filename_to_delete.dat"); + } + +Synchronous Fetches +=================== + +In some scenarios, it would be nice to be able to perform an XHR request or an IndexedDB file operation synchronously in the calling thread. This can make porting applications easier and simplify code flow by avoiding the need to pass a callback. + +All types of Emscripten Fetch API operations (XHRs, IndexedDB accesses) can be performed synchronously by passing the EMSCRIPTEN_FETCH_SYNCHRONOUS flag. When this flag is passed, the calling thread will block to sleep until the fetch operation finishes. See the following example. + + .. code-block:: cpp + + int main() { + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.attributes = EMSCRIPTEN_FETCH_LOAD_TO_MEMORY | EMSCRIPTEN_FETCH_SYNCHRONOUS; + emscripten_fetch_t *fetch = emscripten_fetch(&attr, "file.dat"); // Blocks here until the operation is complete. + if (fetch->status == 200) { + printf("Finished downloading %llu bytes from URL %s.\n", fetch->numBytes, fetch->url); + // The data is now available at fetch->data[0] through fetch->data[fetch->numBytes-1]; + } else { + printf("Downloading %s failed, HTTP failure status code: %d.\n", fetch->url, fetch->status); + } + emscripten_fetch_close(fetch); + } + +In the above code sample, the success and failure callback functions are not used. However, if specified, they will be synchronously called before emscripten_fetch() returns. + +.. note:: + + Synchronous Emscripten Fetch operations are subject to a number of restrictions, depending on which Emscripten build mode (linker flags) is used: + + - **No flags**: Only asynchronous Fetch operations are available. + - **--proxy-to-worker**: Synchronous Fetch operations are allowed for fetches that only do an XHR but do not interact with IndexedDB. + - **-s USE_PTHREADS=1**: Synchronous Fetch operations are available on pthreads, but not on the main thread. + - **--proxy-to-worker** + **-s USE_PTHREADS=1**: Synchronous Fetch operations are available both on the main thread and pthreads. + +Waitable Fetches +================ + +Emscripten Fetch operations can also run in a third mode, called a *waitable* fetch. Waitable fetches start off as asynchronous, but at any point after the fetch has started, the calling thread can issue a wait operation to either wait for the completion of the fetch, or to just poll whether the fetch operation has yet completed. The following code sample illustrates how this works. + + .. code-block:: cpp + + int main() { + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.attributes = EMSCRIPTEN_FETCH_LOAD_TO_MEMORY | EMSCRIPTEN_FETCH_WAITABLE; + emscripten_fetch_t *fetch = emscripten_fetch(&attr, "file.dat"); // Starts as asynchronous. + + EMSCRIPTEN_RESULT ret = EMSCRIPTEN_RESULT_TIMED_OUT; + while(ret == EMSCRIPTEN_RESULT_TIMED_OUT) { + /* possibly do some other work; */ + ret = emscripten_fetch_wait(fetch, 0/*milliseconds to wait, 0 to just poll, INFINITY=wait until completion*/); + } + // The operation has finished, safe to examine the fields of the 'fetch' pointer now. + + if (fetch->status == 200) { + printf("Finished downloading %llu bytes from URL %s.\n", fetch->numBytes, fetch->url); + // The data is now available at fetch->data[0] through fetch->data[fetch->numBytes-1]; + } else { + printf("Downloading %s failed, HTTP failure status code: %d.\n", fetch->url, fetch->status); + } + emscripten_fetch_close(fetch); + } + +Waitable fetches allow interleaving multiple tasks in one thread so that the issuing thread can perform some other work until the fetch completes. + +.. note:: + + Waitable fetches are available only in certain build modes: + + - **No flags** or **--proxy-to-worker**: Waitable fetches are not available. + - **-s USE_PTHREADS=1**: Waitable fetches are available on pthreads, but not on the main thread. + - **--proxy-to-worker** + **-s USE_PTHREADS=1**: Waitable fetches are available on all threads. + +Tracking Progress +==================== + +For robust fetch management, there are several fields available to track the status of an XHR. + +The onprogress callback is called whenever new data has been received. This allows one to measure the download speed and compute an ETA for completion. Additionally, the emscripten_fetch_t structure passes the XHR object fields readyState, status and statusText, which give information about the HTTP loading state of the request. + +The emscripten_fetch_attr_t object has a timeoutMsecs field which allows specifying a timeout duration for the transfer. Additionally, emscripten_fetch_close() can be called at any time for asynchronous and waitable fetches to abort the download. The following example illustrates these fields and the onprogress handler. + + .. code-block:: cpp + + void downloadProgress(emscripten_fetch_t *fetch) { + printf("Downloading %s.. %.2f%% complete.\n", + fetch->url, fetch->dataOffset * 100.0 / fetch->totalBytes); + } + + int main() { + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.attributes = EMSCRIPTEN_FETCH_LOAD_TO_MEMORY; + attr.onsuccess = downloadSucceeded; + attr.onprogress = downloadProgress; + attr.onerror = downloadFailed; + emscripten_fetch(&attr, "myfile.dat"); + } + +Managing Large Files +==================== + +Particular attention should be paid to the memory usage strategy of a fetch. Previous examples have all passed the EMSCRIPTEN_FETCH_LOAD_TO_MEMORY flag, which causes emscripten_fetch() to populate the downloaded file in full in memory in the onsuccess() callback. This is convenient when the whole file is to be immediately accessed afterwards, but for large files, this can be a wasteful strategy in terms of memory usage. If the file is very large, it might not even fit inside the application's heap area. + +The following subsections provide ways to manage large fetches in a memory efficient manner. + +Downloading directly to IndexedDB +--------------------------------- + +If an application wants to download a file for local access, but does not immediately need to use the file, e.g. when preloading data up front for later access, it is a good idea to avoid the EMSCRIPTEN_FETCH_LOAD_TO_MEMORY flag altogether, and only pass the EMSCRIPTEN_FETCH_PERSIST_FILE flag instead. This causes the fetch to download the file directly to IndexedDB, which avoids temporarily populating the file in memory after the download finishes. In this scenario, the onsuccess() handler will only report the total downloaded file size, but will not contain the data bytes to the file. + +Streaming Downloads +------------------- + +If the application does not need random seek access to the file, but is able to process the file in a streaming manner, it can use the EMSCRIPTEN_FETCH_STREAM_DATA flag to stream through the bytes in the file as they are downloaded. If this flag is passed, the downloaded data chunks are passed into the onprogress() callback in coherent file sequential order. See the following snippet for an example. + + .. code-block:: cpp + + void downloadProgress(emscripten_fetch_t *fetch) { + printf("Downloading %s.. %.2f%% complete. HTTP readyState: %d. HTTP status: %d.\n" + "HTTP statusText: %s. Received chunk [%llu, %llu[\n", + fetch->url, (fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes, + fetch->readyState, fetch->status, fetch->statusText, + fetch->dataOffset, fetch->dataOffset + fetch->numBytes); + + // Process the partial data stream fetch->data[0] thru fetch->data[fetch->numBytes-1] + // This buffer represents the file at offset fetch->dataOffset. + for(size_t i = 0; i < fetch->numBytes; ++i) + ; // Process fetch->data[i]; + } + + int main() { + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.attributes = EMSCRIPTEN_FETCH_STREAM_DATA; + attr.onsuccess = downloadSucceeded; + attr.onprogress = downloadProgress; + attr.onerror = downloadFailed; + attr.timeoutMsecs = 2*60; + emscripten_fetch(&attr, "myfile.dat"); + } + +In this case, the onsuccess() handler will not receive the final file buffer at all so memory usage will remain at a minimum. + +Byte Range Downloads +-------------------- + +TODO: Example about how to perform HTTP Byte Range Requests to obtain parts of files. + +TODO To Document +=============== + +TODO for the following tasks/examples: + + - Emscripten_fetch can be used to upload files to remote servers via HTTP PUT + - Emscripten_fetch_attr_t allows setting custom HTTP request headers (e.g. for cache control) + - Document HTTP simple auth + - Document how to populate to a certain filesystem path location in IndexedB, and fopen() afterwards + - Document overriddenMimeType + - Reference documentation of the individual fields + - Example about loading only from IndexedDB without XHRing + - Example about overriding an existing file in IndexedDB with a new XHR + - Example how to preload a whole filesystem + - Example how to persist content as gzipped to IndexedDB and decompress on load diff --git a/tests/fetch/example_async_xhr_to_memory.cpp b/tests/fetch/example_async_xhr_to_memory.cpp new file mode 100644 index 0000000000000..3037ee9f0da17 --- /dev/null +++ b/tests/fetch/example_async_xhr_to_memory.cpp @@ -0,0 +1,27 @@ +#include +#include +#include + +void downloadSucceeded(emscripten_fetch_t *fetch) +{ + printf("Finished downloading %llu bytes from URL %s.\n", fetch->numBytes, fetch->url); + // The data is now available at fetch->data[0] through fetch->data[fetch->numBytes-1]; + emscripten_fetch_close(fetch); // Free data associated with the fetch. +} + +void downloadFailed(emscripten_fetch_t *fetch) +{ + printf("Downloading %s failed, HTTP failure status code: %d.\n", fetch->url, fetch->status); + emscripten_fetch_close(fetch); // Also free data on failure. +} + +int main() +{ + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.attributes = EMSCRIPTEN_FETCH_LOAD_TO_MEMORY; + attr.onsuccess = downloadSucceeded; + attr.onerror = downloadFailed; + emscripten_fetch(&attr, "myfile.dat"); +} diff --git a/tests/fetch/example_async_xhr_to_memory_via_indexeddb.cpp b/tests/fetch/example_async_xhr_to_memory_via_indexeddb.cpp new file mode 100644 index 0000000000000..7dfab76e079a4 --- /dev/null +++ b/tests/fetch/example_async_xhr_to_memory_via_indexeddb.cpp @@ -0,0 +1,27 @@ +#include +#include +#include + +void downloadSucceeded(emscripten_fetch_t *fetch) +{ + printf("Finished downloading %llu bytes from URL %s.\n", fetch->numBytes, fetch->url); + // The data is now available at fetch->data[0] through fetch->data[fetch->numBytes-1]; + emscripten_fetch_close(fetch); // Free data associated with the fetch. +} + +void downloadFailed(emscripten_fetch_t *fetch) +{ + printf("Downloading %s failed, HTTP failure status code: %d.\n", fetch->url, fetch->status); + emscripten_fetch_close(fetch); // Also free data on failure. +} + +int main() +{ + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.attributes = EMSCRIPTEN_FETCH_LOAD_TO_MEMORY | EMSCRIPTEN_FETCH_PERSIST_FILE; + attr.onsuccess = downloadSucceeded; + attr.onerror = downloadFailed; + emscripten_fetch(&attr, "myfile.dat"); +} diff --git a/tests/fetch/example_idb_delete.cpp b/tests/fetch/example_idb_delete.cpp new file mode 100644 index 0000000000000..7d1c5b1659a0b --- /dev/null +++ b/tests/fetch/example_idb_delete.cpp @@ -0,0 +1,22 @@ +#include +#include +#include + +void success(emscripten_fetch_t *fetch) +{ + printf("Deleting file from IDB succeeded.\n"); + emscripten_fetch_close(fetch); +} + +void failure(emscripten_fetch_t *fetch) +{ + printf("Deleting file from IDB failed.\n"); + emscripten_fetch_close(fetch); +} + +int main() +{ + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "EM_IDB_DELETE"); + emscripten_fetch(&attr, "filename_to_delete.dat"); +} diff --git a/tests/fetch/example_idb_store.cpp b/tests/fetch/example_idb_store.cpp new file mode 100644 index 0000000000000..94e4946fe44b8 --- /dev/null +++ b/tests/fetch/example_idb_store.cpp @@ -0,0 +1,40 @@ +#include +#include +#include +#include +#include + +void success(emscripten_fetch_t *fetch) +{ + printf("IDB store succeeded.\n"); + emscripten_fetch_close(fetch); +} + +void failure(emscripten_fetch_t *fetch) +{ + printf("IDB store failed.\n"); + emscripten_fetch_close(fetch); +} + +void persistFileToIndexedDB(const char *outputFilename, uint8_t *data, size_t numBytes) +{ + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "EM_IDB_STORE"); + attr.attributes = EMSCRIPTEN_FETCH_REPLACE | EMSCRIPTEN_FETCH_PERSIST_FILE; + attr.requestData = (char *)data; + attr.requestDataSize = numBytes; + attr.onsuccess = success; + attr.onerror = failure; + emscripten_fetch(&attr, outputFilename); +} + +int main() +{ + // Create data + uint8_t *data = (uint8_t*)malloc(10240); + srand(time(NULL)); + for(int i = 0; i < 10240; ++i) data[i] = (uint8_t)rand(); + + persistFileToIndexedDB("outputfile.dat", data, 10240); +} diff --git a/tests/fetch/example_stream_async_xhr.cpp b/tests/fetch/example_stream_async_xhr.cpp new file mode 100644 index 0000000000000..ecad466d78b9a --- /dev/null +++ b/tests/fetch/example_stream_async_xhr.cpp @@ -0,0 +1,43 @@ +#include +#include +#include + +void downloadSucceeded(emscripten_fetch_t *fetch) +{ + printf("Finished downloading %llu bytes from URL %s.\n", fetch->numBytes, fetch->url); + // The data is now available at fetch->data[0] through fetch->data[fetch->numBytes-1]; + emscripten_fetch_close(fetch); // Free data associated with the fetch. +} + +void downloadFailed(emscripten_fetch_t *fetch) +{ + printf("Downloading %s failed, HTTP failure status code: %d.\n", fetch->url, fetch->status); + emscripten_fetch_close(fetch); // Also free data on failure. +} + +void downloadProgress(emscripten_fetch_t *fetch) +{ + printf("Downloading %s.. %.2f%% complete. HTTP readyState: %d. HTTP status: %d.\n" + "HTTP statusText: %s. Received chunk [%llu, %llu[\n", + fetch->url, (fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes, + fetch->readyState, fetch->status, fetch->statusText, + fetch->dataOffset, fetch->dataOffset + fetch->numBytes); + + // Process the partial data stream fetch->data[0] thru fetch->data[fetch->numBytes-1] + // This buffer represents the file at offset fetch->dataOffset. + for(size_t i = 0; i < fetch->numBytes; ++i) + ; // Process fetch->data[i]; +} + +int main() +{ + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.attributes = EMSCRIPTEN_FETCH_STREAM_DATA; + attr.onsuccess = downloadSucceeded; + attr.onprogress = downloadProgress; + attr.onerror = downloadFailed; + attr.timeoutMsecs = 2*60; + emscripten_fetch(&attr, "myfile.dat"); +} diff --git a/tests/fetch/example_sync_xhr_to_memory.cpp b/tests/fetch/example_sync_xhr_to_memory.cpp new file mode 100644 index 0000000000000..36f9aee47def9 --- /dev/null +++ b/tests/fetch/example_sync_xhr_to_memory.cpp @@ -0,0 +1,20 @@ +#include +#include +#include +#include + +int main() +{ + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.attributes = EMSCRIPTEN_FETCH_LOAD_TO_MEMORY | EMSCRIPTEN_FETCH_SYNCHRONOUS; + emscripten_fetch_t *fetch = emscripten_fetch(&attr, "file.dat"); // Blocks here until the operation is complete. + if (fetch->status == 200) { + printf("Finished downloading %llu bytes from URL %s.\n", fetch->numBytes, fetch->url); + // The data is now available at fetch->data[0] through fetch->data[fetch->numBytes-1]; + } else { + printf("Downloading %s failed, HTTP failure status code: %d.\n", fetch->url, fetch->status); + } + emscripten_fetch_close(fetch); +} diff --git a/tests/fetch/example_synchronous_fetch.cpp b/tests/fetch/example_synchronous_fetch.cpp new file mode 100644 index 0000000000000..bed4a8f6c29a9 --- /dev/null +++ b/tests/fetch/example_synchronous_fetch.cpp @@ -0,0 +1,21 @@ +#include +#include +#include +#include + +int main() +{ + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.attributes = EMSCRIPTEN_FETCH_LOAD_TO_MEMORY | EMSCRIPTEN_FETCH_SYNCHRONOUS; + emscripten_fetch_t *fetch = emscripten_fetch(&attr, "gears.png"); + + if (result == 0) { + result = 2; + printf("emscripten_fetch() failed to run synchronously!\n"); + } +#ifdef REPORT_RESULT + REPORT_RESULT(); +#endif +} diff --git a/tests/fetch/example_waitable_xhr_to_memory.cpp b/tests/fetch/example_waitable_xhr_to_memory.cpp new file mode 100644 index 0000000000000..8326fafda60ae --- /dev/null +++ b/tests/fetch/example_waitable_xhr_to_memory.cpp @@ -0,0 +1,29 @@ +#include +#include +#include +#include + +int main() +{ + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.attributes = EMSCRIPTEN_FETCH_LOAD_TO_MEMORY | EMSCRIPTEN_FETCH_WAITABLE; + emscripten_fetch_t *fetch = emscripten_fetch(&attr, "file.dat"); // Starts as asynchronous. + + EMSCRIPTEN_RESULT ret = EMSCRIPTEN_RESULT_TIMED_OUT; + while(ret == EMSCRIPTEN_RESULT_TIMED_OUT) + { + /* possibly do some other work; */ + ret = emscripten_fetch_wait(fetch, 0/*milliseconds to wait. 0 to just poll, INFINITY=wait until completion*/); + } + // The operation has finished, safe to examine the fields of the 'fetch' pointer now. + + if (fetch->status == 200) { + printf("Finished downloading %llu bytes from URL %s.\n", fetch->numBytes, fetch->url); + // The data is now available at fetch->data[0] through fetch->data[fetch->numBytes-1]; + } else { + printf("Downloading %s failed, HTTP failure status code: %d.\n", fetch->url, fetch->status); + } + emscripten_fetch_close(fetch); +} diff --git a/tests/fetch/example_xhr_progress.cpp b/tests/fetch/example_xhr_progress.cpp new file mode 100644 index 0000000000000..51d37f4f06487 --- /dev/null +++ b/tests/fetch/example_xhr_progress.cpp @@ -0,0 +1,39 @@ +#include +#include +#include + +void downloadSucceeded(emscripten_fetch_t *fetch) +{ + printf("Finished downloading %llu bytes from URL %s.\n", fetch->numBytes, fetch->url); + // The data is now available at fetch->data[0] through fetch->data[fetch->numBytes-1]; + emscripten_fetch_close(fetch); // Free data associated with the fetch. +} + +void downloadFailed(emscripten_fetch_t *fetch) +{ + printf("Downloading %s failed, HTTP failure status code: %d.\n", fetch->url, fetch->status); + emscripten_fetch_close(fetch); // Also free data on failure. +} + +void downloadProgress(emscripten_fetch_t *fetch) +{ + printf("Downloading %s.. %.2f%% complete. HTTP readyState: %d. HTTP status: %d.\n" + "HTTP statusText: %s. Received chunk [%llu, %llu[\n", + fetch->url, + (fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes, + fetch->readyState, fetch->status, fetch->statusText, + fetch->dataOffset, fetch->dataOffset + fetch->numBytes); +} + +int main() +{ + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.attributes = EMSCRIPTEN_FETCH_LOAD_TO_MEMORY; + attr.onsuccess = downloadSucceeded; + attr.onprogress = downloadProgress; + attr.onerror = downloadFailed; + attr.timeoutMsecs = 2*60; + emscripten_fetch(&attr, "myfile.dat"); +} From de4a44716eb4b6de1fb17c527aa2b5b6ae8d6984 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Fri, 9 Sep 2016 15:49:48 +0300 Subject: [PATCH 59/74] Emscripten_fetch() doc update --- site/source/docs/api_reference/fetch.rst | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/site/source/docs/api_reference/fetch.rst b/site/source/docs/api_reference/fetch.rst index 8233569e9b532..15ca1aef48431 100644 --- a/site/source/docs/api_reference/fetch.rst +++ b/site/source/docs/api_reference/fetch.rst @@ -274,20 +274,23 @@ In this case, the onsuccess() handler will not receive the final file buffer at Byte Range Downloads -------------------- +Large files can also be managed in smaller chunks by performing Byte Range downloads on them. This initiates an XHR or IndexedDB transfer that only fetches the desired subrange of the whole file. This is useful for example when a large package file contains multiple smaller ones at certain seek offsets, which can be dealt with separately. + TODO: Example about how to perform HTTP Byte Range Requests to obtain parts of files. TODO To Document =============== -TODO for the following tasks/examples: +Emscripten_fetch() supports the following operations as well, that need documenting: - Emscripten_fetch can be used to upload files to remote servers via HTTP PUT - Emscripten_fetch_attr_t allows setting custom HTTP request headers (e.g. for cache control) - - Document HTTP simple auth - - Document how to populate to a certain filesystem path location in IndexedB, and fopen() afterwards - - Document overriddenMimeType - - Reference documentation of the individual fields - - Example about loading only from IndexedDB without XHRing - - Example about overriding an existing file in IndexedDB with a new XHR - - Example how to preload a whole filesystem - - Example how to persist content as gzipped to IndexedDB and decompress on load + - Document HTTP simple auth fields in Emscripten_fetch_attr_t. + - Document how to populate to a certain filesystem path location in IndexedB, and e.g. fopen() it via ASMFS afterwards. + - Document overriddenMimeType attribute in Emscripten_fetch_attr_t. + - Reference documentation of the individual fields in Emscripten_fetch_attr_t, Emscripten_fetch_t and #defines. + - Example about loading only from IndexedDB without XHRing. + - Example about overriding an existing file in IndexedDB with a new XHR. + - Example how to preload a whole filesystem to IndexedDB for easy replacement of --preload-file. + - Example how to persist content as gzipped to IndexedDB and decompress on load. + - Example how to abort and resume partial transfers to IndexedDB. From 6fb7fc7d6f6a7029364b8a52b1dc56976d78c794 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Thu, 15 Sep 2016 20:15:42 +0300 Subject: [PATCH 60/74] Remove redundant include. Add fcntl-open test for ASMFS. --- system/lib/fetch/asmfs.cpp | 1 - tests/fcntl-open/src.c | 4 ++++ tests/test_browser.py | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/system/lib/fetch/asmfs.cpp b/system/lib/fetch/asmfs.cpp index 31df3b8b19524..28907672c1528 100644 --- a/system/lib/fetch/asmfs.cpp +++ b/system/lib/fetch/asmfs.cpp @@ -5,7 +5,6 @@ #include #include #define __NEED_struct_iovec -#include #include #include #include diff --git a/tests/fcntl-open/src.c b/tests/fcntl-open/src.c index fc5d5c764ab04..d0fa576908670 100644 --- a/tests/fcntl-open/src.c +++ b/tests/fcntl-open/src.c @@ -91,5 +91,9 @@ int main() { signal(SIGABRT, cleanup); setup(); test(); +#ifdef REPORT_RESULT + int result = 0; + REPORT_RESULT(); +#endif return EXIT_SUCCESS; } diff --git a/tests/test_browser.py b/tests/test_browser.py index d95dbf7d96701..206b58c8b5c02 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -3361,3 +3361,6 @@ def test_asmfs_unistd_access(self): def test_asmfs_unistd_unlink(self): # TODO: Once symlinks are supported, remove -DNO_SYMLINK=1 self.btest('unistd/unlink.c', expected='0', args=['-s', 'ASMFS=1', '-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1', '--proxy-to-worker', '-DNO_SYMLINK=1']) + + def test_asmfs_test_fcntl_open(self): + self.btest('fcntl-open/src.c', expected='0', args=['-s', 'ASMFS=1', '-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1', '--proxy-to-worker']) From e7aad14ab1afc748208d140b0a039fa82253d8f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Fri, 16 Sep 2016 20:36:39 +0300 Subject: [PATCH 61/74] Implement stat syscall. Improve open() behavior to make fcntl-open test pass. --- system/lib/fetch/asmfs.cpp | 131 ++++++++++++++++++++++++++++++++++--- tests/fcntl-open/src.c | 66 +++++++++++++++++-- 2 files changed, 183 insertions(+), 14 deletions(-) diff --git a/system/lib/fetch/asmfs.cpp b/system/lib/fetch/asmfs.cpp index 28907672c1528..153cdbe4cec73 100644 --- a/system/lib/fetch/asmfs.cpp +++ b/system/lib/fetch/asmfs.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include "syscall_arch.h" extern "C" { @@ -505,7 +506,16 @@ long __syscall5(int which, ...) // open if ((flags & O_ASYNC)) RETURN_ERRNO(ENOTSUP, "TODO: Opening files with O_ASYNC flag is not supported in ASMFS"); if ((flags & O_DIRECT)) RETURN_ERRNO(ENOTSUP, "TODO: O_DIRECT flag is not supported in ASMFS"); if ((flags & O_DSYNC)) RETURN_ERRNO(ENOTSUP, "TODO: O_DSYNC flag is not supported in ASMFS"); - if ((flags & O_EXCL) && !(flags & O_CREAT)) RETURN_ERRNO(EINVAL, "open() with O_EXCL flag needs to always be paired with O_CREAT"); // Spec says the behavior is undefined, we can just enforce it + + // Spec says that the result of O_EXCL without O_CREAT is undefined. + // We could enforce it as an error condition, as follows: +// if ((flags & O_EXCL) && !(flags & O_CREAT)) RETURN_ERRNO(EINVAL, "open() with O_EXCL flag needs to always be paired with O_CREAT"); + // However existing earlier unit tests in Emscripten expect that O_EXCL is simply ignored when O_CREAT was not passed. So do that for now. + if ((flags & O_EXCL) && !(flags & O_CREAT)) { + EM_ASM_INT({ Module['printErr']('warning: open(pathname="' + Pointer_stringify($0) + '", flags=0x' + ($1).toString(16) + ', mode=0' + ($2).toString(8) + ': flag O_EXCL should always be paired with O_CREAT. Ignoring O_EXCL)') }, pathname, flags, mode); + flags &= ~O_EXCL; + } + if ((flags & (O_NONBLOCK|O_NDELAY))) RETURN_ERRNO(ENOTSUP, "TODO: Opening files with O_NONBLOCK or O_NDELAY flags is not supported in ASMFS"); if ((flags & O_PATH)) RETURN_ERRNO(ENOTSUP, "TODO: Opening files with O_PATH flag is not supported in ASMFS"); if ((flags & O_SYNC)) RETURN_ERRNO(ENOTSUP, "TODO: Opening files with O_SYNC flag is not supported in ASMFS"); @@ -547,10 +557,11 @@ long __syscall5(int which, ...) // open if (!(node->mode & 0444)) RETURN_ERRNO(EACCES, "The requested access to the file is not allowed"); if ((flags & O_CREAT) && (flags & O_EXCL)) RETURN_ERRNO(EEXIST, "pathname already exists and O_CREAT and O_EXCL were used"); if (node->type == INODE_DIR && accessMode != O_RDONLY) RETURN_ERRNO(EISDIR, "pathname refers to a directory and the access requested involved writing (that is, O_WRONLY or O_RDWR is set)"); + if (node->type == INODE_DIR && (flags & O_TRUNC)) RETURN_ERRNO(EISDIR, "pathname refers to a directory and the access flags specified invalid flag O_TRUNC"); if (node->fetch) emscripten_fetch_wait(node->fetch, INFINITY); } - if ((flags & O_CREAT) || (flags & O_TRUNC) || (flags & O_EXCL)) + if ((flags & O_CREAT) && ((flags & O_TRUNC) || (flags & O_EXCL))) { // Create a new empty file or truncate existing one. if (node) @@ -559,7 +570,7 @@ long __syscall5(int which, ...) // open node->fetch = 0; node->size = 0; } - else + else if ((flags & O_CREAT)) { inode *directory = create_directory_hierarchy_for_file(root, relpath, mode); node = create_inode((flags & O_DIRECTORY) ? INODE_DIR : INODE_FILE, mode); @@ -567,7 +578,7 @@ long __syscall5(int which, ...) // open link_inode(node, directory); } } - else if (!node || (!node->fetch && !node->data)) + else if (!node || (node->type == INODE_FILE && !node->fetch && !node->data)) { emscripten_fetch_t *fetch = 0; if (!(flags & O_DIRECTORY) && accessMode != O_WRONLY) @@ -584,7 +595,7 @@ long __syscall5(int which, ...) // open // case synchronous_fopen: emscripten_fetch_wait(fetch, INFINITY); - if (fetch->status != 200 || fetch->totalBytes == 0) + if (!(flags & O_CREAT) && (fetch->status != 200 || fetch->totalBytes == 0)) { emscripten_fetch_close(fetch); RETURN_ERRNO(ENOENT, "O_CREAT is not set and the named file does not exist (attempted emscripten_fetch() XHR to download)"); @@ -1149,9 +1160,113 @@ long __syscall183(int which, ...) // getcwd // TODO: syscall192: mmap2 // TODO: syscall193: truncate64 // TODO: syscall194: ftruncate64 -// TODO: syscall195: SYS_stat64 -// TODO: syscall196: SYS_lstat64 -// TODO: syscall197: SYS_fstat64 + +static long __stat64(inode *node, struct stat *buf) +{ + buf->st_dev = 1; // ID of device containing file: Hardcode 1 for now, no meaning at the moment for Emscripten. + buf->st_ino = (ino_t)node; // TODO: This needs to be an inode ID number proper. + buf->st_mode = node->mode; + switch(node->type) { + case INODE_DIR: buf->st_mode |= S_IFDIR; break; + case INODE_FILE: buf->st_mode |= S_IFREG; break; // Regular file + /* TODO: + case socket: buf->st_mode |= S_IFSOCK; break; + case symlink: buf->st_mode |= S_IFLNK; break; + case block device: buf->st_mode |= S_IFBLK; break; + case character device: buf->st_mode |= S_IFCHR; break; + case FIFO: buf->st_mode |= S_IFIFO; break; + */ + } + buf->st_nlink = 1; // The number of hard links. TODO: Use this for real when links are supported. + buf->st_uid = node->uid; + buf->st_gid = node->gid; + buf->st_rdev = 1; // Device ID (if special file) No meaning right now for Emscripten. + buf->st_size = node->fetch ? node->fetch->totalBytes : 0; + if (node->size > buf->st_size) buf->st_size = node->size; + buf->st_blocks = (buf->st_size + 511) / 512; // The syscall docs state this is hardcoded to # of 512 byte blocks. + buf->st_blksize = 1024*1024; // Specifies the preferred blocksize for efficient disk I/O. + buf->st_atim.tv_sec = node->atime; + buf->st_mtim.tv_sec = node->mtime; + buf->st_ctim.tv_sec = node->ctime; + return 0; +} + +long __syscall195(int which, ...) // SYS_stat64 +{ + va_list vl; + va_start(vl, which); + const char *pathname = va_arg(vl, const char *); + struct stat *buf = va_arg(vl, struct stat *); + va_end(vl); + EM_ASM_INT({ Module['printErr']('SYS_stat64(pathname="' + Pointer_stringify($0) + '", buf=0x' + ($1).toString(16) + ')') }, pathname, buf); + + int len = strlen(pathname); + if (len > MAX_PATHNAME_LENGTH) RETURN_ERRNO(ENAMETOOLONG, "pathname was too long"); + if (len == 0) RETURN_ERRNO(ENOENT, "pathname is empty"); + + // Find if this file exists already in the filesystem? + inode *root = (pathname[0] == '/') ? filesystem_root() : get_cwd(); + const char *relpath = (pathname[0] == '/') ? pathname+1 : pathname; + + int err; + inode *node = find_inode(root, relpath, &err); + if (err == ENOTDIR) RETURN_ERRNO(ENOTDIR, "A component of the path prefix of pathname is not a directory"); + if (err == ELOOP) RETURN_ERRNO(ELOOP, "Too many symbolic links encountered while traversing the path"); + if (err == EACCES) RETURN_ERRNO(EACCES, "Search permission is denied for one of the directories in the path prefix of pathname"); + if (err && err != ENOENT) RETURN_ERRNO(err, "find_inode() error"); + if (err == ENOENT || !node) RETURN_ERRNO(ENOENT, "A component of pathname does not exist"); + + return __stat64(node, buf); +} + +long __syscall196(int which, ...) // SYS_lstat64 +{ + va_list vl; + va_start(vl, which); + const char *pathname = va_arg(vl, const char *); + struct stat *buf = va_arg(vl, struct stat *); + va_end(vl); + EM_ASM_INT({ Module['printErr']('SYS_lstat64(pathname="' + Pointer_stringify($0) + '", buf=0x' + ($1).toString(16) + ')') }, pathname, buf); + + int len = strlen(pathname); + if (len > MAX_PATHNAME_LENGTH) RETURN_ERRNO(ENAMETOOLONG, "pathname was too long"); + if (len == 0) RETURN_ERRNO(ENOENT, "pathname is empty"); + + // Find if this file exists already in the filesystem? + inode *root = (pathname[0] == '/') ? filesystem_root() : get_cwd(); + const char *relpath = (pathname[0] == '/') ? pathname+1 : pathname; + + // TODO: When symbolic links are implemented, make this return info about the symlink itself and not the file it points to. + int err; + inode *node = find_inode(root, relpath, &err); + if (err == ENOTDIR) RETURN_ERRNO(ENOTDIR, "A component of the path prefix of pathname is not a directory"); + if (err == ELOOP) RETURN_ERRNO(ELOOP, "Too many symbolic links encountered while traversing the path"); + if (err == EACCES) RETURN_ERRNO(EACCES, "Search permission is denied for one of the directories in the path prefix of pathname"); + if (err && err != ENOENT) RETURN_ERRNO(err, "find_inode() error"); + if (err == ENOENT || !node) RETURN_ERRNO(ENOENT, "A component of pathname does not exist"); + + return __stat64(node, buf); +} + +long __syscall197(int which, ...) // SYS_fstat64 +{ + va_list vl; + va_start(vl, which); + int fd = va_arg(vl, int); + struct stat *buf = va_arg(vl, struct stat *); + va_end(vl); + EM_ASM_INT({ Module['printErr']('SYS_fstat64(fd="' + Pointer_stringify($0) + '", buf=0x' + ($1).toString(16) + ')') }, fd, buf); + + FileDescriptor *desc = (FileDescriptor*)fd; + if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) RETURN_ERRNO(EBADF, "fd isn't a valid open file descriptor"); + + int err; + inode *node = desc->node; + if (!node) RETURN_ERRNO(ENOENT, "A component of pathname does not exist"); + + return __stat64(node, buf); +} + // TODO: syscall198: lchown // TODO: syscall207: fchown32 // TODO: syscall212: chown32 diff --git a/tests/fcntl-open/src.c b/tests/fcntl-open/src.c index d0fa576908670..116f537a2387c 100644 --- a/tests/fcntl-open/src.c +++ b/tests/fcntl-open/src.c @@ -13,16 +13,20 @@ char nonexistent_name[] = "noexist-##"; void create_file(const char *path, const char *buffer, int mode) { int fd = open(path, O_WRONLY | O_CREAT | O_EXCL, mode); assert(fd >= 0); + assert(!errno); int err = write(fd, buffer, sizeof(char) * strlen(buffer)); assert(err == (sizeof(char) * strlen(buffer))); + assert(!errno); close(fd); + assert(!errno); } void setup() { create_file("test-file", "abcdef", 0777); mkdir("test-folder", 0777); + assert(!errno); } void cleanup() { @@ -35,7 +39,9 @@ void cleanup() { unlink(nonexistent_name); } } + errno = 0; unlink("creat-me"); + assert(!errno); } void test() { @@ -51,19 +57,48 @@ void test() { if (j & 0x8) flags |= O_APPEND; printf("EXISTING FILE %d,%d\n", i, j); - printf("success: %d\n", open("test-file", flags, 0777) != -1); + int success = open("test-file", flags, 0777) != -1; + printf("success: %d\n", success); printf("errno: %d\n", errno); - stat("test-file", &s); + if ((flags & O_CREAT) && (flags & O_EXCL)) { + assert(!success); + assert(errno == EEXIST); + } else { + assert(success); + assert(errno == 0); + } + errno = 0; + + int ret = stat("test-file", &s); + assert(ret == 0); + assert(errno == 0); printf("st_mode: 0%o\n", s.st_mode & 037777777000); + assert((s.st_mode & 037777777000) == 0100000); memset(&s, 0, sizeof s); printf("\n"); errno = 0; printf("EXISTING FOLDER %d,%d\n", i, j); - printf("success: %d\n", open("test-folder", flags, 0777) != -1); + success = open("test-folder", flags, 0777) != -1; + printf("success: %d\n", success); printf("errno: %d\n", errno); - stat("test-folder", &s); + if ((flags & O_CREAT) && (flags & O_EXCL)) { + assert(!success); + assert(errno == EEXIST); + } else if ((flags & O_TRUNC) || i != 0 /*mode != O_RDONLY*/) { + assert(!success); + assert(errno == EISDIR); + } else { + assert(success); + assert(errno == 0); + } + errno = 0; + + ret = stat("test-folder", &s); + assert(ret == 0); + assert(errno == 0); printf("st_mode: 0%o\n", s.st_mode & 037777777000); + assert((s.st_mode & 037777777000) == 040000); memset(&s, 0, sizeof s); printf("\n"); errno = 0; @@ -71,10 +106,28 @@ void test() { nonexistent_name[8] = 'a' + i; nonexistent_name[9] = 'a' + j; printf("NON-EXISTING %d,%d\n", i, j); - printf("success: %d\n", open(nonexistent_name, flags, 0777) != -1); + success = open(nonexistent_name, flags, 0777) != -1; + printf("success: %d\n", success); printf("errno: %d\n", errno); - stat(nonexistent_name, &s); + if ((flags & O_CREAT)) { + assert(success); + assert(errno == 0); + } else { + assert(!success); + assert(errno == ENOENT); + } + + ret = stat(nonexistent_name, &s); printf("st_mode: 0%o\n", s.st_mode & 037777777000); + + if ((flags & O_CREAT)) { + assert(ret == 0); + assert((s.st_mode & 037777777000) == 0100000); + } else { + assert(ret != 0); + assert((s.st_mode & 037777777000) == 0); + } + memset(&s, 0, sizeof s); printf("\n"); errno = 0; @@ -84,6 +137,7 @@ void test() { printf("CREAT\n"); printf("success: %d\n", creat("creat-me", 0777) != -1); printf("errno: %d\n", errno); + errno = 0; } int main() { From c07bec00988d051f861aaabda8deb2baaea304aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Sat, 17 Sep 2016 18:41:57 +0300 Subject: [PATCH 62/74] Implement traversing of . and .. directories in ASMFS --- system/lib/fetch/asmfs.cpp | 134 ++++++++++++++++++++++++++++----- tests/asmfs/relative_paths.cpp | 35 +++++++++ tests/test_browser.py | 3 + 3 files changed, 155 insertions(+), 17 deletions(-) create mode 100644 tests/asmfs/relative_paths.cpp diff --git a/system/lib/fetch/asmfs.cpp b/system/lib/fetch/asmfs.cpp index 153cdbe4cec73..95309e081a370 100644 --- a/system/lib/fetch/asmfs.cpp +++ b/system/lib/fetch/asmfs.cpp @@ -183,12 +183,17 @@ static void unlink_inode(inode *node) // Compares two strings for equality until a '\0' or a '/' is hit. Returns 0 if the strings differ, // or a pointer to the beginning of the next directory component name of s1 if the strings are equal. -static const char *path_cmp(const char *s1, const char *s2) +static const char *path_cmp(const char *s1, const char *s2, bool *is_directory) { + *is_directory = true; while(*s1 == *s2) { if (*s1 == '/') return s1+1; - if (*s1 == '\0') return s1; + if (*s1 == '\0') + { + *is_directory = false; + return s1; + } ++s1; ++s2; } @@ -224,15 +229,49 @@ static inode *create_directory_hierarchy_for_file(inode *root, const char *path_ assert(root); if (!root) return 0; + // Traverse . and .. + while(path_to_file[0] == '.') + { + if (path_to_file[1] == '/') path_to_file += 2; // Skip over redundant "./././././" blocks + else if (path_to_file[1] == '\0') path_to_file += 1; + else if (path_to_file[1] == '.' && (path_to_file[2] == '/' || path_to_file[2] == '\0')) // Go up to parent directories with ".." + { + root = root->parent; + if (!root) return 0; + assert(root->type == INODE_DIR); // Anything that is a parent should automatically be a directory. + path_to_file += (path_to_file[2] == '/') ? 3 : 2; + } + else break; + } + if (path_to_file[0] == '\0') return 0; + inode *node = root->child; while(node) { - const char *child_path = path_cmp(path_to_file, node->name); + bool is_directory = false; + const char *child_path = path_cmp(path_to_file, node->name, &is_directory); EM_ASM_INT( { Module['printErr']('path_cmp ' + Pointer_stringify($0) + ', ' + Pointer_stringify($1) + ', ' + Pointer_stringify($2) + ' .') }, path_to_file, node->name, child_path); if (child_path) { + if (is_directory && node->type != INODE_DIR) return 0; // "A component used as a directory in pathname is not, in fact, a directory" + // The directory name matches. path_to_file = child_path; + + // Traverse . and .. + while(path_to_file[0] == '.') + { + if (path_to_file[1] == '/') path_to_file += 2; // Skip over redundant "./././././" blocks + else if (path_to_file[1] == '\0') path_to_file += 1; + else if (path_to_file[1] == '.' && (path_to_file[2] == '/' || path_to_file[2] == '\0')) // Go up to parent directories with ".." + { + node = node->parent; + if (!node) return 0; + assert(node->type == INODE_DIR); // Anything that is a parent should automatically be a directory. + path_to_file += (path_to_file[2] == '/') ? 3 : 2; + } + else break; + } if (path_to_file[0] == '\0') return node; if (path_to_file[0] == '/' && path_to_file[1] == '\0' /* && node is a directory*/) return node; root = node; @@ -279,7 +318,23 @@ static inode *find_parent_inode(inode *root, const char *path, int *out_errno) assert(out_errno); // Passing in error is mandatory. if (!root) RETURN_NODE_AND_ERRNO(0, ENOENT); - if (!path || path[0] == '\0') RETURN_NODE_AND_ERRNO(0, ENOENT); + if (!path) RETURN_NODE_AND_ERRNO(0, ENOENT); + + // Traverse . and .. + while(path[0] == '.') + { + if (path[1] == '/') path += 2; // Skip over redundant "./././././" blocks + else if (path[1] == '\0') path += 1; + else if (path[1] == '.' && (path[2] == '/' || path[2] == '\0')) // Go up to parent directories with ".." + { + root = root->parent; + if (!root) RETURN_NODE_AND_ERRNO(0, ENOENT); + assert(root->type == INODE_DIR); // Anything that is a parent should automatically be a directory. + path += (path[2] == '/') ? 3 : 2; + } + else break; + } + if (path[0] == '\0') RETURN_NODE_AND_ERRNO(0, ENOENT); if (path[0] == '/' && path[1] == '\0') RETURN_NODE_AND_ERRNO(0, ENOENT); if (root->type != INODE_DIR) RETURN_NODE_AND_ERRNO(0, ENOTDIR); // "A component used as a directory in pathname is not, in fact, a directory" @@ -291,11 +346,30 @@ static inode *find_parent_inode(inode *root, const char *path, int *out_errno) inode *node = root->child; while(node) { - const char *child_path = path_cmp(path, node->name); + bool is_directory = false; + const char *child_path = path_cmp(path, node->name, &is_directory); if (child_path) { + if (is_directory && node->type != INODE_DIR) RETURN_NODE_AND_ERRNO(0, ENOTDIR); // "A component used as a directory in pathname is not, in fact, a directory" + // The directory name matches. path = child_path; + + // Traverse . and .. + while(path[0] == '.') + { + if (path[1] == '/') path += 2; // Skip over redundant "./././././" blocks + else if (path[1] == '\0') path += 1; + else if (path[1] == '.' && (path[2] == '/' || path[2] == '\0')) // Go up to parent directories with ".." + { + node = node->parent; + if (!node) RETURN_NODE_AND_ERRNO(0, ENOENT); + assert(node->type == INODE_DIR); // Anything that is a parent should automatically be a directory. + path += (path[2] == '/') ? 3 : 2; + } + else break; + } + if (path >= basename) RETURN_NODE_AND_ERRNO(node, 0); if (!*path) RETURN_NODE_AND_ERRNO(0, ENOENT); node = node->child; @@ -325,31 +399,57 @@ static inode *find_inode(inode *root, const char *path, int *out_errno) // TODO: RETURN_ERRNO(ELOOP, "Too many symbolic links were encountered in translating pathname"); // TODO: RETURN_ERRNO(EACCES, "one of the directories in the path prefix of pathname did not allow search permission"); - // special-case finding empty string path "" or "/" returns the root searched in. - if (!path || path[0] == '\0') RETURN_NODE_AND_ERRNO(root, 0); - if (path[0] == '/' && path[1] == '\0') + // special-case finding empty string path "", "." or "/" returns the root searched in. + if (root->type != INODE_DIR) RETURN_NODE_AND_ERRNO(0, ENOTDIR); // "A component used as a directory in pathname is not, in fact, a directory" + if (!path) RETURN_NODE_AND_ERRNO(root, 0); + + // Traverse . and .. + while(path[0] == '.') { - if (root->type == INODE_DIR) RETURN_NODE_AND_ERRNO(root, 0); - else RETURN_NODE_AND_ERRNO(0, ENOTDIR); // "A component used as a directory in pathname is not, in fact, a directory" + if (path[1] == '/') path += 2; // Skip over redundant "./././././" blocks + else if (path[1] == '\0') path += 1; + else if (path[1] == '.' && (path[2] == '/' || path[2] == '\0')) // Go up to parent directories with ".." + { + root = root->parent; + if (!root) RETURN_NODE_AND_ERRNO(0, ENOENT); + assert(root->type == INODE_DIR); // Anything that is a parent should automatically be a directory. + path += (path[2] == '/') ? 3 : 2; + } + else break; } - if (root->type != INODE_DIR) RETURN_NODE_AND_ERRNO(0, ENOTDIR); // "A component used as a directory in pathname is not, in fact, a directory" + if (path[0] == '\0') RETURN_NODE_AND_ERRNO(root, 0); inode *node = root->child; while(node) { - const char *child_path = path_cmp(path, node->name); + bool is_directory = false; + const char *child_path = path_cmp(path, node->name, &is_directory); if (child_path) { + if (is_directory && node->type != INODE_DIR) RETURN_NODE_AND_ERRNO(0, ENOTDIR); // "A component used as a directory in pathname is not, in fact, a directory" + // The directory name matches. path = child_path; - // If we arrived to the end of the search, this is the node we were looking for. - if (path[0] == '\0') RETURN_NODE_AND_ERRNO(node, 0); - if (path[0] == '/' && path[1] == '\0') + // Traverse . and .. + while(path[0] == '.') { - if (node->type == INODE_DIR) RETURN_NODE_AND_ERRNO(node, 0); - else RETURN_NODE_AND_ERRNO(0, ENOTDIR); // "A component used as a directory in pathname is not, in fact, a directory" + if (path[1] == '/') path += 2; // Skip over redundant "./././././" blocks + else if (path[1] == '\0') path += 1; + else if (path[1] == '.' && (path[2] == '/' || path[2] == '\0')) // Go up to parent directories with ".." + { + node = node->parent; + if (!node) RETURN_NODE_AND_ERRNO(0, ENOENT); + assert(node->type == INODE_DIR); // Anything that is a parent should automatically be a directory. + path += (path[2] == '/') ? 3 : 2; + } + else break; } + + // If we arrived to the end of the search, this is the node we were looking for. + if (path[0] == '\0') RETURN_NODE_AND_ERRNO(node, 0); + if (path[0] == '/' && node->type != INODE_DIR) RETURN_NODE_AND_ERRNO(0, ENOTDIR); // "A component used as a directory in pathname is not, in fact, a directory" + if (path[0] == '/' && path[1] == '\0') RETURN_NODE_AND_ERRNO(node, 0); node = node->child; } else diff --git a/tests/asmfs/relative_paths.cpp b/tests/asmfs/relative_paths.cpp new file mode 100644 index 0000000000000..09e7179af0386 --- /dev/null +++ b/tests/asmfs/relative_paths.cpp @@ -0,0 +1,35 @@ +#include +#include +#include +#include +#ifdef __EMSCRIPTEN__ +#include +#endif + +#include +#include +#include + +int main() +{ + int ret = mkdir("a", 0777); assert(ret == 0); assert(errno == 0); + ret = mkdir("./././a/b", 0777); assert(ret == 0); assert(errno == 0); + ret = mkdir("a/b/./../b/c", 0777); assert(ret == 0); assert(errno == 0); + FILE *file = fopen("./a/./hello_file.txt", "wb"); + assert(file); + fputs("test", file); + ret = fclose(file); assert(ret == 0); assert(errno == 0); + + ret = chdir("./././a/./b/../b/c/./../c/."); assert(ret == 0); assert(errno == 0); + file = fopen(".././../hello_file.txt", "rb"); + assert(file); assert(errno == 0); + char str[6] = {}; + fread(str, 1, 5, file); assert(errno == 0); + ret = fclose(file); assert(ret == 0); assert(errno == 0); + assert(!strcmp(str, "test")); + +#ifdef REPORT_RESULT + int result = 0; + REPORT_RESULT(); +#endif +} diff --git a/tests/test_browser.py b/tests/test_browser.py index 206b58c8b5c02..0803225601354 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -3364,3 +3364,6 @@ def test_asmfs_unistd_unlink(self): def test_asmfs_test_fcntl_open(self): self.btest('fcntl-open/src.c', expected='0', args=['-s', 'ASMFS=1', '-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1', '--proxy-to-worker']) + + def test_asmfs_relative_paths(self): + self.btest('asmfs/relative_paths.cpp', expected='0', args=['-s', 'ASMFS=1', '-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1', '--proxy-to-worker']) From 6c6bfa9748c08c8c764eb54ff186e89c6b684edf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Sat, 17 Sep 2016 23:48:30 +0300 Subject: [PATCH 63/74] Make stat() download the file from CDN before returning the result. Do llseek just as 32-bit for now as a workaround. Uriencode fetch()ed URL names, add test. --- system/lib/fetch/asmfs.cpp | 63 ++++++++++++++++++++++++++++++-------- tests/asmfs/hello_file.cpp | 2 +- tests/test_browser.py | 3 +- 3 files changed, 54 insertions(+), 14 deletions(-) diff --git a/system/lib/fetch/asmfs.cpp b/system/lib/fetch/asmfs.cpp index 95309e081a370..bbc4e742fa1b7 100644 --- a/system/lib/fetch/asmfs.cpp +++ b/system/lib/fetch/asmfs.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include "syscall_arch.h" extern "C" { @@ -202,6 +203,19 @@ static const char *path_cmp(const char *s1, const char *s2, bool *is_directory) return 0; } +#define HEX_NIBBLE(x) ("0123456789abcdef"[(x)]) +static void uriEncode(char *dst, int dstLengthBytes, const char *src) +{ + char *end = dst + dstLengthBytes - 4; // Use last 4 bytes of dst as a guard area to avoid overflow below. + while(*src && dst < end) + { + if (isalnum(*src) || *src == '-' || *src == '_' || *src == '.' || *src == '~') *dst++ = *src; + else *dst++ = '%', *dst++ = HEX_NIBBLE(*src >> 4), *dst++ = HEX_NIBBLE(*src & 15); + ++src; + } + *dst = '\0'; +} + // Copies string 'path' to 'dst', but stops on the first forward slash '/' character. // Returns number of bytes written, excluding null terminator static int strcpy_inodename(char *dst, const char *path) @@ -590,14 +604,8 @@ long __syscall4(int which, ...) // write return __syscall146(146/*writev*/, fd, &io, 1); } -long __syscall5(int which, ...) // open +static long open(const char *pathname, int flags, int mode) { - va_list vl; - va_start(vl, which); - const char *pathname = va_arg(vl, const char*); - int flags = va_arg(vl, int); - int mode = va_arg(vl, int); - va_end(vl); EM_ASM_INT({ Module['printErr']('open(pathname="' + Pointer_stringify($0) + '", flags=0x' + ($1).toString(16) + ', mode=0' + ($2).toString(8) + ')') }, pathname, flags, mode); @@ -688,7 +696,9 @@ long __syscall5(int which, ...) // open emscripten_fetch_attr_init(&attr); strcpy(attr.requestMethod, "GET"); attr.attributes = EMSCRIPTEN_FETCH_APPEND | EMSCRIPTEN_FETCH_LOAD_TO_MEMORY | EMSCRIPTEN_FETCH_WAITABLE | EMSCRIPTEN_FETCH_PERSIST_FILE; - fetch = emscripten_fetch(&attr, pathname); + char uriEncodedPathName[3*PATH_MAX+4]; // times 3 because uri-encoding can expand the filename at most 3x. + uriEncode(uriEncodedPathName, 3*PATH_MAX+4, pathname); + fetch = emscripten_fetch(&attr, uriEncodedPathName); // switch(fopen_mode) // { @@ -745,12 +755,20 @@ long __syscall5(int which, ...) // open return (long)desc; } -long __syscall6(int which, ...) // close +long __syscall5(int which, ...) // open { va_list vl; va_start(vl, which); - int fd = va_arg(vl, int); + const char *pathname = va_arg(vl, const char*); + int flags = va_arg(vl, int); + int mode = va_arg(vl, int); va_end(vl); + + return open(pathname, flags, mode); +} + +static long close(int fd) +{ EM_ASM_INT({ Module['printErr']('close(fd=' + $0 + ')') }, fd); FileDescriptor *desc = (FileDescriptor*)fd; @@ -767,6 +785,16 @@ long __syscall6(int which, ...) // close return 0; } +long __syscall6(int which, ...) // close +{ + va_list vl; + va_start(vl, which); + int fd = va_arg(vl, int); + va_end(vl); + + return close(fd); +} + long __syscall9(int which, ...) // link { va_list vl; @@ -1082,7 +1110,7 @@ long __syscall140(int which, ...) // llseek off_t *result = va_arg(vl, off_t *); unsigned int whence = va_arg(vl, unsigned int); va_end(vl); - EM_ASM_INT({ Module['printErr']('llseek(fd=' + $0 + ', offset=0x' + (($1<<32)|$2) + ', result=0x' + ($3).toString(16) + ', whence=' + $4 + ')') }, + EM_ASM_INT({ Module['printErr']('llseek(fd=' + $0 + ', offset_high=' + $1 + ', offset_low=' + $2 + ', result=0x' + ($3).toString(16) + ', whence=' + $4 + ')') }, fd, offset_high, offset_low, result, whence); FileDescriptor *desc = (FileDescriptor*)fd; @@ -1090,7 +1118,9 @@ long __syscall140(int which, ...) // llseek if (desc->node->fetch) emscripten_fetch_wait(desc->node->fetch, INFINITY); - int64_t offset = (int64_t)(((uint64_t)offset_high << 32) | (uint64_t)offset_low); +// TODO: The following does not work, for some reason seek is getting called with 32-bit signed offsets? +// int64_t offset = (int64_t)(((uint64_t)offset_high << 32) | (uint64_t)offset_low); + int64_t offset = (int64_t)(int32_t)offset_low; int64_t newPos; switch(whence) { @@ -1310,6 +1340,15 @@ long __syscall195(int which, ...) // SYS_stat64 int err; inode *node = find_inode(root, relpath, &err); + + if (!node && (err == ENOENT || err == ENOTDIR)) + { + // Populate the file from the CDN to the filesystem if it didn't yet exist. + long fd = open(pathname, O_RDONLY, 0777); + if (fd) close(fd); + node = find_inode(root, relpath, &err); + } + if (err == ENOTDIR) RETURN_ERRNO(ENOTDIR, "A component of the path prefix of pathname is not a directory"); if (err == ELOOP) RETURN_ERRNO(ELOOP, "Too many symbolic links encountered while traversing the path"); if (err == EACCES) RETURN_ERRNO(EACCES, "Search permission is denied for one of the directories in the path prefix of pathname"); diff --git a/tests/asmfs/hello_file.cpp b/tests/asmfs/hello_file.cpp index 04ac15dce7450..d76ca54fcdfd6 100644 --- a/tests/asmfs/hello_file.cpp +++ b/tests/asmfs/hello_file.cpp @@ -6,7 +6,7 @@ int main() { - FILE *file = fopen("hello_file.txt", "rb"); + FILE *file = fopen("hello file !#$%&'()+,-.;=@[]^_`{}~ %%.txt", "rb"); assert(file); fseek(file, 0, SEEK_END); diff --git a/tests/test_browser.py b/tests/test_browser.py index 0803225601354..b452a432d065d 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -3333,7 +3333,8 @@ def test_fetch_idb_delete(self): self.btest('fetch/idb_delete.cpp', expected='0', args=['-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1', '-s', 'FETCH=1', '--proxy-to-worker']) def test_asmfs_hello_file(self): - shutil.copyfile(path_from_root('tests', 'asmfs', 'hello_file.txt'), os.path.join(self.get_dir(), 'hello_file.txt')) + # Test basic file loading and the valid character set for files. + shutil.copyfile(path_from_root('tests', 'asmfs', 'hello_file.txt'), os.path.join(self.get_dir(), 'hello file !#$%&\'()+,-.;=@[]^_`{}~ %%.txt')) self.btest('asmfs/hello_file.cpp', expected='0', args=['-s', 'ASMFS=1', '-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1', '--proxy-to-worker']) def test_asmfs_read_file_twice(self): From 502bfec2a21b7a617110ee20bbe60642e78e51a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Sun, 18 Sep 2016 00:20:29 +0300 Subject: [PATCH 64/74] Log out the client and server side reported URLs of the performed fetch on success/failure. Don't uriencode forward slashes in ASMFS, because we need to retain their meaning as path separators. --- src/Fetch.js | 9 +++++---- system/lib/fetch/asmfs.cpp | 5 +++-- tests/asmfs/hello_file.cpp | 2 +- tests/test_browser.py | 3 ++- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/Fetch.js b/src/Fetch.js index 5af76d60a021d..a88b82dc12422 100644 --- a/src/Fetch.js +++ b/src/Fetch.js @@ -300,6 +300,7 @@ function __emscripten_fetch_xhr(fetch, onsuccess, onerror, onprogress) { console.log('fetch: xhr.open(requestMethod="' + requestMethod + '", url: "' + url_ +'", userName: ' + userNameStr + ', password: ' + passwordStr + ');'); #endif xhr.open(requestMethod, url_, !fetchAttrSynchronous, userNameStr, passwordStr); + xhr.url_ = url_; // Save the url for debugging purposes (and for comparing to the responseURL that server side advertised) xhr.responseType = fetchAttrStreamData ? 'moz-chunked-arraybuffer' : 'arraybuffer'; if (overriddenMimeType) { @@ -356,12 +357,12 @@ function __emscripten_fetch_xhr(fetch, onsuccess, onerror, onprogress) { // if (xhr.statusText) stringToUTF8(fetch + 44, xhr.statusText, 64); if (xhr.status == 200) { #if FETCH_DEBUG - console.log('fetch: xhr succeeded with status 200'); + console.log('fetch: xhr of URL "' + xhr.url_ + '" / responseURL "' + xhr.responseURL + '" succeeded with status 200'); #endif if (onsuccess) onsuccess(fetch, xhr, e); } else { #if FETCH_DEBUG - console.error('fetch: xhr failed with status ' + xhr.status); + console.error('fetch: xhr of URL "' + xhr.url_ + '" / responseURL "' + xhr.responseURL + '" failed with status ' + xhr.status); #endif if (onerror) onerror(fetch, xhr, e); } @@ -370,7 +371,7 @@ function __emscripten_fetch_xhr(fetch, onsuccess, onerror, onprogress) { var status = xhr.status; // XXX TODO: Overwriting xhr.status doesn't work here, so don't override anywhere else either. if (xhr.readyState == 4 && status == 0) status = 404; // If no error recorded, pretend it was 404 Not Found. #if FETCH_DEBUG - console.error('fetch: xhr error with readyState ' + xhr.readyState + ' and status ' + status); + console.error('fetch: xhr of URL "' + xhr.url_ + '" / responseURL "' + xhr.responseURL + '" finished with error, readyState ' + xhr.readyState + ' and status ' + status); #endif HEAPU32[fetch + 12 >> 2] = 0; Fetch.setu64(fetch + 16, 0); @@ -382,7 +383,7 @@ function __emscripten_fetch_xhr(fetch, onsuccess, onerror, onprogress) { } xhr.ontimeout = function(e) { #if FETCH_DEBUG - console.error('fetch: xhr timed out with readyState ' + xhr.readyState + ' and status ' + xhr.status); + console.error('fetch: xhr of URL "' + xhr.url_ + '" / responseURL "' + xhr.responseURL + '" timed out, readyState ' + xhr.readyState + ' and status ' + xhr.status); #endif if (onerror) onerror(fetch, xhr, e); } diff --git a/system/lib/fetch/asmfs.cpp b/system/lib/fetch/asmfs.cpp index bbc4e742fa1b7..e51194495b790 100644 --- a/system/lib/fetch/asmfs.cpp +++ b/system/lib/fetch/asmfs.cpp @@ -203,14 +203,15 @@ static const char *path_cmp(const char *s1, const char *s2, bool *is_directory) return 0; } -#define HEX_NIBBLE(x) ("0123456789abcdef"[(x)]) +#define NIBBLE_TO_CHAR(x) ("0123456789abcdef"[(x)]) static void uriEncode(char *dst, int dstLengthBytes, const char *src) { char *end = dst + dstLengthBytes - 4; // Use last 4 bytes of dst as a guard area to avoid overflow below. while(*src && dst < end) { if (isalnum(*src) || *src == '-' || *src == '_' || *src == '.' || *src == '~') *dst++ = *src; - else *dst++ = '%', *dst++ = HEX_NIBBLE(*src >> 4), *dst++ = HEX_NIBBLE(*src & 15); + else if (*src == '/') *dst++ = *src; // NB. forward slashes should generally be uriencoded, but for file path purposes, we want to keep them intact. + else *dst++ = '%', *dst++ = NIBBLE_TO_CHAR(*src >> 4), *dst++ = NIBBLE_TO_CHAR(*src & 15); // This charater needs uriencoding. ++src; } *dst = '\0'; diff --git a/tests/asmfs/hello_file.cpp b/tests/asmfs/hello_file.cpp index d76ca54fcdfd6..8b9a1bcbdd821 100644 --- a/tests/asmfs/hello_file.cpp +++ b/tests/asmfs/hello_file.cpp @@ -6,7 +6,7 @@ int main() { - FILE *file = fopen("hello file !#$%&'()+,-.;=@[]^_`{}~ %%.txt", "rb"); + FILE *file = fopen("dirrey/hello file !#$%&'()+,-.;=@[]^_`{}~ %%.txt", "rb"); assert(file); fseek(file, 0, SEEK_END); diff --git a/tests/test_browser.py b/tests/test_browser.py index b452a432d065d..3e6da8e47a37f 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -3334,7 +3334,8 @@ def test_fetch_idb_delete(self): def test_asmfs_hello_file(self): # Test basic file loading and the valid character set for files. - shutil.copyfile(path_from_root('tests', 'asmfs', 'hello_file.txt'), os.path.join(self.get_dir(), 'hello file !#$%&\'()+,-.;=@[]^_`{}~ %%.txt')) + os.mkdir(os.path.join(self.get_dir(), 'dirrey')) + shutil.copyfile(path_from_root('tests', 'asmfs', 'hello_file.txt'), os.path.join(self.get_dir(), 'dirrey', 'hello file !#$%&\'()+,-.;=@[]^_`{}~ %%.txt')) self.btest('asmfs/hello_file.cpp', expected='0', args=['-s', 'ASMFS=1', '-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1', '--proxy-to-worker']) def test_asmfs_read_file_twice(self): From f940dbea720e0e223df0de3509e198dc991bd2f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Sun, 20 Nov 2016 22:45:11 +0200 Subject: [PATCH 65/74] Fix setting of xhr.timeout in Fetch API to occur only after xhr.open() is called, to appease IE11 behavior (https://msdn.microsoft.com/en-us/library/cc304105(v=vs.85).aspx). Also, setting .timeout can be done for asynchronous XHRs (https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/timeout). --- src/Fetch.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Fetch.js b/src/Fetch.js index a88b82dc12422..f83bedd1d17e0 100644 --- a/src/Fetch.js +++ b/src/Fetch.js @@ -293,13 +293,13 @@ function __emscripten_fetch_xhr(fetch, onsuccess, onerror, onprogress) { var overriddenMimeTypeStr = overriddenMimeType ? Pointer_stringify(overriddenMimeType) : undefined; var xhr = new XMLHttpRequest(); - xhr.timeout = timeoutMsecs; xhr.withCredentials = withCredentials; #if FETCH_DEBUG console.log('fetch: xhr.timeout: ' + xhr.timeout + ', xhr.withCredentials: ' + xhr.withCredentials); console.log('fetch: xhr.open(requestMethod="' + requestMethod + '", url: "' + url_ +'", userName: ' + userNameStr + ', password: ' + passwordStr + ');'); #endif xhr.open(requestMethod, url_, !fetchAttrSynchronous, userNameStr, passwordStr); + if (!fetchAttrSynchronous) xhr.timeout = timeoutMsecs; // XHR timeout field is only accessible in async XHRs, and must be set after .open() but before .send(). xhr.url_ = url_; // Save the url for debugging purposes (and for comparing to the responseURL that server side advertised) xhr.responseType = fetchAttrStreamData ? 'moz-chunked-arraybuffer' : 'arraybuffer'; From 58ad472fbd91f4b244e58c6a647fb376d0bc197d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Sun, 20 Nov 2016 23:55:52 +0200 Subject: [PATCH 66/74] Document the possibility that totalBytes=0 in the Fetch XHR event handlers. --- src/Fetch.js | 3 +++ system/include/emscripten/fetch.h | 1 + 2 files changed, 4 insertions(+) diff --git a/src/Fetch.js b/src/Fetch.js index f83bedd1d17e0..355bd882cd487 100644 --- a/src/Fetch.js +++ b/src/Fetch.js @@ -346,6 +346,9 @@ function __emscripten_fetch_xhr(fetch, onsuccess, onerror, onprogress) { Fetch.setu64(fetch + 16, ptrLen);//{{{ makeSetValue('fetch', 16/*TODO:jsonify*/, 'ptrLen', 'i64')}}}; Fetch.setu64(fetch + 24, 0);//{{{ makeSetValue('fetch', 24/*TODO:jsonify*/, '0', 'i64')}}}; if (len) { + // If the final XHR.onload handler receives the bytedata to compute total length, report that, + // otherwise don't write anything out here, which will retain the latest byte size reported in + // the most recent XHR.onprogress handler. Fetch.setu64(fetch + 32, len);//{{{ makeSetValue('fetch', 32/*TODO:jsonify*/, 'len', 'i64')}}}; } HEAPU16[fetch + 40 >> 1] = xhr.readyState; diff --git a/system/include/emscripten/fetch.h b/system/include/emscripten/fetch.h index 35ee4a9ab8e10..8867c7f3b1e63 100644 --- a/system/include/emscripten/fetch.h +++ b/system/include/emscripten/fetch.h @@ -131,6 +131,7 @@ struct emscripten_fetch_t uint64_t dataOffset; // XXX 24 // Specifies the total number of bytes that the response body will be. + // Note: This field may be zero, if the server does not report the Content-Length field. uint64_t totalBytes; // XXX 32 // Specifies the readyState of the XHR request: From e7341343dc96bfa7a4384c6da84c3e7e988319f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Mon, 21 Nov 2016 00:16:08 +0200 Subject: [PATCH 67/74] Implement emscripten_fetch_close() more properly. --- src/Fetch.js | 12 +++++++++--- system/lib/fetch/emscripten_fetch.cpp | 18 +++++++++++++++++- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/Fetch.js b/src/Fetch.js index 355bd882cd487..1cfbeead0e60e 100644 --- a/src/Fetch.js +++ b/src/Fetch.js @@ -174,7 +174,9 @@ function __emscripten_fetch_load_cached_data(db, fetch, onsuccess, onerror) { console.log('fetch: Loaded file ' + pathStr + ' from IndexedDB, length: ' + len); #endif - var ptr = _malloc(len); // TODO: make emscripten_fetch_close() free() this data. + // The data pointer malloc()ed here has the same lifetime as the emscripten_fetch_t structure itself has, and is + // freed when emscripten_fetch_close() is called. + var ptr = _malloc(len); HEAPU8.set(new Uint8Array(value), ptr); HEAPU32[fetch + 12 >> 2] = ptr;//{{{ makeSetValue('fetch', 12/*TODO:jsonify*/, 'ptr', 'i32')}}}; Fetch.setu64(fetch + 16, len);//{{{ makeSetValue('fetch', 16/*TODO:jsonify*/, 'len', 'i64')}}}; @@ -339,7 +341,9 @@ function __emscripten_fetch_xhr(fetch, onsuccess, onerror, onprogress) { #if FETCH_DEBUG console.log('fetch: allocating ' + ptrLen + ' bytes in Emscripten heap for xhr data'); #endif - ptr = _malloc(ptrLen); // TODO: make emscripten_fetch_close() free() this data. + // The data pointer malloc()ed here has the same lifetime as the emscripten_fetch_t structure itself has, and is + // freed when emscripten_fetch_close() is called. + ptr = _malloc(ptrLen); HEAPU8.set(new Uint8Array(xhr.response), ptr); // TODO: Since DYNAMICTOP is not coherent, this can corrupt } HEAPU32[fetch + 12 >> 2] = ptr;//{{{ makeSetValue('fetch', 12/*TODO:jsonify*/, 'ptr', 'i32')}}}; @@ -397,7 +401,9 @@ function __emscripten_fetch_xhr(fetch, onsuccess, onerror, onprogress) { #if FETCH_DEBUG console.log('fetch: allocating ' + ptrLen + ' bytes in Emscripten heap for xhr data'); #endif - ptr = _malloc(ptrLen); // TODO: make emscripten_fetch_close() free() this data. + // The data pointer malloc()ed here has the same lifetime as the emscripten_fetch_t structure itself has, and is + // freed when emscripten_fetch_close() is called. + ptr = _malloc(ptrLen); HEAPU8.set(new Uint8Array(xhr.response), ptr); } HEAPU32[fetch + 12 >> 2] = ptr;//{{{ makeSetValue('fetch', 12/*TODO:jsonify*/, 'ptr', 'i32')}}}; diff --git a/system/lib/fetch/emscripten_fetch.cpp b/system/lib/fetch/emscripten_fetch.cpp index ed5eed81cb140..62e63346c2d9d 100644 --- a/system/lib/fetch/emscripten_fetch.cpp +++ b/system/lib/fetch/emscripten_fetch.cpp @@ -127,9 +127,25 @@ EMSCRIPTEN_RESULT emscripten_fetch_wait(emscripten_fetch_t *fetch, double timeou EMSCRIPTEN_RESULT emscripten_fetch_close(emscripten_fetch_t *fetch) { + if (!fetch) return EMSCRIPTEN_RESULT_SUCCESS; // Closing null pointer is ok, same as with free(). + #if __EMSCRIPTEN_PTHREADS__ emscripten_atomic_store_u32(&fetch->__proxyState, 0); #endif - free(fetch); // TODO: thread-safety before freeing (what if freeing an operation in progress? explicit emscripten_fetch_abort()?) + // This function frees the fetch pointer so that it is invalid to access it anymore. + // Use a few key fields as an integrity check that we are being passed a good pointer to a valid fetch structure, + // which has not been yet closed. (double close is an error) + if (fetch->id == 0 || fetch->readyState > 4) return EMSCRIPTEN_RESULT_INVALID_PARAM; + + // This fetch is aborted. Call the error handler if the fetch was still in progress and was canceled in flight. + if (fetch->readyState != 4 /*DONE*/ && fetch->__attributes.onerror) + { + fetch->status = (unsigned short)-1; + strcpy(fetch->statusText, "aborted with emscripten_fetch_close()"); + fetch->__attributes.onerror(fetch); + } + fetch->id = 0; + free((void*)fetch->data); + free(fetch); return EMSCRIPTEN_RESULT_SUCCESS; } From fb37935a1404244206d29a03c3add30c99855dce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Mon, 21 Nov 2016 00:37:11 +0200 Subject: [PATCH 68/74] Make Fetch downloads tests and examples prepare for fetch->totalBytes == 0 possibility, so that the presented code shows by a good example of how to properly handle XHR events. --- site/source/docs/api_reference/fetch.rst | 28 ++++++++++++++---------- tests/fetch/cached_xhr.cpp | 12 ++++++++-- tests/fetch/example_stream_async_xhr.cpp | 5 +++-- tests/fetch/example_xhr_progress.cpp | 6 ++--- tests/fetch/stream_file.cpp | 5 +++-- tests/fetch/sync_xhr.cpp | 6 ++++- tests/fetch/to_indexeddb.cpp | 6 ++++- tests/fetch/to_memory.cpp | 6 ++++- 8 files changed, 50 insertions(+), 24 deletions(-) diff --git a/site/source/docs/api_reference/fetch.rst b/site/source/docs/api_reference/fetch.rst index 15ca1aef48431..b4cc7f6f62afb 100644 --- a/site/source/docs/api_reference/fetch.rst +++ b/site/source/docs/api_reference/fetch.rst @@ -210,19 +210,22 @@ The emscripten_fetch_attr_t object has a timeoutMsecs field which allows specify .. code-block:: cpp void downloadProgress(emscripten_fetch_t *fetch) { - printf("Downloading %s.. %.2f%% complete.\n", - fetch->url, fetch->dataOffset * 100.0 / fetch->totalBytes); + if (fetch->totalBytes) { + printf("Downloading %s.. %.2f%% complete.\n", fetch->url, fetch->dataOffset * 100.0 / fetch->totalBytes); + } else { + printf("Downloading %s.. %lld bytes complete.\n", fetch->url, fetch->dataOffset + fetch->numBytes); + } } int main() { - emscripten_fetch_attr_t attr; - emscripten_fetch_attr_init(&attr); - strcpy(attr.requestMethod, "GET"); - attr.attributes = EMSCRIPTEN_FETCH_LOAD_TO_MEMORY; - attr.onsuccess = downloadSucceeded; - attr.onprogress = downloadProgress; - attr.onerror = downloadFailed; - emscripten_fetch(&attr, "myfile.dat"); + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.attributes = EMSCRIPTEN_FETCH_LOAD_TO_MEMORY; + attr.onsuccess = downloadSucceeded; + attr.onprogress = downloadProgress; + attr.onerror = downloadFailed; + emscripten_fetch(&attr, "myfile.dat"); } Managing Large Files @@ -245,9 +248,10 @@ If the application does not need random seek access to the file, but is able to .. code-block:: cpp void downloadProgress(emscripten_fetch_t *fetch) { - printf("Downloading %s.. %.2f%% complete. HTTP readyState: %d. HTTP status: %d.\n" + printf("Downloading %s.. %.2f%%s complete. HTTP readyState: %d. HTTP status: %d.\n" "HTTP statusText: %s. Received chunk [%llu, %llu[\n", - fetch->url, (fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes, + fetch->url, fetch->totalBytes > 0 ? (fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes : (fetch->dataOffset + fetch->numBytes), + fetch->totalBytes > 0 ? "%" : " bytes", fetch->readyState, fetch->status, fetch->statusText, fetch->dataOffset, fetch->dataOffset + fetch->numBytes); diff --git a/tests/fetch/cached_xhr.cpp b/tests/fetch/cached_xhr.cpp index 6750d1da27403..c84d0281e5317 100644 --- a/tests/fetch/cached_xhr.cpp +++ b/tests/fetch/cached_xhr.cpp @@ -25,7 +25,11 @@ void fetchFromIndexedDB() }; attr.onprogress = [](emscripten_fetch_t *fetch) { - printf("Downloading.. %.2f%% complete.\n", (fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes); + if (fetch->totalBytes > 0) { + printf("Downloading.. %.2f%% complete.\n", (fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes); + } else { + printf("Downloading.. %lld bytes complete.\n", fetch->dataOffset + fetch->numBytes); + } }; attr.attributes = EMSCRIPTEN_FETCH_APPEND | EMSCRIPTEN_FETCH_NO_DOWNLOAD; emscripten_fetch_t *fetch = emscripten_fetch(&attr, "gears.png"); @@ -47,7 +51,11 @@ int main() fetchFromIndexedDB(); }; attr.onprogress = [](emscripten_fetch_t *fetch) { - printf("Downloading.. %.2f%% complete.\n", (fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes); + if (fetch->totalBytes > 0) { + printf("Downloading.. %.2f%% complete.\n", (fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes); + } else { + printf("Downloading.. %lld bytes complete.\n", fetch->dataOffset + fetch->numBytes); + } }; attr.attributes = EMSCRIPTEN_FETCH_REPLACE | EMSCRIPTEN_FETCH_LOAD_TO_MEMORY | EMSCRIPTEN_FETCH_PERSIST_FILE; emscripten_fetch_t *fetch = emscripten_fetch(&attr, "gears.png"); diff --git a/tests/fetch/example_stream_async_xhr.cpp b/tests/fetch/example_stream_async_xhr.cpp index ecad466d78b9a..7fcb866cecbe9 100644 --- a/tests/fetch/example_stream_async_xhr.cpp +++ b/tests/fetch/example_stream_async_xhr.cpp @@ -17,9 +17,10 @@ void downloadFailed(emscripten_fetch_t *fetch) void downloadProgress(emscripten_fetch_t *fetch) { - printf("Downloading %s.. %.2f%% complete. HTTP readyState: %d. HTTP status: %d.\n" + printf("Downloading %s.. %.2f%s complete. HTTP readyState: %d. HTTP status: %d.\n" "HTTP statusText: %s. Received chunk [%llu, %llu[\n", - fetch->url, (fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes, + fetch->url, (fetch->totalBytes > 0) ? ((fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes) : (double)(fetch->dataOffset + fetch->numBytes), + (fetch->totalBytes > 0) ? "%" : " bytes", fetch->readyState, fetch->status, fetch->statusText, fetch->dataOffset, fetch->dataOffset + fetch->numBytes); diff --git a/tests/fetch/example_xhr_progress.cpp b/tests/fetch/example_xhr_progress.cpp index 51d37f4f06487..630eeb8ba8136 100644 --- a/tests/fetch/example_xhr_progress.cpp +++ b/tests/fetch/example_xhr_progress.cpp @@ -17,10 +17,10 @@ void downloadFailed(emscripten_fetch_t *fetch) void downloadProgress(emscripten_fetch_t *fetch) { - printf("Downloading %s.. %.2f%% complete. HTTP readyState: %d. HTTP status: %d.\n" + printf("Downloading %s.. %.2f%s complete. HTTP readyState: %d. HTTP status: %d.\n" "HTTP statusText: %s. Received chunk [%llu, %llu[\n", - fetch->url, - (fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes, + fetch->url, (fetch->totalBytes > 0) ? ((fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes) : (double)(fetch->dataOffset + fetch->numBytes), + (fetch->totalBytes > 0) ? "%" : " bytes", fetch->readyState, fetch->status, fetch->statusText, fetch->dataOffset, fetch->dataOffset + fetch->numBytes); } diff --git a/tests/fetch/stream_file.cpp b/tests/fetch/stream_file.cpp index b73213d7da995..2ae931776fe96 100644 --- a/tests/fetch/stream_file.cpp +++ b/tests/fetch/stream_file.cpp @@ -33,8 +33,9 @@ int main() assert(fetch->numBytes > 0); assert(fetch->dataOffset + fetch->numBytes <= fetch->totalBytes); assert(fetch->totalBytes <= 134217728); - printf("Downloading.. %.2f%% complete. Received chunk [%llu, %llu[\n", - (fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes, + printf("Downloading.. %.2f%s complete. Received chunk [%llu, %llu[\n", + (fetch->totalBytes > 0) ? ((fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes) : (double)(fetch->dataOffset + fetch->numBytes), + (fetch->totalBytes > 0) ? "%" : " bytes", fetch->dataOffset, fetch->dataOffset + fetch->numBytes); diff --git a/tests/fetch/sync_xhr.cpp b/tests/fetch/sync_xhr.cpp index 4a38ecb67fb5e..81f436231a5a6 100644 --- a/tests/fetch/sync_xhr.cpp +++ b/tests/fetch/sync_xhr.cpp @@ -30,7 +30,11 @@ int main() }; attr.onprogress = [](emscripten_fetch_t *fetch) { - printf("Downloading.. %.2f%% complete.\n", (fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes); + if (fetch->totalBytes > 0) { + printf("Downloading.. %.2f%% complete.\n", (fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes); + } else { + printf("Downloading.. %lld bytes complete.\n", fetch->dataOffset + fetch->numBytes); + } }; attr.onerror = [](emscripten_fetch_t *fetch) { diff --git a/tests/fetch/to_indexeddb.cpp b/tests/fetch/to_indexeddb.cpp index c61edd26f611b..14155ee9ae35f 100644 --- a/tests/fetch/to_indexeddb.cpp +++ b/tests/fetch/to_indexeddb.cpp @@ -33,7 +33,11 @@ int main() attr.onprogress = [](emscripten_fetch_t *fetch) { assert(fetch); if (fetch->status != 200) return; - printf("Downloading.. %.2f%% complete.\n", (fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes); + if (fetch->totalBytes > 0) { + printf("Downloading.. %.2f%% complete.\n", (fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes); + } else { + printf("Downloading.. %lld bytes complete.\n", fetch->dataOffset + fetch->numBytes); + } }; attr.onerror = [](emscripten_fetch_t *fetch) { diff --git a/tests/fetch/to_memory.cpp b/tests/fetch/to_memory.cpp index 7d8d29599089b..53444883addce 100644 --- a/tests/fetch/to_memory.cpp +++ b/tests/fetch/to_memory.cpp @@ -51,7 +51,11 @@ int main() assert(fetch); if (fetch->status != 200) return; printf("onprogress: dataOffset: %llu, numBytes: %llu, totalBytes: %llu\n", fetch->dataOffset, fetch->numBytes, fetch->totalBytes); - printf("Downloading.. %.2f%% complete.\n", (fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes); + if (fetch->totalBytes > 0) { + printf("Downloading.. %.2f%% complete.\n", (fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes); + } else { + printf("Downloading.. %lld bytes complete.\n", fetch->dataOffset + fetch->numBytes); + } #ifdef FILE_DOES_NOT_EXIST assert(false && "onprogress handler called, but the file should not exist"); // We should not receive progress reports if the file doesn't exist. #endif From b779777c652ef97359197974c8c92ca57e3d2bdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Mon, 21 Nov 2016 00:38:10 +0200 Subject: [PATCH 69/74] Add missing attr object to example_idb_delete.cpp --- tests/fetch/example_idb_delete.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/fetch/example_idb_delete.cpp b/tests/fetch/example_idb_delete.cpp index 7d1c5b1659a0b..cd460ce6145bc 100644 --- a/tests/fetch/example_idb_delete.cpp +++ b/tests/fetch/example_idb_delete.cpp @@ -16,6 +16,7 @@ void failure(emscripten_fetch_t *fetch) int main() { + emscripten_fetch_attr_t attr; emscripten_fetch_attr_init(&attr); strcpy(attr.requestMethod, "EM_IDB_DELETE"); emscripten_fetch(&attr, "filename_to_delete.dat"); From b304b04dff78dbce5615d959ac89f782caf7c3b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Mon, 21 Nov 2016 00:42:50 +0200 Subject: [PATCH 70/74] Document the ownership and lifetime semantics of fetch_attr_t requestData buffer and fetch_t data buffer. --- system/include/emscripten/fetch.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/system/include/emscripten/fetch.h b/system/include/emscripten/fetch.h index 8867c7f3b1e63..a2a8238c04bd5 100644 --- a/system/include/emscripten/fetch.h +++ b/system/include/emscripten/fetch.h @@ -97,6 +97,8 @@ struct emscripten_fetch_attr_t // If non-zero, specified a pointer to the data that is to be passed as the body (payload) of the request // that is being performed. Leave as zero if no request body needs to be sent. + // The memory pointed to by this field is provided by the user, and needs to be valid only until the call to + // emscripten_fetch() returns. const char *requestData; // XXX 84 // Specifies the length of the buffer pointed by 'requestData'. Leave as 0 if no request body needs to be sent. @@ -120,6 +122,8 @@ struct emscripten_fetch_t // In onprogress() handler: // - If the EMSCRIPTEN_FETCH_STREAM_DATA attribute was specified for the transfer, this points to a partial // chunk of bytes related to the transfer. Otherwise this will be null. + // The data buffer provided here has identical lifetime with the emscripten_fetch_t object itself, and is freed by + // calling emscripten_fetch_close() on the emscripten_fetch_t pointer. const char *data; // XXX 12 // Specifies the length of the above data block in bytes. When the download finishes, this field will be valid even if From 319369b74b12ab252ebe68617f6e73b0d7e958e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Mon, 21 Nov 2016 22:00:53 +0200 Subject: [PATCH 71/74] Remove use of hardcoded struct offsets in Fetch implementation. --- src/Fetch.js | 175 +++++++++++++++++------------- src/fetch-worker.js | 8 +- system/include/emscripten/fetch.h | 59 +++++----- 3 files changed, 133 insertions(+), 109 deletions(-) diff --git a/src/Fetch.js b/src/Fetch.js index 1cfbeead0e60e..d161aab548730 100644 --- a/src/Fetch.js +++ b/src/Fetch.js @@ -1,4 +1,33 @@ var Fetch = { + attr_t_offset_requestMethod: 0, + attr_t_offset_userData: 32, + attr_t_offset_onsuccess: 36, + attr_t_offset_onerror: 40, + attr_t_offset_onprogress: 44, + attr_t_offset_attributes: 48, + attr_t_offset_timeoutMSecs: 52, + attr_t_offset_withCredentials: 56, + attr_t_offset_destinationPath: 60, + attr_t_offset_userName: 64, + attr_t_offset_password: 68, + attr_t_offset_requestHeaders: 72, + attr_t_offset_overriddenMimeType: 76, + attr_t_offset_requestData: 80, + attr_t_offset_requestDataSize: 84, + + fetch_t_offset_id: 0, + fetch_t_offset_userData: 4, + fetch_t_offset_url: 8, + fetch_t_offset_data: 12, + fetch_t_offset_numBytes: 16, + fetch_t_offset_dataOffset: 24, + fetch_t_offset_totalBytes: 32, + fetch_t_offset_readyState: 40, + fetch_t_offset_status: 42, + fetch_t_offset_statusText: 44, + fetch_t_offset___proxyState: 108, + fetch_t_offset___attributes: 112, + xhrs: [], // The web worker that runs proxied file I/O requests. worker: undefined, @@ -109,9 +138,9 @@ function __emscripten_fetch_delete_cached_data(db, fetch, onsuccess, onerror) { return; } - var fetch_attr = fetch + 112/*TODO:structs_info*/; - var path = HEAPU32[fetch_attr + 64 >> 2];//{{{ makeGetValue('fetch_attr', 64/*TODO*/, 'i32') }}}; - if (!path) path = HEAPU32[fetch + 8 >> 2];//{{{ makeGetValue('fetch', 8/*TODO:structs_info.jsonify this*/, 'i32') }}}; + var fetch_attr = fetch + Fetch.fetch_t_offset___attributes; + var path = HEAPU32[fetch_attr + Fetch.attr_t_offset_destinationPath >> 2]; + if (!path) path = HEAPU32[fetch + Fetch.fetch_t_offset_url >> 2]; var pathStr = Pointer_stringify(path); try { @@ -123,12 +152,12 @@ function __emscripten_fetch_delete_cached_data(db, fetch, onsuccess, onerror) { #if FETCH_DEBUG console.log('fetch: Deleted file ' + pathStr + ' from IndexedDB'); #endif - HEAPU32[fetch + 12 >> 2] = 0;//{{{ makeSetValue('fetch', 12/*TODO:jsonify*/, 'ptr', 'i32')}}}; - Fetch.setu64(fetch + 16, 0);//{{{ makeSetValue('fetch', 16/*TODO:jsonify*/, 'len', 'i64')}}}; - Fetch.setu64(fetch + 24, 0);//{{{ makeSetValue('fetch', 24/*TODO:jsonify*/, '0', 'i64')}}}; - Fetch.setu64(fetch + 32, 0);//{{{ makeSetValue('fetch', 32/*TODO:jsonify*/, 'len', 'i64')}}}; - HEAPU16[fetch + 40 >> 1] = 4; // Mimic XHR readyState 4 === 'DONE: The operation is complete' - HEAPU16[fetch + 42 >> 1] = 200; // Mimic XHR HTTP status code 200 "OK" + HEAPU32[fetch + Fetch.fetch_t_offset_data >> 2] = 0; + Fetch.setu64(fetch + Fetch.fetch_t_offset_numBytes, 0); + Fetch.setu64(fetch + Fetch.fetch_t_offset_dataOffset, 0); + Fetch.setu64(fetch + Fetch.fetch_t_offset_dataOffset, 0); + HEAPU16[fetch + Fetch.fetch_t_offset_readyState >> 1] = 4; // Mimic XHR readyState 4 === 'DONE: The operation is complete' + HEAPU16[fetch + Fetch.fetch_t_offset_status >> 1] = 200; // Mimic XHR HTTP status code 200 "OK" onsuccess(fetch, 0, value); }; @@ -136,8 +165,8 @@ function __emscripten_fetch_delete_cached_data(db, fetch, onsuccess, onerror) { #if FETCH_DEBUG console.error('fetch: Failed to delete file ' + pathStr + ' from IndexedDB! error: ' + error); #endif - HEAPU16[fetch + 40 >> 1] = 4; // Mimic XHR readyState 4 === 'DONE: The operation is complete' - HEAPU16[fetch + 42 >> 1] = 404; // Mimic XHR HTTP status code 404 "Not Found" + HEAPU16[fetch + Fetch.fetch_t_offset_readyState >> 1] = 4; // Mimic XHR readyState 4 === 'DONE: The operation is complete' + HEAPU16[fetch + Fetch.fetch_t_offset_status >> 1] = 404; // Mimic XHR HTTP status code 404 "Not Found" onerror(fetch, 0, error); }; } catch(e) { @@ -157,9 +186,9 @@ function __emscripten_fetch_load_cached_data(db, fetch, onsuccess, onerror) { return; } - var fetch_attr = fetch + 112/*TODO:structs_info*/; - var path = HEAPU32[fetch_attr + 64 >> 2];//{{{ makeGetValue('fetch_attr', 64/*TODO*/, 'i32') }}}; - if (!path) path = HEAPU32[fetch + 8 >> 2];//{{{ makeGetValue('fetch', 8/*TODO:structs_info.jsonify this*/, 'i32') }}}; + var fetch_attr = fetch + Fetch.fetch_t_offset___attributes; + var path = HEAPU32[fetch_attr + Fetch.attr_t_offset_destinationPath >> 2]; + if (!path) path = HEAPU32[fetch + Fetch.fetch_t_offset_url >> 2]; var pathStr = Pointer_stringify(path); try { @@ -178,12 +207,12 @@ function __emscripten_fetch_load_cached_data(db, fetch, onsuccess, onerror) { // freed when emscripten_fetch_close() is called. var ptr = _malloc(len); HEAPU8.set(new Uint8Array(value), ptr); - HEAPU32[fetch + 12 >> 2] = ptr;//{{{ makeSetValue('fetch', 12/*TODO:jsonify*/, 'ptr', 'i32')}}}; - Fetch.setu64(fetch + 16, len);//{{{ makeSetValue('fetch', 16/*TODO:jsonify*/, 'len', 'i64')}}}; - Fetch.setu64(fetch + 24, 0);//{{{ makeSetValue('fetch', 24/*TODO:jsonify*/, '0', 'i64')}}}; - Fetch.setu64(fetch + 32, len);//{{{ makeSetValue('fetch', 32/*TODO:jsonify*/, 'len', 'i64')}}}; - HEAPU16[fetch + 40 >> 1] = 4; // Mimic XHR readyState 4 === 'DONE: The operation is complete' - HEAPU16[fetch + 42 >> 1] = 200; // Mimic XHR HTTP status code 200 "OK" + HEAPU32[fetch + Fetch.fetch_t_offset_data >> 2] = ptr; + Fetch.setu64(fetch + Fetch.fetch_t_offset_numBytes, len); + Fetch.setu64(fetch + Fetch.fetch_t_offset_dataOffset, 0); + Fetch.setu64(fetch + Fetch.fetch_t_offset_totalBytes, len); + HEAPU16[fetch + Fetch.fetch_t_offset_readyState >> 1] = 4; // Mimic XHR readyState 4 === 'DONE: The operation is complete' + HEAPU16[fetch + Fetch.fetch_t_offset_status >> 1] = 200; // Mimic XHR HTTP status code 200 "OK" onsuccess(fetch, 0, value); } else { @@ -191,8 +220,8 @@ function __emscripten_fetch_load_cached_data(db, fetch, onsuccess, onerror) { #if FETCH_DEBUG console.error('fetch: File ' + pathStr + ' not found in IndexedDB'); #endif - HEAPU16[fetch + 40 >> 1] = 4; // Mimic XHR readyState 4 === 'DONE: The operation is complete' - HEAPU16[fetch + 42 >> 1] = 404; // Mimic XHR HTTP status code 404 "Not Found" + HEAPU16[fetch + Fetch.fetch_t_offset_readyState >> 1] = 4; // Mimic XHR readyState 4 === 'DONE: The operation is complete' + HEAPU16[fetch + Fetch.fetch_t_offset_status >> 1] = 404; // Mimic XHR HTTP status code 404 "Not Found" onerror(fetch, 0, 'no data'); } }; @@ -200,8 +229,8 @@ function __emscripten_fetch_load_cached_data(db, fetch, onsuccess, onerror) { #if FETCH_DEBUG console.error('fetch: Failed to load file ' + pathStr + ' from IndexedDB!'); #endif - HEAPU16[fetch + 40 >> 1] = 4; // Mimic XHR readyState 4 === 'DONE: The operation is complete' - HEAPU16[fetch + 42 >> 1] = 404; // Mimic XHR HTTP status code 404 "Not Found" + HEAPU16[fetch + Fetch.fetch_t_offset_readyState >> 1] = 4; // Mimic XHR readyState 4 === 'DONE: The operation is complete' + HEAPU16[fetch + Fetch.fetch_t_offset_status >> 1] = 404; // Mimic XHR HTTP status code 404 "Not Found" onerror(fetch, 0, error); }; } catch(e) { @@ -221,9 +250,9 @@ function __emscripten_fetch_cache_data(db, fetch, data, onsuccess, onerror) { return; } - var fetch_attr = fetch + 112/*TODO:structs_info*/; - var destinationPath = HEAPU32[fetch_attr + 64 >> 2];//{{{ makeGetValue('fetch_attr', 64/*TODO*/, 'i32') }}}; - if (!destinationPath) destinationPath = HEAPU32[fetch + 8 >> 2];//{{{ makeGetValue('fetch', 8/*TODO*/, 'i32') }}}; + var fetch_attr = fetch + Fetch.fetch_t_offset___attributes; + var destinationPath = HEAPU32[fetch_attr + Fetch.attr_t_offset_destinationPath >> 2]; + if (!destinationPath) destinationPath = HEAPU32[fetch + Fetch.fetch_t_offset_url >> 2]; var destinationPathStr = Pointer_stringify(destinationPath); try { @@ -234,8 +263,8 @@ function __emscripten_fetch_cache_data(db, fetch, data, onsuccess, onerror) { #if FETCH_DEBUG console.log('fetch: Stored file "' + destinationPathStr + '" to IndexedDB cache.'); #endif - HEAPU16[fetch + 40 >> 1] = 4; // Mimic XHR readyState 4 === 'DONE: The operation is complete' - HEAPU16[fetch + 42 >> 1] = 200; // Mimic XHR HTTP status code 200 "OK" + HEAPU16[fetch + Fetch.fetch_t_offset_readyState >> 1] = 4; // Mimic XHR readyState 4 === 'DONE: The operation is complete' + HEAPU16[fetch + Fetch.fetch_t_offset_status >> 1] = 200; // Mimic XHR HTTP status code 200 "OK" onsuccess(fetch, 0, destinationPathStr); }; putRequest.onerror = function(error) { @@ -245,8 +274,8 @@ function __emscripten_fetch_cache_data(db, fetch, data, onsuccess, onerror) { // Most likely we got an error if IndexedDB is unwilling to store any more data for this page. // TODO: Can we identify and break down different IndexedDB-provided errors and convert those // to more HTTP status codes for more information? - HEAPU16[fetch + 40 >> 1] = 4; // Mimic XHR readyState 4 === 'DONE: The operation is complete' - HEAPU16[fetch + 42 >> 1] = 413; // Mimic XHR HTTP status code 413 "Payload Too Large" + HEAPU16[fetch + Fetch.fetch_t_offset_readyState >> 1] = 4; // Mimic XHR readyState 4 === 'DONE: The operation is complete' + HEAPU16[fetch + Fetch.fetch_t_offset_status >> 1] = 413; // Mimic XHR HTTP status code 413 "Payload Too Large" onerror(fetch, 0, error); }; } catch(e) { @@ -258,7 +287,7 @@ function __emscripten_fetch_cache_data(db, fetch, data, onsuccess, onerror) { } function __emscripten_fetch_xhr(fetch, onsuccess, onerror, onprogress) { - var url = HEAPU32[fetch + 8 >> 2];//{{{ makeGetValue('fetch', 8/*TODO:structs_info.jsonify this*/, 'i32') }}}; + var url = HEAPU32[fetch + Fetch.fetch_t_offset_url >> 2]; if (!url) { #if FETCH_DEBUG console.error('fetch: XHR failed, no URL specified!'); @@ -268,18 +297,18 @@ function __emscripten_fetch_xhr(fetch, onsuccess, onerror, onprogress) { } var url_ = Pointer_stringify(url); - var fetch_attr = fetch + 112/*TODO:structs_info*/; + var fetch_attr = fetch + Fetch.fetch_t_offset___attributes; var requestMethod = Pointer_stringify(fetch_attr); if (!requestMethod) requestMethod = 'GET'; - var userData = HEAPU32[fetch_attr + 32 >> 2];//{{{ makeGetValue('fetch_attr', 32/*TODO:structs_info.jsonify this*/, 'i32') }}}; - var fetchAttributes = HEAPU32[fetch_attr + 48 >> 2];//{{{ makeGetValue('fetch_attr', 48/*TODO*/, 'i32') }}}; - var timeoutMsecs = HEAPU32[fetch_attr + 52 >> 2];//{{{ makeGetValue('fetch_attr', 52/*TODO*/, 'i32') }}}; - var withCredentials = !!HEAPU32[fetch_attr + 56 >> 2];//({{{ makeGetValue('fetch_attr', 56/*TODO*/, 'i32') }}}); - var destinationPath = HEAPU32[fetch_attr + 64 >> 2];//{{{ makeGetValue('fetch_attr', 64/*TODO*/, 'i32') }}}; - var userName = HEAPU32[fetch_attr + 68 >> 2];//{{{ makeGetValue('fetch_attr', 68/*TODO*/, 'i32') }}}; - var password = HEAPU32[fetch_attr + 72 >> 2];//{{{ makeGetValue('fetch_attr', 72/*TODO*/, 'i32') }}}; - var requestHeaders = HEAPU32[fetch_attr + 76 >> 2];//{{{ makeGetValue('fetch_attr', 76/*TODO*/, 'i32') }}}; - var overriddenMimeType = HEAPU32[fetch_attr + 80 >> 2];//{{{ makeGetValue('fetch_attr', 80/*TODO*/, 'i32') }}}; + var userData = HEAPU32[fetch_attr + Fetch.attr_t_offset_userData >> 2]; + var fetchAttributes = HEAPU32[fetch_attr + Fetch.attr_t_offset_attributes >> 2]; + var timeoutMsecs = HEAPU32[fetch_attr + Fetch.attr_t_offset_timeoutMSecs >> 2]; + var withCredentials = !!HEAPU32[fetch_attr + Fetch.attr_t_offset_withCredentials >> 2]; + var destinationPath = HEAPU32[fetch_attr + Fetch.attr_t_offset_destinationPath >> 2]; + var userName = HEAPU32[fetch_attr + Fetch.attr_t_offset_userName >> 2]; + var password = HEAPU32[fetch_attr + Fetch.attr_t_offset_password >> 2]; + var requestHeaders = HEAPU32[fetch_attr + Fetch.attr_t_offset_requestHeaders >> 2]; + var overriddenMimeType = HEAPU32[fetch_attr + Fetch.attr_t_offset_overriddenMimeType >> 2]; var fetchAttrLoadToMemory = !!(fetchAttributes & 1/*EMSCRIPTEN_FETCH_LOAD_TO_MEMORY*/); var fetchAttrStreamData = !!(fetchAttributes & 2/*EMSCRIPTEN_FETCH_STREAM_DATA*/); @@ -313,9 +342,9 @@ function __emscripten_fetch_xhr(fetch, onsuccess, onerror, onprogress) { } if (requestHeaders) { for(;;) { - var key = HEAPU32[requestHeaders >> 2];//{{{ makeGetValue('requestHeaders', 0, 'i32') }}}; + var key = HEAPU32[requestHeaders >> 2]; if (!key) break; - var value = HEAPU32[requestHeaders + 4 >> 2];//{{{ makeGetValue('requestHeaders', 4, 'i32') }}}; + var value = HEAPU32[requestHeaders + 4 >> 2]; if (!value) break; requestHeaders += 8; var keyStr = Pointer_stringify(key); @@ -328,7 +357,7 @@ function __emscripten_fetch_xhr(fetch, onsuccess, onerror, onprogress) { } Fetch.xhrs.push(xhr); var id = Fetch.xhrs.length; - HEAPU32[fetch >> 2] = id;//{{{ makeSetValue('fetch', 0/*TODO:jsonify*/, 'id', 'i32')}}}; + HEAPU32[fetch + Fetch.fetch_t_offset_id >> 2] = id; var data = null; // TODO: Support user to pass data to request. // TODO: Support specifying custom headers to the request. @@ -344,24 +373,24 @@ function __emscripten_fetch_xhr(fetch, onsuccess, onerror, onprogress) { // The data pointer malloc()ed here has the same lifetime as the emscripten_fetch_t structure itself has, and is // freed when emscripten_fetch_close() is called. ptr = _malloc(ptrLen); - HEAPU8.set(new Uint8Array(xhr.response), ptr); // TODO: Since DYNAMICTOP is not coherent, this can corrupt + HEAPU8.set(new Uint8Array(xhr.response), ptr); } - HEAPU32[fetch + 12 >> 2] = ptr;//{{{ makeSetValue('fetch', 12/*TODO:jsonify*/, 'ptr', 'i32')}}}; - Fetch.setu64(fetch + 16, ptrLen);//{{{ makeSetValue('fetch', 16/*TODO:jsonify*/, 'ptrLen', 'i64')}}}; - Fetch.setu64(fetch + 24, 0);//{{{ makeSetValue('fetch', 24/*TODO:jsonify*/, '0', 'i64')}}}; + HEAPU32[fetch + Fetch.fetch_t_offset_data >> 2] = ptr; + Fetch.setu64(fetch + Fetch.fetch_t_offset_numBytes, ptrLen); + Fetch.setu64(fetch + Fetch.fetch_t_offset_dataOffset, 0); if (len) { // If the final XHR.onload handler receives the bytedata to compute total length, report that, // otherwise don't write anything out here, which will retain the latest byte size reported in // the most recent XHR.onprogress handler. - Fetch.setu64(fetch + 32, len);//{{{ makeSetValue('fetch', 32/*TODO:jsonify*/, 'len', 'i64')}}}; + Fetch.setu64(fetch + Fetch.fetch_t_offset_totalBytes, len); } - HEAPU16[fetch + 40 >> 1] = xhr.readyState; + HEAPU16[fetch + Fetch.fetch_t_offset_readyState >> 1] = xhr.readyState; if (xhr.readyState === 4 && xhr.status === 0) { if (len > 0) xhr.status = 200; // If loading files from a source that does not give HTTP status code, assume success if we got data bytes. else xhr.status = 404; // Conversely, no data bytes is 404. } - HEAPU16[fetch + 42 >> 1] = xhr.status; -// if (xhr.statusText) stringToUTF8(fetch + 44, xhr.statusText, 64); + HEAPU16[fetch + Fetch.fetch_t_offset_status >> 1] = xhr.status; +// if (xhr.statusText) stringToUTF8(fetch + Fetch.fetch_t_offset_statusText, xhr.statusText, 64); if (xhr.status == 200) { #if FETCH_DEBUG console.log('fetch: xhr of URL "' + xhr.url_ + '" / responseURL "' + xhr.responseURL + '" succeeded with status 200'); @@ -380,12 +409,12 @@ function __emscripten_fetch_xhr(fetch, onsuccess, onerror, onprogress) { #if FETCH_DEBUG console.error('fetch: xhr of URL "' + xhr.url_ + '" / responseURL "' + xhr.responseURL + '" finished with error, readyState ' + xhr.readyState + ' and status ' + status); #endif - HEAPU32[fetch + 12 >> 2] = 0; - Fetch.setu64(fetch + 16, 0); - Fetch.setu64(fetch + 24, 0); - Fetch.setu64(fetch + 32, 0); - HEAPU16[fetch + 40 >> 1] = xhr.readyState; - HEAPU16[fetch + 42 >> 1] = status; + HEAPU32[fetch + Fetch.fetch_t_offset_data >> 2] = 0; + Fetch.setu64(fetch + Fetch.fetch_t_offset_numBytes, 0); + Fetch.setu64(fetch + Fetch.fetch_t_offset_dataOffset, 0); + Fetch.setu64(fetch + Fetch.fetch_t_offset_totalBytes, 0); + HEAPU16[fetch + Fetch.fetch_t_offset_readyState >> 1] = xhr.readyState; + HEAPU16[fetch + Fetch.fetch_t_offset_status >> 1] = status; if (onerror) onerror(fetch, xhr, e); } xhr.ontimeout = function(e) { @@ -406,14 +435,14 @@ function __emscripten_fetch_xhr(fetch, onsuccess, onerror, onprogress) { ptr = _malloc(ptrLen); HEAPU8.set(new Uint8Array(xhr.response), ptr); } - HEAPU32[fetch + 12 >> 2] = ptr;//{{{ makeSetValue('fetch', 12/*TODO:jsonify*/, 'ptr', 'i32')}}}; - Fetch.setu64(fetch + 16, ptrLen);//{{{ makeSetValue('fetch', 16/*TODO:jsonify*/, 'ptrLen', 'i64')}}}; - Fetch.setu64(fetch + 24, e.loaded - ptrLen);//{{{ makeSetValue('fetch', 24/*TODO:jsonify*/, 'e.loaded - ptrLen', 'i64')}}}; - Fetch.setu64(fetch + 32, e.total);//{{{ makeSetValue('fetch', 32/*TODO:jsonify*/, 'e.total', 'i64')}}}; - HEAPU16[fetch + 40 >> 1] = xhr.readyState; + HEAPU32[fetch + Fetch.fetch_t_offset_data >> 2] = ptr; + Fetch.setu64(fetch + Fetch.fetch_t_offset_numBytes, ptrLen); + Fetch.setu64(fetch + Fetch.fetch_t_offset_dataOffset, e.loaded - ptrLen); + Fetch.setu64(fetch + Fetch.fetch_t_offset_totalBytes, e.total); + HEAPU16[fetch + Fetch.fetch_t_offset_readyState >> 1] = xhr.readyState; if (xhr.readyState >= 3 && xhr.status === 0 && e.loaded > 0) xhr.status = 200; // If loading files from a source that does not give HTTP status code, assume success if we get data bytes - HEAPU16[fetch + 42 >> 1] = xhr.status; - if (xhr.statusText) stringToUTF8(fetch + 44, xhr.statusText, 64); + HEAPU16[fetch + Fetch.fetch_t_offset_status >> 1] = xhr.status; + if (xhr.statusText) stringToUTF8(fetch + Fetch.fetch_t_offset_statusText, xhr.statusText, 64); if (onprogress) onprogress(fetch, xhr, e); } #if FETCH_DEBUG @@ -432,12 +461,12 @@ function __emscripten_fetch_xhr(fetch, onsuccess, onerror, onprogress) { function emscripten_start_fetch(fetch, successcb, errorcb, progresscb) { if (typeof Module !== 'undefined') Module['noExitRuntime'] = true; // If we are the main Emscripten runtime, we should not be closing down. - var fetch_attr = fetch + 112/*TODO:structs_info*/; + var fetch_attr = fetch + Fetch.fetch_t_offset___attributes; var requestMethod = Pointer_stringify(fetch_attr); - var onsuccess = HEAPU32[fetch_attr + 36 >> 2];//{{{ makeGetValue('fetch_attr', 36/*TODO:structs_info.jsonify this*/, 'i32') }}}; - var onerror = HEAPU32[fetch_attr + 40 >> 2];//{{{ makeGetValue('fetch_attr', 40/*TODO:structs_info.jsonify this*/, 'i32') }}}; - var onprogress = HEAPU32[fetch_attr + 44 >> 2];//{{{ makeGetValue('fetch_attr', 44/*TODO:structs_info.jsonify this*/, 'i32') }}}; - var fetchAttributes = HEAPU32[fetch_attr + 48 >> 2];//{{{ makeGetValue('fetch_attr', 48/*TODO*/, 'i32') }}}; + var onsuccess = HEAPU32[fetch_attr + Fetch.attr_t_offset_onsuccess >> 2]; + var onerror = HEAPU32[fetch_attr + Fetch.attr_t_offset_onerror >> 2]; + var onprogress = HEAPU32[fetch_attr + Fetch.attr_t_offset_onprogress >> 2]; + var fetchAttributes = HEAPU32[fetch_attr + Fetch.attr_t_offset_attributes >> 2]; var fetchAttrLoadToMemory = !!(fetchAttributes & 1/*EMSCRIPTEN_FETCH_LOAD_TO_MEMORY*/); var fetchAttrStreamData = !!(fetchAttributes & 2/*EMSCRIPTEN_FETCH_STREAM_DATA*/); var fetchAttrPersistFile = !!(fetchAttributes & 4/*EMSCRIPTEN_FETCH_PERSIST_FILE*/); @@ -510,8 +539,8 @@ function emscripten_start_fetch(fetch, successcb, errorcb, progresscb) { } if (requestMethod === 'EM_IDB_STORE') { - var dataPtr = HEAPU32[fetch_attr + 84 >> 2]; - var dataLength = HEAPU32[fetch_attr + 88 >> 2]; + var dataPtr = HEAPU32[fetch_attr + Fetch.attr_t_offset_requestData >> 2]; + var dataLength = HEAPU32[fetch_attr + Fetch.attr_t_offset_requestDataSize >> 2]; var data = HEAPU8.slice(dataPtr, dataPtr + dataLength); // TODO(?): Here we perform a clone of the data, because storing shared typed arrays to IndexedDB does not seem to be allowed. __emscripten_fetch_cache_data(Fetch.dbInstance, fetch, data, reportSuccess, reportError); } else if (requestMethod === 'EM_IDB_DELETE') { diff --git a/src/fetch-worker.js b/src/fetch-worker.js index 336b09f281a96..4980846eaaff5 100644 --- a/src/fetch-worker.js +++ b/src/fetch-worker.js @@ -93,13 +93,13 @@ function processWorkQueue() { console.log('processWorkQueue: starting fetch'); function successcb(fetch) { console.log('FETCH-WORKER: fetch finished on success'); - Atomics.compareExchange(HEAPU32, fetch + 108 >> 2, 1, 2); - Atomics.wake(HEAP32, fetch + 108 >> 2, 1); + Atomics.compareExchange(HEAPU32, fetch + Fetch.fetch_t_offset___proxyState >> 2, 1, 2); + Atomics.wake(HEAP32, fetch + Fetch.fetch_t_offset___proxyState >> 2, 1); } function errorcb(fetch) { console.log('FETCH-WORKER: fetch finished on failure'); - Atomics.compareExchange(HEAPU32, fetch + 108 >> 2, 1, 2); - Atomics.wake(HEAP32, fetch + 108 >> 2, 1); + Atomics.compareExchange(HEAPU32, fetch + Fetch.fetch_t_offset___proxyState >> 2, 1, 2); + Atomics.wake(HEAP32, fetch + Fetch.fetch_t_offset___proxyState >> 2, 1); } function progresscb(fetch) { console.log('FETCH-WORKER: fetch progress..'); diff --git a/system/include/emscripten/fetch.h b/system/include/emscripten/fetch.h index a2a8238c04bd5..d3960cfdc618e 100644 --- a/system/include/emscripten/fetch.h +++ b/system/include/emscripten/fetch.h @@ -48,73 +48,68 @@ struct emscripten_fetch_t; struct emscripten_fetch_attr_t { // 'POST', 'GET', etc. - char requestMethod[32]; // XXX 0 + char requestMethod[32]; // Custom data that can be tagged along the process. - void *userData; // XXX 32 + void *userData; - void (*onsuccess)(emscripten_fetch_t *fetch); // XXX 36 - void (*onerror)(emscripten_fetch_t *fetch); // XXX 40 - void (*onprogress)(emscripten_fetch_t *fetch); // XXX 44 + void (*onsuccess)(emscripten_fetch_t *fetch); + void (*onerror)(emscripten_fetch_t *fetch); + void (*onprogress)(emscripten_fetch_t *fetch); // EMSCRIPTEN_FETCH_* attributes - uint32_t attributes; // XXX 48 + uint32_t attributes; // Specifies the amount of time the request can take before failing due to a timeout. - unsigned long timeoutMSecs; // XXX 52 + unsigned long timeoutMSecs; // Indicates whether cross-site access control requests should be made using credentials. - EM_BOOL withCredentials; // XXX 56 - - // If true, performs a synchronous blocking XHR. The emscripten_fetch() function call does not return until - // the request has completed. Setting this to true in the main browser thread will fail with - // EMSCRIPTEN_RESULT_NOT_SUPPORTED. - EM_BOOL synchronousRequestTODODELETEONCESTRUCTJSONIFIED; // XXX 60 + EM_BOOL withCredentials; // Specifies the destination path in IndexedDB where to store the downloaded content body. If this is empty, the transfer // is not stored to IndexedDB at all. // Note that this struct does not contain space to hold this string, it only carries a pointer. // Calling emscripten_fetch() will make an internal copy of this string. - const char *destinationPath; // XXX 64 + const char *destinationPath; // Specifies the authentication username to use for the request, if necessary. // Note that this struct does not contain space to hold this string, it only carries a pointer. // Calling emscripten_fetch() will make an internal copy of this string. - const char *userName; // XXX 68 + const char *userName; // Specifies the authentication username to use for the request, if necessary. // Note that this struct does not contain space to hold this string, it only carries a pointer. // Calling emscripten_fetch() will make an internal copy of this string. - const char *password; // XXX 72 + const char *password; // Points to an array of strings to pass custom headers to the request. This array takes the form // {"key1", "value1", "key2", "value2", "key3", "value3", ..., 0 }; Note especially that the array // needs to be terminated with a null pointer. - const char * const *requestHeaders; // XXX 76 + const char * const *requestHeaders; // Pass a custom MIME type here to force the browser to treat the received data with the given type. - const char *overriddenMimeType; // XXX 80 + const char *overriddenMimeType; // If non-zero, specified a pointer to the data that is to be passed as the body (payload) of the request // that is being performed. Leave as zero if no request body needs to be sent. // The memory pointed to by this field is provided by the user, and needs to be valid only until the call to // emscripten_fetch() returns. - const char *requestData; // XXX 84 + const char *requestData; // Specifies the length of the buffer pointed by 'requestData'. Leave as 0 if no request body needs to be sent. - size_t requestDataSize; // XXX 88 + size_t requestDataSize; }; struct emscripten_fetch_t { // Unique identifier for this fetch in progress. - unsigned int id; // XXX 0 + unsigned int id; // Custom data that can be tagged along the process. - void *userData; // XXX 4 + void *userData; // The remote URL that is being downloaded. - const char *url; // XXX 8 + const char *url; // In onsuccess() handler: // - If the EMSCRIPTEN_FETCH_LOAD_TO_MEMORY attribute was specified for the transfer, this points to the @@ -124,19 +119,19 @@ struct emscripten_fetch_t // chunk of bytes related to the transfer. Otherwise this will be null. // The data buffer provided here has identical lifetime with the emscripten_fetch_t object itself, and is freed by // calling emscripten_fetch_close() on the emscripten_fetch_t pointer. - const char *data; // XXX 12 + const char *data; // Specifies the length of the above data block in bytes. When the download finishes, this field will be valid even if // EMSCRIPTEN_FETCH_LOAD_TO_MEMORY was not specified. - uint64_t numBytes; // XXX 16 + uint64_t numBytes; // If EMSCRIPTEN_FETCH_STREAM_DATA is being performed, this indicates the byte offset from the start of the stream // that the data block specifies. (for onprogress() streaming XHR transfer, the number of bytes downloaded so far before this chunk) - uint64_t dataOffset; // XXX 24 + uint64_t dataOffset; // Specifies the total number of bytes that the response body will be. // Note: This field may be zero, if the server does not report the Content-Length field. - uint64_t totalBytes; // XXX 32 + uint64_t totalBytes; // Specifies the readyState of the XHR request: // 0: UNSENT: request not sent yet @@ -145,18 +140,18 @@ struct emscripten_fetch_t // 3: LOADING: download in progress. // 4: DONE: download finished. // See https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/readyState - unsigned short readyState; // XXX 40 + unsigned short readyState; // Specifies the status code of the response. - unsigned short status; // XXX 42 + unsigned short status; // Specifies a human-readable form of the status code. - char statusText[64]; // XXX 44 + char statusText[64]; - uint32_t __proxyState; // XXX 108 + uint32_t __proxyState; // For internal use only. - emscripten_fetch_attr_t __attributes; // XXX 112 + emscripten_fetch_attr_t __attributes; }; // Clears the fields of an emscripten_fetch_attr_t structure to their default values in a future-compatible manner. From 8a3766bca994406c6b3b4a64ee84f3d23ff51e38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Wed, 7 Dec 2016 21:42:36 +0200 Subject: [PATCH 72/74] Separate creation of fetch-worker.js to shared.py --- emcc.py | 27 +-------------------------- tools/shared.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 26 deletions(-) diff --git a/emcc.py b/emcc.py index edbe3f34fbf08..9feee2609bd23 100755 --- a/emcc.py +++ b/emcc.py @@ -1771,32 +1771,7 @@ def repl(m): # Generate the fetch-worker.js script for multithreaded emscripten_fetch() support if targeting pthreads. if shared.Settings.FETCH and shared.Settings.USE_PTHREADS: - src = open(final, 'r').read() - funcs_to_import = ['alignMemoryPage', 'getTotalMemory', 'stringToUTF8', 'intArrayFromString', 'lengthBytesUTF8', 'stringToUTF8Array', '_emscripten_is_main_runtime_thread', '_emscripten_futex_wait'] - asm_funcs_to_import = ['_malloc', '_free', '_sbrk', '_pthread_mutex_lock', '_pthread_mutex_unlock'] - function_prologue = '''this.onerror = function(e) { - console.error(e); -} - -''' - asm_start = src.find('// EMSCRIPTEN_START_ASM') - for func in funcs_to_import + asm_funcs_to_import: - loc = src.find('function ' + func + '(', asm_start if func in asm_funcs_to_import else 0) - if loc == -1: - logging.fatal('failed to find function ' + func + '!') - sys.exit(1) - end_loc = src.find('{', loc) + 1 - nesting_level = 1 - while nesting_level > 0: - if src[end_loc] == '{': nesting_level += 1 - if src[end_loc] == '}': nesting_level -= 1 - end_loc += 1 - - func_code = src[loc:end_loc] - function_prologue = function_prologue + '\n' + func_code - - fetch_worker_src = function_prologue + '\n' + shared.clang_preprocess(shared.path_from_root('src', 'fetch-worker.js')) - open(os.path.join(os.path.dirname(os.path.abspath(target)), 'fetch-worker.js'), 'w').write(fetch_worker_src) + shared.make_fetch_worker(final, os.path.join(os.path.dirname(os.path.abspath(target)), 'fetch-worker.js')) if shared.Settings.BINARYEN: # Insert a call to integrate with wasm.js diff --git a/tools/shared.py b/tools/shared.py index b9f35903c7f71..801e2c6c1b22b 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -2338,4 +2338,36 @@ def read_and_preprocess(filename): f = f[:m.start(0)] + included_file + f[m.end(0):] +# Generates a suitable fetch-worker.js script from the given input source JS file (which is an asm.js build output), +# and writes it out to location output_file. fetch-worker.js is the root entry point for a dedicated filesystem web +# worker in -s ASMFS=1 mode. +def make_fetch_worker(source_file, output_file): + src = open(source_file, 'r').read() + funcs_to_import = ['alignMemoryPage', 'getTotalMemory', 'stringToUTF8', 'intArrayFromString', 'lengthBytesUTF8', 'stringToUTF8Array', '_emscripten_is_main_runtime_thread', '_emscripten_futex_wait'] + asm_funcs_to_import = ['_malloc', '_free', '_sbrk', '_pthread_mutex_lock', '_pthread_mutex_unlock'] + function_prologue = '''this.onerror = function(e) { + console.error(e); +} + +''' + asm_start = src.find('// EMSCRIPTEN_START_ASM') + for func in funcs_to_import + asm_funcs_to_import: + loc = src.find('function ' + func + '(', asm_start if func in asm_funcs_to_import else 0) + if loc == -1: + logging.fatal('failed to find function ' + func + '!') + sys.exit(1) + end_loc = src.find('{', loc) + 1 + nesting_level = 1 + while nesting_level > 0: + if src[end_loc] == '{': nesting_level += 1 + if src[end_loc] == '}': nesting_level -= 1 + end_loc += 1 + + func_code = src[loc:end_loc] + function_prologue = function_prologue + '\n' + func_code + + fetch_worker_src = function_prologue + '\n' + clang_preprocess(path_from_root('src', 'fetch-worker.js')) + open(output_file, 'w').write(fetch_worker_src) + + import js_optimizer From a9eb3d304c1b7a6500dd47f2e8b7036caaf1aa9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Wed, 7 Dec 2016 22:19:52 +0200 Subject: [PATCH 73/74] Remove debug prints in fetch-worker.js --- src/fetch-worker.js | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/fetch-worker.js b/src/fetch-worker.js index 4980846eaaff5..ab38d3fa75610 100644 --- a/src/fetch-worker.js +++ b/src/fetch-worker.js @@ -65,8 +65,6 @@ function Pointer_stringify(ptr, /* optional */ length) { return Module['UTF8ToString'](ptr); } - -console.log('fetch worker script loading.'); Fetch.staticInit(); var queuePtr = 0; @@ -85,24 +83,19 @@ function processWorkQueue() { var numQueuedItems = Atomics_load(HEAPU32, queuePtr + 4 >> 2); if (numQueuedItems == 0) return; - console.log('polling work to perform, there are ' + numQueuedItems + ' work items in the queue.'); var queuedOperations = Atomics_load(HEAPU32, queuePtr >> 2); var queueSize = Atomics_load(HEAPU32, queuePtr + 8 >> 2); for(var i = 0; i < numQueuedItems; ++i) { var fetch = Atomics_load(HEAPU32, (queuedOperations >> 2)+i); - console.log('processWorkQueue: starting fetch'); function successcb(fetch) { - console.log('FETCH-WORKER: fetch finished on success'); Atomics.compareExchange(HEAPU32, fetch + Fetch.fetch_t_offset___proxyState >> 2, 1, 2); Atomics.wake(HEAP32, fetch + Fetch.fetch_t_offset___proxyState >> 2, 1); } function errorcb(fetch) { - console.log('FETCH-WORKER: fetch finished on failure'); Atomics.compareExchange(HEAPU32, fetch + Fetch.fetch_t_offset___proxyState >> 2, 1, 2); Atomics.wake(HEAP32, fetch + Fetch.fetch_t_offset___proxyState >> 2, 1); } function progresscb(fetch) { - console.log('FETCH-WORKER: fetch progress..'); } try { emscripten_start_fetch(fetch, successcb, errorcb, progresscb); @@ -134,7 +127,6 @@ this.onmessage = function(e) { HEAPU16 = new Uint16Array(buffer); HEAP32 = new Int32Array(buffer); HEAPU32 = new Uint32Array(buffer); - console.log('fetch: fetch Worker initialized. queue ptr ' + queuePtr + ', heap length: ' + buffer.byteLength); interval = setInterval(processWorkQueue, 100); } } From 8e7ee902f89b0c49e38b07bc0197b2d9484f1c72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Mon, 26 Dec 2016 13:36:46 +0200 Subject: [PATCH 74/74] Update version to 1.37.1 to clear cache for Emscripten Fetch. --- emscripten-version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/emscripten-version.txt b/emscripten-version.txt index a9eeadaa59718..d6f86c3c20b37 100644 --- a/emscripten-version.txt +++ b/emscripten-version.txt @@ -1 +1 @@ -"1.37.0" +"1.37.1"