diff --git a/AUTHORS b/AUTHORS index 0441963dded57..8353c0dfa5a64 100644 --- a/AUTHORS +++ b/AUTHORS @@ -269,3 +269,5 @@ a license to everyone to use it as detailed in LICENSE.) * Vilibald Wanča * Alex Hixon * Vladimir Davidovich +* Christophe Gragnic + diff --git a/emcc.py b/emcc.py index 3b113d063208b..9feee2609bd23 100755 --- a/emcc.py +++ b/emcc.py @@ -1138,6 +1138,21 @@ def check(input_file): # stb_image 2.x need to have STB_IMAGE_IMPLEMENTATION defined to include the implementation when compiling newargs.append('-DSTB_IMAGE_IMPLEMENTATION') + if shared.Settings.ASMFS and final_suffix in JS_CONTAINING_SUFFIXES: + input_files.append((next_arg_index, shared.path_from_root('system', 'lib', 'fetch', 'asmfs.cpp'))) + newargs.append('-D__EMSCRIPTEN_ASMFS__=1') + next_arg_index += 1 + shared.Settings.NO_FILESYSTEM = 1 + shared.Settings.FETCH = 1 + if not shared.Settings.USE_PTHREADS: + logging.error('-s ASMFS=1 requires either -s USE_PTHREADS=1 or -s USE_PTHREADS=2 to be set!') + sys.exit(1) + + if shared.Settings.FETCH and final_suffix in JS_CONTAINING_SUFFIXES: + input_files.append((next_arg_index, shared.path_from_root('system', 'lib', 'fetch', 'emscripten_fetch.cpp'))) + next_arg_index += 1 + js_libraries.append(shared.path_from_root('src', 'library_fetch.js')) + forced_stdlibs = [] if shared.Settings.DEMANGLE_SUPPORT: shared.Settings.EXPORTED_FUNCTIONS += ['___cxa_demangle'] @@ -1754,6 +1769,10 @@ def repl(m): if shared.Settings.USE_PTHREADS: shutil.copyfile(shared.path_from_root('src', 'pthread-main.js'), os.path.join(os.path.dirname(os.path.abspath(target)), 'pthread-main.js')) + # Generate the fetch-worker.js script for multithreaded emscripten_fetch() support if targeting pthreads. + if shared.Settings.FETCH and shared.Settings.USE_PTHREADS: + shared.make_fetch_worker(final, os.path.join(os.path.dirname(os.path.abspath(target)), 'fetch-worker.js')) + if shared.Settings.BINARYEN: # Insert a call to integrate with wasm.js js = open(final).read() diff --git a/emscripten-version.txt b/emscripten-version.txt index a9eeadaa59718..d6f86c3c20b37 100644 --- a/emscripten-version.txt +++ b/emscripten-version.txt @@ -1 +1 @@ -"1.37.0" +"1.37.1" diff --git a/site/source/docs/api_reference/fetch.rst b/site/source/docs/api_reference/fetch.rst new file mode 100644 index 0000000000000..b4cc7f6f62afb --- /dev/null +++ b/site/source/docs/api_reference/fetch.rst @@ -0,0 +1,300 @@ +.. _fetch-api: + +========= +Fetch API +========= + +The Emscripten Fetch API allows native code to transfer files via XHR (HTTP GET, PUT, POST) from remote servers, and to persist the downloaded files locally in browser's IndexedDB storage, so that they can be reaccessed locally on subsequent page visits. The Fetch API is callable from multiple threads, and the network requests can be run either synchronously or asynchronously as desired. + +Introduction +============ + +The use of the Fetch API is quick to illustrate via an example. The following application downloads a file from a web server asynchronously to memory inside the application heap. + + .. code-block:: cpp + + #include + #include + #include + + void downloadSucceeded(emscripten_fetch_t *fetch) { + printf("Finished downloading %llu bytes from URL %s.\n", fetch->numBytes, fetch->url); + // The data is now available at fetch->data[0] through fetch->data[fetch->numBytes-1]; + emscripten_fetch_close(fetch); // Free data associated with the fetch. + } + + void downloadFailed(emscripten_fetch_t *fetch) { + printf("Downloading %s failed, HTTP failure status code: %d.\n", fetch->url, fetch->status); + emscripten_fetch_close(fetch); // Also free data on failure. + } + + int main() { + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.attributes = EMSCRIPTEN_FETCH_LOAD_TO_MEMORY; + attr.onsuccess = downloadSucceeded; + attr.onerror = downloadFailed; + emscripten_fetch(&attr, "myfile.dat"); + } + +If a relative pathname is specified to a call to emscripten_fetch, like in the above example, the XHR is performed relative to the href (URL) of the current page. Passing a fully qualified absolute URL allows downloading files across domains, however these are subject to `HTTP access control (CORS) rules `_. + +By default the Fetch API runs asynchronously, which means that the emscripten_fetch() function call returns immediately and the operation will continue to occur on the background. When the operation finishes, either the success or the failure callback will be invoked. + +Persisting data +=============== + +The XHR requests issued by the Fetch API are subject to the usual browser caching behavior. These caches are transient (temporary) so there is no guarantee that the data will persist in the cache for a given period of time. Additionally, if the files are somewhat large (multiple megabytes), browsers typically don't cache the downloads at all. + +To enable a more explicit control for persisting the downloaded files, the Fetch API interacts with the browser's IndexedDB API, which can load and store large data files that are available on subsequent visits to the page. To enable IndexedDB storage, pass the EMSCRIPTEN_FETCH_PERSIST_FILE flag in the fetch attributes: + + .. code-block:: cpp + + int main() { + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + ... + attr.attributes = EMSCRIPTEN_FETCH_LOAD_TO_MEMORY | EMSCRIPTEN_FETCH_PERSIST_FILE; + ... + emscripten_fetch(&attr, "myfile.dat"); + } + +For a full example, see the file tests/fetch/example_async_xhr_to_memory_via_indexeddb.cpp. + +Persisting data bytes from memory +--------------------------------- + +Sometimes it is useful to persist a range of bytes from application memory to IndexedDB (without having to perform any XHRs). This is possible with the Emscripten Fetch API by passing the special HTTP action verb "EM_IDB_STORE" to the Emscripten Fetch operation. + + .. code-block:: cpp + + void success(emscripten_fetch_t *fetch) { + printf("IDB store succeeded.\n"); + emscripten_fetch_close(fetch); + } + + void failure(emscripten_fetch_t *fetch) { + printf("IDB store failed.\n"); + emscripten_fetch_close(fetch); + } + + void persistFileToIndexedDB(const char *outputFilename, uint8_t *data, size_t numBytes) { + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "EM_IDB_STORE"); + attr.attributes = EMSCRIPTEN_FETCH_REPLACE | EMSCRIPTEN_FETCH_PERSIST_FILE; + attr.requestData = (char *)data; + attr.requestDataSize = numBytes; + attr.onsuccess = success; + attr.onerror = failure; + emscripten_fetch(&attr, outputFilename); + } + + int main() { + // Create data + uint8_t *data = (uint8_t*)malloc(10240); + srand(time(NULL)); + for(int i = 0; i < 10240; ++i) data[i] = (uint8_t)rand(); + + persistFileToIndexedDB("outputfile.dat", data, 10240); + } + +Deleting a file from IndexedDB +------------------------------ + +Files can be cleaned up from IndexedDB by using the HTTP action verb "EM_IDB_DELETE": + + .. code-block:: cpp + + void success(emscripten_fetch_t *fetch) { + printf("Deleting file from IDB succeeded.\n"); + emscripten_fetch_close(fetch); + } + + void failure(emscripten_fetch_t *fetch) { + printf("Deleting file from IDB failed.\n"); + emscripten_fetch_close(fetch); + } + + int main() { + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "EM_IDB_DELETE"); + emscripten_fetch(&attr, "filename_to_delete.dat"); + } + +Synchronous Fetches +=================== + +In some scenarios, it would be nice to be able to perform an XHR request or an IndexedDB file operation synchronously in the calling thread. This can make porting applications easier and simplify code flow by avoiding the need to pass a callback. + +All types of Emscripten Fetch API operations (XHRs, IndexedDB accesses) can be performed synchronously by passing the EMSCRIPTEN_FETCH_SYNCHRONOUS flag. When this flag is passed, the calling thread will block to sleep until the fetch operation finishes. See the following example. + + .. code-block:: cpp + + int main() { + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.attributes = EMSCRIPTEN_FETCH_LOAD_TO_MEMORY | EMSCRIPTEN_FETCH_SYNCHRONOUS; + emscripten_fetch_t *fetch = emscripten_fetch(&attr, "file.dat"); // Blocks here until the operation is complete. + if (fetch->status == 200) { + printf("Finished downloading %llu bytes from URL %s.\n", fetch->numBytes, fetch->url); + // The data is now available at fetch->data[0] through fetch->data[fetch->numBytes-1]; + } else { + printf("Downloading %s failed, HTTP failure status code: %d.\n", fetch->url, fetch->status); + } + emscripten_fetch_close(fetch); + } + +In the above code sample, the success and failure callback functions are not used. However, if specified, they will be synchronously called before emscripten_fetch() returns. + +.. note:: + + Synchronous Emscripten Fetch operations are subject to a number of restrictions, depending on which Emscripten build mode (linker flags) is used: + + - **No flags**: Only asynchronous Fetch operations are available. + - **--proxy-to-worker**: Synchronous Fetch operations are allowed for fetches that only do an XHR but do not interact with IndexedDB. + - **-s USE_PTHREADS=1**: Synchronous Fetch operations are available on pthreads, but not on the main thread. + - **--proxy-to-worker** + **-s USE_PTHREADS=1**: Synchronous Fetch operations are available both on the main thread and pthreads. + +Waitable Fetches +================ + +Emscripten Fetch operations can also run in a third mode, called a *waitable* fetch. Waitable fetches start off as asynchronous, but at any point after the fetch has started, the calling thread can issue a wait operation to either wait for the completion of the fetch, or to just poll whether the fetch operation has yet completed. The following code sample illustrates how this works. + + .. code-block:: cpp + + int main() { + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.attributes = EMSCRIPTEN_FETCH_LOAD_TO_MEMORY | EMSCRIPTEN_FETCH_WAITABLE; + emscripten_fetch_t *fetch = emscripten_fetch(&attr, "file.dat"); // Starts as asynchronous. + + EMSCRIPTEN_RESULT ret = EMSCRIPTEN_RESULT_TIMED_OUT; + while(ret == EMSCRIPTEN_RESULT_TIMED_OUT) { + /* possibly do some other work; */ + ret = emscripten_fetch_wait(fetch, 0/*milliseconds to wait, 0 to just poll, INFINITY=wait until completion*/); + } + // The operation has finished, safe to examine the fields of the 'fetch' pointer now. + + if (fetch->status == 200) { + printf("Finished downloading %llu bytes from URL %s.\n", fetch->numBytes, fetch->url); + // The data is now available at fetch->data[0] through fetch->data[fetch->numBytes-1]; + } else { + printf("Downloading %s failed, HTTP failure status code: %d.\n", fetch->url, fetch->status); + } + emscripten_fetch_close(fetch); + } + +Waitable fetches allow interleaving multiple tasks in one thread so that the issuing thread can perform some other work until the fetch completes. + +.. note:: + + Waitable fetches are available only in certain build modes: + + - **No flags** or **--proxy-to-worker**: Waitable fetches are not available. + - **-s USE_PTHREADS=1**: Waitable fetches are available on pthreads, but not on the main thread. + - **--proxy-to-worker** + **-s USE_PTHREADS=1**: Waitable fetches are available on all threads. + +Tracking Progress +==================== + +For robust fetch management, there are several fields available to track the status of an XHR. + +The onprogress callback is called whenever new data has been received. This allows one to measure the download speed and compute an ETA for completion. Additionally, the emscripten_fetch_t structure passes the XHR object fields readyState, status and statusText, which give information about the HTTP loading state of the request. + +The emscripten_fetch_attr_t object has a timeoutMsecs field which allows specifying a timeout duration for the transfer. Additionally, emscripten_fetch_close() can be called at any time for asynchronous and waitable fetches to abort the download. The following example illustrates these fields and the onprogress handler. + + .. code-block:: cpp + + void downloadProgress(emscripten_fetch_t *fetch) { + if (fetch->totalBytes) { + printf("Downloading %s.. %.2f%% complete.\n", fetch->url, fetch->dataOffset * 100.0 / fetch->totalBytes); + } else { + printf("Downloading %s.. %lld bytes complete.\n", fetch->url, fetch->dataOffset + fetch->numBytes); + } + } + + int main() { + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.attributes = EMSCRIPTEN_FETCH_LOAD_TO_MEMORY; + attr.onsuccess = downloadSucceeded; + attr.onprogress = downloadProgress; + attr.onerror = downloadFailed; + emscripten_fetch(&attr, "myfile.dat"); + } + +Managing Large Files +==================== + +Particular attention should be paid to the memory usage strategy of a fetch. Previous examples have all passed the EMSCRIPTEN_FETCH_LOAD_TO_MEMORY flag, which causes emscripten_fetch() to populate the downloaded file in full in memory in the onsuccess() callback. This is convenient when the whole file is to be immediately accessed afterwards, but for large files, this can be a wasteful strategy in terms of memory usage. If the file is very large, it might not even fit inside the application's heap area. + +The following subsections provide ways to manage large fetches in a memory efficient manner. + +Downloading directly to IndexedDB +--------------------------------- + +If an application wants to download a file for local access, but does not immediately need to use the file, e.g. when preloading data up front for later access, it is a good idea to avoid the EMSCRIPTEN_FETCH_LOAD_TO_MEMORY flag altogether, and only pass the EMSCRIPTEN_FETCH_PERSIST_FILE flag instead. This causes the fetch to download the file directly to IndexedDB, which avoids temporarily populating the file in memory after the download finishes. In this scenario, the onsuccess() handler will only report the total downloaded file size, but will not contain the data bytes to the file. + +Streaming Downloads +------------------- + +If the application does not need random seek access to the file, but is able to process the file in a streaming manner, it can use the EMSCRIPTEN_FETCH_STREAM_DATA flag to stream through the bytes in the file as they are downloaded. If this flag is passed, the downloaded data chunks are passed into the onprogress() callback in coherent file sequential order. See the following snippet for an example. + + .. code-block:: cpp + + void downloadProgress(emscripten_fetch_t *fetch) { + printf("Downloading %s.. %.2f%%s complete. HTTP readyState: %d. HTTP status: %d.\n" + "HTTP statusText: %s. Received chunk [%llu, %llu[\n", + fetch->url, fetch->totalBytes > 0 ? (fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes : (fetch->dataOffset + fetch->numBytes), + fetch->totalBytes > 0 ? "%" : " bytes", + fetch->readyState, fetch->status, fetch->statusText, + fetch->dataOffset, fetch->dataOffset + fetch->numBytes); + + // Process the partial data stream fetch->data[0] thru fetch->data[fetch->numBytes-1] + // This buffer represents the file at offset fetch->dataOffset. + for(size_t i = 0; i < fetch->numBytes; ++i) + ; // Process fetch->data[i]; + } + + int main() { + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.attributes = EMSCRIPTEN_FETCH_STREAM_DATA; + attr.onsuccess = downloadSucceeded; + attr.onprogress = downloadProgress; + attr.onerror = downloadFailed; + attr.timeoutMsecs = 2*60; + emscripten_fetch(&attr, "myfile.dat"); + } + +In this case, the onsuccess() handler will not receive the final file buffer at all so memory usage will remain at a minimum. + +Byte Range Downloads +-------------------- + +Large files can also be managed in smaller chunks by performing Byte Range downloads on them. This initiates an XHR or IndexedDB transfer that only fetches the desired subrange of the whole file. This is useful for example when a large package file contains multiple smaller ones at certain seek offsets, which can be dealt with separately. + +TODO: Example about how to perform HTTP Byte Range Requests to obtain parts of files. + +TODO To Document +=============== + +Emscripten_fetch() supports the following operations as well, that need documenting: + + - Emscripten_fetch can be used to upload files to remote servers via HTTP PUT + - Emscripten_fetch_attr_t allows setting custom HTTP request headers (e.g. for cache control) + - Document HTTP simple auth fields in Emscripten_fetch_attr_t. + - Document how to populate to a certain filesystem path location in IndexedB, and e.g. fopen() it via ASMFS afterwards. + - Document overriddenMimeType attribute in Emscripten_fetch_attr_t. + - Reference documentation of the individual fields in Emscripten_fetch_attr_t, Emscripten_fetch_t and #defines. + - Example about loading only from IndexedDB without XHRing. + - Example about overriding an existing file in IndexedDB with a new XHR. + - Example how to preload a whole filesystem to IndexedDB for easy replacement of --preload-file. + - Example how to persist content as gzipped to IndexedDB and decompress on load. + - Example how to abort and resume partial transfers to IndexedDB. diff --git a/site/source/docs/api_reference/index.rst b/site/source/docs/api_reference/index.rst index d94a8a23275e3..833040ef77129 100644 --- a/site/source/docs/api_reference/index.rst +++ b/site/source/docs/api_reference/index.rst @@ -18,6 +18,9 @@ This section lists Emscripten's public API, organised by header file. At a very - :ref:`Filesystem-API` (**library_fs.js**): APIs for managing file systems and synchronous file operations. +- :ref:`Fetch API`: + API for managing accesses to network XHR and IndexedDB. + - :ref:`Module`: Global JavaScript object that can be used to control code execution and access exported methods. @@ -41,6 +44,7 @@ This section lists Emscripten's public API, organised by header file. At a very html5.h preamble.js Filesystem-API + fetch module val.h bind.h diff --git a/site/source/docs/api_reference/preamble.js.rst b/site/source/docs/api_reference/preamble.js.rst index 2b5d0ce92fa7b..17c9105f65a92 100644 --- a/site/source/docs/api_reference/preamble.js.rst +++ b/site/source/docs/api_reference/preamble.js.rst @@ -23,7 +23,9 @@ Calling compiled C functions from JavaScript Call a compiled C function from JavaScript. The function executes a compiled C function from JavaScript and returns the result. C++ name mangling means that "normal" C++ functions cannot be called; the function must either be defined in a **.c** file or be a C++ function defined with ``extern "C"``. - + + ``returnType`` and ``argTypes`` let you specify the types of parameters and the return value. The possible types are ``"number"``, ``"string"`` or ``"array"``, which correspond to the appropriate JavaScript types. Use ``"number"`` for any numeric type or C pointer, ``string`` for C ``char*`` that represent strings, and ``"array"`` for JavaScript arrays and typed arrays; for typed arrays, it must be a Uint8Array or Int8Array. + .. code-block:: javascript // Call C from JavaScript @@ -51,11 +53,11 @@ Calling compiled C functions from JavaScript :param ident: The name of the C function to be called. - :param returnType: The return type of the function. This can be ``"number"``, ``"string"`` or ``"array"``, which correspond to the appropriate JavaScript types (use ``"number"`` for any C pointer, and ``"array"`` for JavaScript arrays and typed arrays; note that arrays are 8-bit), or for a void function it can be ``null`` (note: the JavaScript ``null`` value, not a string containing the word "null"). + :param returnType: The return type of the function. Note that ``array`` is not supported as there is no way for us to know the length of the array. For a void function this can be ``null`` (note: the JavaScript ``null`` value, not a string containing the word "null"). .. note:: 64-bit integers become two 32-bit parameters, for the low and high bits (since 64-bit integers cannot be represented in JavaScript numbers). - :param argTypes: An array of the types of arguments for the function (if there are no arguments, this can be omitted). Types are as in ``returnType``, except that ``array`` is not supported as there is no way for us to know the length of the array). + :param argTypes: An array of the types of arguments for the function (if there are no arguments, this can be omitted). :param args: An array of the arguments to the function, as native JavaScript values (as in ``returnType``). Note that string arguments will be stored on the stack (the JavaScript string will become a C string on the stack). :returns: The result of the function call as a native JavaScript value (as in ``returnType``). :opts: An optional options object. It can contain the following properties: diff --git a/site/source/docs/porting/connecting_cpp_and_javascript/Interacting-with-code.rst b/site/source/docs/porting/connecting_cpp_and_javascript/Interacting-with-code.rst index efef3e7ca6d7e..160ac96666b2f 100644 --- a/site/source/docs/porting/connecting_cpp_and_javascript/Interacting-with-code.rst +++ b/site/source/docs/porting/connecting_cpp_and_javascript/Interacting-with-code.rst @@ -88,8 +88,9 @@ following JavaScript:: The first parameter is the name of the function to be wrapped, the second is the return type of the function (or a JavaScript `null` value if there isn't one), and the third is an array of parameter types (which may be omitted if there are no parameters). The types are -native JavaScript types, "number" (for a C integer, float, or general -pointer) or "string" (for a C ``char*`` that represents a string). +"number" (for a JavaScript number corresponding to a C integer, float, or general +pointer), "string" (for a JavaScript string that corresponds to a C ``char*`` that represents a string) or +"array" (for a JavaScript array or typed array that corresponds to a C array; for typed arrays, it must be a Uint8Array or Int8Array). You can run this yourself by first opening the generated page **function.html** in a web browser (nothing will happen on page diff --git a/src/Fetch.js b/src/Fetch.js new file mode 100644 index 0000000000000..d161aab548730 --- /dev/null +++ b/src/Fetch.js @@ -0,0 +1,568 @@ +var Fetch = { + attr_t_offset_requestMethod: 0, + attr_t_offset_userData: 32, + attr_t_offset_onsuccess: 36, + attr_t_offset_onerror: 40, + attr_t_offset_onprogress: 44, + attr_t_offset_attributes: 48, + attr_t_offset_timeoutMSecs: 52, + attr_t_offset_withCredentials: 56, + attr_t_offset_destinationPath: 60, + attr_t_offset_userName: 64, + attr_t_offset_password: 68, + attr_t_offset_requestHeaders: 72, + attr_t_offset_overriddenMimeType: 76, + attr_t_offset_requestData: 80, + attr_t_offset_requestDataSize: 84, + + fetch_t_offset_id: 0, + fetch_t_offset_userData: 4, + fetch_t_offset_url: 8, + fetch_t_offset_data: 12, + fetch_t_offset_numBytes: 16, + fetch_t_offset_dataOffset: 24, + fetch_t_offset_totalBytes: 32, + fetch_t_offset_readyState: 40, + fetch_t_offset_status: 42, + fetch_t_offset_statusText: 44, + fetch_t_offset___proxyState: 108, + fetch_t_offset___attributes: 112, + + xhrs: [], + // The web worker that runs proxied file I/O requests. + worker: undefined, + // Specifies an instance to the IndexedDB database. The database is opened + // as a preload step before the Emscripten application starts. + dbInstance: undefined, + + setu64: function(addr, val) { + HEAPU32[addr >> 2] = val; + HEAPU32[addr + 4 >> 2] = (val / 4294967296)|0; + }, + + openDatabase: function(dbname, dbversion, onsuccess, onerror) { + try { +#if FETCH_DEBUG + console.log('fetch: indexedDB.open(dbname="' + dbname + '", dbversion="' + dbversion + '");'); +#endif + var openRequest = indexedDB.open(dbname, dbversion); + } catch (e) { return onerror(e); } + + openRequest.onupgradeneeded = function(event) { +#if FETCH_DEBUG + console.log('fetch: IndexedDB upgrade needed. Clearing database.'); +#endif + var db = event.target.result; + if (db.objectStoreNames.contains('FILES')) { + db.deleteObjectStore('FILES'); + } + db.createObjectStore('FILES'); + }; + openRequest.onsuccess = function(event) { onsuccess(event.target.result); }; + openRequest.onerror = function(error) { onerror(error); }; + }, + + initFetchWorker: function() { + var stackSize = 128*1024; + var stack = allocate(stackSize>>2, "i32*", ALLOC_DYNAMIC); + Fetch.worker.postMessage({cmd: 'init', TOTAL_MEMORY: TOTAL_MEMORY, DYNAMICTOP_PTR: DYNAMICTOP_PTR, STACKTOP: stack, STACK_MAX: stack + stackSize, queuePtr: _fetch_work_queue, buffer: HEAPU8.buffer}, [HEAPU8.buffer]); + }, + + staticInit: function() { +#if USE_PTHREADS + var isMainThread = (typeof ENVIRONMENT_IS_FETCH_WORKER === 'undefined' && !ENVIRONMENT_IS_PTHREAD); +#else + var isMainThread = (typeof ENVIRONMENT_IS_FETCH_WORKER === 'undefined'); +#endif + + var onsuccess = function(db) { +#if FETCH_DEBUG + console.log('fetch: IndexedDB successfully opened.'); +#endif + Fetch.dbInstance = db; + +#if USE_PTHREADS + if (isMainThread) { + Fetch.initFetchWorker(); + removeRunDependency('library_fetch_init'); + } +#else + if (typeof ENVIRONMENT_IS_FETCH_WORKER === 'undefined' || !ENVIRONMENT_IS_FETCH_WORKER) removeRunDependency('library_fetch_init'); +#endif + }; + var onerror = function() { +#if FETCH_DEBUG + console.error('fetch: IndexedDB open failed.'); +#endif + Fetch.dbInstance = false; + +#if USE_PTHREADS + if (isMainThread) { + Fetch.initFetchWorker(); + removeRunDependency('library_fetch_init'); + } +#endif + }; + Fetch.openDatabase('emscripten_filesystem', 1, onsuccess, onerror); + +#if USE_PTHREADS + if (isMainThread) { + addRunDependency('library_fetch_init'); + + var fetchJs = 'fetch-worker.js'; + // Allow HTML module to configure the location where the 'pthread-main.js' file will be loaded from, + // either via Module.locateFile() function, or via Module.pthreadMainPrefixURL string. If neither + // of these are passed, then the default URL 'pthread-main.js' relative to the main html file is loaded. + if (typeof Module['locateFile'] === 'function') fetchJs = Module['locateFile'](fetchJs); + else if (Module['pthreadMainPrefixURL']) fetchJs = Module['pthreadMainPrefixURL'] + fetchJs; + Fetch.worker = new Worker(fetchJs); + Fetch.worker.onmessage = function(e) { + Module['print']('fetch-worker sent a message: ' + e.filename + ':' + e.lineno + ': ' + e.message); + }; + Fetch.worker.onerror = function(e) { + Module['printErr']('fetch-worker sent an error! ' + e.filename + ':' + e.lineno + ': ' + e.message); + }; + } +#else + if (typeof ENVIRONMENT_IS_FETCH_WORKER === 'undefined' || !ENVIRONMENT_IS_FETCH_WORKER) addRunDependency('library_fetch_init'); +#endif + } +} + +function __emscripten_fetch_delete_cached_data(db, fetch, onsuccess, onerror) { + if (!db) { +#if FETCH_DEBUG + console.error('fetch: IndexedDB not available!'); +#endif + onerror(fetch, 0, 'IndexedDB not available!'); + return; + } + + var fetch_attr = fetch + Fetch.fetch_t_offset___attributes; + var path = HEAPU32[fetch_attr + Fetch.attr_t_offset_destinationPath >> 2]; + if (!path) path = HEAPU32[fetch + Fetch.fetch_t_offset_url >> 2]; + var pathStr = Pointer_stringify(path); + + try { + var transaction = db.transaction(['FILES'], 'readwrite'); + var packages = transaction.objectStore('FILES'); + var request = packages.delete(pathStr); + request.onsuccess = function(event) { + var value = event.target.result; +#if FETCH_DEBUG + console.log('fetch: Deleted file ' + pathStr + ' from IndexedDB'); +#endif + HEAPU32[fetch + Fetch.fetch_t_offset_data >> 2] = 0; + Fetch.setu64(fetch + Fetch.fetch_t_offset_numBytes, 0); + Fetch.setu64(fetch + Fetch.fetch_t_offset_dataOffset, 0); + Fetch.setu64(fetch + Fetch.fetch_t_offset_dataOffset, 0); + HEAPU16[fetch + Fetch.fetch_t_offset_readyState >> 1] = 4; // Mimic XHR readyState 4 === 'DONE: The operation is complete' + HEAPU16[fetch + Fetch.fetch_t_offset_status >> 1] = 200; // Mimic XHR HTTP status code 200 "OK" + + onsuccess(fetch, 0, value); + }; + request.onerror = function(error) { +#if FETCH_DEBUG + console.error('fetch: Failed to delete file ' + pathStr + ' from IndexedDB! error: ' + error); +#endif + HEAPU16[fetch + Fetch.fetch_t_offset_readyState >> 1] = 4; // Mimic XHR readyState 4 === 'DONE: The operation is complete' + HEAPU16[fetch + Fetch.fetch_t_offset_status >> 1] = 404; // Mimic XHR HTTP status code 404 "Not Found" + onerror(fetch, 0, error); + }; + } catch(e) { +#if FETCH_DEBUG + console.error('fetch: Failed to load file ' + pathStr + ' from IndexedDB! Got exception ' + e); +#endif + onerror(fetch, 0, e); + } +} + +function __emscripten_fetch_load_cached_data(db, fetch, onsuccess, onerror) { + if (!db) { +#if FETCH_DEBUG + console.error('fetch: IndexedDB not available!'); +#endif + onerror(fetch, 0, 'IndexedDB not available!'); + return; + } + + var fetch_attr = fetch + Fetch.fetch_t_offset___attributes; + var path = HEAPU32[fetch_attr + Fetch.attr_t_offset_destinationPath >> 2]; + if (!path) path = HEAPU32[fetch + Fetch.fetch_t_offset_url >> 2]; + var pathStr = Pointer_stringify(path); + + try { + var transaction = db.transaction(['FILES'], 'readonly'); + var packages = transaction.objectStore('FILES'); + var getRequest = packages.get(pathStr); + getRequest.onsuccess = function(event) { + if (event.target.result) { + var value = event.target.result; + var len = value.byteLength || value.length; +#if FETCH_DEBUG + console.log('fetch: Loaded file ' + pathStr + ' from IndexedDB, length: ' + len); +#endif + + // The data pointer malloc()ed here has the same lifetime as the emscripten_fetch_t structure itself has, and is + // freed when emscripten_fetch_close() is called. + var ptr = _malloc(len); + HEAPU8.set(new Uint8Array(value), ptr); + HEAPU32[fetch + Fetch.fetch_t_offset_data >> 2] = ptr; + Fetch.setu64(fetch + Fetch.fetch_t_offset_numBytes, len); + Fetch.setu64(fetch + Fetch.fetch_t_offset_dataOffset, 0); + Fetch.setu64(fetch + Fetch.fetch_t_offset_totalBytes, len); + HEAPU16[fetch + Fetch.fetch_t_offset_readyState >> 1] = 4; // Mimic XHR readyState 4 === 'DONE: The operation is complete' + HEAPU16[fetch + Fetch.fetch_t_offset_status >> 1] = 200; // Mimic XHR HTTP status code 200 "OK" + + onsuccess(fetch, 0, value); + } else { + // Succeeded to load, but the load came back with the value of undefined, treat that as an error since we never store undefined in db. +#if FETCH_DEBUG + console.error('fetch: File ' + pathStr + ' not found in IndexedDB'); +#endif + HEAPU16[fetch + Fetch.fetch_t_offset_readyState >> 1] = 4; // Mimic XHR readyState 4 === 'DONE: The operation is complete' + HEAPU16[fetch + Fetch.fetch_t_offset_status >> 1] = 404; // Mimic XHR HTTP status code 404 "Not Found" + onerror(fetch, 0, 'no data'); + } + }; + getRequest.onerror = function(error) { +#if FETCH_DEBUG + console.error('fetch: Failed to load file ' + pathStr + ' from IndexedDB!'); +#endif + HEAPU16[fetch + Fetch.fetch_t_offset_readyState >> 1] = 4; // Mimic XHR readyState 4 === 'DONE: The operation is complete' + HEAPU16[fetch + Fetch.fetch_t_offset_status >> 1] = 404; // Mimic XHR HTTP status code 404 "Not Found" + onerror(fetch, 0, error); + }; + } catch(e) { +#if FETCH_DEBUG + console.error('fetch: Failed to load file ' + pathStr + ' from IndexedDB! Got exception ' + e); +#endif + onerror(fetch, 0, e); + } +} + +function __emscripten_fetch_cache_data(db, fetch, data, onsuccess, onerror) { + if (!db) { +#if FETCH_DEBUG + console.error('fetch: IndexedDB not available!'); +#endif + onerror(fetch, 0, 'IndexedDB not available!'); + return; + } + + var fetch_attr = fetch + Fetch.fetch_t_offset___attributes; + var destinationPath = HEAPU32[fetch_attr + Fetch.attr_t_offset_destinationPath >> 2]; + if (!destinationPath) destinationPath = HEAPU32[fetch + Fetch.fetch_t_offset_url >> 2]; + var destinationPathStr = Pointer_stringify(destinationPath); + + try { + var transaction = db.transaction(['FILES'], 'readwrite'); + var packages = transaction.objectStore('FILES'); + var putRequest = packages.put(data, destinationPathStr); + putRequest.onsuccess = function(event) { +#if FETCH_DEBUG + console.log('fetch: Stored file "' + destinationPathStr + '" to IndexedDB cache.'); +#endif + HEAPU16[fetch + Fetch.fetch_t_offset_readyState >> 1] = 4; // Mimic XHR readyState 4 === 'DONE: The operation is complete' + HEAPU16[fetch + Fetch.fetch_t_offset_status >> 1] = 200; // Mimic XHR HTTP status code 200 "OK" + onsuccess(fetch, 0, destinationPathStr); + }; + putRequest.onerror = function(error) { +#if FETCH_DEBUG + console.error('fetch: Failed to store file "' + destinationPathStr + '" to IndexedDB cache!'); +#endif + // Most likely we got an error if IndexedDB is unwilling to store any more data for this page. + // TODO: Can we identify and break down different IndexedDB-provided errors and convert those + // to more HTTP status codes for more information? + HEAPU16[fetch + Fetch.fetch_t_offset_readyState >> 1] = 4; // Mimic XHR readyState 4 === 'DONE: The operation is complete' + HEAPU16[fetch + Fetch.fetch_t_offset_status >> 1] = 413; // Mimic XHR HTTP status code 413 "Payload Too Large" + onerror(fetch, 0, error); + }; + } catch(e) { +#if FETCH_DEBUG + console.error('fetch: Failed to store file "' + destinationPathStr + '" to IndexedDB cache! Exception: ' + e); +#endif + onerror(fetch, 0, e); + } +} + +function __emscripten_fetch_xhr(fetch, onsuccess, onerror, onprogress) { + var url = HEAPU32[fetch + Fetch.fetch_t_offset_url >> 2]; + if (!url) { +#if FETCH_DEBUG + console.error('fetch: XHR failed, no URL specified!'); +#endif + onerror(fetch, 0, 'no url specified!'); + return; + } + var url_ = Pointer_stringify(url); + + var fetch_attr = fetch + Fetch.fetch_t_offset___attributes; + var requestMethod = Pointer_stringify(fetch_attr); + if (!requestMethod) requestMethod = 'GET'; + var userData = HEAPU32[fetch_attr + Fetch.attr_t_offset_userData >> 2]; + var fetchAttributes = HEAPU32[fetch_attr + Fetch.attr_t_offset_attributes >> 2]; + var timeoutMsecs = HEAPU32[fetch_attr + Fetch.attr_t_offset_timeoutMSecs >> 2]; + var withCredentials = !!HEAPU32[fetch_attr + Fetch.attr_t_offset_withCredentials >> 2]; + var destinationPath = HEAPU32[fetch_attr + Fetch.attr_t_offset_destinationPath >> 2]; + var userName = HEAPU32[fetch_attr + Fetch.attr_t_offset_userName >> 2]; + var password = HEAPU32[fetch_attr + Fetch.attr_t_offset_password >> 2]; + var requestHeaders = HEAPU32[fetch_attr + Fetch.attr_t_offset_requestHeaders >> 2]; + var overriddenMimeType = HEAPU32[fetch_attr + Fetch.attr_t_offset_overriddenMimeType >> 2]; + + var fetchAttrLoadToMemory = !!(fetchAttributes & 1/*EMSCRIPTEN_FETCH_LOAD_TO_MEMORY*/); + var fetchAttrStreamData = !!(fetchAttributes & 2/*EMSCRIPTEN_FETCH_STREAM_DATA*/); + var fetchAttrPersistFile = !!(fetchAttributes & 4/*EMSCRIPTEN_FETCH_PERSIST_FILE*/); + var fetchAttrAppend = !!(fetchAttributes & 8/*EMSCRIPTEN_FETCH_APPEND*/); + var fetchAttrReplace = !!(fetchAttributes & 16/*EMSCRIPTEN_FETCH_REPLACE*/); + var fetchAttrNoDownload = !!(fetchAttributes & 32/*EMSCRIPTEN_FETCH_NO_DOWNLOAD*/); + var fetchAttrSynchronous = !!(fetchAttributes & 64/*EMSCRIPTEN_FETCH_SYNCHRONOUS*/); + var fetchAttrWaitable = !!(fetchAttributes & 128/*EMSCRIPTEN_FETCH_WAITABLE*/); + + var userNameStr = userName ? Pointer_stringify(userName) : undefined; + var passwordStr = password ? Pointer_stringify(password) : undefined; + var overriddenMimeTypeStr = overriddenMimeType ? Pointer_stringify(overriddenMimeType) : undefined; + + var xhr = new XMLHttpRequest(); + xhr.withCredentials = withCredentials; +#if FETCH_DEBUG + console.log('fetch: xhr.timeout: ' + xhr.timeout + ', xhr.withCredentials: ' + xhr.withCredentials); + console.log('fetch: xhr.open(requestMethod="' + requestMethod + '", url: "' + url_ +'", userName: ' + userNameStr + ', password: ' + passwordStr + ');'); +#endif + xhr.open(requestMethod, url_, !fetchAttrSynchronous, userNameStr, passwordStr); + if (!fetchAttrSynchronous) xhr.timeout = timeoutMsecs; // XHR timeout field is only accessible in async XHRs, and must be set after .open() but before .send(). + xhr.url_ = url_; // Save the url for debugging purposes (and for comparing to the responseURL that server side advertised) + xhr.responseType = fetchAttrStreamData ? 'moz-chunked-arraybuffer' : 'arraybuffer'; + + if (overriddenMimeType) { +#if FETCH_DEBUG + console.log('fetch: xhr.overrideMimeType("' + overriddenMimeTypeStr + '");'); +#endif + xhr.overrideMimeType(overriddenMimeTypeStr); + } + if (requestHeaders) { + for(;;) { + var key = HEAPU32[requestHeaders >> 2]; + if (!key) break; + var value = HEAPU32[requestHeaders + 4 >> 2]; + if (!value) break; + requestHeaders += 8; + var keyStr = Pointer_stringify(key); + var valueStr = Pointer_stringify(value); +#if FETCH_DEBUG + console.log('fetch: xhr.setRequestHeader("' + keyStr + '", "' + valueStr + '");'); +#endif + xhr.setRequestHeader(keyStr, valueStr); + } + } + Fetch.xhrs.push(xhr); + var id = Fetch.xhrs.length; + HEAPU32[fetch + Fetch.fetch_t_offset_id >> 2] = id; + var data = null; // TODO: Support user to pass data to request. + // TODO: Support specifying custom headers to the request. + + xhr.onload = function(e) { + var len = xhr.response ? xhr.response.byteLength : 0; + var ptr = 0; + var ptrLen = 0; + if (fetchAttrLoadToMemory && !fetchAttrStreamData) { + ptrLen = len; +#if FETCH_DEBUG + console.log('fetch: allocating ' + ptrLen + ' bytes in Emscripten heap for xhr data'); +#endif + // The data pointer malloc()ed here has the same lifetime as the emscripten_fetch_t structure itself has, and is + // freed when emscripten_fetch_close() is called. + ptr = _malloc(ptrLen); + HEAPU8.set(new Uint8Array(xhr.response), ptr); + } + HEAPU32[fetch + Fetch.fetch_t_offset_data >> 2] = ptr; + Fetch.setu64(fetch + Fetch.fetch_t_offset_numBytes, ptrLen); + Fetch.setu64(fetch + Fetch.fetch_t_offset_dataOffset, 0); + if (len) { + // If the final XHR.onload handler receives the bytedata to compute total length, report that, + // otherwise don't write anything out here, which will retain the latest byte size reported in + // the most recent XHR.onprogress handler. + Fetch.setu64(fetch + Fetch.fetch_t_offset_totalBytes, len); + } + HEAPU16[fetch + Fetch.fetch_t_offset_readyState >> 1] = xhr.readyState; + if (xhr.readyState === 4 && xhr.status === 0) { + if (len > 0) xhr.status = 200; // If loading files from a source that does not give HTTP status code, assume success if we got data bytes. + else xhr.status = 404; // Conversely, no data bytes is 404. + } + HEAPU16[fetch + Fetch.fetch_t_offset_status >> 1] = xhr.status; +// if (xhr.statusText) stringToUTF8(fetch + Fetch.fetch_t_offset_statusText, xhr.statusText, 64); + if (xhr.status == 200) { +#if FETCH_DEBUG + console.log('fetch: xhr of URL "' + xhr.url_ + '" / responseURL "' + xhr.responseURL + '" succeeded with status 200'); +#endif + if (onsuccess) onsuccess(fetch, xhr, e); + } else { +#if FETCH_DEBUG + console.error('fetch: xhr of URL "' + xhr.url_ + '" / responseURL "' + xhr.responseURL + '" failed with status ' + xhr.status); +#endif + if (onerror) onerror(fetch, xhr, e); + } + } + xhr.onerror = function(e) { + var status = xhr.status; // XXX TODO: Overwriting xhr.status doesn't work here, so don't override anywhere else either. + if (xhr.readyState == 4 && status == 0) status = 404; // If no error recorded, pretend it was 404 Not Found. +#if FETCH_DEBUG + console.error('fetch: xhr of URL "' + xhr.url_ + '" / responseURL "' + xhr.responseURL + '" finished with error, readyState ' + xhr.readyState + ' and status ' + status); +#endif + HEAPU32[fetch + Fetch.fetch_t_offset_data >> 2] = 0; + Fetch.setu64(fetch + Fetch.fetch_t_offset_numBytes, 0); + Fetch.setu64(fetch + Fetch.fetch_t_offset_dataOffset, 0); + Fetch.setu64(fetch + Fetch.fetch_t_offset_totalBytes, 0); + HEAPU16[fetch + Fetch.fetch_t_offset_readyState >> 1] = xhr.readyState; + HEAPU16[fetch + Fetch.fetch_t_offset_status >> 1] = status; + if (onerror) onerror(fetch, xhr, e); + } + xhr.ontimeout = function(e) { +#if FETCH_DEBUG + console.error('fetch: xhr of URL "' + xhr.url_ + '" / responseURL "' + xhr.responseURL + '" timed out, readyState ' + xhr.readyState + ' and status ' + xhr.status); +#endif + if (onerror) onerror(fetch, xhr, e); + } + xhr.onprogress = function(e) { + var ptrLen = (fetchAttrLoadToMemory && fetchAttrStreamData && xhr.response) ? xhr.response.byteLength : 0; + var ptr = 0; + if (fetchAttrLoadToMemory && fetchAttrStreamData) { +#if FETCH_DEBUG + console.log('fetch: allocating ' + ptrLen + ' bytes in Emscripten heap for xhr data'); +#endif + // The data pointer malloc()ed here has the same lifetime as the emscripten_fetch_t structure itself has, and is + // freed when emscripten_fetch_close() is called. + ptr = _malloc(ptrLen); + HEAPU8.set(new Uint8Array(xhr.response), ptr); + } + HEAPU32[fetch + Fetch.fetch_t_offset_data >> 2] = ptr; + Fetch.setu64(fetch + Fetch.fetch_t_offset_numBytes, ptrLen); + Fetch.setu64(fetch + Fetch.fetch_t_offset_dataOffset, e.loaded - ptrLen); + Fetch.setu64(fetch + Fetch.fetch_t_offset_totalBytes, e.total); + HEAPU16[fetch + Fetch.fetch_t_offset_readyState >> 1] = xhr.readyState; + if (xhr.readyState >= 3 && xhr.status === 0 && e.loaded > 0) xhr.status = 200; // If loading files from a source that does not give HTTP status code, assume success if we get data bytes + HEAPU16[fetch + Fetch.fetch_t_offset_status >> 1] = xhr.status; + if (xhr.statusText) stringToUTF8(fetch + Fetch.fetch_t_offset_statusText, xhr.statusText, 64); + if (onprogress) onprogress(fetch, xhr, e); + } +#if FETCH_DEBUG + console.log('fetch: xhr.send(data=' + data + ')'); +#endif + try { + xhr.send(data); + } catch(e) { +#if FETCH_DEBUG + console.error('fetch: xhr failed with exception: ' + e); +#endif + if (onerror) onerror(fetch, xhr, e); + } +} + +function emscripten_start_fetch(fetch, successcb, errorcb, progresscb) { + if (typeof Module !== 'undefined') Module['noExitRuntime'] = true; // If we are the main Emscripten runtime, we should not be closing down. + + var fetch_attr = fetch + Fetch.fetch_t_offset___attributes; + var requestMethod = Pointer_stringify(fetch_attr); + var onsuccess = HEAPU32[fetch_attr + Fetch.attr_t_offset_onsuccess >> 2]; + var onerror = HEAPU32[fetch_attr + Fetch.attr_t_offset_onerror >> 2]; + var onprogress = HEAPU32[fetch_attr + Fetch.attr_t_offset_onprogress >> 2]; + var fetchAttributes = HEAPU32[fetch_attr + Fetch.attr_t_offset_attributes >> 2]; + var fetchAttrLoadToMemory = !!(fetchAttributes & 1/*EMSCRIPTEN_FETCH_LOAD_TO_MEMORY*/); + var fetchAttrStreamData = !!(fetchAttributes & 2/*EMSCRIPTEN_FETCH_STREAM_DATA*/); + var fetchAttrPersistFile = !!(fetchAttributes & 4/*EMSCRIPTEN_FETCH_PERSIST_FILE*/); + var fetchAttrAppend = !!(fetchAttributes & 8/*EMSCRIPTEN_FETCH_APPEND*/); + var fetchAttrReplace = !!(fetchAttributes & 16/*EMSCRIPTEN_FETCH_REPLACE*/); + var fetchAttrNoDownload = !!(fetchAttributes & 32/*EMSCRIPTEN_FETCH_NO_DOWNLOAD*/); + + var reportSuccess = function(fetch, xhr, e) { +#if FETCH_DEBUG + console.log('fetch: operation success. e: ' + e); +#endif + if (onsuccess && Runtime.dynCall) Runtime.dynCall('vi', onsuccess, [fetch]); + else if (successcb) successcb(fetch); + }; + + var cacheResultAndReportSuccess = function(fetch, xhr, e) { +#if FETCH_DEBUG + console.log('fetch: operation success. Caching result.. e: ' + e); +#endif + var storeSuccess = function() { +#if FETCH_DEBUG + console.log('fetch: IndexedDB store succeeded.'); +#endif + }; + var storeError = function() { +#if FETCH_DEBUG + console.error('fetch: IndexedDB store failed.'); +#endif + }; + __emscripten_fetch_cache_data(Fetch.dbInstance, fetch, xhr.response, storeSuccess, storeError); + if (onsuccess && Runtime.dynCall) Runtime.dynCall('vi', onsuccess, [fetch]); + else if (successcb) successcb(fetch); + }; + + var reportProgress = function(fetch, xhr, e) { + if (onprogress && Runtime.dynCall) Runtime.dynCall('vi', onprogress, [fetch]); + else if (progresscb) progresscb(fetch); + }; + + var reportError = function(fetch, xhr, e) { +#if FETCH_DEBUG + console.error('fetch: operation failed: ' + e); +#endif + if (onerror && Runtime.dynCall) Runtime.dynCall('vi', onerror, [fetch]); + else if (errorcb) errorcb(fetch); + }; + + var performUncachedXhr = function(fetch, xhr, e) { +#if FETCH_DEBUG + console.error('fetch: starting (uncached) XHR: ' + e); +#endif + __emscripten_fetch_xhr(fetch, reportSuccess, reportError, reportProgress); + }; + + var performCachedXhr = function(fetch, xhr, e) { +#if FETCH_DEBUG + console.error('fetch: starting (cached) XHR: ' + e); +#endif + __emscripten_fetch_xhr(fetch, cacheResultAndReportSuccess, reportError, reportProgress); + }; + + // Should we try IndexedDB first? + if (!fetchAttrReplace || requestMethod === 'EM_IDB_STORE' || requestMethod === 'EM_IDB_DELETE') { + if (!Fetch.dbInstance) { +#if FETCH_DEBUG + console.error('fetch: failed to read IndexedDB! Database is not open.'); +#endif + reportError(fetch, 0, 'IndexedDB is not open'); + return 0; // todo: free + } + + if (requestMethod === 'EM_IDB_STORE') { + var dataPtr = HEAPU32[fetch_attr + Fetch.attr_t_offset_requestData >> 2]; + var dataLength = HEAPU32[fetch_attr + Fetch.attr_t_offset_requestDataSize >> 2]; + var data = HEAPU8.slice(dataPtr, dataPtr + dataLength); // TODO(?): Here we perform a clone of the data, because storing shared typed arrays to IndexedDB does not seem to be allowed. + __emscripten_fetch_cache_data(Fetch.dbInstance, fetch, data, reportSuccess, reportError); + } else if (requestMethod === 'EM_IDB_DELETE') { + __emscripten_fetch_delete_cached_data(Fetch.dbInstance, fetch, reportSuccess, reportError); + } else if (fetchAttrNoDownload) { + __emscripten_fetch_load_cached_data(Fetch.dbInstance, fetch, reportSuccess, reportError); + } else if (fetchAttrPersistFile) { + __emscripten_fetch_load_cached_data(Fetch.dbInstance, fetch, reportSuccess, performCachedXhr); + } else { + __emscripten_fetch_load_cached_data(Fetch.dbInstance, fetch, reportSuccess, performUncachedXhr); + } + } else if (!fetchAttrNoDownload) { + if (fetchAttrPersistFile) { + __emscripten_fetch_xhr(fetch, cacheResultAndReportSuccess, reportError, reportProgress); + } else { + __emscripten_fetch_xhr(fetch, reportSuccess, reportError, reportProgress); + } + } else { +#if FETCH_DEBUG + console.error('fetch: Invalid combination of flags passed.'); +#endif + return 0; // todo: free + } + return fetch; +} diff --git a/src/fetch-worker.js b/src/fetch-worker.js new file mode 100644 index 0000000000000..ab38d3fa75610 --- /dev/null +++ b/src/fetch-worker.js @@ -0,0 +1,132 @@ +#include "Fetch.js" + +var Atomics_add = Atomics.add; +var Atomics_and = Atomics.and; +var Atomics_compareExchange = Atomics.compareExchange; +var Atomics_exchange = Atomics.exchange; +var Atomics_wait = Atomics.wait; +var Atomics_wake = Atomics.wake; +var Atomics_wakeOrRequeue = Atomics.wakeOrRequeue; +var Atomics_isLockFree = Atomics.isLockFree; +var Atomics_load = Atomics.load; +var Atomics_or = Atomics.or; +var Atomics_store = Atomics.store; +var Atomics_sub = Atomics.sub; +var Atomics_xor = Atomics.xor; + +var ENVIRONMENT_IS_FETCH_WORKER = true; +var ENVIRONMENT_IS_WORKER = true; +var ENVIRONMENT_IS_PTHREAD = true; +var __pthread_is_main_runtime_thread=0; +var DYNAMICTOP_PTR = 0; +var TOTAL_MEMORY = 0; +function enlargeMemory() { + abort('Cannot enlarge memory arrays, since compiling with pthreads support enabled (-s USE_PTHREADS=1).'); +} +var nan = NaN; +var inf = Infinity; + +function _emscripten_asm_const_v() {} + +function assert(condition) { + if (!condition) console.error('assert failure!'); +} + +/// TODO: DO SOMETHING ABOUT ME. +function Pointer_stringify(ptr, /* optional */ length) { + if (length === 0 || !ptr) return ""; + // TODO: use TextDecoder + // Find the length, and check for UTF while doing so + var hasUtf = 0; + var t; + var i = 0; + while (1) { + t = HEAPU8[(((ptr)+(i))>>0)]; + hasUtf |= t; + if (t == 0 && !length) break; + i++; + if (length && i == length) break; + } + if (!length) length = i; + + var ret = ""; + + if (hasUtf < 128) { + var MAX_CHUNK = 1024; // split up into chunks, because .apply on a huge string can overflow the stack + var curr; + while (length > 0) { + curr = String.fromCharCode.apply(String, HEAPU8.subarray(ptr, ptr + Math.min(length, MAX_CHUNK))); + ret = ret ? ret + curr : curr; + ptr += MAX_CHUNK; + length -= MAX_CHUNK; + } + return ret; + } + return Module['UTF8ToString'](ptr); +} + +Fetch.staticInit(); + +var queuePtr = 0; +var buffer = null; +var STACKTOP = 0; +var STACK_MAX = 0; +var HEAP8 = null; +var HEAPU8 = null; +var HEAP16 = null; +var HEAPU16 = null; +var HEAP32 = null; +var HEAPU32 = null; + +function processWorkQueue() { + if (!queuePtr) return; + var numQueuedItems = Atomics_load(HEAPU32, queuePtr + 4 >> 2); + if (numQueuedItems == 0) return; + + var queuedOperations = Atomics_load(HEAPU32, queuePtr >> 2); + var queueSize = Atomics_load(HEAPU32, queuePtr + 8 >> 2); + for(var i = 0; i < numQueuedItems; ++i) { + var fetch = Atomics_load(HEAPU32, (queuedOperations >> 2)+i); + function successcb(fetch) { + Atomics.compareExchange(HEAPU32, fetch + Fetch.fetch_t_offset___proxyState >> 2, 1, 2); + Atomics.wake(HEAP32, fetch + Fetch.fetch_t_offset___proxyState >> 2, 1); + } + function errorcb(fetch) { + Atomics.compareExchange(HEAPU32, fetch + Fetch.fetch_t_offset___proxyState >> 2, 1, 2); + Atomics.wake(HEAP32, fetch + Fetch.fetch_t_offset___proxyState >> 2, 1); + } + function progresscb(fetch) { + } + try { + emscripten_start_fetch(fetch, successcb, errorcb, progresscb); + } catch(e) { + console.error(e); + } + /* + if (interval != undefined) { + clearInterval(interval); + interval = undefined; + } + */ + } + Atomics_store(HEAPU32, queuePtr + 4 >> 2, 0); +} + +interval = 0; +this.onmessage = function(e) { + if (e.data.cmd == 'init') { + queuePtr = e.data.queuePtr; + buffer = e.data.buffer; + STACKTOP = e.data.STACKTOP; + STACK_MAX = e.data.STACK_MAX; + DYNAMICTOP_PTR = e.data.DYNAMICTOP_PTR; + TOTAL_MEMORY = e.data.TOTAL_MEMORY; + HEAP8 = new Int8Array(buffer); + HEAPU8 = new Uint8Array(buffer); + HEAP16 = new Int16Array(buffer); + HEAPU16 = new Uint16Array(buffer); + HEAP32 = new Int32Array(buffer); + HEAPU32 = new Uint32Array(buffer); + interval = setInterval(processWorkQueue, 100); + } +} diff --git a/src/library_fetch.js b/src/library_fetch.js new file mode 100644 index 0000000000000..29411d845f253 --- /dev/null +++ b/src/library_fetch.js @@ -0,0 +1,24 @@ +#include Fetch.js + +var LibraryFetch = { +#if USE_PTHREADS + $Fetch__postset: 'if (!ENVIRONMENT_IS_PTHREAD) Fetch.staticInit();', + fetch_work_queue: '; if (ENVIRONMENT_IS_PTHREAD) _fetch_work_queue = PthreadWorkerInit._fetch_work_queue; else PthreadWorkerInit._fetch_work_queue = _fetch_work_queue = allocate(12, "i32*", ALLOC_STATIC)', +#else + $Fetch__postset: 'Fetch.staticInit();', + fetch_work_queue: 'allocate(12, "i32*", ALLOC_STATIC)', +#endif + $Fetch: Fetch, + _emscripten_get_fetch_work_queue__deps: ['fetch_work_queue'], + _emscripten_get_fetch_work_queue: function() { + return _fetch_work_queue; + }, + + $__emscripten_fetch_load_cached_data: __emscripten_fetch_load_cached_data, + $__emscripten_fetch_cache_data: __emscripten_fetch_cache_data, + $__emscripten_fetch_xhr: __emscripten_fetch_xhr, + emscripten_start_fetch__deps: ['$Fetch', '$__emscripten_fetch_xhr', '$__emscripten_fetch_cache_data', '$__emscripten_fetch_load_cached_data', '_emscripten_get_fetch_work_queue', 'emscripten_is_main_runtime_thread', 'pthread_mutex_lock', 'pthread_mutex_unlock'], + emscripten_start_fetch: emscripten_start_fetch +}; + +mergeInto(LibraryManager.library, LibraryFetch); diff --git a/src/library_proxyfs.js b/src/library_proxyfs.js new file mode 100644 index 0000000000000..1fa0737fd8785 --- /dev/null +++ b/src/library_proxyfs.js @@ -0,0 +1,212 @@ +mergeInto(LibraryManager.library, { + $PROXYFS__deps: ['$FS', '$PATH'], + $PROXYFS: { + mount: function (mount) { + return PROXYFS.createNode(null, '/', mount.opts.fs.lstat(mount.opts.root).mode, 0); + }, + createNode: function (parent, name, mode, dev) { + if (!FS.isDir(mode) && !FS.isFile(mode) && !FS.isLink(mode)) { + throw new FS.ErrnoError(ERRNO_CODES.EINVAL); + } + var node = FS.createNode(parent, name, mode); + node.node_ops = PROXYFS.node_ops; + node.stream_ops = PROXYFS.stream_ops; + return node; + }, + realPath: function (node) { + var parts = []; + while (node.parent !== node) { + parts.push(node.name); + node = node.parent; + } + parts.push(node.mount.opts.root); + parts.reverse(); + return PATH.join.apply(null, parts); + }, + node_ops: { + getattr: function(node) { + var path = PROXYFS.realPath(node); + var stat; + try { + stat = node.mount.opts.fs.lstat(path); + } catch (e) { + if (!e.code) throw e; + throw new FS.ErrnoError(ERRNO_CODES[e.code]); + } + return { + dev: stat.dev, + ino: stat.ino, + mode: stat.mode, + nlink: stat.nlink, + uid: stat.uid, + gid: stat.gid, + rdev: stat.rdev, + size: stat.size, + atime: stat.atime, + mtime: stat.mtime, + ctime: stat.ctime, + blksize: stat.blksize, + blocks: stat.blocks + }; + }, + setattr: function(node, attr) { + var path = PROXYFS.realPath(node); + try { + if (attr.mode !== undefined) { + node.mount.opts.fs.chmod(path, attr.mode); + // update the common node structure mode as well + node.mode = attr.mode; + } + if (attr.timestamp !== undefined) { + var date = new Date(attr.timestamp); + node.mount.opts.fs.utime(path, date, date); + } + if (attr.size !== undefined) { + node.mount.opts.fs.truncate(path, attr.size); + } + } catch (e) { + if (!e.code) throw e; + throw new FS.ErrnoError(ERRNO_CODES[e.code]); + } + }, + lookup: function (parent, name) { + try { + var path = PATH.join2(PROXYFS.realPath(parent), name); + var mode = parent.mount.opts.fs.lstat(path).mode; + var node = PROXYFS.createNode(parent, name, mode); + return node; + } catch(e) { + if (!e.code) throw e; + throw new FS.ErrnoError(ERRNO_CODES[e.code]); + } + }, + mknod: function (parent, name, mode, dev) { + var node = PROXYFS.createNode(parent, name, mode, dev); + // create the backing node for this in the fs root as well + var path = PROXYFS.realPath(node); + try { + if (FS.isDir(node.mode)) { + node.mount.opts.fs.mkdir(path, node.mode); + } else { + node.mount.opts.fs.writeFile(path, '', { mode: node.mode }); + } + } catch (e) { + if (!e.code) throw e; + throw new FS.ErrnoError(ERRNO_CODES[e.code]); + } + return node; + }, + rename: function (oldNode, newDir, newName) { + var oldPath = PROXYFS.realPath(oldNode); + var newPath = PATH.join2(PROXYFS.realPath(newDir), newName); + try { + oldNode.mount.opts.fs.rename(oldPath, newPath); + } catch(e) { + if (!e.code) throw e; + throw new FS.ErrnoError(ERRNO_CODES[e.code]); + } + }, + unlink: function(parent, name) { + var path = PATH.join2(PROXYFS.realPath(parent), name); + try { + parent.mount.opts.fs.unlink(path); + } catch(e) { + if (!e.code) throw e; + throw new FS.ErrnoError(ERRNO_CODES[e.code]); + } + }, + rmdir: function(parent, name) { + var path = PATH.join2(PROXYFS.realPath(parent), name); + try { + parent.mount.opts.fs.rmdir(path); + } catch(e) { + if (!e.code) throw e; + throw new FS.ErrnoError(ERRNO_CODES[e.code]); + } + }, + readdir: function(node) { + var path = PROXYFS.realPath(node); + try { + return node.mount.opts.fs.readdir(path); + } catch(e) { + if (!e.code) throw e; + throw new FS.ErrnoError(ERRNO_CODES[e.code]); + } + }, + symlink: function(parent, newName, oldPath) { + var newPath = PATH.join2(PROXYFS.realPath(parent), newName); + try { + parent.mount.opts.fs.symlink(oldPath, newPath); + } catch(e) { + if (!e.code) throw e; + throw new FS.ErrnoError(ERRNO_CODES[e.code]); + } + }, + readlink: function(node) { + var path = PROXYFS.realPath(node); + try { + return node.mount.opts.fs.readlink(path); + } catch(e) { + if (!e.code) throw e; + throw new FS.ErrnoError(ERRNO_CODES[e.code]); + } + }, + }, + stream_ops: { + open: function (stream) { + var path = PROXYFS.realPath(stream.node); + try { + stream.nfd = stream.node.mount.opts.fs.open(path,stream.flags); + } catch(e) { + if (!e.code) throw e; + throw new FS.ErrnoError(ERRNO_CODES[e.code]); + } + }, + close: function (stream) { + try { + stream.node.mount.opts.fs.close(stream.nfd); + } catch(e) { + if (!e.code) throw e; + throw new FS.ErrnoError(ERRNO_CODES[e.code]); + } + }, + read: function (stream, buffer, offset, length, position) { + try { + return stream.node.mount.opts.fs.read(stream.nfd, buffer, offset, length, position); + } catch(e) { + if (!e.code) throw e; + throw new FS.ErrnoError(ERRNO_CODES[e.code]); + } + }, + write: function (stream, buffer, offset, length, position) { + try { + return stream.node.mount.opts.fs.write(stream.nfd, buffer, offset, length, position); + } catch(e) { + if (!e.code) throw e; + throw new FS.ErrnoError(ERRNO_CODES[e.code]); + } + }, + llseek: function (stream, offset, whence) { + var position = offset; + if (whence === 1) { // SEEK_CUR. + position += stream.position; + } else if (whence === 2) { // SEEK_END. + if (FS.isFile(stream.node.mode)) { + try { + var stat = stream.node.mount.opts.fs.fstat(stream.nfd); + position += stat.size; + } catch (e) { + throw new FS.ErrnoError(ERRNO_CODES[e.code]); + } + } + } + + if (position < 0) { + throw new FS.ErrnoError(ERRNO_CODES.EINVAL); + } + + return position; + } + } + } +}); diff --git a/src/library_pthread.js b/src/library_pthread.js index 9cf5649e0071d..d9b1b3b2a8af5 100644 --- a/src/library_pthread.js +++ b/src/library_pthread.js @@ -308,7 +308,7 @@ var LibraryPThread = { }; worker.onerror = function(e) { - Module['printErr']('pthread sent an error! ' + e.message); + Module['printErr']('pthread sent an error! ' + e.filename + ':' + e.lineno + ': ' + e.message); }; // Allocate tempDoublePtr for the worker. This is done here on the worker's behalf, since we may need to do this statically diff --git a/src/modules.js b/src/modules.js index 927f12c8d48f2..1bb6d9eae6cd0 100644 --- a/src/modules.js +++ b/src/modules.js @@ -120,6 +120,7 @@ var LibraryManager = { libraries = libraries.concat([ 'library_idbfs.js', 'library_nodefs.js', + 'library_proxyfs.js', 'library_sockfs.js', 'library_workerfs.js', 'library_lz4.js', diff --git a/src/preamble.js b/src/preamble.js index c68b11266414a..120b8a97f9346 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -590,7 +590,7 @@ function UTF8ToString(ptr) { // Copies the given Javascript String object 'str' to the given byte array at address 'outIdx', // encoded in UTF8 form and null-terminated. The copy will require at most str.length*4+1 bytes of space in the HEAP. -// Use the function lengthBytesUTF8() to compute the exact number of bytes (excluding null terminator) that this function will write. +// Use the function lengthBytesUTF8 to compute the exact number of bytes (excluding null terminator) that this function will write. // Parameters: // str: the Javascript string to copy. // outU8Array: the array to copy to. Each index in this array is assumed to be one 8-byte element. @@ -655,7 +655,7 @@ function stringToUTF8Array(str, outU8Array, outIdx, maxBytesToWrite) { // Copies the given Javascript String object 'str' to the emscripten HEAP at address 'outPtr', // null-terminated and encoded in UTF8 form. The copy will require at most str.length*4+1 bytes of space in the HEAP. -// Use the function lengthBytesUTF8() to compute the exact number of bytes (excluding null terminator) that this function will write. +// Use the function lengthBytesUTF8 to compute the exact number of bytes (excluding null terminator) that this function will write. // Returns the number of bytes written, EXCLUDING the null terminator. function stringToUTF8(str, outPtr, maxBytesToWrite) { @@ -1180,6 +1180,7 @@ if (typeof Atomics === 'undefined') { Atomics['add'] = function(t, i, v) { var w = t[i]; t[i] += v; return w; } Atomics['and'] = function(t, i, v) { var w = t[i]; t[i] &= v; return w; } Atomics['compareExchange'] = function(t, i, e, r) { var w = t[i]; if (w == e) t[i] = r; return w; } + Atomics['exchange'] = function(t, i, v) { var w = t[i]; t[i] = v; return w; } Atomics['wait'] = function(t, i, v, o) { if (t[i] != v) abort('Multithreading is not supported, cannot sleep to wait for futex!'); } Atomics['wake'] = function(t, i, c) { return 0; } Atomics['wakeOrRequeue'] = function(t, i1, c, i2, v) { return 0; } @@ -1713,7 +1714,10 @@ function writeStringToMemory(string, buffer, dontAddNull) { {{{ maybeExport('writeStringToMemory') }}} function writeArrayToMemory(array, buffer) { - HEAP8.set(array, buffer); +#if ASSERTIONS + assert(array.length >= 0, 'writeArrayToMemory array must have a length (should be an array or typed array)') +#endif + HEAP8.set(array, buffer); } {{{ maybeExport('writeArrayToMemory') }}} diff --git a/src/settings.js b/src/settings.js index badd9e527a3c5..fa6506f93cb0c 100644 --- a/src/settings.js +++ b/src/settings.js @@ -795,8 +795,15 @@ var BUNDLED_CD_DEBUG_FILE = ""; // Path to the CyberDWARF debug file passed to t var TEXTDECODER = 1; // Is enabled, use the JavaScript TextDecoder API for string marshalling. // Enabled by default, set this to 0 to disable. + var OFFSCREENCANVAS_SUPPORT = 0; // If set to 1, enables support for transferring canvases to pthreads and creating WebGL contexts in them, // as well as explicit swap control for GL contexts. This needs browser support for the OffscreenCanvas // specification. +var FETCH_DEBUG = 0; // If nonzero, prints out debugging information in library_fetch.js + +var FETCH = 0; // If nonzero, enables emscripten_fetch API. + +var ASMFS = 0; // If set to 1, uses the multithreaded filesystem that is implemented within the asm.js module, using emscripten_fetch. Implies -s FETCH=1. + // Reserved: variables containing POINTER_MASKING. diff --git a/src/shell.html b/src/shell.html index ec4895f074895..7b494f1a7f034 100644 --- a/src/shell.html +++ b/src/shell.html @@ -103,7 +103,7 @@ - +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +// Emscripten fetch attributes: +// If passed, the body of the request will be present in full in the onsuccess() handler. +#define EMSCRIPTEN_FETCH_LOAD_TO_MEMORY 1 + +// If passed, the intermediate streamed bytes will be passed in to the onprogress() handler. If not specified, the +// onprogress() handler will still be called, but without data bytes. +#define EMSCRIPTEN_FETCH_STREAM_DATA 2 + +// If passed, the final download will be stored in IndexedDB. If not specified, the file will only reside in browser memory. +#define EMSCRIPTEN_FETCH_PERSIST_FILE 4 + +// If the file already exists in IndexedDB, it is returned without redownload. If a partial transfer exists in IndexedDB, +// the download will resume from where it left off and run to completion. +// EMSCRIPTEN_FETCH_APPEND, EMSCRIPTEN_FETCH_REPLACE and EMSCRIPTEN_FETCH_NO_DOWNLOAD are mutually exclusive. +#define EMSCRIPTEN_FETCH_APPEND 8 + +// If the file already exists in IndexedDB, the old file will be deleted and a new download is started. +// EMSCRIPTEN_FETCH_APPEND, EMSCRIPTEN_FETCH_REPLACE and EMSCRIPTEN_FETCH_NO_DOWNLOAD are mutually exclusive. +#define EMSCRIPTEN_FETCH_REPLACE 16 + +// If specified, the file will only be looked up in IndexedDB, but if it does not exist, it is not attempted to be downloaded +// over the network but an error is raised. +// EMSCRIPTEN_FETCH_APPEND, EMSCRIPTEN_FETCH_REPLACE and EMSCRIPTEN_FETCH_NO_DOWNLOAD are mutually exclusive. +#define EMSCRIPTEN_FETCH_NO_DOWNLOAD 32 + +// If specified, emscripten_fetch() will synchronously run to completion before returning. +// The callback handlers will be called from within emscripten_fetch() while the operation is in progress. +#define EMSCRIPTEN_FETCH_SYNCHRONOUS 64 + +// If specified, it will be possible to call emscripten_fetch_wait() on the fetch +// to test or wair for its completion. +#define EMSCRIPTEN_FETCH_WAITABLE 128 + +struct emscripten_fetch_t; + +// Specifies the parameters for a newly initiated fetch operation. +struct emscripten_fetch_attr_t +{ + // 'POST', 'GET', etc. + char requestMethod[32]; + + // Custom data that can be tagged along the process. + void *userData; + + void (*onsuccess)(emscripten_fetch_t *fetch); + void (*onerror)(emscripten_fetch_t *fetch); + void (*onprogress)(emscripten_fetch_t *fetch); + + // EMSCRIPTEN_FETCH_* attributes + uint32_t attributes; + + // Specifies the amount of time the request can take before failing due to a timeout. + unsigned long timeoutMSecs; + + // Indicates whether cross-site access control requests should be made using credentials. + EM_BOOL withCredentials; + + // Specifies the destination path in IndexedDB where to store the downloaded content body. If this is empty, the transfer + // is not stored to IndexedDB at all. + // Note that this struct does not contain space to hold this string, it only carries a pointer. + // Calling emscripten_fetch() will make an internal copy of this string. + const char *destinationPath; + + // Specifies the authentication username to use for the request, if necessary. + // Note that this struct does not contain space to hold this string, it only carries a pointer. + // Calling emscripten_fetch() will make an internal copy of this string. + const char *userName; + + // Specifies the authentication username to use for the request, if necessary. + // Note that this struct does not contain space to hold this string, it only carries a pointer. + // Calling emscripten_fetch() will make an internal copy of this string. + const char *password; + + // Points to an array of strings to pass custom headers to the request. This array takes the form + // {"key1", "value1", "key2", "value2", "key3", "value3", ..., 0 }; Note especially that the array + // needs to be terminated with a null pointer. + const char * const *requestHeaders; + + // Pass a custom MIME type here to force the browser to treat the received data with the given type. + const char *overriddenMimeType; + + // If non-zero, specified a pointer to the data that is to be passed as the body (payload) of the request + // that is being performed. Leave as zero if no request body needs to be sent. + // The memory pointed to by this field is provided by the user, and needs to be valid only until the call to + // emscripten_fetch() returns. + const char *requestData; + + // Specifies the length of the buffer pointed by 'requestData'. Leave as 0 if no request body needs to be sent. + size_t requestDataSize; +}; + +struct emscripten_fetch_t +{ + // Unique identifier for this fetch in progress. + unsigned int id; + + // Custom data that can be tagged along the process. + void *userData; + + // The remote URL that is being downloaded. + const char *url; + + // In onsuccess() handler: + // - If the EMSCRIPTEN_FETCH_LOAD_TO_MEMORY attribute was specified for the transfer, this points to the + // body of the downloaded data. Otherwise this will be null. + // In onprogress() handler: + // - If the EMSCRIPTEN_FETCH_STREAM_DATA attribute was specified for the transfer, this points to a partial + // chunk of bytes related to the transfer. Otherwise this will be null. + // The data buffer provided here has identical lifetime with the emscripten_fetch_t object itself, and is freed by + // calling emscripten_fetch_close() on the emscripten_fetch_t pointer. + const char *data; + + // Specifies the length of the above data block in bytes. When the download finishes, this field will be valid even if + // EMSCRIPTEN_FETCH_LOAD_TO_MEMORY was not specified. + uint64_t numBytes; + + // If EMSCRIPTEN_FETCH_STREAM_DATA is being performed, this indicates the byte offset from the start of the stream + // that the data block specifies. (for onprogress() streaming XHR transfer, the number of bytes downloaded so far before this chunk) + uint64_t dataOffset; + + // Specifies the total number of bytes that the response body will be. + // Note: This field may be zero, if the server does not report the Content-Length field. + uint64_t totalBytes; + + // Specifies the readyState of the XHR request: + // 0: UNSENT: request not sent yet + // 1: OPENED: emscripten_fetch has been called. + // 2: HEADERS_RECEIVED: emscripten_fetch has been called, and headers and status are available. + // 3: LOADING: download in progress. + // 4: DONE: download finished. + // See https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/readyState + unsigned short readyState; + + // Specifies the status code of the response. + unsigned short status; + + // Specifies a human-readable form of the status code. + char statusText[64]; + + uint32_t __proxyState; + + // For internal use only. + emscripten_fetch_attr_t __attributes; +}; + +// Clears the fields of an emscripten_fetch_attr_t structure to their default values in a future-compatible manner. +void emscripten_fetch_attr_init(emscripten_fetch_attr_t *fetch_attr); + +// Initiates a new Emscripten fetch operation, which downloads data from the given URL or from IndexedDB database. +emscripten_fetch_t *emscripten_fetch(emscripten_fetch_attr_t *fetch_attr, const char *url); + +// Synchronously blocks to wait for the given fetch operation to complete. This operation is not allowed in the main browser +// thread, in which case it will return EMSCRIPTEN_RESULT_NOT_SUPPORTED. Pass timeoutMSecs=infinite to wait indefinitely. If +// the wait times out, the return value will be EMSCRIPTEN_RESULT_TIMEOUT. +// The onsuccess()/onerror()/onprogress() handlers will be called in the calling thread from within this function before +// this function returns. +EMSCRIPTEN_RESULT emscripten_fetch_wait(emscripten_fetch_t *fetch, double timeoutMSecs); + +// Closes a finished or an executing fetch operation and frees up all memory. If the fetch operation was still executing, the +// onerror() handler will be called in the calling thread before this function returns. +EMSCRIPTEN_RESULT emscripten_fetch_close(emscripten_fetch_t *fetch); + +#ifdef __cplusplus +} +#endif + +// ~__emscripten_fetch_h__ +#endif diff --git a/system/lib/fetch/asmfs.cpp b/system/lib/fetch/asmfs.cpp new file mode 100644 index 0000000000000..e51194495b790 --- /dev/null +++ b/system/lib/fetch/asmfs.cpp @@ -0,0 +1,1508 @@ +#include +#include +#include +#include +#include +#include +#define __NEED_struct_iovec +#include +#include +#include +#include +#include +#include +#include +#include +#include "syscall_arch.h" + +extern "C" { + +// http://stackoverflow.com/questions/417142/what-is-the-maximum-length-of-a-url-in-different-browsers +#define MAX_PATHNAME_LENGTH 2000 + +#define INODE_TYPE uint32_t +#define INODE_FILE 1 +#define INODE_DIR 2 + +struct inode +{ + char name[NAME_MAX+1]; // NAME_MAX actual bytes + one byte for null termination. + inode *parent; // ID of the parent node + inode *sibling; // ID of a sibling node (these form a singular linked list that specifies the content under a directory) + inode *child; // ID of the first child node in a chain of children (the root of a linked list of inodes) + uint32_t uid; // User ID of the owner + uint32_t gid; // Group ID of the owning group + uint32_t mode; // r/w/x modes + time_t ctime; // Time when the inode was last modified + time_t mtime; // Time when the content was last modified + time_t atime; // Time when the content was last accessed + size_t size; // Size of the file in bytes + size_t capacity; // Amount of bytes allocated to pointer data + uint8_t *data; // The actual file contents. + + INODE_TYPE type; + + emscripten_fetch_t *fetch; +}; + +#define EM_FILEDESCRIPTOR_MAGIC 0x64666d65U // 'emfd' +struct FileDescriptor +{ + uint32_t magic; + ssize_t file_pos; + uint32_t mode; + uint32_t flags; + + inode *node; +}; + +static inode *create_inode(INODE_TYPE type, int mode) +{ + inode *i = (inode*)malloc(sizeof(inode)); + memset(i, 0, sizeof(inode)); + i->ctime = i->mtime = i->atime = time(0); + i->type = type; + i->mode = mode; + return i; +} + +// The current working directory of the application process. +static inode *cwd_inode = 0; + +static inode *filesystem_root() +{ + static inode *root_node = create_inode(INODE_DIR, 0777); + return root_node; +} + +static inode *get_cwd() +{ + if (!cwd_inode) cwd_inode = filesystem_root(); + return cwd_inode; +} + +static void set_cwd(inode *node) +{ + cwd_inode = node; +} + +static void inode_abspath(inode *node, char *dst, int dstLen) +{ + if (!node) + { + assert(dstLen >= strlen("(null)")+1); + strcpy(dst, "(null)"); + return; + } + if (node == filesystem_root()) + { + assert(dstLen >= strlen("/")+1); + strcpy(dst, "/"); + return; + } +#define MAX_DIRECTORY_DEPTH 512 + inode *stack[MAX_DIRECTORY_DEPTH]; + int depth = 0; + while(node->parent && depth < MAX_DIRECTORY_DEPTH) + { + stack[depth++] = node; + node = node->parent; + } + char *dstEnd = dst + dstLen; + *dstEnd-- = '\0'; + while(depth > 0 && dst < dstEnd) + { + if (dst < dstEnd) *dst++ = '/'; + --depth; + int len = strlen(stack[depth]->name); + if (len > dstEnd - dst) len = dstEnd - dst; + strncpy(dst, stack[depth]->name, len); + dst += len; + } +} + +static void delete_inode(inode *node) +{ + free(node); +} + +// Makes node the child of parent. +static void link_inode(inode *node, inode *parent) +{ + char parentName[PATH_MAX]; + inode_abspath(parent, parentName, PATH_MAX); + EM_ASM_INT( { Module['printErr']('link_inode: node "' + Pointer_stringify($0) + '" to parent "' + Pointer_stringify($1) + '".') }, + node->name, parentName); + // When linking a node, it can't be part of the filesystem tree (but it can have children of its own) + assert(!node->parent); + assert(!node->sibling); + + // The inode pointed by 'node' is not yet part of the filesystem, so it's not shared memory and only this thread + // is accessing it. Therefore setting the node's parent here is not yet racy, do that operation first. + node->parent = parent; + + // This node is to become the first child of the parent, and the old first child of the parent should + // become the sibling of this node, i.e. + // 1) node->sibling = parent->child; + // 2) parent->child = node; + // However these two operations need to occur atomically in order to be coherent. To ensure that, run the two + // operations in a CAS loop, which is possible because the first operation is not racy until the node is 'published' + // to the filesystem tree by the compare_exchange operation. + do { __atomic_load(&parent->child, &node->sibling, __ATOMIC_SEQ_CST); // node->sibling <- parent->child + } while (!__atomic_compare_exchange(&parent->child, &node->sibling, &node, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)); // parent->child <- node if it had not raced to change value in between +} + +// Traverse back in sibling linked list, or 0 if no such node exist. +static inode *find_predecessor_sibling(inode *node, inode *parent) +{ + inode *child = parent->child; + if (child == node) return 0; + while(child && child->sibling != node) child = child->sibling; + if (!child->sibling) return 0; + return child; +} + +static void unlink_inode(inode *node) +{ + EM_ASM_INT( { Module['printErr']('unlink_inode: node ' + Pointer_stringify($0) + ' from its parent ' + Pointer_stringify($1) + '.') }, + node->name, node->parent->name); + inode *parent = node->parent; + if (!parent) return; + node->parent = 0; + + if (parent->child == node) + { + parent->child = node->sibling; + } + else + { + inode *predecessor = find_predecessor_sibling(node, parent); + if (predecessor) predecessor->sibling = node->sibling; + } + node->parent = node->sibling = 0; +} + +// Compares two strings for equality until a '\0' or a '/' is hit. Returns 0 if the strings differ, +// or a pointer to the beginning of the next directory component name of s1 if the strings are equal. +static const char *path_cmp(const char *s1, const char *s2, bool *is_directory) +{ + *is_directory = true; + while(*s1 == *s2) + { + if (*s1 == '/') return s1+1; + if (*s1 == '\0') + { + *is_directory = false; + return s1; + } + ++s1; + ++s2; + } + if (*s1 == '/' && *s2 == '\0') return s1+1; + if (*s1 == '\0' && *s2 == '/') return s1; + return 0; +} + +#define NIBBLE_TO_CHAR(x) ("0123456789abcdef"[(x)]) +static void uriEncode(char *dst, int dstLengthBytes, const char *src) +{ + char *end = dst + dstLengthBytes - 4; // Use last 4 bytes of dst as a guard area to avoid overflow below. + while(*src && dst < end) + { + if (isalnum(*src) || *src == '-' || *src == '_' || *src == '.' || *src == '~') *dst++ = *src; + else if (*src == '/') *dst++ = *src; // NB. forward slashes should generally be uriencoded, but for file path purposes, we want to keep them intact. + else *dst++ = '%', *dst++ = NIBBLE_TO_CHAR(*src >> 4), *dst++ = NIBBLE_TO_CHAR(*src & 15); // This charater needs uriencoding. + ++src; + } + *dst = '\0'; +} + +// Copies string 'path' to 'dst', but stops on the first forward slash '/' character. +// Returns number of bytes written, excluding null terminator +static int strcpy_inodename(char *dst, const char *path) +{ + char *d = dst; + while(*path && *path != '/') *dst++ = *path++; + *dst = '\0'; + return dst - d; +} + +// Returns a pointer to the basename part of the string, i.e. the string after the last occurrence of a forward slash character +static const char *basename_part(const char *path) +{ + const char *s = path; + while(*path) + { + if (*path == '/') s = path+1; + ++path; + } + return s; +} + +static inode *create_directory_hierarchy_for_file(inode *root, const char *path_to_file, unsigned int mode) +{ + assert(root); + if (!root) return 0; + + // Traverse . and .. + while(path_to_file[0] == '.') + { + if (path_to_file[1] == '/') path_to_file += 2; // Skip over redundant "./././././" blocks + else if (path_to_file[1] == '\0') path_to_file += 1; + else if (path_to_file[1] == '.' && (path_to_file[2] == '/' || path_to_file[2] == '\0')) // Go up to parent directories with ".." + { + root = root->parent; + if (!root) return 0; + assert(root->type == INODE_DIR); // Anything that is a parent should automatically be a directory. + path_to_file += (path_to_file[2] == '/') ? 3 : 2; + } + else break; + } + if (path_to_file[0] == '\0') return 0; + + inode *node = root->child; + while(node) + { + bool is_directory = false; + const char *child_path = path_cmp(path_to_file, node->name, &is_directory); + EM_ASM_INT( { Module['printErr']('path_cmp ' + Pointer_stringify($0) + ', ' + Pointer_stringify($1) + ', ' + Pointer_stringify($2) + ' .') }, path_to_file, node->name, child_path); + if (child_path) + { + if (is_directory && node->type != INODE_DIR) return 0; // "A component used as a directory in pathname is not, in fact, a directory" + + // The directory name matches. + path_to_file = child_path; + + // Traverse . and .. + while(path_to_file[0] == '.') + { + if (path_to_file[1] == '/') path_to_file += 2; // Skip over redundant "./././././" blocks + else if (path_to_file[1] == '\0') path_to_file += 1; + else if (path_to_file[1] == '.' && (path_to_file[2] == '/' || path_to_file[2] == '\0')) // Go up to parent directories with ".." + { + node = node->parent; + if (!node) return 0; + assert(node->type == INODE_DIR); // Anything that is a parent should automatically be a directory. + path_to_file += (path_to_file[2] == '/') ? 3 : 2; + } + else break; + } + if (path_to_file[0] == '\0') return node; + if (path_to_file[0] == '/' && path_to_file[1] == '\0' /* && node is a directory*/) return node; + root = node; + node = node->child; + } + else + { + node = node->sibling; + } + } + EM_ASM_INT( { Module['printErr']('path_to_file ' + Pointer_stringify($0) + ' .') }, path_to_file); + const char *basename_pos = basename_part(path_to_file); + EM_ASM_INT( { Module['printErr']('basename_pos ' + Pointer_stringify($0) + ' .') }, basename_pos); + while(*path_to_file && path_to_file < basename_pos) + { + node = create_inode(INODE_DIR, mode); + path_to_file += strcpy_inodename(node->name, path_to_file) + 1; + link_inode(node, root); + EM_ASM_INT( { Module['print']('create_directory_hierarchy_for_file: created directory ' + Pointer_stringify($0) + ' under parent ' + Pointer_stringify($1) + '.') }, + node->name, node->parent->name); + root = node; + } + return root; +} +// Same as above, but the root node is deduced from 'path'. (either absolute if path starts with "/", or relative) +static inode *create_directory_hierarchy_for_file(const char *path, unsigned int mode) +{ + inode *root; + if (path[0] == '/') root = filesystem_root(), ++path; + else root = get_cwd(); + return create_directory_hierarchy_for_file(root, path, mode); +} + +#define RETURN_NODE_AND_ERRNO(node, errno) do { *out_errno = (errno); return (node); } while(0) + +// Given a pathname to a file/directory, finds the inode of the directory that would contain the file/directory, or 0 if the intermediate path doesn't exist. +// Note that the file/directory pointed to by path does not need to exist, only its parent does. +static inode *find_parent_inode(inode *root, const char *path, int *out_errno) +{ + char rootName[PATH_MAX]; + inode_abspath(root, rootName, PATH_MAX); + EM_ASM_INT({ Module['printErr']('find_parent_inode(root="' + Pointer_stringify($0) + '", path="' + Pointer_stringify($1) + '")') }, rootName, path); + + assert(out_errno); // Passing in error is mandatory. + + if (!root) RETURN_NODE_AND_ERRNO(0, ENOENT); + if (!path) RETURN_NODE_AND_ERRNO(0, ENOENT); + + // Traverse . and .. + while(path[0] == '.') + { + if (path[1] == '/') path += 2; // Skip over redundant "./././././" blocks + else if (path[1] == '\0') path += 1; + else if (path[1] == '.' && (path[2] == '/' || path[2] == '\0')) // Go up to parent directories with ".." + { + root = root->parent; + if (!root) RETURN_NODE_AND_ERRNO(0, ENOENT); + assert(root->type == INODE_DIR); // Anything that is a parent should automatically be a directory. + path += (path[2] == '/') ? 3 : 2; + } + else break; + } + if (path[0] == '\0') RETURN_NODE_AND_ERRNO(0, ENOENT); + if (path[0] == '/' && path[1] == '\0') RETURN_NODE_AND_ERRNO(0, ENOENT); + if (root->type != INODE_DIR) RETURN_NODE_AND_ERRNO(0, ENOTDIR); // "A component used as a directory in pathname is not, in fact, a directory" + + // TODO: RETURN_ERRNO(ELOOP, "Too many symbolic links were encountered in translating pathname"); + // TODO: RETURN_ERRNO(EACCES, "one of the directories in the path prefix of pathname did not allow search permission"); + + const char *basename = basename_part(path); + if (path == basename) RETURN_NODE_AND_ERRNO(root, 0); + inode *node = root->child; + while(node) + { + bool is_directory = false; + const char *child_path = path_cmp(path, node->name, &is_directory); + if (child_path) + { + if (is_directory && node->type != INODE_DIR) RETURN_NODE_AND_ERRNO(0, ENOTDIR); // "A component used as a directory in pathname is not, in fact, a directory" + + // The directory name matches. + path = child_path; + + // Traverse . and .. + while(path[0] == '.') + { + if (path[1] == '/') path += 2; // Skip over redundant "./././././" blocks + else if (path[1] == '\0') path += 1; + else if (path[1] == '.' && (path[2] == '/' || path[2] == '\0')) // Go up to parent directories with ".." + { + node = node->parent; + if (!node) RETURN_NODE_AND_ERRNO(0, ENOENT); + assert(node->type == INODE_DIR); // Anything that is a parent should automatically be a directory. + path += (path[2] == '/') ? 3 : 2; + } + else break; + } + + if (path >= basename) RETURN_NODE_AND_ERRNO(node, 0); + if (!*path) RETURN_NODE_AND_ERRNO(0, ENOENT); + node = node->child; + if (node->type != INODE_DIR) RETURN_NODE_AND_ERRNO(0, ENOTDIR); // "A component used as a directory in pathname is not, in fact, a directory" + } + else + { + node = node->sibling; + } + } + RETURN_NODE_AND_ERRNO(0, ENOTDIR); // "A component used as a directory in pathname is not, in fact, a directory" +} + +// Given a root inode of the filesystem and a path relative to it, e.g. "some/directory/dir_or_file", +// returns the inode that corresponds to "dir_or_file", or 0 if it doesn't exist. +// If the parameter out_closest_parent is specified, the closest (grand)parent node will be returned. +static inode *find_inode(inode *root, const char *path, int *out_errno) +{ + char rootName[PATH_MAX]; + inode_abspath(root, rootName, PATH_MAX); + EM_ASM_INT({ Module['printErr']('find_inode(root="' + Pointer_stringify($0) + '", path="' + Pointer_stringify($1) + '")') }, rootName, path); + + assert(out_errno); // Passing in error is mandatory. + + if (!root) RETURN_NODE_AND_ERRNO(0, ENOENT); + + // TODO: RETURN_ERRNO(ELOOP, "Too many symbolic links were encountered in translating pathname"); + // TODO: RETURN_ERRNO(EACCES, "one of the directories in the path prefix of pathname did not allow search permission"); + + // special-case finding empty string path "", "." or "/" returns the root searched in. + if (root->type != INODE_DIR) RETURN_NODE_AND_ERRNO(0, ENOTDIR); // "A component used as a directory in pathname is not, in fact, a directory" + if (!path) RETURN_NODE_AND_ERRNO(root, 0); + + // Traverse . and .. + while(path[0] == '.') + { + if (path[1] == '/') path += 2; // Skip over redundant "./././././" blocks + else if (path[1] == '\0') path += 1; + else if (path[1] == '.' && (path[2] == '/' || path[2] == '\0')) // Go up to parent directories with ".." + { + root = root->parent; + if (!root) RETURN_NODE_AND_ERRNO(0, ENOENT); + assert(root->type == INODE_DIR); // Anything that is a parent should automatically be a directory. + path += (path[2] == '/') ? 3 : 2; + } + else break; + } + if (path[0] == '\0') RETURN_NODE_AND_ERRNO(root, 0); + + inode *node = root->child; + while(node) + { + bool is_directory = false; + const char *child_path = path_cmp(path, node->name, &is_directory); + if (child_path) + { + if (is_directory && node->type != INODE_DIR) RETURN_NODE_AND_ERRNO(0, ENOTDIR); // "A component used as a directory in pathname is not, in fact, a directory" + + // The directory name matches. + path = child_path; + + // Traverse . and .. + while(path[0] == '.') + { + if (path[1] == '/') path += 2; // Skip over redundant "./././././" blocks + else if (path[1] == '\0') path += 1; + else if (path[1] == '.' && (path[2] == '/' || path[2] == '\0')) // Go up to parent directories with ".." + { + node = node->parent; + if (!node) RETURN_NODE_AND_ERRNO(0, ENOENT); + assert(node->type == INODE_DIR); // Anything that is a parent should automatically be a directory. + path += (path[2] == '/') ? 3 : 2; + } + else break; + } + + // If we arrived to the end of the search, this is the node we were looking for. + if (path[0] == '\0') RETURN_NODE_AND_ERRNO(node, 0); + if (path[0] == '/' && node->type != INODE_DIR) RETURN_NODE_AND_ERRNO(0, ENOTDIR); // "A component used as a directory in pathname is not, in fact, a directory" + if (path[0] == '/' && path[1] == '\0') RETURN_NODE_AND_ERRNO(node, 0); + node = node->child; + } + else + { + node = node->sibling; + } + } + RETURN_NODE_AND_ERRNO(0, ENOENT); +} + +// Same as above, but the root node is deduced from 'path'. (either absolute if path starts with "/", or relative) +static inode *find_inode(const char *path, int *out_errno) +{ + inode *root; + if (path[0] == '/') root = filesystem_root(), ++path; + else root = get_cwd(); + return find_inode(root, path, out_errno); +} + +// Debug function that dumps out the filesystem tree to console. +void emscripten_dump_fs_tree(inode *root, char *path) +{ + char str[256]; + sprintf(str,"%s:", path); + EM_ASM_INT( { Module['print'](Pointer_stringify($0)) }, str); + + // Print out: + // file mode | number of links | owner name | group name | file size in bytes | file last modified time | path name + // which aligns with "ls -AFTRl" on console + inode *child = root->child; + uint64_t totalSize = 0; + while(child) + { + sprintf(str,"%c%c%c%c%c%c%c%c%c%c %d user%u group%u %u Jan 1 1970 %s%c", + child->type == INODE_DIR ? 'd' : '-', + (child->mode & S_IRUSR) ? 'r' : '-', + (child->mode & S_IWUSR) ? 'w' : '-', + (child->mode & S_IXUSR) ? 'x' : '-', + (child->mode & S_IRGRP) ? 'r' : '-', + (child->mode & S_IWGRP) ? 'w' : '-', + (child->mode & S_IXGRP) ? 'x' : '-', + (child->mode & S_IROTH) ? 'r' : '-', + (child->mode & S_IWOTH) ? 'w' : '-', + (child->mode & S_IXOTH) ? 'x' : '-', + 1, // number of links to this file + child->uid, + child->gid, + child->size, + child->name, + child->type == INODE_DIR ? '/' : ' '); + EM_ASM_INT( { Module['print'](Pointer_stringify($0)) }, str); + + totalSize += child->size; + child = child->sibling; + } + + sprintf(str, "total %llu bytes\n", totalSize); + EM_ASM_INT( { Module['print'](Pointer_stringify($0)) }, str); + + child = root->child; + char *path_end = path + strlen(path); + while(child) + { + if (child->type == INODE_DIR) + { + strcpy(path_end, child->name); + strcat(path_end, "/"); + emscripten_dump_fs_tree(child, path); + } + child = child->sibling; + } +} + +void emscripten_dump_fs_root() +{ + EM_ASM({ Module['printErr']('emscripten_dump_fs_root()') }); + char path[PATH_MAX] = "/"; + emscripten_dump_fs_tree(filesystem_root(), path); +} + +#define RETURN_ERRNO(errno, error_reason) do { \ + EM_ASM_INT({ Module['printErr'](Pointer_stringify($0) + '() returned errno ' + #errno + '(' + $1 + '): ' + error_reason + '!')}, __FUNCTION__, errno); \ + return -errno; \ + } while(0) + +static char stdout_buffer[4096] = {}; +static int stdout_buffer_end = 0; +static char stderr_buffer[4096] = {}; +static int stderr_buffer_end = 0; + +static void print_stream(void *bytes, int numBytes, bool stdout) +{ + char *buffer = stdout ? stdout_buffer : stderr_buffer; + int &buffer_end = stdout ? stdout_buffer_end : stderr_buffer_end; + + memcpy(buffer + buffer_end, bytes, numBytes); + buffer_end += numBytes; + int new_buffer_start = 0; + for(int i = 0; i < buffer_end; ++i) + { + if (buffer[i] == '\n') + { + buffer[i] = 0; + EM_ASM_INT( { Module['print'](Pointer_stringify($0)) }, buffer+new_buffer_start); + new_buffer_start = i+1; + } + } + size_t new_buffer_size = buffer_end - new_buffer_start; + memmove(buffer, buffer + new_buffer_start, new_buffer_size); + buffer_end = new_buffer_size; +} + +long __syscall3(int which, ...) // read +{ + va_list vl; + va_start(vl, which); + int fd = va_arg(vl, int); + void *buf = va_arg(vl, void *); + size_t count = va_arg(vl, size_t); + va_end(vl); + EM_ASM_INT({ Module['printErr']('read(fd=' + $0 + ', buf=0x' + ($1).toString(16) + ', count=' + $2 + ')') }, fd, buf, count); + + iovec io = { buf, count }; + return __syscall145(145/*readv*/, fd, &io, 1); +} + +long __syscall4(int which, ...) // write +{ + va_list vl; + va_start(vl, which); + int fd = va_arg(vl, int); + void *buf = va_arg(vl, void *); + size_t count = va_arg(vl, size_t); + va_end(vl); + EM_ASM_INT({ Module['printErr']('write(fd=' + $0 + ', buf=0x' + ($1).toString(16) + ', count=' + $2 + ')') }, fd, buf, count); + + iovec io = { buf, count }; + return __syscall146(146/*writev*/, fd, &io, 1); +} + +static long open(const char *pathname, int flags, int mode) +{ + EM_ASM_INT({ Module['printErr']('open(pathname="' + Pointer_stringify($0) + '", flags=0x' + ($1).toString(16) + ', mode=0' + ($2).toString(8) + ')') }, + pathname, flags, mode); + + int accessMode = (flags & O_ACCMODE); + + if ((flags & O_ASYNC)) RETURN_ERRNO(ENOTSUP, "TODO: Opening files with O_ASYNC flag is not supported in ASMFS"); + if ((flags & O_DIRECT)) RETURN_ERRNO(ENOTSUP, "TODO: O_DIRECT flag is not supported in ASMFS"); + if ((flags & O_DSYNC)) RETURN_ERRNO(ENOTSUP, "TODO: O_DSYNC flag is not supported in ASMFS"); + + // Spec says that the result of O_EXCL without O_CREAT is undefined. + // We could enforce it as an error condition, as follows: +// if ((flags & O_EXCL) && !(flags & O_CREAT)) RETURN_ERRNO(EINVAL, "open() with O_EXCL flag needs to always be paired with O_CREAT"); + // However existing earlier unit tests in Emscripten expect that O_EXCL is simply ignored when O_CREAT was not passed. So do that for now. + if ((flags & O_EXCL) && !(flags & O_CREAT)) { + EM_ASM_INT({ Module['printErr']('warning: open(pathname="' + Pointer_stringify($0) + '", flags=0x' + ($1).toString(16) + ', mode=0' + ($2).toString(8) + ': flag O_EXCL should always be paired with O_CREAT. Ignoring O_EXCL)') }, pathname, flags, mode); + flags &= ~O_EXCL; + } + + if ((flags & (O_NONBLOCK|O_NDELAY))) RETURN_ERRNO(ENOTSUP, "TODO: Opening files with O_NONBLOCK or O_NDELAY flags is not supported in ASMFS"); + if ((flags & O_PATH)) RETURN_ERRNO(ENOTSUP, "TODO: Opening files with O_PATH flag is not supported in ASMFS"); + if ((flags & O_SYNC)) RETURN_ERRNO(ENOTSUP, "TODO: Opening files with O_SYNC flag is not supported in ASMFS"); + + // The flags:O_CLOEXEC flag is ignored, doesn't have meaning for Emscripten + + // TODO: the flags:O_DIRECT flag seems like a great way to let applications explicitly control XHR/IndexedDB read/write buffering behavior? + + // The flags:O_LARGEFILE flag is ignored, we should always be largefile-compatible + + // TODO: The flags:O_NOATIME is ignored, file access times have not been implemented yet + // The flags O_NOCTTY, O_NOFOLLOW + + if ((flags & O_TMPFILE)) + { + if (accessMode != O_WRONLY && accessMode != O_RDWR) RETURN_ERRNO(EINVAL, "O_TMPFILE was specified in flags, but neither O_WRONLY nor O_RDWR was specified"); + else RETURN_ERRNO(EOPNOTSUPP, "TODO: The filesystem containing pathname does not support O_TMPFILE"); + } + + // TODO: if (too_many_files_open) RETURN_ERRNO(EMFILE, "The per-process limit on the number of open file descriptors has been reached, see getrlimit(RLIMIT_NOFILE)"); + + int len = strlen(pathname); + if (len > MAX_PATHNAME_LENGTH) RETURN_ERRNO(ENAMETOOLONG, "pathname was too long"); + if (len == 0) RETURN_ERRNO(ENOENT, "pathname is empty"); + + // Find if this file exists already in the filesystem? + inode *root = (pathname[0] == '/') ? filesystem_root() : get_cwd(); + const char *relpath = (pathname[0] == '/') ? pathname+1 : pathname; + + int err; + inode *node = find_inode(root, relpath, &err); + if (err == ENOTDIR) RETURN_ERRNO(ENOTDIR, "A component used as a directory in pathname is not, in fact, a directory"); + if (err == ELOOP) RETURN_ERRNO(ELOOP, "Too many symbolic links were encountered in resolving pathname"); + if (err == EACCES) RETURN_ERRNO(EACCES, "Search permission is denied for one of the directories in the path prefix of pathname"); + if (err && err != ENOENT) RETURN_ERRNO(err, "find_inode() error"); + if (node) + { + if ((flags & O_DIRECTORY) && node->type != INODE_DIR) RETURN_ERRNO(ENOTDIR, "O_DIRECTORY was specified and pathname was not a directory"); + if (!(node->mode & 0444)) RETURN_ERRNO(EACCES, "The requested access to the file is not allowed"); + if ((flags & O_CREAT) && (flags & O_EXCL)) RETURN_ERRNO(EEXIST, "pathname already exists and O_CREAT and O_EXCL were used"); + if (node->type == INODE_DIR && accessMode != O_RDONLY) RETURN_ERRNO(EISDIR, "pathname refers to a directory and the access requested involved writing (that is, O_WRONLY or O_RDWR is set)"); + if (node->type == INODE_DIR && (flags & O_TRUNC)) RETURN_ERRNO(EISDIR, "pathname refers to a directory and the access flags specified invalid flag O_TRUNC"); + if (node->fetch) emscripten_fetch_wait(node->fetch, INFINITY); + } + + if ((flags & O_CREAT) && ((flags & O_TRUNC) || (flags & O_EXCL))) + { + // Create a new empty file or truncate existing one. + if (node) + { + if (node->fetch) emscripten_fetch_close(node->fetch); + node->fetch = 0; + node->size = 0; + } + else if ((flags & O_CREAT)) + { + inode *directory = create_directory_hierarchy_for_file(root, relpath, mode); + node = create_inode((flags & O_DIRECTORY) ? INODE_DIR : INODE_FILE, mode); + strcpy(node->name, basename_part(pathname)); + link_inode(node, directory); + } + } + else if (!node || (node->type == INODE_FILE && !node->fetch && !node->data)) + { + emscripten_fetch_t *fetch = 0; + if (!(flags & O_DIRECTORY) && accessMode != O_WRONLY) + { + // If not, we'll need to fetch it. + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.attributes = EMSCRIPTEN_FETCH_APPEND | EMSCRIPTEN_FETCH_LOAD_TO_MEMORY | EMSCRIPTEN_FETCH_WAITABLE | EMSCRIPTEN_FETCH_PERSIST_FILE; + char uriEncodedPathName[3*PATH_MAX+4]; // times 3 because uri-encoding can expand the filename at most 3x. + uriEncode(uriEncodedPathName, 3*PATH_MAX+4, pathname); + fetch = emscripten_fetch(&attr, uriEncodedPathName); + + // switch(fopen_mode) + // { + // case synchronous_fopen: + emscripten_fetch_wait(fetch, INFINITY); + + if (!(flags & O_CREAT) && (fetch->status != 200 || fetch->totalBytes == 0)) + { + emscripten_fetch_close(fetch); + RETURN_ERRNO(ENOENT, "O_CREAT is not set and the named file does not exist (attempted emscripten_fetch() XHR to download)"); + } + // break; + // case asynchronous_fopen: + // break; + // } + } + + if (node) + { + // If we had an existing inode entry, just associate the entry with the newly fetched data. + if (node->type == INODE_FILE) node->fetch = fetch; + } + else if ((flags & O_CREAT) // If the filesystem entry did not exist, but we have a create flag, ... + || (!node && fetch)) // ... or if it did not exist in our fs, but it could be found via fetch(), ... + { + // ... add it as a new entry to the fs. + inode *directory = create_directory_hierarchy_for_file(root, relpath, mode); + node = create_inode((flags & O_DIRECTORY) ? INODE_DIR : INODE_FILE, mode); + strcpy(node->name, basename_part(pathname)); + node->fetch = fetch; + link_inode(node, directory); + } + else + { + if (fetch) emscripten_fetch_close(fetch); + RETURN_ERRNO(ENOENT, "O_CREAT is not set and the named file does not exist"); + } + node->size = node->fetch->totalBytes; + emscripten_dump_fs_root(); + } + + FileDescriptor *desc = (FileDescriptor*)malloc(sizeof(FileDescriptor)); + desc->magic = EM_FILEDESCRIPTOR_MAGIC; + desc->node = node; + desc->file_pos = ((flags & O_APPEND) && node->fetch) ? node->fetch->totalBytes : 0; + desc->mode = mode; + desc->flags = flags; + + // TODO: The file descriptor needs to be a small number, man page: + // "a small, nonnegative integer for use in subsequent system calls + // (read(2), write(2), lseek(2), fcntl(2), etc.). The file descriptor + // returned by a successful call will be the lowest-numbered file + // descriptor not currently open for the process." + return (long)desc; +} + +long __syscall5(int which, ...) // open +{ + va_list vl; + va_start(vl, which); + const char *pathname = va_arg(vl, const char*); + int flags = va_arg(vl, int); + int mode = va_arg(vl, int); + va_end(vl); + + return open(pathname, flags, mode); +} + +static long close(int fd) +{ + EM_ASM_INT({ Module['printErr']('close(fd=' + $0 + ')') }, fd); + + FileDescriptor *desc = (FileDescriptor*)fd; + if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) RETURN_ERRNO(EBADF, "fd isn't a valid open file descriptor"); + + if (desc->node && desc->node->fetch) + { + emscripten_fetch_wait(desc->node->fetch, INFINITY); // TODO: This should not be necessary- test this out + emscripten_fetch_close(desc->node->fetch); + desc->node->fetch = 0; + } + desc->magic = 0; + free(desc); + return 0; +} + +long __syscall6(int which, ...) // close +{ + va_list vl; + va_start(vl, which); + int fd = va_arg(vl, int); + va_end(vl); + + return close(fd); +} + +long __syscall9(int which, ...) // link +{ + va_list vl; + va_start(vl, which); + const char *oldpath = va_arg(vl, const char *); + const char *newpath = va_arg(vl, const char *); + va_end(vl); + EM_ASM_INT({ Module['printErr']('link(oldpath="' + Pointer_stringify($0) + '", newpath="' + Pointer_stringify($1) + '")') }, oldpath, newpath); + + RETURN_ERRNO(ENOTSUP, "TODO: link() is a stub and not yet implemented in ASMFS"); +} + +long __syscall10(int which, ...) // unlink +{ + va_list vl; + va_start(vl, which); + const char *pathname = va_arg(vl, const char *); + va_end(vl); + EM_ASM_INT({ Module['printErr']('unlink(pathname="' + Pointer_stringify($0) + '")') }, pathname); + + int len = strlen(pathname); + if (len > MAX_PATHNAME_LENGTH) RETURN_ERRNO(ENAMETOOLONG, "pathname was too long"); + if (len == 0) RETURN_ERRNO(ENOENT, "pathname is empty"); + + int err; + inode *node = find_inode(pathname, &err); + if (err == ENOTDIR) RETURN_ERRNO(ENOTDIR, "A component used as a directory in pathname is not, in fact, a directory"); + if (err == ELOOP) RETURN_ERRNO(ELOOP, "Too many symbolic links were encountered in translating pathname"); + if (err == EACCES) RETURN_ERRNO(EACCES, "One of the directories in the path prefix of pathname did not allow search permission"); + if (err == ENOENT) RETURN_ERRNO(ENOENT, "A component in pathname does not exist or is a dangling symbolic link"); + if (err) RETURN_ERRNO(err, "find_inode() error"); + + if (!node) RETURN_ERRNO(ENOENT, "file does not exist"); + + inode *parent = node->parent; + + if (parent && !(parent->mode & 0222)) RETURN_ERRNO(EACCES, "Write access to the directory containing pathname is not allowed for the process's effective UID"); + + // TODO: RETURN_ERRNO(EPERM, "The directory containing pathname has the sticky bit (S_ISVTX) set and the process's effective user ID is neither the user ID of the file to be deleted nor that of the directory containing it, and the process is not privileged"); + // TODO: RETURN_ERRNO(EROFS, "pathname refers to a file on a read-only filesystem"); + + if (!(node->mode & 0222)) + { + if (node->type == INODE_DIR) RETURN_ERRNO(EISDIR, "directory deletion not permitted"); // Linux quirk: Return EISDIR error for not having permission to delete a directory. + else RETURN_ERRNO(EPERM, "file deletion not permitted"); // but return EPERM error for no permission to delete a file. + } + + if (node->child) RETURN_ERRNO(EISDIR, "directory is not empty"); // Linux quirk: Return EISDIR error if not being able to delete a nonempty directory. + + unlink_inode(node); + + return 0; +} + +long __syscall12(int which, ...) // chdir +{ + va_list vl; + va_start(vl, which); + const char *pathname = va_arg(vl, const char *); + va_end(vl); + EM_ASM_INT({ Module['printErr']('chdir(pathname="' + Pointer_stringify($0) + '")') }, pathname); + + int len = strlen(pathname); + if (len > MAX_PATHNAME_LENGTH) RETURN_ERRNO(ENAMETOOLONG, "pathname was too long"); + if (len == 0) RETURN_ERRNO(ENOENT, "pathname is empty"); + + int err; + inode *node = find_inode(pathname, &err); + if (err == ENOTDIR) RETURN_ERRNO(ENOTDIR, "A component used as a directory in pathname is not, in fact, a directory"); + if (err == ELOOP) RETURN_ERRNO(ELOOP, "Too many symbolic links were encountered in resolving path"); + if (err == EACCES) RETURN_ERRNO(EACCES, "Search permission is denied for one of the components of path"); + if (err == ENOENT) RETURN_ERRNO(ENOENT, "Directory component in pathname does not exist or is a dangling symbolic link"); + if (err) RETURN_ERRNO(err, "find_inode() error"); + if (!node) RETURN_ERRNO(ENOENT, "The directory specified in path does not exist"); + if (node->type != INODE_DIR) RETURN_ERRNO(ENOTDIR, "Path is not a directory"); + + set_cwd(node); + return 0; +} + +long __syscall14(int which, ...) // mknod +{ + va_list vl; + va_start(vl, which); + const char *pathname = va_arg(vl, const char *); + mode_t mode = va_arg(vl, mode_t); + int dev = va_arg(vl, int); + va_end(vl); + EM_ASM_INT({ Module['printErr']('mknod(pathname="' + Pointer_stringify($0) + '", mode=0' + ($1).toString(8) + ', dev=' + $2 + ')') }, pathname, mode, dev); + + RETURN_ERRNO(ENOTSUP, "TODO: mknod() is a stub and not yet implemented in ASMFS"); +} + +long __syscall15(int which, ...) // chmod +{ + va_list vl; + va_start(vl, which); + const char *pathname = va_arg(vl, const char *); + int mode = va_arg(vl, int); + va_end(vl); + EM_ASM_INT({ Module['printErr']('chmod(pathname="' + Pointer_stringify($0) + '", mode=0' + ($1).toString(8) + ')') }, pathname, mode); + + int len = strlen(pathname); + if (len > MAX_PATHNAME_LENGTH) RETURN_ERRNO(ENAMETOOLONG, "pathname was too long"); + if (len == 0) RETURN_ERRNO(ENOENT, "pathname is empty"); + + int err; + inode *node = find_inode(pathname, &err); + if (err == ENOTDIR) RETURN_ERRNO(ENOTDIR, "A component used as a directory in pathname is not, in fact, a directory"); + if (err == ELOOP) RETURN_ERRNO(ELOOP, "Too many symbolic links were encountered in resolving pathname"); + if (err == EACCES) RETURN_ERRNO(EACCES, "Search permission is denied on a component of the path prefix"); + if (err == ENOENT) RETURN_ERRNO(ENOENT, "Directory component in pathname does not exist or is a dangling symbolic link"); + if (err) RETURN_ERRNO(err, "find_inode() error"); + if (!node) RETURN_ERRNO(ENOENT, "The file does not exist"); + + // TODO: if (not allowed) RETURN_ERRNO(EPERM, "The effective UID does not match the owner of the file"); + // TODO: read-only filesystems: if (fs is read-only) RETURN_ERRNO(EROFS, "The named file resides on a read-only filesystem"); + + node->mode = mode; + return 0; +} + +long __syscall33(int which, ...) // access +{ + va_list vl; + va_start(vl, which); + const char *pathname = va_arg(vl, const char *); + int mode = va_arg(vl, int); + va_end(vl); + EM_ASM_INT({ Module['printErr']('access(pathname="' + Pointer_stringify($0) + '", mode=0' + ($1).toString(8) + ')') }, pathname, mode); + + int len = strlen(pathname); + if (len > MAX_PATHNAME_LENGTH) RETURN_ERRNO(ENAMETOOLONG, "pathname was too long"); + if (len == 0) RETURN_ERRNO(ENOENT, "pathname is empty"); + + if ((mode & F_OK) && (mode & (R_OK | W_OK | X_OK))) RETURN_ERRNO(EINVAL, "mode was incorrectly specified"); + + int err; + inode *node = find_inode(pathname, &err); + if (err == ENOTDIR) RETURN_ERRNO(ENOTDIR, "A component used as a directory in pathname is not, in fact, a directory"); + if (err == ELOOP) RETURN_ERRNO(ELOOP, "Too many symbolic links were encountered in resolving pathname"); + if (err == EACCES) RETURN_ERRNO(EACCES, "Search permission is denied for one of the directories in the path prefix of pathname"); + if (err == ENOENT) RETURN_ERRNO(ENOENT, "A component of pathname does not exist or is a dangling symbolic link"); + if (err) RETURN_ERRNO(err, "find_inode() error"); + if (!node) RETURN_ERRNO(ENOENT, "Pathname does not exist"); + + // Just testing if a file exists? + if ((mode & F_OK)) return 0; + + // TODO: RETURN_ERRNO(EROFS, "Write permission was requested for a file on a read-only filesystem"); + + if ((mode & R_OK) && !(node->mode & 0444)) RETURN_ERRNO(EACCES, "Read access would be denied to the file"); + if ((mode & W_OK) && !(node->mode & 0222)) RETURN_ERRNO(EACCES, "Write access would be denied to the file"); + if ((mode & X_OK) && !(node->mode & 0111)) RETURN_ERRNO(EACCES, "Execute access would be denied to the file"); + + return 0; +} + +long __syscall36(int which, ...) // sync +{ + EM_ASM({ Module['printErr']('sync()') }); + + // Spec mandates that "sync() is always successful". + return 0; +} + +// TODO: syscall38, int rename(const char *oldpath, const char *newpath); + +long __syscall39(int which, ...) // mkdir +{ + va_list vl; + va_start(vl, which); + const char *pathname = va_arg(vl, const char *); + mode_t mode = va_arg(vl, mode_t); + va_end(vl); + EM_ASM_INT({ Module['printErr']('mkdir(pathname="' + Pointer_stringify($0) + '", mode=0' + ($1).toString(8) + ')') }, pathname, mode); + + int len = strlen(pathname); + if (len > MAX_PATHNAME_LENGTH) RETURN_ERRNO(ENAMETOOLONG, "pathname was too long"); + if (len == 0) RETURN_ERRNO(ENOENT, "pathname is empty"); + + inode *root = (pathname[0] == '/') ? filesystem_root() : get_cwd(); + const char *relpath = (pathname[0] == '/') ? pathname+1 : pathname; + int err; + inode *parent_dir = find_parent_inode(root, relpath, &err); + if (err == ENOTDIR) RETURN_ERRNO(ENOTDIR, "A component used as a directory in pathname is not, in fact, a directory"); + if (err == ELOOP) RETURN_ERRNO(ELOOP, "Too many symbolic links were encountered in resolving pathname"); + if (err == EACCES) RETURN_ERRNO(EACCES, "One of the directories in pathname did not allow search permission"); + if (err) RETURN_ERRNO(err, "find_inode() error"); + if (!parent_dir) RETURN_ERRNO(ENOENT, "A directory component in pathname does not exist or is a dangling symbolic link"); + + // TODO: if (component of path wasn't actually a directory) RETURN_ERRNO(ENOTDIR, "A component used as a directory in pathname is not, in fact, a directory"); + + inode *existing = find_inode(parent_dir, basename_part(pathname), &err); + if (err == ENOTDIR) RETURN_ERRNO(ENOTDIR, "A component used as a directory in pathname is not, in fact, a directory"); + if (err == ELOOP) RETURN_ERRNO(ELOOP, "Too many symbolic links were encountered in resolving pathname"); + if (err == EACCES) RETURN_ERRNO(EACCES, "One of the directories in pathname did not allow search permission"); + if (err && err != ENOENT) RETURN_ERRNO(err, "find_inode() error"); + if (existing) RETURN_ERRNO(EEXIST, "pathname already exists (not necessarily as a directory)"); + if (!(parent_dir->mode & 0222)) RETURN_ERRNO(EACCES, "The parent directory does not allow write permission to the process"); + + // TODO: read-only filesystems: if (fs is read-only) RETURN_ERRNO(EROFS, "Pathname refers to a file on a read-only filesystem"); + + inode *directory = create_inode(INODE_DIR, mode); + strcpy(directory->name, basename_part(pathname)); + link_inode(directory, parent_dir); + return 0; +} + +long __syscall40(int which, ...) // rmdir +{ + va_list vl; + va_start(vl, which); + const char *pathname = va_arg(vl, const char *); + va_end(vl); + EM_ASM_INT({ Module['printErr']('rmdir(pathname="' + Pointer_stringify($0) + '")') }, pathname); + + int len = strlen(pathname); + if (len > MAX_PATHNAME_LENGTH) RETURN_ERRNO(ENAMETOOLONG, "pathname was too long"); + if (len == 0) RETURN_ERRNO(ENOENT, "pathname is empty"); + + if (!strcmp(pathname, ".") || (len >= 2 && !strcmp(pathname+len-2, "/."))) RETURN_ERRNO(EINVAL, "pathname has . as last component"); + if (!strcmp(pathname, "..") || (len >= 3 && !strcmp(pathname+len-3, "/.."))) RETURN_ERRNO(ENOTEMPTY, "pathname has .. as its final component"); + + int err; + inode *node = find_inode(pathname, &err); + if (err == ENOTDIR) RETURN_ERRNO(ENOTDIR, "A component used as a directory in pathname is not, in fact, a directory"); + if (err == ELOOP) RETURN_ERRNO(ELOOP, "Too many symbolic links were encountered in resolving pathname"); + if (err == EACCES) RETURN_ERRNO(EACCES, "one of the directories in the path prefix of pathname did not allow search permission"); + if (err == ENOENT) RETURN_ERRNO(ENOENT, "A directory component in pathname does not exist or is a dangling symbolic link"); + if (err) RETURN_ERRNO(err, "find_inode() error"); + if (!node) RETURN_ERRNO(ENOENT, "directory does not exist"); + if (node == filesystem_root() || node == get_cwd()) RETURN_ERRNO(EBUSY, "pathname is currently in use by the system or some process that prevents its removal (pathname is currently used as a mount point or is the root directory of the calling process)"); + if (node->parent && !(node->parent->mode & 0222)) RETURN_ERRNO(EACCES, "Write access to the directory containing pathname was not allowed"); + if (node->type != INODE_DIR) RETURN_ERRNO(ENOTDIR, "pathname is not a directory"); + if (node->child) RETURN_ERRNO(ENOTEMPTY, "pathname contains entries other than . and .."); + + // TODO: RETURN_ERRNO(EPERM, "The directory containing pathname has the sticky bit (S_ISVTX) set and the process's effective user ID is neither the user ID of the file to be deleted nor that of the directory containing it, and the process is not privileged"); + // TODO: RETURN_ERRNO(EROFS, "pathname refers to a directory on a read-only filesystem"); + + unlink_inode(node); + + return 0; +} + +long __syscall41(int which, ...) // dup +{ + va_list vl; + va_start(vl, which); + unsigned int fd = va_arg(vl, unsigned int); + va_end(vl); + EM_ASM_INT({ Module['printErr']('dup(fd=' + $0 + ')') }, fd); + + FileDescriptor *desc = (FileDescriptor*)fd; + if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) RETURN_ERRNO(EBADF, "fd isn't a valid open file descriptor"); + + inode *node = desc->node; + if (!node) RETURN_ERRNO(-1, "ASMFS internal error: file descriptor points to a nonexisting file"); + + // TODO: RETURN_ERRNO(EMFILE, "The per-process limit on the number of open file descriptors has been reached (see RLIMIT_NOFILE)"); + + RETURN_ERRNO(ENOTSUP, "TODO: dup() is a stub and not yet implemented in ASMFS"); +} + +// TODO: syscall42: int pipe(int pipefd[2]); + +long __syscall54(int which, ...) // ioctl/sysctl +{ + va_list vl; + va_start(vl, which); + int fd = va_arg(vl, int); + int request = va_arg(vl, int); + char *argp = va_arg(vl, char *); + va_end(vl); + EM_ASM_INT({ Module['printErr']('ioctl(fd=' + $0 + ', request=' + $1 + ', argp=0x' + $2 + ')') }, fd, request, argp); + RETURN_ERRNO(ENOTSUP, "TODO: ioctl() is a stub and not yet implemented in ASMFS"); +} + +// TODO: syscall60: mode_t umask(mode_t mask); +// TODO: syscall63: dup2 +// TODO: syscall83: symlink +// TODO: syscall85: readlink +// TODO: syscall91: munmap +// TODO: syscall94: fchmod +// TODO: syscall102: socketcall + + +long __syscall118(int which, ...) // fsync +{ + va_list vl; + va_start(vl, which); + unsigned int fd = va_arg(vl, unsigned int); + va_end(vl); + + FileDescriptor *desc = (FileDescriptor*)fd; + if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) RETURN_ERRNO(EBADF, "fd isn't a valid open file descriptor"); + + inode *node = desc->node; + if (!node) RETURN_ERRNO(-1, "ASMFS internal error: file descriptor points to a non-file"); + + return 0; +} + +// TODO: syscall133: fchdir + +long __syscall140(int which, ...) // llseek +{ + va_list vl; + va_start(vl, which); + unsigned int fd = va_arg(vl, unsigned int); + unsigned long offset_high = va_arg(vl, unsigned long); + unsigned long offset_low = va_arg(vl, unsigned long); + off_t *result = va_arg(vl, off_t *); + unsigned int whence = va_arg(vl, unsigned int); + va_end(vl); + EM_ASM_INT({ Module['printErr']('llseek(fd=' + $0 + ', offset_high=' + $1 + ', offset_low=' + $2 + ', result=0x' + ($3).toString(16) + ', whence=' + $4 + ')') }, + fd, offset_high, offset_low, result, whence); + + FileDescriptor *desc = (FileDescriptor*)fd; + if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) RETURN_ERRNO(EBADF, "fd isn't a valid open file descriptor"); + + if (desc->node->fetch) emscripten_fetch_wait(desc->node->fetch, INFINITY); + +// TODO: The following does not work, for some reason seek is getting called with 32-bit signed offsets? +// int64_t offset = (int64_t)(((uint64_t)offset_high << 32) | (uint64_t)offset_low); + int64_t offset = (int64_t)(int32_t)offset_low; + int64_t newPos; + switch(whence) + { + case SEEK_SET: newPos = offset; break; + case SEEK_CUR: newPos = desc->file_pos + offset; break; + case SEEK_END: newPos = (desc->node->fetch ? desc->node->fetch->numBytes : desc->node->size) + offset; break; + case 3/*SEEK_DATA*/: RETURN_ERRNO(EINVAL, "whence is invalid (sparse files, whence=SEEK_DATA, is not supported"); + case 4/*SEEK_HOLE*/: RETURN_ERRNO(EINVAL, "whence is invalid (sparse files, whence=SEEK_HOLE, is not supported"); + default: RETURN_ERRNO(EINVAL, "whence is invalid"); + } + if (newPos < 0) RETURN_ERRNO(EINVAL, "The resulting file offset would be negative"); + if (newPos > 0x7FFFFFFFLL) RETURN_ERRNO(EOVERFLOW, "The resulting file offset cannot be represented in an off_t"); + + desc->file_pos = newPos; + + if (result) *result = desc->file_pos; + return 0; +} + +// TODO: syscall144 msync + +long __syscall145(int which, ...) // readv +{ + va_list vl; + va_start(vl, which); + int fd = va_arg(vl, int); + const iovec *iov = va_arg(vl, const iovec*); + int iovcnt = va_arg(vl, int); + va_end(vl); + EM_ASM_INT({ Module['printErr']('readv(fd=' + $0 + ', iov=0x' + ($1).toString(16) + ', iovcnt=' + $2 + ')') }, fd, iov, iovcnt); + + FileDescriptor *desc = (FileDescriptor*)fd; + if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) RETURN_ERRNO(EBADF, "fd isn't a valid open file descriptor"); + + inode *node = desc->node; + if (!node) RETURN_ERRNO(-1, "ASMFS internal error: file descriptor points to a non-file"); + if (node->type == INODE_DIR) RETURN_ERRNO(EISDIR, "fd refers to a directory"); + if (node->type != INODE_FILE /* TODO: && node->type != socket */) RETURN_ERRNO(EINVAL, "fd is attached to an object which is unsuitable for reading"); + + // TODO: if (node->type == INODE_FILE && desc has O_NONBLOCK && read would block) RETURN_ERRNO(EAGAIN, "The file descriptor fd refers to a file other than a socket and has been marked nonblocking (O_NONBLOCK), and the read would block"); + // TODO: if (node->type == socket && desc has O_NONBLOCK && read would block) RETURN_ERRNO(EWOULDBLOCK, "The file descriptor fd refers to a socket and has been marked nonblocking (O_NONBLOCK), and the read would block"); + + if (node->fetch) emscripten_fetch_wait(node->fetch, INFINITY); + + if (node->size > 0 && !node->data && (!node->fetch || !node->fetch->data)) RETURN_ERRNO(-1, "ASMFS internal error: no file data available"); + if (iovcnt < 0) RETURN_ERRNO(EINVAL, "The vector count, iovcnt, is less than zero"); + + ssize_t total_read_amount = 0; + for(int i = 0; i < iovcnt; ++i) + { + ssize_t n = total_read_amount + iov[i].iov_len; + if (n < total_read_amount) RETURN_ERRNO(EINVAL, "The sum of the iov_len values overflows an ssize_t value"); + if (!iov[i].iov_base && iov[i].iov_len > 0) RETURN_ERRNO(EINVAL, "iov_len specifies a positive length buffer but iov_base is a null pointer"); + total_read_amount = n; + } + + size_t offset = desc->file_pos; + uint8_t *data = node->data ? node->data : (node->fetch ? (uint8_t *)node->fetch->data : 0); + for(int i = 0; i < iovcnt; ++i) + { + ssize_t dataLeft = node->size - offset; + if (dataLeft <= 0) break; + size_t bytesToCopy = (size_t)dataLeft < iov[i].iov_len ? dataLeft : iov[i].iov_len; + memcpy(iov[i].iov_base, &data[offset], bytesToCopy); + offset += bytesToCopy; + } + ssize_t numRead = offset - desc->file_pos; + desc->file_pos = offset; + return numRead; +} + +long __syscall146(int which, ...) // writev +{ + va_list vl; + va_start(vl, which); + int fd = va_arg(vl, int); + const iovec *iov = va_arg(vl, const iovec*); + int iovcnt = va_arg(vl, int); + va_end(vl); + EM_ASM_INT({ Module['printErr']('writev(fd=' + $0 + ', iov=0x' + ($1).toString(16) + ', iovcnt=' + $2 + ')') }, fd, iov, iovcnt); + + FileDescriptor *desc = (FileDescriptor*)fd; + if (fd != 1/*stdout*/ && fd != 2/*stderr*/) // TODO: Resolve the hardcoding of stdin,stdout & stderr + { + if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) RETURN_ERRNO(EBADF, "fd isn't a valid open file descriptor"); + } + + if (iovcnt < 0) RETURN_ERRNO(EINVAL, "The vector count, iovcnt, is less than zero"); + + ssize_t total_write_amount = 0; + for(int i = 0; i < iovcnt; ++i) + { + ssize_t n = total_write_amount + iov[i].iov_len; + if (n < total_write_amount) RETURN_ERRNO(EINVAL, "The sum of the iov_len values overflows an ssize_t value"); + if (!iov[i].iov_base && iov[i].iov_len > 0) RETURN_ERRNO(EINVAL, "iov_len specifies a positive length buffer but iov_base is a null pointer"); + total_write_amount = n; + } + + if (fd == 1/*stdout*/ || fd == 2/*stderr*/) + { + ssize_t bytesWritten = 0; + for(int i = 0; i < iovcnt; ++i) + { + print_stream(iov[i].iov_base, iov[i].iov_len, fd == 1); + bytesWritten += iov[i].iov_len; + } + return bytesWritten; + } + else + { + // Enlarge the file in memory to fit space for the new data + size_t newSize = desc->file_pos + total_write_amount; + inode *node = desc->node; + if (node->capacity < newSize) + { + size_t newCapacity = (newSize > (size_t)(node->capacity*1.25) ? newSize : (size_t)(node->capacity*1.25)); // Geometric increases in size for amortized O(1) behavior + uint8_t *newData = (uint8_t *)realloc(node->data, newCapacity); + if (!newData) + { + newData = (uint8_t *)malloc(newCapacity); + memcpy(newData, node->data, node->size); + // TODO: init gaps with zeroes. + free(node->data); + } + node->data = newData; + node->size = newSize; + node->capacity = newCapacity; + } + + for(int i = 0; i < iovcnt; ++i) + { + memcpy((uint8_t*)node->data + desc->file_pos, iov[i].iov_base, iov[i].iov_len); + desc->file_pos += iov[i].iov_len; + } + } + return total_write_amount; +} + +// TODO: syscall148: fdatasync +// TODO: syscall168: poll + +// TODO: syscall180: pread64 +// TODO: syscall181: pwrite64 + +long __syscall183(int which, ...) // getcwd +{ + va_list vl; + va_start(vl, which); + char *buf = va_arg(vl, char *); + size_t size = va_arg(vl, size_t); + va_end(vl); + EM_ASM_INT({ Module['printErr']('getcwd(buf=0x' + $0 + ', size= ' + $1 + ')') }, buf, size); + + if (!buf && size > 0) RETURN_ERRNO(EFAULT, "buf points to a bad address"); + if (buf && size == 0) RETURN_ERRNO(EINVAL, "The size argument is zero and buf is not a null pointer"); + + inode *cwd = get_cwd(); + if (!cwd) RETURN_ERRNO(-1, "ASMFS internal error: no current working directory?!"); + // TODO: RETURN_ERRNO(ENOENT, "The current working directory has been unlinked"); + // TODO: RETURN_ERRNO(EACCES, "Permission to read or search a component of the filename was denied"); + inode_abspath(cwd, buf, size); + if (strlen(buf) >= size-1) RETURN_ERRNO(ERANGE, "The size argument is less than the length of the absolute pathname of the working directory, including the terminating null byte. You need to allocate a bigger array and try again"); + + return 0; +} + +// TODO: syscall192: mmap2 +// TODO: syscall193: truncate64 +// TODO: syscall194: ftruncate64 + +static long __stat64(inode *node, struct stat *buf) +{ + buf->st_dev = 1; // ID of device containing file: Hardcode 1 for now, no meaning at the moment for Emscripten. + buf->st_ino = (ino_t)node; // TODO: This needs to be an inode ID number proper. + buf->st_mode = node->mode; + switch(node->type) { + case INODE_DIR: buf->st_mode |= S_IFDIR; break; + case INODE_FILE: buf->st_mode |= S_IFREG; break; // Regular file + /* TODO: + case socket: buf->st_mode |= S_IFSOCK; break; + case symlink: buf->st_mode |= S_IFLNK; break; + case block device: buf->st_mode |= S_IFBLK; break; + case character device: buf->st_mode |= S_IFCHR; break; + case FIFO: buf->st_mode |= S_IFIFO; break; + */ + } + buf->st_nlink = 1; // The number of hard links. TODO: Use this for real when links are supported. + buf->st_uid = node->uid; + buf->st_gid = node->gid; + buf->st_rdev = 1; // Device ID (if special file) No meaning right now for Emscripten. + buf->st_size = node->fetch ? node->fetch->totalBytes : 0; + if (node->size > buf->st_size) buf->st_size = node->size; + buf->st_blocks = (buf->st_size + 511) / 512; // The syscall docs state this is hardcoded to # of 512 byte blocks. + buf->st_blksize = 1024*1024; // Specifies the preferred blocksize for efficient disk I/O. + buf->st_atim.tv_sec = node->atime; + buf->st_mtim.tv_sec = node->mtime; + buf->st_ctim.tv_sec = node->ctime; + return 0; +} + +long __syscall195(int which, ...) // SYS_stat64 +{ + va_list vl; + va_start(vl, which); + const char *pathname = va_arg(vl, const char *); + struct stat *buf = va_arg(vl, struct stat *); + va_end(vl); + EM_ASM_INT({ Module['printErr']('SYS_stat64(pathname="' + Pointer_stringify($0) + '", buf=0x' + ($1).toString(16) + ')') }, pathname, buf); + + int len = strlen(pathname); + if (len > MAX_PATHNAME_LENGTH) RETURN_ERRNO(ENAMETOOLONG, "pathname was too long"); + if (len == 0) RETURN_ERRNO(ENOENT, "pathname is empty"); + + // Find if this file exists already in the filesystem? + inode *root = (pathname[0] == '/') ? filesystem_root() : get_cwd(); + const char *relpath = (pathname[0] == '/') ? pathname+1 : pathname; + + int err; + inode *node = find_inode(root, relpath, &err); + + if (!node && (err == ENOENT || err == ENOTDIR)) + { + // Populate the file from the CDN to the filesystem if it didn't yet exist. + long fd = open(pathname, O_RDONLY, 0777); + if (fd) close(fd); + node = find_inode(root, relpath, &err); + } + + if (err == ENOTDIR) RETURN_ERRNO(ENOTDIR, "A component of the path prefix of pathname is not a directory"); + if (err == ELOOP) RETURN_ERRNO(ELOOP, "Too many symbolic links encountered while traversing the path"); + if (err == EACCES) RETURN_ERRNO(EACCES, "Search permission is denied for one of the directories in the path prefix of pathname"); + if (err && err != ENOENT) RETURN_ERRNO(err, "find_inode() error"); + if (err == ENOENT || !node) RETURN_ERRNO(ENOENT, "A component of pathname does not exist"); + + return __stat64(node, buf); +} + +long __syscall196(int which, ...) // SYS_lstat64 +{ + va_list vl; + va_start(vl, which); + const char *pathname = va_arg(vl, const char *); + struct stat *buf = va_arg(vl, struct stat *); + va_end(vl); + EM_ASM_INT({ Module['printErr']('SYS_lstat64(pathname="' + Pointer_stringify($0) + '", buf=0x' + ($1).toString(16) + ')') }, pathname, buf); + + int len = strlen(pathname); + if (len > MAX_PATHNAME_LENGTH) RETURN_ERRNO(ENAMETOOLONG, "pathname was too long"); + if (len == 0) RETURN_ERRNO(ENOENT, "pathname is empty"); + + // Find if this file exists already in the filesystem? + inode *root = (pathname[0] == '/') ? filesystem_root() : get_cwd(); + const char *relpath = (pathname[0] == '/') ? pathname+1 : pathname; + + // TODO: When symbolic links are implemented, make this return info about the symlink itself and not the file it points to. + int err; + inode *node = find_inode(root, relpath, &err); + if (err == ENOTDIR) RETURN_ERRNO(ENOTDIR, "A component of the path prefix of pathname is not a directory"); + if (err == ELOOP) RETURN_ERRNO(ELOOP, "Too many symbolic links encountered while traversing the path"); + if (err == EACCES) RETURN_ERRNO(EACCES, "Search permission is denied for one of the directories in the path prefix of pathname"); + if (err && err != ENOENT) RETURN_ERRNO(err, "find_inode() error"); + if (err == ENOENT || !node) RETURN_ERRNO(ENOENT, "A component of pathname does not exist"); + + return __stat64(node, buf); +} + +long __syscall197(int which, ...) // SYS_fstat64 +{ + va_list vl; + va_start(vl, which); + int fd = va_arg(vl, int); + struct stat *buf = va_arg(vl, struct stat *); + va_end(vl); + EM_ASM_INT({ Module['printErr']('SYS_fstat64(fd="' + Pointer_stringify($0) + '", buf=0x' + ($1).toString(16) + ')') }, fd, buf); + + FileDescriptor *desc = (FileDescriptor*)fd; + if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) RETURN_ERRNO(EBADF, "fd isn't a valid open file descriptor"); + + int err; + inode *node = desc->node; + if (!node) RETURN_ERRNO(ENOENT, "A component of pathname does not exist"); + + return __stat64(node, buf); +} + +// TODO: syscall198: lchown +// TODO: syscall207: fchown32 +// TODO: syscall212: chown32 + +long __syscall220(int which, ...) // getdents64 (get directory entries 64-bit) +{ + va_list vl; + va_start(vl, which); + unsigned int fd = va_arg(vl, unsigned int); + dirent *de = va_arg(vl, dirent*); + unsigned int count = va_arg(vl, unsigned int); + va_end(vl); + unsigned int dirents_size = count / sizeof(de); // The number of dirent structures that can fit into the provided buffer. + dirent *de_end = de + dirents_size; + EM_ASM_INT({ Module['printErr']('getdents64(fd=' + $0 + ', de=0x' + ($1).toString(16) + ', count=' + $2 + ')') }, fd, de, count); + + FileDescriptor *desc = (FileDescriptor*)fd; + if (!desc || desc->magic != EM_FILEDESCRIPTOR_MAGIC) RETURN_ERRNO(EBADF, "Invalid file descriptor fd"); + + inode *node = desc->node; + if (!node) RETURN_ERRNO(ENOENT, "No such directory"); + if (dirents_size == 0) RETURN_ERRNO(EINVAL, "Result buffer is too small"); + if (node->type != INODE_DIR) RETURN_ERRNO(ENOTDIR, "File descriptor does not refer to a directory"); + + inode *dotdot = node->parent ? node->parent : node; // In "/", the directory ".." refers to itself. + + ssize_t orig_file_pos = desc->file_pos; + ssize_t file_pos = 0; + // There are always two hardcoded directories "." and ".." + if (de >= de_end) return desc->file_pos - orig_file_pos; + if (desc->file_pos <= file_pos) + { + de->d_ino = (ino_t)node; // TODO: Create inode numbers instead of using pointers + de->d_off = file_pos; + de->d_reclen = sizeof(dirent); + de->d_type = DT_DIR; + strcpy(de->d_name, "."); + ++de; + desc->file_pos += sizeof(dirent); + } + file_pos += sizeof(dirent); + + if (de >= de_end) return desc->file_pos - orig_file_pos; + if (desc->file_pos <= file_pos) + { + de->d_ino = (ino_t)dotdot; // TODO: Create inode numbers instead of using pointers + de->d_off = file_pos; + de->d_reclen = sizeof(dirent); + de->d_type = DT_DIR; + strcpy(de->d_name, ".."); + ++de; + desc->file_pos += sizeof(dirent); + } + file_pos += sizeof(dirent); + + node = node->child; + while(node && de < de_end) + { + if (desc->file_pos <= file_pos) + { + de->d_ino = (ino_t)node; // TODO: Create inode numbers instead of using pointers + de->d_off = file_pos; + de->d_reclen = sizeof(dirent); + de->d_type = (node->type == INODE_DIR) ? DT_DIR : DT_REG /*Regular file*/; + de->d_name[255] = 0; + strncpy(de->d_name, node->name, 255); + ++de; + desc->file_pos += sizeof(dirent); + } + node = node->sibling; + file_pos += sizeof(dirent); + } + + return desc->file_pos - orig_file_pos; +} + +// TODO: syscall221: fcntl64 +// TODO: syscall268: statfs64 +// TODO: syscall269: fstatfs64 +// TODO: syscall295: openat +// TODO: syscall296: mkdirat +// TODO: syscall297: mknodat +// TODO: syscall298: fchownat +// TODO: syscall300: fstatat64 +// TODO: syscall301: unlinkat +// TODO: syscall302: renameat +// TODO: syscall303: linkat +// TODO: syscall304: symlinkat +// TODO: syscall305: readlinkat +// TODO: syscall306: fchmodat +// TODO: syscall307: faccessat +// TODO: syscall320: utimensat +// TODO: syscall324: fallocate +// TODO: syscall330: dup3 +// TODO: syscall331: pipe2 +// TODO: syscall333: preadv +// TODO: syscall334: pwritev + +} // ~extern "C" diff --git a/system/lib/fetch/emscripten_fetch.cpp b/system/lib/fetch/emscripten_fetch.cpp new file mode 100644 index 0000000000000..62e63346c2d9d --- /dev/null +++ b/system/lib/fetch/emscripten_fetch.cpp @@ -0,0 +1,151 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +struct __emscripten_fetch_queue +{ + emscripten_fetch_t **queuedOperations; + int numQueuedItems; + int queueSize; +}; + +extern "C" { + void emscripten_start_fetch(emscripten_fetch_t *fetch); + __emscripten_fetch_queue *_emscripten_get_fetch_work_queue(); + + __emscripten_fetch_queue *_emscripten_get_fetch_queue() + { + __emscripten_fetch_queue *queue = _emscripten_get_fetch_work_queue(); + if (!queue->queuedOperations) + { + queue->queueSize = 64; + queue->numQueuedItems = 0; + queue->queuedOperations = (emscripten_fetch_t**)malloc(sizeof(emscripten_fetch_t*) * queue->queueSize); + } + return queue; + } +} + +void emscripten_proxy_fetch(emscripten_fetch_t *fetch) +{ + // TODO: mutex lock + __emscripten_fetch_queue *queue = _emscripten_get_fetch_queue(); +// TODO handle case when queue->numQueuedItems >= queue->queueSize + queue->queuedOperations[queue->numQueuedItems++] = fetch; + EM_ASM_INT( { console.log('Queued fetch to fetch-worker to process. There are now ' + $0 + ' operations in the queue.') }, + queue->numQueuedItems); + // TODO: mutex unlock +} + +void emscripten_fetch_attr_init(emscripten_fetch_attr_t *fetch_attr) +{ + memset(fetch_attr, 0, sizeof(emscripten_fetch_attr_t)); +} + +static int globalFetchIdCounter = 1; +emscripten_fetch_t *emscripten_fetch(emscripten_fetch_attr_t *fetch_attr, const char *url) +{ + if (!fetch_attr) return 0; + if (!url) return 0; + + const bool waitable = (fetch_attr->attributes & EMSCRIPTEN_FETCH_WAITABLE) != 0; + const bool synchronous = (fetch_attr->attributes & EMSCRIPTEN_FETCH_SYNCHRONOUS) != 0; + const bool readFromIndexedDB = (fetch_attr->attributes & (EMSCRIPTEN_FETCH_APPEND | EMSCRIPTEN_FETCH_NO_DOWNLOAD)) != 0; + const bool writeToIndexedDB = (fetch_attr->attributes & EMSCRIPTEN_FETCH_PERSIST_FILE) != 0 || !strncmp(fetch_attr->requestMethod, "EM_IDB_", strlen("EM_IDB_")); + const bool performXhr = (fetch_attr->attributes & EMSCRIPTEN_FETCH_NO_DOWNLOAD) == 0; + const bool isMainBrowserThread = emscripten_is_main_browser_thread() != 0; + if (isMainBrowserThread && synchronous && (performXhr || readFromIndexedDB || writeToIndexedDB)) + { + EM_ASM_INT( { Module['printErr']('emscripten_fetch("' + Pointer_stringify($0) + '") failed! Synchronous blocking XHRs and IndexedDB operations are not supported on the main browser thread. Try dropping the EMSCRIPTEN_FETCH_SYNCHRONOUS flag, or run with the linker flag --proxy-to-worker to decouple main C runtime thread from the main browser thread.') }, + url); + return 0; + } + + emscripten_fetch_t *fetch = (emscripten_fetch_t *)malloc(sizeof(emscripten_fetch_t)); + memset(fetch, 0, sizeof(emscripten_fetch_t)); + fetch->id = globalFetchIdCounter++; // TODO: make this thread-safe! + fetch->userData = fetch_attr->userData; + fetch->url = strdup(url); // TODO: free + fetch->__attributes = *fetch_attr; + fetch->__attributes.destinationPath = fetch->__attributes.destinationPath ? strdup(fetch->__attributes.destinationPath) : 0; // TODO: free + fetch->__attributes.userName = fetch->__attributes.userName ? strdup(fetch->__attributes.userName) : 0; // TODO: free + fetch->__attributes.password = fetch->__attributes.password ? strdup(fetch->__attributes.password) : 0; // TODO: free + fetch->__attributes.requestHeaders = 0;// TODO:strdup(fetch->__attributes.requestHeaders); + fetch->__attributes.overriddenMimeType = fetch->__attributes.overriddenMimeType ? strdup(fetch->__attributes.overriddenMimeType) : 0; // TODO: free + +#if __EMSCRIPTEN_PTHREADS__ + // Depending on the type of fetch, we can either perform it in the same Worker/thread than the caller, or we might need + // to run it in a separate Worker. There is a dedicated fetch worker that is available for the fetch, but in some scenarios + // it might be desirable to run in the same Worker as the caller, so deduce here whether to run the fetch in this thread, + // or if we need to use the fetch-worker instead. + if (waitable // Waitable fetches can be synchronously waited on, so must always be proxied + || (synchronous && (readFromIndexedDB || writeToIndexedDB))) // Synchronous IndexedDB access needs proxying + { + emscripten_atomic_store_u32(&fetch->__proxyState, 1); // sent to proxy worker. + emscripten_proxy_fetch(fetch); + + if (synchronous) emscripten_fetch_wait(fetch, INFINITY); + } + else +#endif + emscripten_start_fetch(fetch); + return fetch; +} + +EMSCRIPTEN_RESULT emscripten_fetch_wait(emscripten_fetch_t *fetch, double timeoutMsecs) +{ +#if __EMSCRIPTEN_PTHREADS__ + if (!fetch) return EMSCRIPTEN_RESULT_INVALID_PARAM; + uint32_t proxyState = emscripten_atomic_load_u32(&fetch->__proxyState); + if (proxyState == 2) return EMSCRIPTEN_RESULT_SUCCESS; // already finished. + if (proxyState != 1) return EMSCRIPTEN_RESULT_INVALID_PARAM; // the fetch should be ongoing? +// #ifdef FETCH_DEBUG + EM_ASM({ console.log('fetch: emscripten_fetch_wait..') }); +// #endif + // TODO: timeoutMsecs is currently ignored. Return EMSCRIPTEN_RESULT_TIMED_OUT on timeout. + while(proxyState == 1/*sent to proxy worker*/) + { + emscripten_futex_wait(&fetch->__proxyState, proxyState, 100 /*TODO HACK:Sleep sometimes doesn't wake up?*/);//timeoutMsecs); + proxyState = emscripten_atomic_load_u32(&fetch->__proxyState); + } +// #ifdef FETCH_DEBUG + EM_ASM({ console.log('fetch: emscripten_fetch_wait done..') }); +// #endif + + if (proxyState == 2) return EMSCRIPTEN_RESULT_SUCCESS; + else return EMSCRIPTEN_RESULT_FAILED; +#else + EM_ASM({ console.error('fetch: emscripten_fetch_wait is not available when building without pthreads!') }); + return EMSCRIPTEN_RESULT_FAILED; +#endif +} + +EMSCRIPTEN_RESULT emscripten_fetch_close(emscripten_fetch_t *fetch) +{ + if (!fetch) return EMSCRIPTEN_RESULT_SUCCESS; // Closing null pointer is ok, same as with free(). + +#if __EMSCRIPTEN_PTHREADS__ + emscripten_atomic_store_u32(&fetch->__proxyState, 0); +#endif + // This function frees the fetch pointer so that it is invalid to access it anymore. + // Use a few key fields as an integrity check that we are being passed a good pointer to a valid fetch structure, + // which has not been yet closed. (double close is an error) + if (fetch->id == 0 || fetch->readyState > 4) return EMSCRIPTEN_RESULT_INVALID_PARAM; + + // This fetch is aborted. Call the error handler if the fetch was still in progress and was canceled in flight. + if (fetch->readyState != 4 /*DONE*/ && fetch->__attributes.onerror) + { + fetch->status = (unsigned short)-1; + strcpy(fetch->statusText, "aborted with emscripten_fetch_close()"); + fetch->__attributes.onerror(fetch); + } + fetch->id = 0; + free((void*)fetch->data); + free(fetch); + return EMSCRIPTEN_RESULT_SUCCESS; +} diff --git a/system/lib/libc/musl/arch/emscripten/syscall_arch.h b/system/lib/libc/musl/arch/emscripten/syscall_arch.h index b4d5fcfdc32b6..931599112aac7 100644 --- a/system/lib/libc/musl/arch/emscripten/syscall_arch.h +++ b/system/lib/libc/musl/arch/emscripten/syscall_arch.h @@ -26,6 +26,10 @@ // static syscalls. we must have one non-variadic argument before the rest due to ISO C. +#ifdef __cplusplus +extern "C" { +#endif + long __syscall1(int which, ...); long __syscall3(int which, ...); long __syscall4(int which, ...); @@ -138,3 +142,6 @@ long __syscall333(int which, ...); long __syscall334(int which, ...); long __syscall340(int which, ...); +#ifdef __cplusplus +} +#endif diff --git a/tests/asmfs/fopen_write.cpp b/tests/asmfs/fopen_write.cpp new file mode 100644 index 0000000000000..0ec0c2e4d3f20 --- /dev/null +++ b/tests/asmfs/fopen_write.cpp @@ -0,0 +1,35 @@ +#include +#include +#include +#include +#include + +void create_file() +{ + FILE *file = fopen("hello_file.txt", "wb"); + assert(file); + const char *data = "Hello data!"; + fwrite(data, 1, strlen(data), file); + fclose(file); +} + +void read_file() +{ + FILE *file = fopen("hello_file.txt", "rb"); + char buffer[128] = {}; + size_t read = fread(buffer, 1, sizeof(buffer), file); + printf("read %u bytes. Result: %s\n", (unsigned int)read, buffer); + assert(read == strlen("Hello data!")); + assert(!strcmp(buffer, "Hello data!")); +} + +int main() +{ + create_file(); + read_file(); + +#ifdef REPORT_RESULT + int result = 0; + REPORT_RESULT(); +#endif +} diff --git a/tests/asmfs/hello_file.cpp b/tests/asmfs/hello_file.cpp new file mode 100644 index 0000000000000..8b9a1bcbdd821 --- /dev/null +++ b/tests/asmfs/hello_file.cpp @@ -0,0 +1,32 @@ +#include +#include +#include +#include +#include + +int main() +{ + FILE *file = fopen("dirrey/hello file !#$%&'()+,-.;=@[]^_`{}~ %%.txt", "rb"); + assert(file); + fseek(file, 0, SEEK_END); + + long size = ftell(file); + rewind(file); + printf("size: %ld\n", size); + + char *buffer = (char*) malloc (sizeof(char)*(size+1)); + assert(buffer); + buffer[size] = '\0'; + + size_t read = fread(buffer, 1, size, file); + printf("File contents: %s\n", buffer); + assert(size == 6); + assert(!strcmp(buffer, "Hello!")); + + fclose(file); + +#ifdef REPORT_RESULT + int result = 0; + REPORT_RESULT(); +#endif +} diff --git a/tests/asmfs/hello_file.txt b/tests/asmfs/hello_file.txt new file mode 100644 index 0000000000000..05a682bd4e7c7 --- /dev/null +++ b/tests/asmfs/hello_file.txt @@ -0,0 +1 @@ +Hello! \ No newline at end of file diff --git a/tests/asmfs/read_file_twice.cpp b/tests/asmfs/read_file_twice.cpp new file mode 100644 index 0000000000000..94a4fa629bf3a --- /dev/null +++ b/tests/asmfs/read_file_twice.cpp @@ -0,0 +1,37 @@ +#include +#include +#include +#include +#include + +void read_file() +{ + FILE *file = fopen("hello_file.txt", "rb"); + assert(file); + fseek(file, 0, SEEK_END); + + long size = ftell(file); + rewind(file); + printf("size: %ld\n", size); + + char *buffer = (char*) malloc (sizeof(char)*(size+1)); + assert(buffer); + buffer[size] = '\0'; + + size_t read = fread(buffer, 1, size, file); + printf("File contents: %s\n", buffer); + assert(size == 6); + assert(!strcmp(buffer, "Hello!")); + + fclose(file); +} + +int main() +{ + read_file(); + read_file(); +#ifdef REPORT_RESULT + int result = 0; + REPORT_RESULT(); +#endif +} diff --git a/tests/asmfs/relative_paths.cpp b/tests/asmfs/relative_paths.cpp new file mode 100644 index 0000000000000..09e7179af0386 --- /dev/null +++ b/tests/asmfs/relative_paths.cpp @@ -0,0 +1,35 @@ +#include +#include +#include +#include +#ifdef __EMSCRIPTEN__ +#include +#endif + +#include +#include +#include + +int main() +{ + int ret = mkdir("a", 0777); assert(ret == 0); assert(errno == 0); + ret = mkdir("./././a/b", 0777); assert(ret == 0); assert(errno == 0); + ret = mkdir("a/b/./../b/c", 0777); assert(ret == 0); assert(errno == 0); + FILE *file = fopen("./a/./hello_file.txt", "wb"); + assert(file); + fputs("test", file); + ret = fclose(file); assert(ret == 0); assert(errno == 0); + + ret = chdir("./././a/./b/../b/c/./../c/."); assert(ret == 0); assert(errno == 0); + file = fopen(".././../hello_file.txt", "rb"); + assert(file); assert(errno == 0); + char str[6] = {}; + fread(str, 1, 5, file); assert(errno == 0); + ret = fclose(file); assert(ret == 0); assert(errno == 0); + assert(!strcmp(str, "test")); + +#ifdef REPORT_RESULT + int result = 0; + REPORT_RESULT(); +#endif +} diff --git a/tests/core/test_ccall.out b/tests/core/test_ccall.out index 85b5162b065df..4dca7681bbb7e 100644 --- a/tests/core/test_ccall.out +++ b/tests/core/test_ccall.out @@ -10,6 +10,8 @@ cheez undefined arr-ay undefined +arr-ay +undefined more number,10 650 diff --git a/tests/cstdio/test_remove.cpp b/tests/cstdio/test_remove.cpp index 6bc14bd2a1958..be310407ce4ff 100644 --- a/tests/cstdio/test_remove.cpp +++ b/tests/cstdio/test_remove.cpp @@ -59,5 +59,10 @@ int main() { atexit(cleanup); setup(); test(); + +#ifdef REPORT_RESULT + int result = 0; + REPORT_RESULT(); +#endif return EXIT_SUCCESS; } diff --git a/tests/dirent/test_readdir.c b/tests/dirent/test_readdir.c index e5bb9d7581e2e..85920a78c4112 100644 --- a/tests/dirent/test_readdir.c +++ b/tests/dirent/test_readdir.c @@ -175,5 +175,10 @@ int main() { setup(); test(); test_scandir(); + +#ifdef REPORT_RESULT + int result = 0; + REPORT_RESULT(); +#endif return EXIT_SUCCESS; } diff --git a/tests/dirent/test_readdir_empty.c b/tests/dirent/test_readdir_empty.c index 00102733102ff..66ba79cf38d73 100644 --- a/tests/dirent/test_readdir_empty.c +++ b/tests/dirent/test_readdir_empty.c @@ -42,6 +42,11 @@ int main(int argc, char** argv) { closedir(dir); printf("success\n"); + +#ifdef REPORT_RESULT + int result = 0; + REPORT_RESULT(); +#endif return 0; } diff --git a/tests/fcntl-open/src.c b/tests/fcntl-open/src.c index fc5d5c764ab04..116f537a2387c 100644 --- a/tests/fcntl-open/src.c +++ b/tests/fcntl-open/src.c @@ -13,16 +13,20 @@ char nonexistent_name[] = "noexist-##"; void create_file(const char *path, const char *buffer, int mode) { int fd = open(path, O_WRONLY | O_CREAT | O_EXCL, mode); assert(fd >= 0); + assert(!errno); int err = write(fd, buffer, sizeof(char) * strlen(buffer)); assert(err == (sizeof(char) * strlen(buffer))); + assert(!errno); close(fd); + assert(!errno); } void setup() { create_file("test-file", "abcdef", 0777); mkdir("test-folder", 0777); + assert(!errno); } void cleanup() { @@ -35,7 +39,9 @@ void cleanup() { unlink(nonexistent_name); } } + errno = 0; unlink("creat-me"); + assert(!errno); } void test() { @@ -51,19 +57,48 @@ void test() { if (j & 0x8) flags |= O_APPEND; printf("EXISTING FILE %d,%d\n", i, j); - printf("success: %d\n", open("test-file", flags, 0777) != -1); + int success = open("test-file", flags, 0777) != -1; + printf("success: %d\n", success); printf("errno: %d\n", errno); - stat("test-file", &s); + if ((flags & O_CREAT) && (flags & O_EXCL)) { + assert(!success); + assert(errno == EEXIST); + } else { + assert(success); + assert(errno == 0); + } + errno = 0; + + int ret = stat("test-file", &s); + assert(ret == 0); + assert(errno == 0); printf("st_mode: 0%o\n", s.st_mode & 037777777000); + assert((s.st_mode & 037777777000) == 0100000); memset(&s, 0, sizeof s); printf("\n"); errno = 0; printf("EXISTING FOLDER %d,%d\n", i, j); - printf("success: %d\n", open("test-folder", flags, 0777) != -1); + success = open("test-folder", flags, 0777) != -1; + printf("success: %d\n", success); printf("errno: %d\n", errno); - stat("test-folder", &s); + if ((flags & O_CREAT) && (flags & O_EXCL)) { + assert(!success); + assert(errno == EEXIST); + } else if ((flags & O_TRUNC) || i != 0 /*mode != O_RDONLY*/) { + assert(!success); + assert(errno == EISDIR); + } else { + assert(success); + assert(errno == 0); + } + errno = 0; + + ret = stat("test-folder", &s); + assert(ret == 0); + assert(errno == 0); printf("st_mode: 0%o\n", s.st_mode & 037777777000); + assert((s.st_mode & 037777777000) == 040000); memset(&s, 0, sizeof s); printf("\n"); errno = 0; @@ -71,10 +106,28 @@ void test() { nonexistent_name[8] = 'a' + i; nonexistent_name[9] = 'a' + j; printf("NON-EXISTING %d,%d\n", i, j); - printf("success: %d\n", open(nonexistent_name, flags, 0777) != -1); + success = open(nonexistent_name, flags, 0777) != -1; + printf("success: %d\n", success); printf("errno: %d\n", errno); - stat(nonexistent_name, &s); + if ((flags & O_CREAT)) { + assert(success); + assert(errno == 0); + } else { + assert(!success); + assert(errno == ENOENT); + } + + ret = stat(nonexistent_name, &s); printf("st_mode: 0%o\n", s.st_mode & 037777777000); + + if ((flags & O_CREAT)) { + assert(ret == 0); + assert((s.st_mode & 037777777000) == 0100000); + } else { + assert(ret != 0); + assert((s.st_mode & 037777777000) == 0); + } + memset(&s, 0, sizeof s); printf("\n"); errno = 0; @@ -84,6 +137,7 @@ void test() { printf("CREAT\n"); printf("success: %d\n", creat("creat-me", 0777) != -1); printf("errno: %d\n", errno); + errno = 0; } int main() { @@ -91,5 +145,9 @@ int main() { signal(SIGABRT, cleanup); setup(); test(); +#ifdef REPORT_RESULT + int result = 0; + REPORT_RESULT(); +#endif return EXIT_SUCCESS; } diff --git a/tests/fetch/cached_xhr.cpp b/tests/fetch/cached_xhr.cpp new file mode 100644 index 0000000000000..c84d0281e5317 --- /dev/null +++ b/tests/fetch/cached_xhr.cpp @@ -0,0 +1,64 @@ +#include +#include +#include +#include +#include + +int result = 0; + +// Fetch file without XHRing. +void fetchFromIndexedDB() +{ + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.onsuccess = [](emscripten_fetch_t *fetch) { + assert(fetch->numBytes == fetch->totalBytes); + assert(fetch->data != 0); + printf("Finished downloading %llu bytes\n", fetch->numBytes); + emscripten_fetch_close(fetch); + +#ifdef REPORT_RESULT + result = 1; + REPORT_RESULT(); +#endif + + }; + attr.onprogress = [](emscripten_fetch_t *fetch) { + if (fetch->totalBytes > 0) { + printf("Downloading.. %.2f%% complete.\n", (fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes); + } else { + printf("Downloading.. %lld bytes complete.\n", fetch->dataOffset + fetch->numBytes); + } + }; + attr.attributes = EMSCRIPTEN_FETCH_APPEND | EMSCRIPTEN_FETCH_NO_DOWNLOAD; + emscripten_fetch_t *fetch = emscripten_fetch(&attr, "gears.png"); +} + +// XHR and store to cache. +int main() +{ + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.onsuccess = [](emscripten_fetch_t *fetch) { + assert(fetch->numBytes == fetch->totalBytes); + assert(fetch->data != 0); + printf("Finished downloading %llu bytes\n", fetch->numBytes); + emscripten_fetch_close(fetch); + + // Test that the file now exists: + fetchFromIndexedDB(); + }; + attr.onprogress = [](emscripten_fetch_t *fetch) { + if (fetch->totalBytes > 0) { + printf("Downloading.. %.2f%% complete.\n", (fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes); + } else { + printf("Downloading.. %lld bytes complete.\n", fetch->dataOffset + fetch->numBytes); + } + }; + attr.attributes = EMSCRIPTEN_FETCH_REPLACE | EMSCRIPTEN_FETCH_LOAD_TO_MEMORY | EMSCRIPTEN_FETCH_PERSIST_FILE; + emscripten_fetch_t *fetch = emscripten_fetch(&attr, "gears.png"); + assert(fetch != 0); + memset(&attr, 0, sizeof(attr)); // emscripten_fetch() must be able to operate without referencing to this structure after the call. +} diff --git a/tests/fetch/example_async_xhr_to_memory.cpp b/tests/fetch/example_async_xhr_to_memory.cpp new file mode 100644 index 0000000000000..3037ee9f0da17 --- /dev/null +++ b/tests/fetch/example_async_xhr_to_memory.cpp @@ -0,0 +1,27 @@ +#include +#include +#include + +void downloadSucceeded(emscripten_fetch_t *fetch) +{ + printf("Finished downloading %llu bytes from URL %s.\n", fetch->numBytes, fetch->url); + // The data is now available at fetch->data[0] through fetch->data[fetch->numBytes-1]; + emscripten_fetch_close(fetch); // Free data associated with the fetch. +} + +void downloadFailed(emscripten_fetch_t *fetch) +{ + printf("Downloading %s failed, HTTP failure status code: %d.\n", fetch->url, fetch->status); + emscripten_fetch_close(fetch); // Also free data on failure. +} + +int main() +{ + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.attributes = EMSCRIPTEN_FETCH_LOAD_TO_MEMORY; + attr.onsuccess = downloadSucceeded; + attr.onerror = downloadFailed; + emscripten_fetch(&attr, "myfile.dat"); +} diff --git a/tests/fetch/example_async_xhr_to_memory_via_indexeddb.cpp b/tests/fetch/example_async_xhr_to_memory_via_indexeddb.cpp new file mode 100644 index 0000000000000..7dfab76e079a4 --- /dev/null +++ b/tests/fetch/example_async_xhr_to_memory_via_indexeddb.cpp @@ -0,0 +1,27 @@ +#include +#include +#include + +void downloadSucceeded(emscripten_fetch_t *fetch) +{ + printf("Finished downloading %llu bytes from URL %s.\n", fetch->numBytes, fetch->url); + // The data is now available at fetch->data[0] through fetch->data[fetch->numBytes-1]; + emscripten_fetch_close(fetch); // Free data associated with the fetch. +} + +void downloadFailed(emscripten_fetch_t *fetch) +{ + printf("Downloading %s failed, HTTP failure status code: %d.\n", fetch->url, fetch->status); + emscripten_fetch_close(fetch); // Also free data on failure. +} + +int main() +{ + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.attributes = EMSCRIPTEN_FETCH_LOAD_TO_MEMORY | EMSCRIPTEN_FETCH_PERSIST_FILE; + attr.onsuccess = downloadSucceeded; + attr.onerror = downloadFailed; + emscripten_fetch(&attr, "myfile.dat"); +} diff --git a/tests/fetch/example_idb_delete.cpp b/tests/fetch/example_idb_delete.cpp new file mode 100644 index 0000000000000..cd460ce6145bc --- /dev/null +++ b/tests/fetch/example_idb_delete.cpp @@ -0,0 +1,23 @@ +#include +#include +#include + +void success(emscripten_fetch_t *fetch) +{ + printf("Deleting file from IDB succeeded.\n"); + emscripten_fetch_close(fetch); +} + +void failure(emscripten_fetch_t *fetch) +{ + printf("Deleting file from IDB failed.\n"); + emscripten_fetch_close(fetch); +} + +int main() +{ + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "EM_IDB_DELETE"); + emscripten_fetch(&attr, "filename_to_delete.dat"); +} diff --git a/tests/fetch/example_idb_store.cpp b/tests/fetch/example_idb_store.cpp new file mode 100644 index 0000000000000..94e4946fe44b8 --- /dev/null +++ b/tests/fetch/example_idb_store.cpp @@ -0,0 +1,40 @@ +#include +#include +#include +#include +#include + +void success(emscripten_fetch_t *fetch) +{ + printf("IDB store succeeded.\n"); + emscripten_fetch_close(fetch); +} + +void failure(emscripten_fetch_t *fetch) +{ + printf("IDB store failed.\n"); + emscripten_fetch_close(fetch); +} + +void persistFileToIndexedDB(const char *outputFilename, uint8_t *data, size_t numBytes) +{ + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "EM_IDB_STORE"); + attr.attributes = EMSCRIPTEN_FETCH_REPLACE | EMSCRIPTEN_FETCH_PERSIST_FILE; + attr.requestData = (char *)data; + attr.requestDataSize = numBytes; + attr.onsuccess = success; + attr.onerror = failure; + emscripten_fetch(&attr, outputFilename); +} + +int main() +{ + // Create data + uint8_t *data = (uint8_t*)malloc(10240); + srand(time(NULL)); + for(int i = 0; i < 10240; ++i) data[i] = (uint8_t)rand(); + + persistFileToIndexedDB("outputfile.dat", data, 10240); +} diff --git a/tests/fetch/example_stream_async_xhr.cpp b/tests/fetch/example_stream_async_xhr.cpp new file mode 100644 index 0000000000000..7fcb866cecbe9 --- /dev/null +++ b/tests/fetch/example_stream_async_xhr.cpp @@ -0,0 +1,44 @@ +#include +#include +#include + +void downloadSucceeded(emscripten_fetch_t *fetch) +{ + printf("Finished downloading %llu bytes from URL %s.\n", fetch->numBytes, fetch->url); + // The data is now available at fetch->data[0] through fetch->data[fetch->numBytes-1]; + emscripten_fetch_close(fetch); // Free data associated with the fetch. +} + +void downloadFailed(emscripten_fetch_t *fetch) +{ + printf("Downloading %s failed, HTTP failure status code: %d.\n", fetch->url, fetch->status); + emscripten_fetch_close(fetch); // Also free data on failure. +} + +void downloadProgress(emscripten_fetch_t *fetch) +{ + printf("Downloading %s.. %.2f%s complete. HTTP readyState: %d. HTTP status: %d.\n" + "HTTP statusText: %s. Received chunk [%llu, %llu[\n", + fetch->url, (fetch->totalBytes > 0) ? ((fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes) : (double)(fetch->dataOffset + fetch->numBytes), + (fetch->totalBytes > 0) ? "%" : " bytes", + fetch->readyState, fetch->status, fetch->statusText, + fetch->dataOffset, fetch->dataOffset + fetch->numBytes); + + // Process the partial data stream fetch->data[0] thru fetch->data[fetch->numBytes-1] + // This buffer represents the file at offset fetch->dataOffset. + for(size_t i = 0; i < fetch->numBytes; ++i) + ; // Process fetch->data[i]; +} + +int main() +{ + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.attributes = EMSCRIPTEN_FETCH_STREAM_DATA; + attr.onsuccess = downloadSucceeded; + attr.onprogress = downloadProgress; + attr.onerror = downloadFailed; + attr.timeoutMsecs = 2*60; + emscripten_fetch(&attr, "myfile.dat"); +} diff --git a/tests/fetch/example_sync_xhr_to_memory.cpp b/tests/fetch/example_sync_xhr_to_memory.cpp new file mode 100644 index 0000000000000..36f9aee47def9 --- /dev/null +++ b/tests/fetch/example_sync_xhr_to_memory.cpp @@ -0,0 +1,20 @@ +#include +#include +#include +#include + +int main() +{ + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.attributes = EMSCRIPTEN_FETCH_LOAD_TO_MEMORY | EMSCRIPTEN_FETCH_SYNCHRONOUS; + emscripten_fetch_t *fetch = emscripten_fetch(&attr, "file.dat"); // Blocks here until the operation is complete. + if (fetch->status == 200) { + printf("Finished downloading %llu bytes from URL %s.\n", fetch->numBytes, fetch->url); + // The data is now available at fetch->data[0] through fetch->data[fetch->numBytes-1]; + } else { + printf("Downloading %s failed, HTTP failure status code: %d.\n", fetch->url, fetch->status); + } + emscripten_fetch_close(fetch); +} diff --git a/tests/fetch/example_synchronous_fetch.cpp b/tests/fetch/example_synchronous_fetch.cpp new file mode 100644 index 0000000000000..bed4a8f6c29a9 --- /dev/null +++ b/tests/fetch/example_synchronous_fetch.cpp @@ -0,0 +1,21 @@ +#include +#include +#include +#include + +int main() +{ + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.attributes = EMSCRIPTEN_FETCH_LOAD_TO_MEMORY | EMSCRIPTEN_FETCH_SYNCHRONOUS; + emscripten_fetch_t *fetch = emscripten_fetch(&attr, "gears.png"); + + if (result == 0) { + result = 2; + printf("emscripten_fetch() failed to run synchronously!\n"); + } +#ifdef REPORT_RESULT + REPORT_RESULT(); +#endif +} diff --git a/tests/fetch/example_waitable_xhr_to_memory.cpp b/tests/fetch/example_waitable_xhr_to_memory.cpp new file mode 100644 index 0000000000000..8326fafda60ae --- /dev/null +++ b/tests/fetch/example_waitable_xhr_to_memory.cpp @@ -0,0 +1,29 @@ +#include +#include +#include +#include + +int main() +{ + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.attributes = EMSCRIPTEN_FETCH_LOAD_TO_MEMORY | EMSCRIPTEN_FETCH_WAITABLE; + emscripten_fetch_t *fetch = emscripten_fetch(&attr, "file.dat"); // Starts as asynchronous. + + EMSCRIPTEN_RESULT ret = EMSCRIPTEN_RESULT_TIMED_OUT; + while(ret == EMSCRIPTEN_RESULT_TIMED_OUT) + { + /* possibly do some other work; */ + ret = emscripten_fetch_wait(fetch, 0/*milliseconds to wait. 0 to just poll, INFINITY=wait until completion*/); + } + // The operation has finished, safe to examine the fields of the 'fetch' pointer now. + + if (fetch->status == 200) { + printf("Finished downloading %llu bytes from URL %s.\n", fetch->numBytes, fetch->url); + // The data is now available at fetch->data[0] through fetch->data[fetch->numBytes-1]; + } else { + printf("Downloading %s failed, HTTP failure status code: %d.\n", fetch->url, fetch->status); + } + emscripten_fetch_close(fetch); +} diff --git a/tests/fetch/example_xhr_progress.cpp b/tests/fetch/example_xhr_progress.cpp new file mode 100644 index 0000000000000..630eeb8ba8136 --- /dev/null +++ b/tests/fetch/example_xhr_progress.cpp @@ -0,0 +1,39 @@ +#include +#include +#include + +void downloadSucceeded(emscripten_fetch_t *fetch) +{ + printf("Finished downloading %llu bytes from URL %s.\n", fetch->numBytes, fetch->url); + // The data is now available at fetch->data[0] through fetch->data[fetch->numBytes-1]; + emscripten_fetch_close(fetch); // Free data associated with the fetch. +} + +void downloadFailed(emscripten_fetch_t *fetch) +{ + printf("Downloading %s failed, HTTP failure status code: %d.\n", fetch->url, fetch->status); + emscripten_fetch_close(fetch); // Also free data on failure. +} + +void downloadProgress(emscripten_fetch_t *fetch) +{ + printf("Downloading %s.. %.2f%s complete. HTTP readyState: %d. HTTP status: %d.\n" + "HTTP statusText: %s. Received chunk [%llu, %llu[\n", + fetch->url, (fetch->totalBytes > 0) ? ((fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes) : (double)(fetch->dataOffset + fetch->numBytes), + (fetch->totalBytes > 0) ? "%" : " bytes", + fetch->readyState, fetch->status, fetch->statusText, + fetch->dataOffset, fetch->dataOffset + fetch->numBytes); +} + +int main() +{ + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.attributes = EMSCRIPTEN_FETCH_LOAD_TO_MEMORY; + attr.onsuccess = downloadSucceeded; + attr.onprogress = downloadProgress; + attr.onerror = downloadFailed; + attr.timeoutMsecs = 2*60; + emscripten_fetch(&attr, "myfile.dat"); +} diff --git a/tests/fetch/idb_delete.cpp b/tests/fetch/idb_delete.cpp new file mode 100644 index 0000000000000..90c68d38c9a6b --- /dev/null +++ b/tests/fetch/idb_delete.cpp @@ -0,0 +1,54 @@ +#include +#include +#include +#include +#include + +int main() +{ + // 1. Populate file to IndexedDB by a GET. + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.attributes = EMSCRIPTEN_FETCH_REPLACE | EMSCRIPTEN_FETCH_SYNCHRONOUS | EMSCRIPTEN_FETCH_PERSIST_FILE; + emscripten_fetch_t *fetch = emscripten_fetch(&attr, "gears.png"); + assert(fetch->status == 200 && "Initial XHR GET of gears.png should have succeeded"); + emscripten_fetch_close(fetch); + + // 2. Make sure it is there. + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.attributes = EMSCRIPTEN_FETCH_NO_DOWNLOAD | EMSCRIPTEN_FETCH_SYNCHRONOUS | EMSCRIPTEN_FETCH_LOAD_TO_MEMORY; + fetch = emscripten_fetch(&attr, "gears.png"); + assert(fetch->status == 200 && "emscripten_fetch from IndexedDB should have found the file"); + assert(fetch->numBytes > 0); + assert(fetch->data != 0); + emscripten_fetch_close(fetch); + + // 3. Delete the file from IndexedDB by invoking an Emscripten-specific request "EM_IDB_DELETE". + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "EM_IDB_DELETE"); + attr.attributes = EMSCRIPTEN_FETCH_SYNCHRONOUS; + fetch = emscripten_fetch(&attr, "gears.png"); + assert(fetch->status == 200 && "Deleting the file from IndexedDB should have succeeded"); + assert(fetch->numBytes == 0); + assert(fetch->data == 0); + emscripten_fetch_close(fetch); + + // 4. Now try to get the file again from IndexedDB, and ensure it is no longer available. + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.attributes = EMSCRIPTEN_FETCH_NO_DOWNLOAD | EMSCRIPTEN_FETCH_SYNCHRONOUS | EMSCRIPTEN_FETCH_LOAD_TO_MEMORY; + fetch = emscripten_fetch(&attr, "gears.png"); + assert(fetch->status == 404 && "Attempting to GET the file from IndexedDB again should have failed after the file has been deleted"); + assert(fetch->numBytes == 0); + assert(fetch->data == 0); + emscripten_fetch_close(fetch); + + printf("Test succeeded!\n"); + +#ifdef REPORT_RESULT + int result = 0; + REPORT_RESULT(); +#endif +} diff --git a/tests/fetch/idb_store.cpp b/tests/fetch/idb_store.cpp new file mode 100644 index 0000000000000..8d92de8668ebb --- /dev/null +++ b/tests/fetch/idb_store.cpp @@ -0,0 +1,67 @@ +#include +#include +#include +#include +#include +#include +#include + +#define TEST_SIZE 512000 + +int main() +{ + // 1. Populate an IndexedDB file entry with custom data bytes in memory. + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "EM_IDB_STORE"); + attr.attributes = EMSCRIPTEN_FETCH_REPLACE | EMSCRIPTEN_FETCH_SYNCHRONOUS | EMSCRIPTEN_FETCH_PERSIST_FILE; + uint8_t *data = (uint8_t*)malloc(TEST_SIZE); + srand(time(NULL)); + for(int i = 0; i < TEST_SIZE; ++i) + data[i] = (uint8_t)rand() | 0x40; + attr.requestData = (char *)data; + attr.requestDataSize = TEST_SIZE; + emscripten_fetch_t *fetch = emscripten_fetch(&attr, "myfile.dat"); + assert(fetch->status == 200 && "Initial IndexedDB store of myfile.dat should have succeeded"); + emscripten_fetch_close(fetch); + + // 2. Make sure it is there. + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.attributes = EMSCRIPTEN_FETCH_NO_DOWNLOAD | EMSCRIPTEN_FETCH_SYNCHRONOUS | EMSCRIPTEN_FETCH_LOAD_TO_MEMORY; + fetch = emscripten_fetch(&attr, "myfile.dat"); + assert(fetch->status == 200 && "emscripten_fetch from IndexedDB should have found the file"); + assert(fetch->numBytes == TEST_SIZE); + assert(fetch->totalBytes == TEST_SIZE); + assert(fetch->data != 0); + assert(fetch->data != (char *)data); // We should really have obtained a new copy of the memory. + assert(!memcmp((char *)data, fetch->data, TEST_SIZE)); + emscripten_fetch_close(fetch); + + // 3. Delete the file from IndexedDB by invoking an Emscripten-specific request "EM_IDB_DELETE". + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "EM_IDB_DELETE"); + attr.attributes = EMSCRIPTEN_FETCH_SYNCHRONOUS; + fetch = emscripten_fetch(&attr, "myfile.dat"); + assert(fetch->status == 200 && "Deleting the file from IndexedDB should have succeeded"); + assert(fetch->numBytes == 0); + assert(fetch->data == 0); + emscripten_fetch_close(fetch); + + // 4. Now try to get the file again from IndexedDB, and ensure it is no longer available. + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.attributes = EMSCRIPTEN_FETCH_NO_DOWNLOAD | EMSCRIPTEN_FETCH_SYNCHRONOUS | EMSCRIPTEN_FETCH_LOAD_TO_MEMORY; + fetch = emscripten_fetch(&attr, "myfile.dat"); + assert(fetch->status == 404 && "Attempting to GET the file from IndexedDB again should have failed after the file has been deleted"); + assert(fetch->numBytes == 0); + assert(fetch->data == 0); + emscripten_fetch_close(fetch); + + printf("Test succeeded!\n"); + +#ifdef REPORT_RESULT + int result = 0; + REPORT_RESULT(); +#endif +} diff --git a/tests/fetch/stream_file.cpp b/tests/fetch/stream_file.cpp new file mode 100644 index 0000000000000..2ae931776fe96 --- /dev/null +++ b/tests/fetch/stream_file.cpp @@ -0,0 +1,47 @@ +#include +#include +#include +#include +#include + +// Compute rudimentary checksum of data +uint32_t checksum = 0; + +int result = 0; + +int main() +{ + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.onsuccess = [](emscripten_fetch_t *fetch) { + assert(fetch->data == 0); // The data was streamed via onprogress, no bytes available here. + assert(fetch->numBytes == 0); + assert(fetch->totalBytes == 134217728); + printf("Finished downloading %llu bytes\n", fetch->totalBytes); + printf("Data checksum: %08X\n", checksum); + assert(checksum == 0xA7F8E858U); + emscripten_fetch_close(fetch); + +#ifdef REPORT_RESULT + result = 1; + REPORT_RESULT(); +#endif + }; + attr.onprogress = [](emscripten_fetch_t *fetch) { + assert(fetch->data != 0); + assert(fetch->numBytes > 0); + assert(fetch->dataOffset + fetch->numBytes <= fetch->totalBytes); + assert(fetch->totalBytes <= 134217728); + printf("Downloading.. %.2f%s complete. Received chunk [%llu, %llu[\n", + (fetch->totalBytes > 0) ? ((fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes) : (double)(fetch->dataOffset + fetch->numBytes), + (fetch->totalBytes > 0) ? "%" : " bytes", + fetch->dataOffset, + fetch->dataOffset + fetch->numBytes); + + for(int i = 0; i < fetch->numBytes; ++i) + checksum = ((checksum << 8) | (checksum >> 24)) * fetch->data[i] + fetch->data[i]; + }; + attr.attributes = EMSCRIPTEN_FETCH_LOAD_TO_MEMORY | EMSCRIPTEN_FETCH_APPEND | EMSCRIPTEN_FETCH_STREAM_DATA; + emscripten_fetch_t *fetch = emscripten_fetch(&attr, "largefile.txt"); +} diff --git a/tests/fetch/sync_fetch_in_main_thread.cpp b/tests/fetch/sync_fetch_in_main_thread.cpp new file mode 100644 index 0000000000000..33689306582d9 --- /dev/null +++ b/tests/fetch/sync_fetch_in_main_thread.cpp @@ -0,0 +1,36 @@ +#include +#include +#include +#include +#include + +int result = 0; + +int main() +{ + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + attr.attributes = EMSCRIPTEN_FETCH_REPLACE | EMSCRIPTEN_FETCH_LOAD_TO_MEMORY | EMSCRIPTEN_FETCH_WAITABLE; + emscripten_fetch_t *fetch = emscripten_fetch(&attr, "gears.png"); + assert(fetch != 0); + memset(&attr, 0, sizeof(attr)); // emscripten_fetch() must be able to operate without referencing to this structure after the call. + printf("Main thread waiting for fetch to finish...\n"); + emscripten_fetch_wait(fetch, INFINITY); + printf("Main thread waiting for fetch to finish done...\n"); + assert(fetch->data != 0); + assert(fetch->numBytes > 0); + assert(fetch->totalBytes == fetch->numBytes); + assert(fetch->readyState == 4/*DONE*/); + assert(fetch->status == 200); + + uint8_t checksum = 0; + for(int i = 0; i < fetch->numBytes; ++i) + checksum ^= fetch->data[i]; + printf("Data checksum: %02X\n", checksum); + assert(checksum == 0x08); + emscripten_fetch_close(fetch); + +#ifdef REPORT_RESULT + REPORT_RESULT(); +#endif +} diff --git a/tests/fetch/sync_xhr.cpp b/tests/fetch/sync_xhr.cpp new file mode 100644 index 0000000000000..81f436231a5a6 --- /dev/null +++ b/tests/fetch/sync_xhr.cpp @@ -0,0 +1,53 @@ +#include +#include +#include +#include +#include + +int result = 0; + +int main() +{ + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.attributes = EMSCRIPTEN_FETCH_REPLACE | EMSCRIPTEN_FETCH_LOAD_TO_MEMORY | EMSCRIPTEN_FETCH_SYNCHRONOUS; + + attr.onsuccess = [](emscripten_fetch_t *fetch) { + printf("Finished downloading %llu bytes\n", fetch->numBytes); + // Compute rudimentary checksum of data + uint8_t checksum = 0; + for(int i = 0; i < fetch->numBytes; ++i) + checksum ^= fetch->data[i]; + printf("Data checksum: %02X\n", checksum); + assert(checksum == 0x08); + emscripten_fetch_close(fetch); + + if (result == 0) result = 1; +#ifdef REPORT_RESULT + REPORT_RESULT(); +#endif + }; + + attr.onprogress = [](emscripten_fetch_t *fetch) { + if (fetch->totalBytes > 0) { + printf("Downloading.. %.2f%% complete.\n", (fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes); + } else { + printf("Downloading.. %lld bytes complete.\n", fetch->dataOffset + fetch->numBytes); + } + }; + + attr.onerror = [](emscripten_fetch_t *fetch) { + printf("Download failed!\n"); + assert(false && "Shouldn't fail!"); + }; + + emscripten_fetch_t *fetch = emscripten_fetch(&attr, "gears.png"); + if (result == 0) { + result = 2; + printf("emscripten_fetch() failed to run synchronously!\n"); + } +#ifdef REPORT_RESULT + REPORT_RESULT(); +#endif +} diff --git a/tests/fetch/to_indexeddb.cpp b/tests/fetch/to_indexeddb.cpp new file mode 100644 index 0000000000000..14155ee9ae35f --- /dev/null +++ b/tests/fetch/to_indexeddb.cpp @@ -0,0 +1,50 @@ +#include +#include +#include +#include +#include + +int result = 0; + +int main() +{ + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.userData = (void*)0x12345678; + attr.attributes = EMSCRIPTEN_FETCH_REPLACE | EMSCRIPTEN_FETCH_PERSIST_FILE; + + attr.onsuccess = [](emscripten_fetch_t *fetch) { + assert(fetch); + printf("Finished downloading %llu bytes\n", fetch->totalBytes); + assert(fetch->url); + assert(!strcmp(fetch->url, "gears.png")); + assert(fetch->id != 0); + assert((uintptr_t)fetch->userData == 0x12345678); + assert(fetch->totalBytes == 6407); + emscripten_fetch_close(fetch); + +#ifdef REPORT_RESULT + result = 1; + REPORT_RESULT(); +#endif + }; + + attr.onprogress = [](emscripten_fetch_t *fetch) { + assert(fetch); + if (fetch->status != 200) return; + if (fetch->totalBytes > 0) { + printf("Downloading.. %.2f%% complete.\n", (fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes); + } else { + printf("Downloading.. %lld bytes complete.\n", fetch->dataOffset + fetch->numBytes); + } + }; + + attr.onerror = [](emscripten_fetch_t *fetch) { + printf("Download failed!\n"); + }; + + emscripten_fetch_t *fetch = emscripten_fetch(&attr, "gears.png"); + assert(fetch != 0); + memset(&attr, 0, sizeof(attr)); // emscripten_fetch() must be able to operate without referencing to this structure after the call. +} diff --git a/tests/fetch/to_memory.cpp b/tests/fetch/to_memory.cpp new file mode 100644 index 0000000000000..53444883addce --- /dev/null +++ b/tests/fetch/to_memory.cpp @@ -0,0 +1,92 @@ +#include +#include +#include +#include +#include + +int result = 0; + +// This test is run in two modes: if FILE_DOES_NOT_EXIST defined, +// then testing an XHR of a missing file. +// #define FILE_DOES_NOT_EXIST + +int main() +{ + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.userData = (void*)0x12345678; + attr.attributes = EMSCRIPTEN_FETCH_REPLACE | EMSCRIPTEN_FETCH_LOAD_TO_MEMORY; + + attr.onsuccess = [](emscripten_fetch_t *fetch) { +#if FILE_DOES_NOT_EXIST + assert(false && "onsuccess handler called, but the file shouldn't exist"); // Shouldn't reach here if the file doesn't exist +#endif + assert(fetch); + printf("Finished downloading %llu bytes\n", fetch->numBytes); + assert(fetch->url); + assert(!strcmp(fetch->url, "gears.png")); + assert(fetch->id != 0); + assert((uintptr_t)fetch->userData == 0x12345678); + assert(fetch->totalBytes == 6407); + assert(fetch->numBytes == fetch->totalBytes); + assert(fetch->data != 0); + // Compute rudimentary checksum of data + uint8_t checksum = 0; + for(int i = 0; i < fetch->numBytes; ++i) + checksum ^= fetch->data[i]; + printf("Data checksum: %02X\n", checksum); + assert(checksum == 0x08); + emscripten_fetch_close(fetch); + +#ifdef REPORT_RESULT +#ifndef FILE_DOES_NOT_EXIST + result = 1; +#endif + REPORT_RESULT(); +#endif + }; + + attr.onprogress = [](emscripten_fetch_t *fetch) { + assert(fetch); + if (fetch->status != 200) return; + printf("onprogress: dataOffset: %llu, numBytes: %llu, totalBytes: %llu\n", fetch->dataOffset, fetch->numBytes, fetch->totalBytes); + if (fetch->totalBytes > 0) { + printf("Downloading.. %.2f%% complete.\n", (fetch->dataOffset + fetch->numBytes) * 100.0 / fetch->totalBytes); + } else { + printf("Downloading.. %lld bytes complete.\n", fetch->dataOffset + fetch->numBytes); + } +#ifdef FILE_DOES_NOT_EXIST + assert(false && "onprogress handler called, but the file should not exist"); // We should not receive progress reports if the file doesn't exist. +#endif + // We must receive a call to the onprogress handler with 100% completion. + if (fetch->dataOffset + fetch->numBytes == fetch->totalBytes) ++result; + assert(fetch->dataOffset + fetch->numBytes <= fetch->totalBytes); + assert(fetch->url); + assert(!strcmp(fetch->url, "gears.png")); + assert(fetch->id != 0); + assert((uintptr_t)fetch->userData == 0x12345678); + }; + + attr.onerror = [](emscripten_fetch_t *fetch) { + printf("Download failed!\n"); +#ifndef FILE_DOES_NOT_EXIST + assert(false && "onerror handler called, but the transfer should have succeeded!"); // The file exists, shouldn't reach here. +#endif + assert(fetch); + assert(fetch->id != 0); + assert(!strcmp(fetch->url, "gears.png")); + assert((uintptr_t)fetch->userData == 0x12345678); + +#ifdef REPORT_RESULT +#ifdef FILE_DOES_NOT_EXIST + result = 1; +#endif + REPORT_RESULT(); +#endif + }; + + emscripten_fetch_t *fetch = emscripten_fetch(&attr, "gears.png"); + assert(fetch != 0); + memset(&attr, 0, sizeof(attr)); // emscripten_fetch() must be able to operate without referencing to this structure after the call. +} diff --git a/tests/test_browser.py b/tests/test_browser.py index 11e42c0b07110..3e6da8e47a37f 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -3292,3 +3292,80 @@ def test_webgl_offscreen_canvas_in_mainthread_after_pthread(self): # In this build mode, the -s TOTAL_MEMORY=xxx option will be ignored. def test_preallocated_heap(self): self.btest('test_preallocated_heap.cpp', expected='1', args=['-s', 'TOTAL_MEMORY='+str(16*1024*1024), '-s', 'ABORTING_MALLOC=0', '--shell-file', path_from_root('tests', 'test_preallocated_heap_shell.html')]) + + # Tests emscripten_fetch() usage to XHR data directly to memory without persisting results to IndexedDB. + def test_fetch_to_memory(self): + # Test error reporting in the negative case when the file URL doesn't exist. (http 404) + self.btest('fetch/to_memory.cpp', expected='1', args=['--std=c++11', '-s', 'FETCH_DEBUG=1', '-s', 'FETCH=1', '-DFILE_DOES_NOT_EXIST']) + + # Test the positive case when the file URL exists. (http 200) + shutil.copyfile(path_from_root('tests', 'gears.png'), os.path.join(self.get_dir(), 'gears.png')) + self.btest('fetch/to_memory.cpp', expected='1', args=['--std=c++11', '-s', 'FETCH_DEBUG=1', '-s', 'FETCH=1']) + + # Tests emscripten_fetch() usage to persist an XHR into IndexedDB and subsequently load up from there. + def test_fetch_cached_xhr(self): + shutil.copyfile(path_from_root('tests', 'gears.png'), os.path.join(self.get_dir(), 'gears.png')) + self.btest('fetch/cached_xhr.cpp', expected='1', args=['--std=c++11', '-s', 'FETCH_DEBUG=1', '-s', 'FETCH=1']) + + # Test emscripten_fetch() usage to stream a XHR in to memory without storing the full file in memory + def test_fetch_stream_file(self): + # Strategy: create a large 128MB file, and compile with a small 16MB Emscripten heap, so that the tested file + # won't fully fit in the heap. This verifies that streaming works properly. + f = open('largefile.txt', 'w') + s = '12345678' + for i in xrange(14): + s = s[::-1] + s # length of str will be 2^17=128KB + for i in xrange(1024): + f.write(s) + f.close() + self.btest('fetch/stream_file.cpp', expected='1', args=['--std=c++11', '-s', 'FETCH_DEBUG=1', '-s', 'FETCH=1', '-s', 'TOTAL_MEMORY=536870912']) + + # Tests emscripten_fetch() usage in synchronous mode. + def test_fetch_sync_xhr(self): + shutil.copyfile(path_from_root('tests', 'gears.png'), os.path.join(self.get_dir(), 'gears.png')) + self.btest('fetch/sync_xhr.cpp', expected='1', args=['--std=c++11', '-s', 'FETCH_DEBUG=1', '-s', 'FETCH=1', '--proxy-to-worker', '-s', 'USE_PTHREADS=1']) + + def test_fetch_idb_store(self): + self.btest('fetch/idb_store.cpp', expected='0', args=['-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1', '-s', 'FETCH=1', '--proxy-to-worker']) + + def test_fetch_idb_delete(self): + shutil.copyfile(path_from_root('tests', 'gears.png'), os.path.join(self.get_dir(), 'gears.png')) + self.btest('fetch/idb_delete.cpp', expected='0', args=['-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1', '-s', 'FETCH=1', '--proxy-to-worker']) + + def test_asmfs_hello_file(self): + # Test basic file loading and the valid character set for files. + os.mkdir(os.path.join(self.get_dir(), 'dirrey')) + shutil.copyfile(path_from_root('tests', 'asmfs', 'hello_file.txt'), os.path.join(self.get_dir(), 'dirrey', 'hello file !#$%&\'()+,-.;=@[]^_`{}~ %%.txt')) + self.btest('asmfs/hello_file.cpp', expected='0', args=['-s', 'ASMFS=1', '-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1', '--proxy-to-worker']) + + def test_asmfs_read_file_twice(self): + shutil.copyfile(path_from_root('tests', 'asmfs', 'hello_file.txt'), os.path.join(self.get_dir(), 'hello_file.txt')) + self.btest('asmfs/read_file_twice.cpp', expected='0', args=['-s', 'ASMFS=1', '-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1', '--proxy-to-worker']) + + def test_asmfs_fopen_write(self): + self.btest('asmfs/fopen_write.cpp', expected='0', args=['-s', 'ASMFS=1', '-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1', '--proxy-to-worker']) + + def test_asmfs_mkdir_create_unlink_rmdir(self): + self.btest('cstdio/test_remove.cpp', expected='0', args=['-s', 'ASMFS=1', '-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1', '--proxy-to-worker']) + + def test_asmfs_dirent_test_readdir(self): + self.btest('dirent/test_readdir.c', expected='0', args=['-s', 'ASMFS=1', '-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1', '--proxy-to-worker']) + + def test_asmfs_dirent_test_readdir_empty(self): + self.btest('dirent/test_readdir_empty.c', expected='0', args=['-s', 'ASMFS=1', '-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1', '--proxy-to-worker']) + + def test_asmfs_unistd_close(self): + self.btest('unistd/close.c', expected='0', args=['-s', 'ASMFS=1', '-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1', '--proxy-to-worker']) + + def test_asmfs_unistd_access(self): + self.btest('unistd/access.c', expected='0', args=['-s', 'ASMFS=1', '-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1', '--proxy-to-worker']) + + def test_asmfs_unistd_unlink(self): + # TODO: Once symlinks are supported, remove -DNO_SYMLINK=1 + self.btest('unistd/unlink.c', expected='0', args=['-s', 'ASMFS=1', '-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1', '--proxy-to-worker', '-DNO_SYMLINK=1']) + + def test_asmfs_test_fcntl_open(self): + self.btest('fcntl-open/src.c', expected='0', args=['-s', 'ASMFS=1', '-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1', '--proxy-to-worker']) + + def test_asmfs_relative_paths(self): + self.btest('asmfs/relative_paths.cpp', expected='0', args=['-s', 'ASMFS=1', '-s', 'USE_PTHREADS=1', '-s', 'FETCH_DEBUG=1', '--proxy-to-worker']) diff --git a/tests/test_core.py b/tests/test_core.py index 9c658f84088d5..94e5711a27958 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -5766,7 +5766,8 @@ def process(filename): ret = ccall('print_int', null, ['number'], [12]); Module.print(typeof ret); ret = ccall('print_float', null, ['number'], [14.56]); Module.print(typeof ret); ret = ccall('print_string', null, ['string'], ["cheez"]); Module.print(typeof ret); - ret = ccall('print_string', null, ['array'], [[97, 114, 114, 45, 97, 121, 0]]); Module.print(typeof ret); + ret = ccall('print_string', null, ['array'], [[97, 114, 114, 45, 97, 121, 0]]); Module.print(typeof ret); // JS array + ret = ccall('print_string', null, ['array'], [new Uint8Array([97, 114, 114, 45, 97, 121, 0])]); Module.print(typeof ret); // typed array ret = ccall('multi', 'number', ['number', 'number', 'number', 'string'], [2, 1.4, 3, 'more']); Module.print([typeof ret, ret].join(',')); var p = ccall('malloc', 'pointer', ['number'], [4]); setValue(p, 650, 'i32'); diff --git a/tests/test_other.py b/tests/test_other.py index ff804942d3a7f..5fba610c6cbb4 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -2669,6 +2669,251 @@ def test_fs_stream_proto(self): out = run_js('a.out.js', engine=engine, stderr=PIPE, full_output=True) self.assertContained('File size: 724', out) + def test_proxyfs(self): + # This test supposes that 3 different programs share the same directory and files. + # The same JS object is not used for each of them + # But 'require' function caches JS objects. + # If we just load same js-file multiple times like following code, + # these programs (m0,m1,m2) share the same JS object. + # + # var m0 = require('./proxyfs_test.js'); + # var m1 = require('./proxyfs_test.js'); + # var m2 = require('./proxyfs_test.js'); + # + # To separate js-objects for each of them, following 'require' use different js-files. + # + # var m0 = require('./proxyfs_test.js'); + # var m1 = require('./proxyfs_test1.js'); + # var m2 = require('./proxyfs_test2.js'); + # + open('proxyfs_test_main.js', 'w').write(r''' +var m0 = require('./proxyfs_test.js'); +var m1 = require('./proxyfs_test1.js'); +var m2 = require('./proxyfs_test2.js'); + +var section; +function print(str){ + process.stdout.write(section+":"+str+":"); +} + +m0.FS.mkdir('/working'); +m0.FS.mount(m0.PROXYFS,{root:'/',fs:m1.FS},'/working'); +m0.FS.mkdir('/working2'); +m0.FS.mount(m0.PROXYFS,{root:'/',fs:m2.FS},'/working2'); + +section = "child m1 reads and writes local file."; +print("m1 read embed"); +m1.ccall('myreade','number',[],[]); +print("m1 write");console.log(""); +m1.ccall('mywrite0','number',['number'],[1]); +print("m1 read"); +m1.ccall('myread0','number',[],[]); + + +section = "child m2 reads and writes local file."; +print("m2 read embed"); +m2.ccall('myreade','number',[],[]); +print("m2 write");console.log(""); +m2.ccall('mywrite0','number',['number'],[2]); +print("m2 read"); +m2.ccall('myread0','number',[],[]); + +section = "child m1 reads local file."; +print("m1 read"); +m1.ccall('myread0','number',[],[]); + +section = "parent m0 reads and writes local and children's file."; +print("m0 read embed"); +m0.ccall('myreade','number',[],[]); +print("m0 read m1"); +m0.ccall('myread1','number',[],[]); +print("m0 read m2"); +m0.ccall('myread2','number',[],[]); + +section = "m0,m1 and m2 verify local files."; +print("m0 write");console.log(""); +m0.ccall('mywrite0','number',['number'],[0]); +print("m0 read"); +m0.ccall('myread0','number',[],[]); +print("m1 read"); +m1.ccall('myread0','number',[],[]); +print("m2 read"); +m2.ccall('myread0','number',[],[]); + +print("m0 read embed"); +m0.ccall('myreade','number',[],[]); +print("m1 read embed"); +m1.ccall('myreade','number',[],[]); +print("m2 read embed"); +m2.ccall('myreade','number',[],[]); + +section = "parent m0 writes and reads children's files."; +print("m0 write m1");console.log(""); +m0.ccall('mywrite1','number',[],[]); +print("m0 read m1"); +m0.ccall('myread1','number',[],[]); +print("m0 write m2");console.log(""); +m0.ccall('mywrite2','number',[],[]); +print("m0 read m2"); +m0.ccall('myread2','number',[],[]); +print("m1 read"); +m1.ccall('myread0','number',[],[]); +print("m2 read"); +m2.ccall('myread0','number',[],[]); +print("m0 read m0"); +m0.ccall('myread0','number',[],[]); +''') + + open('proxyfs_pre.js', 'w').write(r''' +if (typeof Module === 'undefined') Module = {}; +Module["noInitialRun"]=true; +Module["noExitRuntime"]=true; +''') + + open('proxyfs_embed.txt', 'w').write(r'''test +''') + + open('proxyfs_test.c', 'w').write(r''' +#include + +int +mywrite1(){ + FILE* out = fopen("/working/hoge.txt","w"); + fprintf(out,"test1\n"); + fclose(out); + return 0; +} + +int +myread1(){ + FILE* in = fopen("/working/hoge.txt","r"); + char buf[1024]; + int len; + if(in==NULL) + printf("open failed\n"); + + while(! feof(in)){ + if(fgets(buf,sizeof(buf),in)==buf){ + printf("%s",buf); + } + } + fclose(in); + return 0; +} +int +mywrite2(){ + FILE* out = fopen("/working2/hoge.txt","w"); + fprintf(out,"test2\n"); + fclose(out); + return 0; +} + +int +myread2(){ + { + FILE* in = fopen("/working2/hoge.txt","r"); + char buf[1024]; + int len; + if(in==NULL) + printf("open failed\n"); + + while(! feof(in)){ + if(fgets(buf,sizeof(buf),in)==buf){ + printf("%s",buf); + } + } + fclose(in); + } + return 0; +} + +int +mywrite0(int i){ + FILE* out = fopen("hoge.txt","w"); + fprintf(out,"test0_%d\n",i); + fclose(out); + return 0; +} + +int +myread0(){ + { + FILE* in = fopen("hoge.txt","r"); + char buf[1024]; + int len; + if(in==NULL) + printf("open failed\n"); + + while(! feof(in)){ + if(fgets(buf,sizeof(buf),in)==buf){ + printf("%s",buf); + } + } + fclose(in); + } + return 0; +} + +int +myreade(){ + { + FILE* in = fopen("proxyfs_embed.txt","r"); + char buf[1024]; + int len; + if(in==NULL) + printf("open failed\n"); + + while(! feof(in)){ + if(fgets(buf,sizeof(buf),in)==buf){ + printf("%s",buf); + } + } + fclose(in); + } + return 0; +} +''') + + Popen([PYTHON, EMCC, + '-o', 'proxyfs_test.js', 'proxyfs_test.c', + '--embed-file', 'proxyfs_embed.txt', '--pre-js', 'proxyfs_pre.js', + '-s', 'MAIN_MODULE=1']).communicate() + # Following shutil.copyfile just prevent 'require' of node.js from caching js-object. + # See https://nodejs.org/api/modules.html + shutil.copyfile('proxyfs_test.js', 'proxyfs_test1.js') + shutil.copyfile('proxyfs_test.js', 'proxyfs_test2.js') + out = run_js('proxyfs_test_main.js') + section="child m1 reads and writes local file." + self.assertContained(section+":m1 read embed:test", out) + self.assertContained(section+":m1 write:", out) + self.assertContained(section+":m1 read:test0_1", out) + section="child m2 reads and writes local file." + self.assertContained(section+":m2 read embed:test", out) + self.assertContained(section+":m2 write:", out) + self.assertContained(section+":m2 read:test0_2", out) + section="child m1 reads local file." + self.assertContained(section+":m1 read:test0_1", out) + section="parent m0 reads and writes local and children's file." + self.assertContained(section+":m0 read embed:test", out) + self.assertContained(section+":m0 read m1:test0_1", out) + self.assertContained(section+":m0 read m2:test0_2", out) + section="m0,m1 and m2 verify local files." + self.assertContained(section+":m0 write:", out) + self.assertContained(section+":m0 read:test0_0", out) + self.assertContained(section+":m1 read:test0_1", out) + self.assertContained(section+":m2 read:test0_2", out) + self.assertContained(section+":m0 read embed:test", out) + self.assertContained(section+":m1 read embed:test", out) + self.assertContained(section+":m2 read embed:test", out) + section="parent m0 writes and reads children's files." + self.assertContained(section+":m0 write m1:", out) + self.assertContained(section+":m0 read m1:test1", out) + self.assertContained(section+":m0 write m2:", out) + self.assertContained(section+":m0 read m2:test2", out) + self.assertContained(section+":m1 read:test1", out) + self.assertContained(section+":m2 read:test2", out) + self.assertContained(section+":m0 read m0:test0_0", out) + def check_simd(self, expected_simds, expected_out): if SPIDERMONKEY_ENGINE in JS_ENGINES: out = run_js('a.out.js', engine=SPIDERMONKEY_ENGINE, stderr=PIPE, full_output=True) diff --git a/tests/unistd/access.c b/tests/unistd/access.c index 92b52e5ef1c78..d90d7938b5482 100644 --- a/tests/unistd/access.c +++ b/tests/unistd/access.c @@ -2,8 +2,18 @@ #include #include #include +#include +#include int main() { +#ifdef __EMSCRIPTEN_ASMFS__ + mkdir("working", 0777); + chdir("working"); + close(open("forbidden", O_WRONLY|O_CREAT|O_TRUNC|O_EXCL, 0000)); + close(open("readable", O_WRONLY|O_CREAT|O_TRUNC|O_EXCL, 0444)); + close(open("writeable", O_WRONLY|O_CREAT|O_TRUNC|O_EXCL, 0222)); + close(open("allaccess", O_WRONLY|O_CREAT|O_TRUNC|O_EXCL, 0777)); +#else EM_ASM( FS.mkdir('working'); #if NODEFS @@ -15,7 +25,7 @@ int main() { FS.writeFile('writeable', ''); FS.chmod('writeable', 0222); FS.writeFile('allaccess', ''); FS.chmod('allaccess', 0777); ); - +#endif char* files[] = {"readable", "writeable", "allaccess", "forbidden", "nonexistent"}; for (int i = 0; i < sizeof files / sizeof files[0]; i++) { @@ -34,6 +44,7 @@ int main() { printf("\n"); } +#ifndef __EMSCRIPTEN_ASMFS__ // Restore full permissions on all created files so that python test runner rmtree // won't have problems on deleting the files. On Windows, calling shutil.rmtree() // will fail if any of the files are read-only. @@ -43,5 +54,12 @@ int main() { FS.chmod('writeable', 0777); FS.chmod('allaccess', 0777); ); +#endif + +#ifdef REPORT_RESULT + int result = 0; + REPORT_RESULT(); +#endif + return 0; } diff --git a/tests/unistd/close.c b/tests/unistd/close.c index 7110d18a7f04f..212c63374ec2e 100644 --- a/tests/unistd/close.c +++ b/tests/unistd/close.c @@ -2,25 +2,42 @@ #include #include #include +#include int main() { int f = open("/", O_RDONLY); - printf("fsync(opened): %d\n", fsync(f)); + int ret = fsync(f); + printf("fsync(opened): %d\n", ret); printf("errno: %d\n", errno); + assert(ret == 0); + assert(errno == 0); errno = 0; - printf("close(opened): %d\n", close(f)); + ret = close(f); + printf("close(opened): %d\n", ret); printf("errno: %d\n", errno); + assert(ret == 0); + assert(errno == 0); errno = 0; - printf("fsync(closed): %d\n", fsync(f)); + ret = fsync(f); + printf("fsync(closed): %d\n", ret); printf("errno: %d\n", errno); + assert(ret == -1); + assert(errno == EBADF); errno = 0; - printf("close(closed): %d\n", close(f)); + ret = close(f); + printf("close(closed): %d\n", ret); printf("errno: %d\n", errno); + assert(ret == -1); + assert(errno == EBADF); errno = 0; +#ifdef REPORT_RESULT + int result = 0; + REPORT_RESULT(); +#endif return 0; } diff --git a/tests/unistd/unlink.c b/tests/unistd/unlink.c index 68051c1370649..1d2803b1442fc 100644 --- a/tests/unistd/unlink.c +++ b/tests/unistd/unlink.c @@ -24,11 +24,16 @@ static void create_file(const char *path, const char *buffer, int mode) { void setup() { mkdir("working", 0777); #ifdef __EMSCRIPTEN__ + +#ifdef __EMSCRIPTEN_ASMFS__ + mkdir("working", 0777); +#else EM_ASM( #if NODEFS FS.mount(NODEFS, { root: '.' }, 'working'); #endif ); +#endif #endif chdir("working"); create_file("file", "test", 0777); @@ -163,5 +168,10 @@ int main() { signal(SIGABRT, cleanup); setup(); test(); + +#ifdef REPORT_RESULT + int result = 0; + REPORT_RESULT(); +#endif return EXIT_SUCCESS; } diff --git a/tools/shared.py b/tools/shared.py index 9dd73e4fb5a5d..801e2c6c1b22b 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -2322,6 +2322,10 @@ def safe_copy(src, dst): if dst == '/dev/null': return shutil.copyfile(src, dst) +def clang_preprocess(filename): + # TODO: REMOVE HACK AND PASS PREPROCESSOR FLAGS TO CLANG. + return subprocess.check_output([CLANG_CC, '-DFETCH_DEBUG=1', '-E', '-P', '-C', '-x', 'c', filename]) + def read_and_preprocess(filename): f = open(filename, 'r').read() pos = 0 @@ -2334,4 +2338,36 @@ def read_and_preprocess(filename): f = f[:m.start(0)] + included_file + f[m.end(0):] +# Generates a suitable fetch-worker.js script from the given input source JS file (which is an asm.js build output), +# and writes it out to location output_file. fetch-worker.js is the root entry point for a dedicated filesystem web +# worker in -s ASMFS=1 mode. +def make_fetch_worker(source_file, output_file): + src = open(source_file, 'r').read() + funcs_to_import = ['alignMemoryPage', 'getTotalMemory', 'stringToUTF8', 'intArrayFromString', 'lengthBytesUTF8', 'stringToUTF8Array', '_emscripten_is_main_runtime_thread', '_emscripten_futex_wait'] + asm_funcs_to_import = ['_malloc', '_free', '_sbrk', '_pthread_mutex_lock', '_pthread_mutex_unlock'] + function_prologue = '''this.onerror = function(e) { + console.error(e); +} + +''' + asm_start = src.find('// EMSCRIPTEN_START_ASM') + for func in funcs_to_import + asm_funcs_to_import: + loc = src.find('function ' + func + '(', asm_start if func in asm_funcs_to_import else 0) + if loc == -1: + logging.fatal('failed to find function ' + func + '!') + sys.exit(1) + end_loc = src.find('{', loc) + 1 + nesting_level = 1 + while nesting_level > 0: + if src[end_loc] == '{': nesting_level += 1 + if src[end_loc] == '}': nesting_level -= 1 + end_loc += 1 + + func_code = src[loc:end_loc] + function_prologue = function_prologue + '\n' + func_code + + fetch_worker_src = function_prologue + '\n' + clang_preprocess(path_from_root('src', 'fetch-worker.js')) + open(output_file, 'w').write(fetch_worker_src) + + import js_optimizer