Allow writing to MEMFS through mmap by implementing msync

Author: janjongboom

src/library.js

@@ -2568,7 +2568,7 @@ LibraryManager.library = {
       }
     }
 
-    _mmap.mappings[ptr] = { malloc: ptr, num: num, allocated: allocated };
+    _mmap.mappings[ptr] = { malloc: ptr, num: num, allocated: allocated, fd: fd, flags: flags };
     return ptr;
   },
 
@@ -2578,6 +2578,13 @@ LibraryManager.library = {
     var info = _mmap.mappings[start];
     if (!info) return 0;
     if (num == info.num) {
+      // At the Linux man page, it says:
+      // "The file may not actually be updated until msync(2) or munmap(2) are called."
+      // I guess that means we need to call msync when doing munmap
+      _msync(start, num); // todo: which flags?
+
+      FS.munmap(FS.getStream(info.fd));
+
       _mmap.mappings[start] = null;
       if (info.allocated) {
         _free(info.malloc);
@@ -2598,7 +2605,14 @@ LibraryManager.library = {
   msync: function(addr, len, flags) {
     // int msync(void *addr, size_t len, int flags);
     // http://pubs.opengroup.org/onlinepubs/009696799/functions/msync.html
-    // Pretend to succeed
+    // TODO: support sync'ing parts of allocations
+    var info = _mmap.mappings[addr];
+    if (!info) return 0;
+    if (len == info.num) {
+      var buffer = new Uint8Array(HEAPU8.buffer, addr, len);
+      return FS.msync(FS.getStream(info.fd), buffer, 0, len, info.flags);
+    }
+
     return 0;
   },
 

src/library_fs.js

@@ -1154,6 +1154,15 @@ mergeInto(LibraryManager.library, {
       }
       return stream.stream_ops.mmap(stream, buffer, offset, length, position, prot, flags);
     },
+    msync: function(stream, buffer, offset, length, mmapFlags) {
+      if (!stream.stream_ops.msync) {
+        return 0;
+      }
+      return stream.stream_ops.msync(stream, buffer, offset, length, mmapFlags);
+    },
+    munmap: function(stream) {
+      return 0;
+    },
     ioctl: function(stream, cmd, arg) {
       if (!stream.stream_ops.ioctl) {
         throw new FS.ErrnoError(ERRNO_CODES.ENOTTY);

src/library_memfs.js

@@ -38,7 +38,8 @@ mergeInto(LibraryManager.library, {
               read: MEMFS.stream_ops.read,
               write: MEMFS.stream_ops.write,
               allocate: MEMFS.stream_ops.allocate,
-              mmap: MEMFS.stream_ops.mmap
+              mmap: MEMFS.stream_ops.mmap,
+              msync: MEMFS.stream_ops.msync
             }
           },
           link: {
@@ -373,6 +374,19 @@ mergeInto(LibraryManager.library, {
         }
         return { ptr: ptr, allocated: allocated };
       },
+      msync: function(stream, buffer, offset, length, mmapFlags) {
+        if (!FS.isFile(stream.node.mode)) {
+          throw new FS.ErrnoError(ERRNO_CODES.ENODEV);
+        }
+        if (mmapFlags & {{{ cDefine('MAP_PRIVATE') }}}) {
+          // MAP_PRIVATE calls need not to be synced back to underlying fs
+          return 0;
+        }
+
+        var bytesWritten = MEMFS.stream_ops.write(stream, buffer, 0, length, offset, false);
+        // should we check if bytesWritten and length are the same?
+        return 0;
+      }
     }
   }
 });

tests/fs/test_mmap.c

@@ -0,0 +1,123 @@
+#include <stdio.h>
+#include <sys/mman.h>
+#include <emscripten.h>
+#include <string.h>
+#include <assert.h>
+#include <stdlib.h>
+#include <fcntl.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <sys/io.h>
+#include <sys/mman.h>
+
+int main() {
+  EM_ASM(
+    FS.mkdir('yolo');
+    FS.writeFile('/yolo/in.txt', 'mmap ftw!');
+  );
+
+  // Use mmap to read in.txt
+  {
+    const char* path = "/yolo/in.txt";
+    int fd = open(path, O_RDONLY);
+    assert(fd != -1);
+
+    int filesize = 9;
+    char* map = (char*)mmap(NULL, filesize, PROT_READ, MAP_PRIVATE, fd, 0);
+    assert(map != MAP_FAILED);
+
+    printf("/yolo/in.txt content=");
+    for (int i = 0; i < filesize; i++) {
+        printf("%c", map[i]);
+    }
+    printf("\n");
+
+    int rc = munmap(map, filesize);
+    assert(rc == 0);
+
+    close(fd);
+  }
+
+  // Use mmap to write out.txt
+  {
+    const char* text = "written mmap";
+    const char* path = "/yolo/out.txt";
+
+    int fd = open(path, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600);
+    assert(fd != -1);
+
+    size_t textsize = strlen(text) + 1; // + \0 null character
+    assert(lseek(fd, textsize - 1, SEEK_SET) != -1);
+
+    // need to write something first to allow us to mmap
+    assert(write(fd, "", 1) != -1);
+
+    char *map = (char*)mmap(0, textsize, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+    assert(map != MAP_FAILED);
+
+    for (size_t i = 0; i < textsize; i++) {
+      map[i] = text[i];
+    }
+
+    assert(msync(map, textsize, MS_SYNC) != -1);
+
+    assert(munmap(map, textsize) != -1);
+
+    close(fd);
+  }
+
+  {
+    FILE* fd = fopen("/yolo/out.txt", "r");
+    if (fd == NULL) {
+      printf("failed to open /yolo/out.txt\n");
+      return 1;
+    }
+    char buffer[13];
+    fread(buffer, 1, 14, fd);
+    printf("/yolo/out.txt content=%s\n", buffer);
+    fclose(fd);
+  }
+
+  // MAP_PRIVATE
+  {
+    const char* text = "written mmap";
+    const char* path = "/yolo/private.txt";
+
+    int fd = open(path, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600);
+    assert(fd != -1);
+
+    size_t textsize = strlen(text) + 1; // + \0 null character
+    assert(lseek(fd, textsize - 1, SEEK_SET) != -1);
+
+    // need to write something first to allow us to mmap
+    assert(write(fd, "", 1) != -1);
+
+    char *map = (char*)mmap(0, textsize, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
+    assert(map != MAP_FAILED);
+
+    for (size_t i = 0; i < textsize; i++) {
+      map[i] = text[i];
+    }
+
+    assert(msync(map, textsize, MS_SYNC) != -1);
+
+    assert(munmap(map, textsize) != -1);
+
+    close(fd);
+  }
+
+  {
+    FILE* fd = fopen("/yolo/private.txt", "r");
+    if (fd == NULL) {
+      printf("failed to open /yolo/private.txt\n");
+      return 1;
+    }
+    char buffer[13];
+    fread(buffer, 1, 14, fd);
+    printf("/yolo/private.txt content=%s\n", buffer);
+    fclose(fd);
+  }
+
+  return 0;
+}

tests/fs/test_mmap.out

@@ -0,0 +1,3 @@
+/yolo/in.txt content=mmap ftw!
+/yolo/out.txt content=written mmap
+/yolo/private.txt content=

tests/test_core.py

@@ -4698,6 +4698,15 @@ def test_fs_append(self):
     src = open(path_from_root('tests', 'fs', 'test_append.c'), 'r').read()
     self.do_run(src, 'success', force_c=True)
 
+  def test_fs_mmap(self):
+    if self.emcc_args is None: return self.skip('requires emcc')
+    orig_compiler_opts = Building.COMPILER_TEST_OPTS[:]
+    for fs in ['MEMFS']:
+      src = path_from_root('tests', 'fs', 'test_mmap.c')
+      out = path_from_root('tests', 'fs', 'test_mmap.out')
+      Building.COMPILER_TEST_OPTS = orig_compiler_opts + ['-D' + fs]
+      self.do_run_from_file(src, out)
+
   def test_unistd_access(self):
     self.clear()
     if not self.is_emscripten_abi(): return self.skip('asmjs-unknown-emscripten needed for inline js')