Issue python#23152: Implement _Py_fstat() to support files larger than 2 GB on Windows.

fstat() may fail with EOVERFLOW on files larger than 2 GB because the file size type is an signed 32-bit integer.

Author: zooba

Include/fileutils.h

@@ -21,11 +21,40 @@ PyAPI_FUNC(int) _Py_wstat(
     struct stat *buf);
 #endif
 
+#if defined(HAVE_FSTAT) || defined(MS_WINDOWS)
+
+#ifdef MS_WINDOWS
+struct _Py_stat_struct {
+    unsigned long st_dev;
+    __int64 st_ino;
+    unsigned short st_mode;
+    int st_nlink;
+    int st_uid;
+    int st_gid;
+    unsigned long st_rdev;
+    __int64 st_size;
+    time_t st_atime;
+    int st_atime_nsec;
+    time_t st_mtime;
+    int st_mtime_nsec;
+    time_t st_ctime;
+    int st_ctime_nsec;
+    unsigned long st_file_attributes;
+};
+#else
+#  define _Py_stat_struct stat
+#endif
+
+PyAPI_FUNC(int) _Py_fstat(
+    int fd,
+    struct _Py_stat_struct *stat);
+#endif   /* HAVE_FSTAT || MS_WINDOWS */
+
 #ifdef HAVE_STAT
 PyAPI_FUNC(int) _Py_stat(
     PyObject *path,
     struct stat *statbuf);
-#endif
+#endif   /* HAVE_STAT */
 
 #ifndef Py_LIMITED_API
 PyAPI_FUNC(int) _Py_open(

Modules/_io/fileio.c

@@ -180,9 +180,9 @@ fileio_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
 static int
 check_fd(int fd)
 {
-#if defined(HAVE_FSTAT)
-    struct stat buf;
-    if (!_PyVerify_fd(fd) || (fstat(fd, &buf) < 0 && errno == EBADF)) {
+#if defined(HAVE_FSTAT) || defined(MS_WINDOWS)
+    struct _Py_stat_struct buf;
+    if (!_PyVerify_fd(fd) || (_Py_fstat(fd, &buf) < 0 && errno == EBADF)) {
         PyObject *exc;
         char *msg = strerror(EBADF);
         exc = PyObject_CallFunction(PyExc_OSError, "(is)",
@@ -222,8 +222,8 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds)
 #elif !defined(MS_WINDOWS)
     int *atomic_flag_works = NULL;
 #endif
-#ifdef HAVE_FSTAT
-    struct stat fdfstat;
+#if defined(HAVE_FSTAT) || defined(MS_WINDOWS)
+    struct _Py_stat_struct fdfstat;
 #endif
     int async_err = 0;
 
@@ -420,9 +420,11 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds)
     }
 
     self->blksize = DEFAULT_BUFFER_SIZE;
-#ifdef HAVE_FSTAT
-    if (fstat(self->fd, &fdfstat) < 0)
+#if defined(HAVE_FSTAT) || defined(MS_WINDOWS)
+    if (_Py_fstat(self->fd, &fdfstat) < 0) {
+        PyErr_SetFromErrno(PyExc_OSError);
         goto error;
+    }
 #if defined(S_ISDIR) && defined(EISDIR)
     /* On Unix, open will succeed for directories.
        In Python, there should be no file objects referring to
@@ -437,7 +439,7 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds)
     if (fdfstat.st_blksize > 1)
         self->blksize = fdfstat.st_blksize;
 #endif /* HAVE_STRUCT_STAT_ST_BLKSIZE */
-#endif /* HAVE_FSTAT */
+#endif /* HAVE_FSTAT || MS_WINDOWS */
 
 #if defined(MS_WINDOWS) || defined(__CYGWIN__)
     /* don't translate newlines (\r\n <=> \n) */
@@ -603,17 +605,7 @@ fileio_readinto(fileio *self, PyObject *args)
     return PyLong_FromSsize_t(n);
 }
 
-#ifndef HAVE_FSTAT
-
-static PyObject *
-fileio_readall(fileio *self)
-{
-    _Py_IDENTIFIER(readall);
-    return _PyObject_CallMethodId((PyObject*)&PyRawIOBase_Type,
-                                  &PyId_readall, "O", self);
-}
-
-#else
+#if defined(HAVE_FSTAT) || defined(MS_WINDOWS)
 
 static size_t
 new_buffersize(fileio *self, size_t currentsize)
@@ -637,7 +629,7 @@ new_buffersize(fileio *self, size_t currentsize)
 static PyObject *
 fileio_readall(fileio *self)
 {
-    struct stat st;
+    struct _Py_stat_struct st;
     Py_off_t pos, end;
     PyObject *result;
     Py_ssize_t bytes_read = 0;
@@ -655,7 +647,7 @@ fileio_readall(fileio *self)
 #else
     pos = lseek(self->fd, 0L, SEEK_CUR);
 #endif
-    if (fstat(self->fd, &st) == 0)
+    if (_Py_fstat(self->fd, &st) == 0)
         end = st.st_size;
     else
         end = (Py_off_t)-1;
@@ -729,7 +721,17 @@ fileio_readall(fileio *self)
     return result;
 }
 
-#endif /* HAVE_FSTAT */
+#else
+
+static PyObject *
+fileio_readall(fileio *self)
+{
+    _Py_IDENTIFIER(readall);
+    return _PyObject_CallMethodId((PyObject*)&PyRawIOBase_Type,
+                                  &PyId_readall, "O", self);
+}
+
+#endif /* HAVE_FSTAT || MS_WINDOWS */
 
 static PyObject *
 fileio_read(fileio *self, PyObject *args)

Modules/main.c

@@ -752,9 +752,8 @@ Py_Main(int argc, wchar_t **argv)
                 }
             }
             {
-                /* XXX: does this work on Win/Win64? (see posix_fstat) */
-                struct stat sb;
-                if (fstat(fileno(fp), &sb) == 0 &&
+                struct _Py_stat_struct sb;
+                if (_Py_fstat(fileno(fp), &sb) == 0 &&
                     S_ISDIR(sb.st_mode)) {
                     fprintf(stderr, "%ls: '%ls' is a directory, cannot continue\n", argv[0], filename);
                     fclose(fp);

Modules/mmapmodule.c

@@ -459,8 +459,8 @@ mmap_size_method(mmap_object *self,
 
 #ifdef UNIX
     {
-        struct stat buf;
-        if (-1 == fstat(self->fd, &buf)) {
+        struct _Py_stat_struct buf;
+        if (-1 == _Py_fstat(self->fd, &buf)) {
             PyErr_SetFromErrno(PyExc_OSError);
             return NULL;
         }
@@ -1107,7 +1107,7 @@ static PyObject *
 new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict)
 {
 #ifdef HAVE_FSTAT
-    struct stat st;
+    struct _Py_stat_struct st;
 #endif
     mmap_object *m_obj;
     PyObject *map_size_obj = NULL;
@@ -1174,7 +1174,7 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict)
         (void)fcntl(fd, F_FULLFSYNC);
 #endif
 #ifdef HAVE_FSTAT
-    if (fd != -1 && fstat(fd, &st) == 0 && S_ISREG(st.st_mode)) {
+    if (fd != -1 && _Py_fstat(fd, &st) == 0 && S_ISREG(st.st_mode)) {
         if (map_size == 0) {
             if (st.st_size == 0) {
                 PyErr_SetString(PyExc_ValueError,