bpo-20443: _PyConfig_Read() gets the absolute path of run_filename (GH-14053)

Python now gets the absolute path of the script filename specified on
the command line (ex: "python3 script.py"): the __file__ attribute of
the __main__ module, sys.argv[0] and sys.path[0] become an absolute
path, rather than a relative path.

* Add _Py_isabs() and _Py_abspath() functions.
* _PyConfig_Read() now tries to get the absolute path of
  run_filename, but keeps the relative path if _Py_abspath() fails.
* Reimplement os._getfullpathname() using _Py_abspath().
* Use _Py_isabs() in getpath.c.

Author: vstinner

Diff for: Doc/whatsnew/3.9.rst

@@ -75,6 +75,14 @@ New Features
 Other Language Changes
 ======================
 
+* Python now gets the absolute path of the script filename specified on
+  the command line (ex: ``python3 script.py``): the ``__file__`` attribute of
+  the ``__main__`` module, ``sys.argv[0]`` and ``sys.path[0]`` become an
+  absolute path, rather than a relative path. These paths now remain valid
+  after the current directory is changed by :func:`os.chdir`. As a side effect,
+  a traceback also displays the absolute path for ``__main__`` module frames in
+  this case.
+  (Contributed by Victor Stinner in :issue:`20443`.)
 
 
 New Modules

Diff for: Include/fileutils.h

@@ -154,6 +154,12 @@ PyAPI_FUNC(wchar_t*) _Py_wrealpath(
     size_t resolved_path_len);
 #endif
 
+#ifndef MS_WINDOWS
+PyAPI_FUNC(int) _Py_isabs(const wchar_t *path);
+#endif
+
+PyAPI_FUNC(int) _Py_abspath(const wchar_t *path, wchar_t **abspath_p);
+
 PyAPI_FUNC(wchar_t*) _Py_wgetcwd(
     wchar_t *buf,
     /* Number of characters of 'buf' buffer

Diff for: Lib/test/test_cmd_line_script.py

@@ -217,6 +217,18 @@ def test_basic_script(self):
         with support.temp_dir() as script_dir:
             script_name = _make_test_script(script_dir, 'script')
             self._check_script(script_name, script_name, script_name,
+                               script_dir, None,
+                               importlib.machinery.SourceFileLoader,
+                               expected_cwd=script_dir)
+
+    def test_script_abspath(self):
+        # pass the script using the relative path, expect the absolute path
+        # in __file__ and sys.argv[0]
+        with support.temp_cwd() as script_dir:
+            self.assertTrue(os.path.isabs(script_dir), script_dir)
+
+            script_name = _make_test_script(script_dir, 'script')
+            self._check_script(os.path.basename(script_name), script_name, script_name,
                                script_dir, None,
                                importlib.machinery.SourceFileLoader)
 
@@ -542,7 +554,7 @@ def test_non_ascii(self):
 
         # Issue #16218
         source = 'print(ascii(__file__))\n'
-        script_name = _make_test_script(os.curdir, name, source)
+        script_name = _make_test_script(os.getcwd(), name, source)
         self.addCleanup(support.unlink, script_name)
         rc, stdout, stderr = assert_python_ok(script_name)
         self.assertEqual(

Diff for: Lib/test/test_embed.py

@@ -805,9 +805,10 @@ def test_preinit_parse_argv(self):
         preconfig = {
             'allocator': PYMEM_ALLOCATOR_DEBUG,
         }
+        script_abspath = os.path.abspath('script.py')
         config = {
-            'argv': ['script.py'],
-            'run_filename': 'script.py',
+            'argv': [script_abspath],
+            'run_filename': script_abspath,
             'dev_mode': 1,
             'faulthandler': 1,
             'warnoptions': ['default'],

Diff for: Lib/test/test_warnings/__init__.py

@@ -926,27 +926,26 @@ def run(*args):
             return stderr
 
         # tracemalloc disabled
+        filename = os.path.abspath(support.TESTFN)
         stderr = run('-Wd', support.TESTFN)
-        expected = textwrap.dedent('''
-            {fname}:5: ResourceWarning: unclosed file <...>
+        expected = textwrap.dedent(f'''
+            {filename}:5: ResourceWarning: unclosed file <...>
               f = None
             ResourceWarning: Enable tracemalloc to get the object allocation traceback
-        ''')
-        expected = expected.format(fname=support.TESTFN).strip()
+        ''').strip()
         self.assertEqual(stderr, expected)
 
         # tracemalloc enabled
         stderr = run('-Wd', '-X', 'tracemalloc=2', support.TESTFN)
-        expected = textwrap.dedent('''
-            {fname}:5: ResourceWarning: unclosed file <...>
+        expected = textwrap.dedent(f'''
+            {filename}:5: ResourceWarning: unclosed file <...>
               f = None
             Object allocated at (most recent call last):
-              File "{fname}", lineno 7
+              File "{filename}", lineno 7
                 func()
-              File "{fname}", lineno 3
+              File "{filename}", lineno 3
                 f = open(__file__)
-        ''')
-        expected = expected.format(fname=support.TESTFN).strip()
+        ''').strip()
         self.assertEqual(stderr, expected)
 
 

Diff for: Misc/NEWS.d/next/Core and Builtins/2019-06-13-12-55-38.bpo-20443.bQWAxg.rst

@@ -0,0 +1,3 @@
+Python now gets the absolute path of the script filename specified on the
+command line (ex: "python3 script.py"): the __file__ attribute of the __main__
+module and sys.path[0] become an absolute path, rather than a relative path.

Diff for: Modules/getpath.c

@@ -240,7 +240,7 @@ static PyStatus
 joinpath(wchar_t *buffer, const wchar_t *stuff, size_t buflen)
 {
     size_t n, k;
-    if (stuff[0] != SEP) {
+    if (!_Py_isabs(stuff)) {
         n = wcslen(buffer);
         if (n >= buflen) {
             return PATHLEN_ERR();
@@ -283,7 +283,7 @@ safe_wcscpy(wchar_t *dst, const wchar_t *src, size_t n)
 static PyStatus
 copy_absolute(wchar_t *path, const wchar_t *p, size_t pathlen)
 {
-    if (p[0] == SEP) {
+    if (_Py_isabs(p)) {
         if (safe_wcscpy(path, p, pathlen) < 0) {
             return PATHLEN_ERR();
         }
@@ -312,7 +312,7 @@ copy_absolute(wchar_t *path, const wchar_t *p, size_t pathlen)
 static PyStatus
 absolutize(wchar_t *path, size_t path_len)
 {
-    if (path[0] == SEP) {
+    if (_Py_isabs(path)) {
         return _PyStatus_OK();
     }
 
@@ -761,7 +761,7 @@ calculate_program_full_path(const PyConfig *config,
       * absolutize() should help us out below
       */
     else if(0 == _NSGetExecutablePath(execpath, &nsexeclength) &&
-            execpath[0] == SEP)
+            _Py_isabs(execpath))
     {
         size_t len;
         wchar_t *path = Py_DecodeLocale(execpath, &len);
@@ -815,7 +815,7 @@ calculate_program_full_path(const PyConfig *config,
     else {
         program_full_path[0] = '\0';
     }
-    if (program_full_path[0] != SEP && program_full_path[0] != '\0') {
+    if (!_Py_isabs(program_full_path) && program_full_path[0] != '\0') {
         status = absolutize(program_full_path, program_full_path_len);
         if (_PyStatus_EXCEPTION(status)) {
             return status;
@@ -916,7 +916,7 @@ calculate_argv0_path(PyCalculatePath *calculate, const wchar_t *program_full_pat
     const size_t buflen = Py_ARRAY_LENGTH(tmpbuffer);
     int linklen = _Py_wreadlink(program_full_path, tmpbuffer, buflen);
     while (linklen != -1) {
-        if (tmpbuffer[0] == SEP) {
+        if (_Py_isabs(tmpbuffer)) {
             /* tmpbuffer should never be longer than MAXPATHLEN,
                but extra check does not hurt */
             if (safe_wcscpy(calculate->argv0_path, tmpbuffer, argv0_path_len) < 0) {
@@ -1046,7 +1046,7 @@ calculate_module_search_path(const PyConfig *config,
     while (1) {
         wchar_t *delim = wcschr(defpath, DELIM);
 
-        if (defpath[0] != SEP) {
+        if (!_Py_isabs(defpath)) {
             /* Paths are relative to prefix */
             bufsz += prefixsz;
         }
@@ -1088,7 +1088,7 @@ calculate_module_search_path(const PyConfig *config,
     while (1) {
         wchar_t *delim = wcschr(defpath, DELIM);
 
-        if (defpath[0] != SEP) {
+        if (!_Py_isabs(defpath)) {
             wcscat(buf, prefix);
             if (prefixsz >= 2 && prefix[prefixsz - 2] != SEP &&
                 defpath[0] != (delim ? DELIM : L'\0'))

Diff for: Modules/posixmodule.c

@@ -3784,29 +3784,25 @@ static PyObject *
 os__getfullpathname_impl(PyObject *module, path_t *path)
 /*[clinic end generated code: output=bb8679d56845bc9b input=332ed537c29d0a3e]*/
 {
-    wchar_t woutbuf[MAX_PATH], *woutbufp = woutbuf;
-    wchar_t *wtemp;
-    DWORD result;
-    PyObject *v;
+    wchar_t *abspath;
 
-    result = GetFullPathNameW(path->wide,
-                              Py_ARRAY_LENGTH(woutbuf),
-                              woutbuf, &wtemp);
-    if (result > Py_ARRAY_LENGTH(woutbuf)) {
-        woutbufp = PyMem_New(wchar_t, result);
-        if (!woutbufp)
-            return PyErr_NoMemory();
-        result = GetFullPathNameW(path->wide, result, woutbufp, &wtemp);
+    /* _Py_abspath() is implemented with GetFullPathNameW() on Windows */
+    if (_Py_abspath(path->wide, &abspath) < 0) {
+        return win32_error_object("GetFullPathNameW", path->object);
     }
-    if (result) {
-        v = PyUnicode_FromWideChar(woutbufp, wcslen(woutbufp));
-        if (path->narrow)
-            Py_SETREF(v, PyUnicode_EncodeFSDefault(v));
-    } else
-        v = win32_error_object("GetFullPathNameW", path->object);
-    if (woutbufp != woutbuf)
-        PyMem_Free(woutbufp);
-    return v;
+    if (abspath == NULL) {
+        return PyErr_NoMemory();
+    }
+
+    PyObject *str = PyUnicode_FromWideChar(abspath, wcslen(abspath));
+    PyMem_RawFree(abspath);
+    if (str == NULL) {
+        return NULL;
+    }
+    if (path->narrow) {
+        Py_SETREF(str, PyUnicode_EncodeFSDefault(str));
+    }
+    return str;
 }
 
 

Diff for: Python/fileutils.c

@@ -1734,6 +1734,103 @@ _Py_wrealpath(const wchar_t *path,
 }
 #endif
 
+
+#ifndef MS_WINDOWS
+int
+_Py_isabs(const wchar_t *path)
+{
+    return (path[0] == SEP);
+}
+#endif
+
+
+/* Get an absolute path.
+   On error (ex: fail to get the current directory), return -1.
+   On memory allocation failure, set *abspath_p to NULL and return 0.
+   On success, return a newly allocated to *abspath_p to and return 0.
+   The string must be freed by PyMem_RawFree(). */
+int
+_Py_abspath(const wchar_t *path, wchar_t **abspath_p)
+{
+#ifdef MS_WINDOWS
+    wchar_t woutbuf[MAX_PATH], *woutbufp = woutbuf;
+    DWORD result;
+
+    result = GetFullPathNameW(path,
+                              Py_ARRAY_LENGTH(woutbuf), woutbuf,
+                              NULL);
+    if (!result) {
+        return -1;
+    }
+
+    if (result > Py_ARRAY_LENGTH(woutbuf)) {
+        if ((size_t)result <= (size_t)PY_SSIZE_T_MAX / sizeof(wchar_t)) {
+            woutbufp = PyMem_RawMalloc((size_t)result * sizeof(wchar_t));
+        }
+        else {
+            woutbufp = NULL;
+        }
+        if (!woutbufp) {
+            *abspath_p = NULL;
+            return 0;
+        }
+
+        result = GetFullPathNameW(path, result, woutbufp, NULL);
+        if (!result) {
+            PyMem_RawFree(woutbufp);
+            return -1;
+        }
+    }
+
+    if (woutbufp != woutbuf) {
+        *abspath_p = woutbufp;
+        return 0;
+    }
+
+    *abspath_p = _PyMem_RawWcsdup(woutbufp);
+    return 0;
+#else
+    if (_Py_isabs(path)) {
+        *abspath_p = _PyMem_RawWcsdup(path);
+        return 0;
+    }
+
+    wchar_t cwd[MAXPATHLEN + 1];
+    cwd[Py_ARRAY_LENGTH(cwd) - 1] = 0;
+    if (!_Py_wgetcwd(cwd, Py_ARRAY_LENGTH(cwd) - 1)) {
+        /* unable to get the current directory */
+        return -1;
+    }
+
+    size_t cwd_len = wcslen(cwd);
+    size_t path_len = wcslen(path);
+    size_t len = cwd_len + 1 + path_len + 1;
+    if (len <= (size_t)PY_SSIZE_T_MAX / sizeof(wchar_t)) {
+        *abspath_p = PyMem_RawMalloc(len * sizeof(wchar_t));
+    }
+    else {
+        *abspath_p = NULL;
+    }
+    if (*abspath_p == NULL) {
+        return 0;
+    }
+
+    wchar_t *abspath = *abspath_p;
+    memcpy(abspath, cwd, cwd_len * sizeof(wchar_t));
+    abspath += cwd_len;
+
+    *abspath = (wchar_t)SEP;
+    abspath++;
+
+    memcpy(abspath, path, path_len * sizeof(wchar_t));
+    abspath += path_len;
+
+    *abspath = 0;
+    return 0;
+#endif
+}
+
+
 /* Get the current directory. buflen is the buffer size in wide characters
    including the null character. Decode the path from the locale encoding.