#7301: decode $PYTHONWARNINGS in the same way as argv, test non-ascii values

Author: pjenvey

Include/Python.h

@@ -123,6 +123,9 @@
 /* _Py_Mangle is defined in compile.c */
 PyAPI_FUNC(PyObject*) _Py_Mangle(PyObject *p, PyObject *name);
 
+/* _Py_char2wchar lives in python.c */
+PyAPI_FUNC(wchar_t *) _Py_char2wchar(char *);
+
 /* Convert a possibly signed character to a nonnegative int */
 /* XXX This assumes characters are 8 bits wide */
 #ifdef __CHAR_UNSIGNED__

Lib/test/test_warnings.py

@@ -718,6 +718,19 @@ def test_envvar_and_command_line(self):
                 b"['ignore::UnicodeWarning', 'ignore::DeprecationWarning']")
         self.assertEqual(p.wait(), 0)
 
+    @unittest.skipUnless(sys.getfilesystemencoding() != 'ascii',
+                         'requires non-ascii filesystemencoding')
+    def test_nonascii(self):
+        newenv = os.environ.copy()
+        newenv["PYTHONWARNINGS"] = "ignore:DeprecaciónWarning"
+        newenv["PYTHONIOENCODING"] = "utf-8"
+        p = subprocess.Popen([sys.executable,
+                "-c", "import sys; sys.stdout.write(str(sys.warnoptions))"],
+                stdout=subprocess.PIPE, env=newenv)
+        self.assertEqual(p.communicate()[0],
+                "['ignore:DeprecaciónWarning']".encode('utf-8'))
+        self.assertEqual(p.wait(), 0)
+
 class CEnvironmentVariableTests(EnvironmentVariableTests):
     module = c_warnings
 

Modules/main.c

@@ -266,6 +266,9 @@ Py_Main(int argc, wchar_t **argv)
 	wchar_t *module = NULL;
 	FILE *fp = stdin;
 	char *p;
+#ifdef MS_WINDOWS
+	wchar_t *wp;
+#endif
 	int skipfirstline = 0;
 	int stdin_is_interactive = 0;
 	int help = 0;
@@ -402,29 +405,48 @@ Py_Main(int argc, wchar_t **argv)
 	    (p = Py_GETENV("PYTHONNOUSERSITE")) && *p != '\0')
 		Py_NoUserSiteDirectory = 1;
 
+#ifdef MS_WINDOWS
+	if (!Py_IgnoreEnvironmentFlag && (wp = _wgetenv(L"PYTHONWARNINGS")) &&
+	    *wp != L'\0') {
+		wchar_t *buf, *warning;
+
+		buf = (wchar_t *)malloc((wcslen(wp) + 1) * sizeof(wchar_t));
+		if (buf == NULL)
+			Py_FatalError(
+			   "not enough memory to copy PYTHONWARNINGS");
+		wcscpy(buf, wp);
+		for (warning = wcstok(buf, L",");
+		     warning != NULL;
+		     warning = wcstok(NULL, L",")) {
+			PySys_AddWarnOption(warning);
+		}
+		free(buf);
+	}
+#else
 	if ((p = Py_GETENV("PYTHONWARNINGS")) && *p != '\0') {
-		char *buf, *warning;
+		char *buf, *oldloc;
+		wchar_t *warning;
 
+		/* settle for strtok here as there's no one standard
+		   C89 wcstok */
 		buf = (char *)malloc(strlen(p) + 1);
 		if (buf == NULL)
 			Py_FatalError(
 			   "not enough memory to copy PYTHONWARNINGS");
 		strcpy(buf, p);
-		for (warning = strtok(buf, ",");
-		     warning != NULL;
-		     warning = strtok(NULL, ",")) {
-			wchar_t *wide_warning;
-			size_t len = strlen(buf);
-			wide_warning = (wchar_t *)malloc((len + 1) * sizeof(wchar_t));
-			if (wide_warning == NULL)
-				Py_FatalError(
-				   "not enough memory to copy PYTHONWARNINGS");
-			mbstowcs(wide_warning, warning, len);
-			PySys_AddWarnOption(wide_warning);
-			free(wide_warning);
+		oldloc = strdup(setlocale(LC_ALL, NULL));
+		setlocale(LC_ALL, "");
+		for (p = strtok(buf, ","); p != NULL; p = strtok(NULL, ",")) {
+			if ((warning = _Py_char2wchar(p)) != NULL) {
+				PySys_AddWarnOption(warning);
+				free(warning);
+			}
 		}
+		setlocale(LC_ALL, oldloc);
+		free(oldloc);
 		free(buf);
 	}
+#endif
 
 	if (command == NULL && module == NULL && _PyOS_optind < argc &&
 	    wcscmp(argv[_PyOS_optind], L"-") != 0)

Modules/python.c

@@ -14,8 +14,8 @@ wmain(int argc, wchar_t **argv)
 	return Py_Main(argc, argv);
 }
 #else
-static wchar_t*
-char2wchar(char* arg)
+wchar_t*
+_Py_char2wchar(char* arg)
 {
 	wchar_t *res;
 #ifdef HAVE_BROKEN_MBSTOWCS
@@ -143,7 +143,7 @@ main(int argc, char **argv)
 	oldloc = strdup(setlocale(LC_ALL, NULL));
 	setlocale(LC_ALL, "");
 	for (i = 0; i < argc; i++) {
-		argv_copy2[i] = argv_copy[i] = char2wchar(argv[i]);
+		argv_copy2[i] = argv_copy[i] = _Py_char2wchar(argv[i]);
 		if (!argv_copy[i])
 			return 1;
 	}