Note: only the relevant parts of r79474 are merged.

Merged revisions 78793,78798-78799,78977,79095,79196,79474 via svnmerge from
svn+ssh://pythondev@svn.python.org/python/branches/py3k

................
  r78793 | florent.xicluna | 2010-03-08 13:25:35 +0100 (lun, 08 mar 2010) | 2 lines

  Fix macpath to deal with bytes
................
  r78798 | florent.xicluna | 2010-03-08 14:32:17 +0100 (lun, 08 mar 2010) | 18 lines

  Merged revisions 78777,78787,78790 via svnmerge from
  svn+ssh://pythondev@svn.python.org/python/trunk

  ........
    r78777 | florent.xicluna | 2010-03-08 00:49:03 +0100 (lun, 08 mar 2010) | 4 lines

    Backport the Popen.poll() protection from subprocess to multiprocessing. See #1731717.
    It should fix transient failures on test_multiprocessing.
  ........
    r78787 | florent.xicluna | 2010-03-08 08:21:16 +0100 (lun, 08 mar 2010) | 2 lines

    Don't fail on a debug() statement, if the worker PID is (still) None.
  ........
    r78790 | florent.xicluna | 2010-03-08 12:01:39 +0100 (lun, 08 mar 2010) | 2 lines

    On finalize, don't try to join not started process.
  ........
................
  r78799 | florent.xicluna | 2010-03-08 15:44:41 +0100 (lun, 08 mar 2010) | 2 lines

  Fix ntpath abspath to deal with bytes.
................
  r78977 | florent.xicluna | 2010-03-15 14:14:39 +0100 (lun, 15 mar 2010) | 2 lines

  Fix \xhh specs, #1889.  (an oversight of r60193, r60210).
................
  r79095 | florent.xicluna | 2010-03-19 15:40:31 +0100 (ven, 19 mar 2010) | 2 lines

  Rename test.test_support to test.support for 3.x.
................
  r79196 | florent.xicluna | 2010-03-21 13:29:50 +0100 (dim, 21 mar 2010) | 9 lines

  Merged revisions 79195 via svnmerge from
  svn+ssh://pythondev@svn.python.org/python/trunk

  ........
    r79195 | florent.xicluna | 2010-03-21 13:27:20 +0100 (dim, 21 mar 2010) | 2 lines

    Issue #8179: Fix macpath.realpath() on a non-existing path.
  ........
................
  r79474 | florent.xicluna | 2010-03-28 01:25:02 +0100 (dim, 28 mar 2010) | 33 lines

  Merged revisions 79297,79310,79382,79425-79427,79450 via svnmerge from
  svn+ssh://pythondev@svn.python.org/python/trunk

  ........
    r79297 | florent.xicluna | 2010-03-22 18:18:18 +0100 (lun, 22 mar 2010) | 2 lines

    #7668: Fix test_httpservers failure when sys.executable contains non-ASCII bytes.
  ........
    r79310 | florent.xicluna | 2010-03-22 23:52:11 +0100 (lun, 22 mar 2010) | 2 lines

    Issue #8205: Remove the "Modules" directory from sys.path when Python is running from the build directory (POSIX only).
  ........
    r79382 | florent.xicluna | 2010-03-24 20:33:25 +0100 (mer, 24 mar 2010) | 2 lines

    Skip tests which depend on multiprocessing.sharedctypes, if _ctypes is not available.
  ........
    r79425 | florent.xicluna | 2010-03-25 21:32:07 +0100 (jeu, 25 mar 2010) | 2 lines

    Syntax cleanup `== None` -> `is None`
  ........
    r79426 | florent.xicluna | 2010-03-25 21:33:49 +0100 (jeu, 25 mar 2010) | 2 lines

    #8207: Fix test_pep277 on OS X
  ........
    r79427 | florent.xicluna | 2010-03-25 21:39:10 +0100 (jeu, 25 mar 2010) | 2 lines

    Fix test_unittest and test_warnings when running "python -Werror -m test.regrtest"
  ........
    r79450 | florent.xicluna | 2010-03-26 20:32:44 +0100 (ven, 26 mar 2010) | 2 lines

    Ensure that the failed or unexpected tests are sorted before printing.
  ........
................

Author: florentx

Doc/reference/lexical_analysis.rst

@@ -503,7 +503,7 @@ Notes:
    As in Standard C, up to three octal digits are accepted.
 
 (2)
-   Unlike in Standard C, at most two hex digits are accepted.
+   Unlike in Standard C, exactly two hex digits are required.
 
 (3)
    In a bytes literal, hexadecimal and octal escapes denote the byte with the

Lib/idlelib/PyShell.py

@@ -347,7 +347,7 @@ def __init__(self, tkconsole):
     rpcpid = None
 
     def spawn_subprocess(self):
-        if self.subprocess_arglist == None:
+        if self.subprocess_arglist is None:
             self.subprocess_arglist = self.build_subprocess_arglist()
         args = self.subprocess_arglist
         self.rpcpid = os.spawnv(os.P_NOWAIT, sys.executable, args)

Lib/macpath.py

@@ -172,7 +172,11 @@ def normpath(s):
 def abspath(path):
     """Return an absolute path."""
     if not isabs(path):
-        path = join(os.getcwd(), path)
+        if isinstance(path, bytes):
+            cwd = os.getcwdb()
+        else:
+            cwd = os.getcwd()
+        path = join(cwd, path)
     return normpath(path)
 
 # realpath is a no-op on systems without islink support
@@ -189,7 +193,10 @@ def realpath(path):
     path = components[0] + colon
     for c in components[1:]:
         path = join(path, c)
-        path = Carbon.File.FSResolveAliasFile(path, 1)[0].as_pathname()
+        try:
+            path = Carbon.File.FSResolveAliasFile(path, 1)[0].as_pathname()
+        except Carbon.File.Error:
+            pass
     return path
 
 supports_unicode_filenames = False

Lib/multiprocessing/forking.py

@@ -104,7 +104,12 @@ def __init__(self, process_obj):
 
         def poll(self, flag=os.WNOHANG):
             if self.returncode is None:
-                pid, sts = os.waitpid(self.pid, flag)
+                try:
+                    pid, sts = os.waitpid(self.pid, flag)
+                except os.error:
+                    # Child process not yet created. See #1731717
+                    # e.errno == errno.ECHILD == 10
+                    return None
                 if pid == self.pid:
                     if os.WIFSIGNALED(sts):
                         self.returncode = -os.WTERMSIG(sts)

Lib/multiprocessing/pool.py

@@ -387,7 +387,10 @@ def _terminate_pool(cls, taskqueue, inqueue, outqueue, pool,
         if pool and hasattr(pool[0], 'terminate'):
             debug('joining pool workers')
             for p in pool:
-                p.join()
+                if p.is_alive():
+                    # worker has not yet exited
+                    debug('cleaning up worker %d' % p.pid)
+                    p.join()
 
 #
 # Class whose instances are returned by `Pool.apply_async()`

Lib/multiprocessing/process.py

@@ -179,7 +179,7 @@ def exitcode(self):
     @property
     def ident(self):
         '''
-        Return indentifier (PID) of process or `None` if it has yet to start
+        Return identifier (PID) of process or `None` if it has yet to start
         '''
         if self is _current_process:
             return os.getpid()

Lib/ntpath.py

@@ -561,6 +561,8 @@ def abspath(path):
                 path = _getfullpathname(path)
             except WindowsError:
                 pass # Bad path - return unchanged.
+        elif isinstance(path, bytes):
+            path = os.getcwdb()
         else:
             path = os.getcwd()
         return normpath(path)

Lib/test/regrtest.py

@@ -453,11 +453,6 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, generate=False,
             if module not in save_modules and module.startswith("test."):
                 support.unload(module)
 
-    # The lists won't be sorted if running with -r
-    good.sort()
-    bad.sort()
-    skipped.sort()
-
     if good and not quiet:
         if not bad and not skipped and len(good) > 1:
             print("All", end=' ')
@@ -893,7 +888,8 @@ def printlist(x, width=70, indent=4):
 
     from textwrap import fill
     blanks = ' ' * indent
-    print(fill(' '.join(map(str, x)), width,
+    # Print the sorted list: 'x' may be a '--random' list or a set()
+    print(fill(' '.join(str(elt) for elt in sorted(x)), width,
                initial_indent=blanks, subsequent_indent=blanks))
 
 # Map sys.platform to a string containing the basenames of tests

Lib/test/test_httpservers.py

@@ -294,14 +294,22 @@ def setUp(self):
         self.cgi_dir = os.path.join(self.parent_dir, 'cgi-bin')
         os.mkdir(self.cgi_dir)
 
+        # The shebang line should be pure ASCII: use symlink if possible.
+        # See issue #7668.
+        if hasattr(os, 'symlink'):
+            self.pythonexe = os.path.join(self.parent_dir, 'python')
+            os.symlink(sys.executable, self.pythonexe)
+        else:
+            self.pythonexe = sys.executable
+
         self.file1_path = os.path.join(self.cgi_dir, 'file1.py')
         with open(self.file1_path, 'w') as file1:
-            file1.write(cgi_file1 % sys.executable)
+            file1.write(cgi_file1 % self.pythonexe)
         os.chmod(self.file1_path, 0o777)
 
         self.file2_path = os.path.join(self.cgi_dir, 'file2.py')
         with open(self.file2_path, 'w') as file2:
-            file2.write(cgi_file2 % sys.executable)
+            file2.write(cgi_file2 % self.pythonexe)
         os.chmod(self.file2_path, 0o777)
 
         self.cwd = os.getcwd()
@@ -310,6 +318,8 @@ def setUp(self):
     def tearDown(self):
         try:
             os.chdir(self.cwd)
+            if self.pythonexe != sys.executable:
+                os.remove(self.pythonexe)
             os.remove(self.file1_path)
             os.remove(self.file2_path)
             os.rmdir(self.cgi_dir)

Lib/test/test_multiprocessing.py

@@ -63,6 +63,16 @@ def latin(s):
 
 WIN32 = (sys.platform == "win32")
 
+#
+# Some tests require ctypes
+#
+
+try:
+    from ctypes import Structure, Value, copy, c_int, c_double
+except ImportError:
+    Structure = object
+    c_int = c_double = None
+
 #
 # Creates a wrapper for a function which records the time it takes to finish
 #
@@ -506,7 +516,7 @@ def test_task_done(self):
         queue = self.JoinableQueue()
 
         if sys.version_info < (2, 5) and not hasattr(queue, 'task_done'):
-            return
+            self.skipTest("requires 'queue.task_done()' method")
 
         workers = [self.Process(target=self._test_task_done, args=(queue,))
                    for i in range(4)]
@@ -783,6 +793,8 @@ def test_event(self):
 
 class _TestValue(BaseTestCase):
 
+    ALLOWED_TYPES = ('processes',)
+
     codes_values = [
         ('i', 4343, 24234),
         ('d', 3.625, -4.25),
@@ -795,10 +807,8 @@ def _test(self, values):
             sv.value = cv[2]
 
 
+    @unittest.skipIf(c_int is None, "requires _ctypes")
     def test_value(self, raw=False):
-        if self.TYPE != 'processes':
-            return
-
         if raw:
             values = [self.RawValue(code, value)
                       for code, value, _ in self.codes_values]
@@ -816,13 +826,12 @@ def test_value(self, raw=False):
         for sv, cv in zip(values, self.codes_values):
             self.assertEqual(sv.value, cv[2])
 
+    @unittest.skipIf(c_int is None, "requires _ctypes")
     def test_rawvalue(self):
         self.test_value(raw=True)
 
+    @unittest.skipIf(c_int is None, "requires _ctypes")
     def test_getobj_getlock(self):
-        if self.TYPE != 'processes':
-            return
-
         val1 = self.Value('i', 5)
         lock1 = val1.get_lock()
         obj1 = val1.get_obj()
@@ -850,14 +859,14 @@ def test_getobj_getlock(self):
 
 class _TestArray(BaseTestCase):
 
+    ALLOWED_TYPES = ('processes',)
+
     def f(self, seq):
         for i in range(1, len(seq)):
             seq[i] += seq[i-1]
 
+    @unittest.skipIf(c_int is None, "requires _ctypes")
     def test_array(self, raw=False):
-        if self.TYPE != 'processes':
-            return
-
         seq = [680, 626, 934, 821, 150, 233, 548, 982, 714, 831]
         if raw:
             arr = self.RawArray('i', seq)
@@ -880,13 +889,12 @@ def test_array(self, raw=False):
 
         self.assertEqual(list(arr[:]), seq)
 
+    @unittest.skipIf(c_int is None, "requires _ctypes")
     def test_rawarray(self):
         self.test_array(raw=True)
 
+    @unittest.skipIf(c_int is None, "requires _ctypes")
     def test_getobj_getlock_obj(self):
-        if self.TYPE != 'processes':
-            return
-
         arr1 = self.Array('i', list(range(10)))
         lock1 = arr1.get_lock()
         obj1 = arr1.get_obj()
@@ -1538,12 +1546,6 @@ def test_heap(self):
 #
 #
 
-try:
-    from ctypes import Structure, Value, copy, c_int, c_double
-except ImportError:
-    Structure = object
-    c_int = c_double = None
-
 class _Foo(Structure):
     _fields_ = [
         ('x', c_int),
@@ -1563,10 +1565,8 @@ def _double(self, x, y, foo, arr, string):
         for i in range(len(arr)):
             arr[i] *= 2
 
+    @unittest.skipIf(c_int is None, "requires _ctypes")
     def test_sharedctypes(self, lock=False):
-        if c_int is None:
-            return
-
         x = Value('i', 7, lock=lock)
         y = Value(ctypes.c_double, 1.0/3.0, lock=lock)
         foo = Value(_Foo, 3, 2, lock=lock)
@@ -1589,10 +1589,8 @@ def test_sharedctypes(self, lock=False):
     def test_synchronize(self):
         self.test_sharedctypes(lock=True)
 
+    @unittest.skipIf(c_int is None, "requires _ctypes")
     def test_copy(self):
-        if c_int is None:
-            return
-
         foo = _Foo(2, 5.0)
         bar = copy(foo)
         foo.x = 0
@@ -1664,13 +1662,17 @@ class _TestImportStar(BaseTestCase):
     ALLOWED_TYPES = ('processes',)
 
     def test_import(self):
-        modules = (
+        modules = [
             'multiprocessing', 'multiprocessing.connection',
             'multiprocessing.heap', 'multiprocessing.managers',
             'multiprocessing.pool', 'multiprocessing.process',
-            'multiprocessing.reduction', 'multiprocessing.sharedctypes',
+            'multiprocessing.reduction',
             'multiprocessing.synchronize', 'multiprocessing.util'
-            )
+            ]
+
+        if c_int is not None:
+            # This module requires _ctypes
+            modules.append('multiprocessing.sharedctypes')
 
         for name in modules:
             __import__(name)
@@ -1730,12 +1732,12 @@ def test_level(self):
 
 class TestInvalidHandle(unittest.TestCase):
 
+    @unittest.skipIf(WIN32, "skipped on Windows")
     def test_invalid_handles(self):
-        if WIN32:
-            return
         conn = _multiprocessing.Connection(44977608)
         self.assertRaises(IOError, conn.poll)
         self.assertRaises(IOError, _multiprocessing.Connection, -1)
+
 #
 # Functions used to create test cases from the base ones in this module
 #
@@ -1752,7 +1754,7 @@ def get_attributes(Source, names):
 def create_test_cases(Mixin, type):
     result = {}
     glob = globals()
-    Type = type[0].upper() + type[1:]
+    Type = type.capitalize()
 
     for name in list(glob.keys()):
         if name.startswith('_Test'):

Lib/test/test_warnings.py

@@ -27,11 +27,15 @@ def warnings_state(module):
     except NameError:
         pass
     original_warnings = warning_tests.warnings
+    original_filters = module.filters
     try:
+        module.filters = original_filters[:]
+        module.simplefilter("once")
         warning_tests.warnings = module
         yield
     finally:
         warning_tests.warnings = original_warnings
+        module.filters = original_filters
 
 
 class BaseTest(unittest.TestCase):
@@ -194,6 +198,7 @@ class WarnTests(unittest.TestCase):
     def test_message(self):
         with original_warnings.catch_warnings(record=True,
                 module=self.module) as w:
+            self.module.simplefilter("once")
             for i in range(4):
                 text = 'multi %d' %i  # Different text on each call.
                 self.module.warn(text)
@@ -206,6 +211,7 @@ def test_warn_nonstandard_types(self):
         for ob in (Warning, None, 42):
             with original_warnings.catch_warnings(record=True,
                     module=self.module) as w:
+                self.module.simplefilter("once")
                 self.module.warn(ob)
                 # Don't directly compare objects since
                 # ``Warning() != Warning()``.