Merged revisions 80936 via svnmerge from

svn+ssh://pythondev@svn.python.org/python/branches/py3k

................
  r80936 | benjamin.peterson | 2010-05-07 14:10:11 -0500 (Fri, 07 May 2010) | 76 lines

  Merged revisions 80934 via svnmerge from
  svn+ssh://pythondev@svn.python.org/python/trunk

  ................
    r80934 | benjamin.peterson | 2010-05-07 13:58:23 -0500 (Fri, 07 May 2010) | 69 lines

    Merged revisions 79911,79916-79917,80018,80418,80572-80573,80635-80639,80668,80922 via svnmerge from
    svn+ssh://pythondev@svn.python.org/sandbox/trunk/2to3/lib2to3

    ........
      r79911 | benjamin.peterson | 2010-04-09 15:38:53 -0500 (Fri, 09 Apr 2010) | 1 line

      use absolute import
    ........
      r79916 | benjamin.peterson | 2010-04-09 16:05:21 -0500 (Fri, 09 Apr 2010) | 1 line

      generalize detection of __future__ imports and attach them to the tree
    ........
      r79917 | benjamin.peterson | 2010-04-09 16:11:44 -0500 (Fri, 09 Apr 2010) | 1 line

      don't try to 'fix' relative imports when absolute_import is enabled #8858
    ........
      r80018 | benjamin.peterson | 2010-04-12 16:12:12 -0500 (Mon, 12 Apr 2010) | 4 lines

      prevent diffs from being mangled is multiprocess mode #6409

      Patch by George Boutsioukis.
    ........
      r80418 | benjamin.peterson | 2010-04-23 16:00:03 -0500 (Fri, 23 Apr 2010) | 1 line

      remove unhelpful description
    ........
      r80572 | benjamin.peterson | 2010-04-27 20:33:54 -0500 (Tue, 27 Apr 2010) | 1 line

      use unicode literals
    ........
      r80573 | jeffrey.yasskin | 2010-04-27 23:08:27 -0500 (Tue, 27 Apr 2010) | 6 lines

      Don't transform imports that are already relative.  2to3 turned
        from . import refactor
      into
        from .. import refactor
      which broke the transformation of 2to3 itself.
    ........
      r80635 | benjamin.peterson | 2010-04-29 16:02:23 -0500 (Thu, 29 Apr 2010) | 1 line

      remove imports
    ........
      r80636 | benjamin.peterson | 2010-04-29 16:02:41 -0500 (Thu, 29 Apr 2010) | 1 line

      unicode literal
    ........
      r80637 | benjamin.peterson | 2010-04-29 16:03:42 -0500 (Thu, 29 Apr 2010) | 1 line

      must pass a string to Number
    ........
      r80638 | benjamin.peterson | 2010-04-29 16:05:34 -0500 (Thu, 29 Apr 2010) | 1 line

      unicode literals
    ........
      r80639 | benjamin.peterson | 2010-04-29 16:06:09 -0500 (Thu, 29 Apr 2010) | 1 line

      pass string to Number
    ........
      r80668 | jeffrey.yasskin | 2010-04-30 18:02:47 -0500 (Fri, 30 Apr 2010) | 4 lines

      Make 2to3 run under Python 2.5 so that the benchmark suite at
      http://hg.python.org/benchmarks/ can use it and still run on implementations
      that haven't gotten to 2.6 yet.  Fixes issue 8566.
    ........
      r80922 | benjamin.peterson | 2010-05-07 11:06:25 -0500 (Fri, 07 May 2010) | 1 line

      prevent xrange transformation from wrapping range calls it produces in list
    ........
  ................
................

Author: benjaminp

Lib/lib2to3/fixes/fix_import.py

@@ -43,7 +43,13 @@ class FixImport(fixer_base.BaseFix):
     import_name< 'import' imp=any >
     """
 
+    def start_tree(self, tree, name):
+        super(FixImport, self).start_tree(tree, name)
+        self.skip = "absolute_import" in tree.future_features
+
     def transform(self, node, results):
+        if self.skip:
+            return
         imp = results['imp']
 
         if node.type == syms.import_from:
@@ -71,19 +77,22 @@ def transform(self, node, results):
                     self.warning(node, "absolute and local imports together")
                 return
 
-            new = FromImport('.', [imp])
+            new = FromImport(".", [imp])
             new.prefix = node.prefix
             return new
 
     def probably_a_local_import(self, imp_name):
-        imp_name = imp_name.split('.', 1)[0]
+        if imp_name.startswith("."):
+            # Relative imports are certainly not local imports.
+            return False
+        imp_name = imp_name.split(".", 1)[0]
         base_path = dirname(self.filename)
         base_path = join(base_path, imp_name)
         # If there is no __init__.py next to the file its not in a package
         # so can't be a relative import.
-        if not exists(join(dirname(base_path), '__init__.py')):
+        if not exists(join(dirname(base_path), "__init__.py")):
             return False
-        for ext in ['.py', sep, '.pyc', '.so', '.sl', '.pyd']:
+        for ext in [".py", sep, ".pyc", ".so", ".sl", ".pyd"]:
             if exists(base_path + ext):
                 return True
         return False

Lib/lib2to3/fixes/fix_operator.py

@@ -14,10 +14,10 @@ class FixOperator(fixer_base.BaseFix):
     func = "'(' func=any ')'"
     PATTERN = """
               power< module='operator'
-                trailer< '.' {methods} > trailer< {func} > >
+                trailer< '.' %(methods)s > trailer< %(func)s > >
               |
-              power< {methods} trailer< {func} > >
-              """.format(methods=methods, func=func)
+              power< %(methods)s trailer< %(func)s > >
+              """ % dict(methods=methods, func=func)
 
     def transform(self, node, results):
         method = results["method"][0]

Lib/lib2to3/fixes/fix_reduce.py

@@ -7,9 +7,8 @@
 used in that module.
 """
 
-from .. import pytree
-from .. import fixer_base
-from ..fixer_util import Name, Attr, touch_import
+from lib2to3 import fixer_base
+from lib2to3.fixer_util import touch_import
 
 
 

Lib/lib2to3/fixes/fix_tuple_params.py

@@ -154,7 +154,7 @@ def map_to_index(param_list, prefix=[], d=None):
     if d is None:
         d = {}
     for i, obj in enumerate(param_list):
-        trailer = [Subscript(Number(i))]
+        trailer = [Subscript(Number(str(i)))]
         if isinstance(obj, list):
             map_to_index(obj, trailer, d=d)
         else:

Lib/lib2to3/fixes/fix_xrange.py

@@ -17,6 +17,13 @@ class FixXrange(fixer_base.BaseFix):
               rest=any* >
               """
 
+    def start_tree(self, tree, filename):
+        super(FixXrange, self).start_tree(tree, filename)
+        self.transformed_xranges = set()
+
+    def finish_tree(self, tree, filename):
+        self.transformed_xranges = None
+
     def transform(self, node, results):
         name = results["name"]
         if name.value == "xrange":
@@ -29,9 +36,12 @@ def transform(self, node, results):
     def transform_xrange(self, node, results):
         name = results["name"]
         name.replace(Name("range", prefix=name.prefix))
+        # This prevents the new range call from being wrapped in a list later.
+        self.transformed_xranges.add(id(node))
 
     def transform_range(self, node, results):
-        if not self.in_special_context(node):
+        if (id(node) not in self.transformed_xranges and
+            not self.in_special_context(node)):
             range_call = Call(Name("range"), [results["args"].clone()])
             # Encase the range call in list().
             list_call = Call(Name("list"), [range_call],

Lib/lib2to3/main.py

@@ -2,6 +2,8 @@
 Main program for 2to3.
 """
 
+from __future__ import with_statement
+
 import sys
 import os
 import difflib
@@ -62,8 +64,14 @@ def print_output(self, old, new, filename, equal):
             if self.show_diffs:
                 diff_lines = diff_texts(old, new, filename)
                 try:
-                    for line in diff_lines:
-                        print(line)
+                    if self.output_lock is not None:
+                        with self.output_lock:
+                            for line in diff_lines:
+                                print(line)
+                            sys.stdout.flush()
+                    else:
+                        for line in diff_lines:
+                            print(line)
                 except UnicodeEncodeError:
                     warn("couldn't encode %s's diff for your terminal" %
                          (filename,))
@@ -94,7 +102,7 @@ def main(fixer_pkg, args=None):
     parser.add_option("-x", "--nofix", action="append", default=[],
                       help="Prevent a fixer from being run.")
     parser.add_option("-l", "--list-fixes", action="store_true",
-                      help="List available transformations (fixes/fix_*.py)")
+                      help="List available transformations")
     parser.add_option("-p", "--print-function", action="store_true",
                       help="Modify the grammar so that print() is a function")
     parser.add_option("-v", "--verbose", action="store_true",

Lib/lib2to3/pgen2/tokenize.py

@@ -38,6 +38,13 @@
            "generate_tokens", "untokenize"]
 del token
 
+try:
+    bytes
+except NameError:
+    # Support bytes type in Python <= 2.5, so 2to3 turns itself into
+    # valid Python 3 code.
+    bytes = str
+
 def group(*choices): return '(' + '|'.join(choices) + ')'
 def any(*choices): return group(*choices) + '*'
 def maybe(*choices): return group(*choices) + '?'
@@ -267,7 +274,7 @@ def read_or_stop():
         try:
             return readline()
         except StopIteration:
-            return b''
+            return bytes()
 
     def find_cookie(line):
         try:

Lib/lib2to3/pytree.py

@@ -289,20 +289,20 @@ def pre_order(self):
             for node in child.post_order():
                 yield node
 
-    @property
-    def prefix(self):
+    def _prefix_getter(self):
         """
         The whitespace and comments preceding this node in the input.
         """
         if not self.children:
             return ""
         return self.children[0].prefix
 
-    @prefix.setter
-    def prefix(self, prefix):
+    def _prefix_setter(self, prefix):
         if self.children:
             self.children[0].prefix = prefix
 
+    prefix = property(_prefix_getter, _prefix_setter)
+
     def set_child(self, i, child):
         """
         Equivalent to 'node.children[i] = child'. This method also sets the
@@ -390,18 +390,17 @@ def pre_order(self):
         """Return a pre-order iterator for the tree."""
         yield self
 
-    @property
-    def prefix(self):
+    def _prefix_getter(self):
         """
         The whitespace and comments preceding this token in the input.
         """
         return self._prefix
 
-    @prefix.setter
-    def prefix(self, prefix):
+    def _prefix_setter(self, prefix):
         self.changed()
         self._prefix = prefix
 
+    prefix = property(_prefix_getter, _prefix_setter)
 
 def convert(gr, raw_node):
     """

Lib/lib2to3/refactor.py

@@ -8,6 +8,8 @@
 provides infrastructure to write your own refactoring tool.
 """
 
+from __future__ import with_statement
+
 __author__ = "Guido van Rossum <guido@python.org>"
 
 
@@ -122,13 +124,14 @@ def _to_system_newlines(input):
     _to_system_newlines = _identity
 
 
-def _detect_future_print(source):
+def _detect_future_features(source):
     have_docstring = False
     gen = tokenize.generate_tokens(io.StringIO(source).readline)
     def advance():
         tok = next(gen)
         return tok[0], tok[1]
     ignore = frozenset((token.NEWLINE, tokenize.NL, token.COMMENT))
+    features = set()
     try:
         while True:
             tp, value = advance()
@@ -140,26 +143,25 @@ def advance():
                 have_docstring = True
             elif tp == token.NAME and value == "from":
                 tp, value = advance()
-                if tp != token.NAME and value != "__future__":
+                if tp != token.NAME or value != "__future__":
                     break
                 tp, value = advance()
-                if tp != token.NAME and value != "import":
+                if tp != token.NAME or value != "import":
                     break
                 tp, value = advance()
                 if tp == token.OP and value == "(":
                     tp, value = advance()
                 while tp == token.NAME:
-                    if value == "print_function":
-                        return True
+                    features.add(value)
                     tp, value = advance()
-                    if tp != token.OP and value != ",":
+                    if tp != token.OP or value != ",":
                         break
                     tp, value = advance()
             else:
                 break
     except StopIteration:
         pass
-    return False
+    return frozenset(features)
 
 
 class FixerError(Exception):
@@ -341,7 +343,8 @@ def refactor_string(self, data, name):
             An AST corresponding to the refactored input stream; None if
             there were errors during the parse.
         """
-        if _detect_future_print(data):
+        features = _detect_future_features(data)
+        if "print_function" in features:
             self.driver.grammar = pygram.python_grammar_no_print_statement
         try:
             tree = self.driver.parse_string(data)
@@ -351,6 +354,7 @@ def refactor_string(self, data, name):
             return
         finally:
             self.driver.grammar = self.grammar
+        tree.future_features = features
         self.log_debug("Refactoring %s", name)
         self.refactor_tree(tree, name)
         return tree
@@ -605,6 +609,7 @@ class MultiprocessRefactoringTool(RefactoringTool):
     def __init__(self, *args, **kwargs):
         super(MultiprocessRefactoringTool, self).__init__(*args, **kwargs)
         self.queue = None
+        self.output_lock = None
 
     def refactor(self, items, write=False, doctests_only=False,
                  num_processes=1):
@@ -618,6 +623,7 @@ def refactor(self, items, write=False, doctests_only=False,
         if self.queue is not None:
             raise RuntimeError("already doing multiple processes")
         self.queue = multiprocessing.JoinableQueue()
+        self.output_lock = multiprocessing.Lock()
         processes = [multiprocessing.Process(target=self._child)
                      for i in range(num_processes)]
         try:

Lib/lib2to3/tests/test_fixers.py

@@ -1497,6 +1497,17 @@ def test_in_consuming_context(self):
         for call in fixer_util.consuming_calls:
             self.unchanged("a = %s(range(10))" % call)
 
+class Test_xrange_with_reduce(FixerTestCase):
+
+    def setUp(self):
+        super(Test_xrange_with_reduce, self).setUp(["xrange", "reduce"])
+
+    def test_double_transform(self):
+        b = """reduce(x, xrange(5))"""
+        a = """from functools import reduce
+reduce(x, range(5))"""
+        self.check(b, a)
+
 class Test_raw_input(FixerTestCase):
     fixer = "raw_input"
 
@@ -3679,7 +3690,7 @@ def fake_exists(name):
             self.files_checked.append(name)
             return self.always_exists or (name in self.present_files)
 
-        from ..fixes import fix_import
+        from lib2to3.fixes import fix_import
         fix_import.exists = fake_exists
 
     def tearDown(self):
@@ -3722,6 +3733,12 @@ def test_not_in_package(self):
         self.present_files = set(["bar.py"])
         self.unchanged(s)
 
+    def test_with_absolute_import_enabled(self):
+        s = "from __future__ import absolute_import\nimport bar"
+        self.always_exists = False
+        self.present_files = set(["__init__.py", "bar.py"])
+        self.unchanged(s)
+
     def test_in_package(self):
         b = "import bar"
         a = "from . import bar"
@@ -3736,6 +3753,10 @@ def test_import_from_package(self):
         self.present_files = set(["__init__.py", "bar" + os.path.sep])
         self.check(b, a)
 
+    def test_already_relative_import(self):
+        s = "from . import bar"
+        self.unchanged(s)
+
     def test_comments_and_indent(self):
         b = "import bar # Foo"
         a = "from . import bar # Foo"

Lib/lib2to3/tests/test_parser.py

@@ -6,6 +6,8 @@
 test_grammar.py files from both Python 2 and Python 3.
 """
 
+from __future__ import with_statement
+
 # Testing imports
 from . import support
 from .support import driver, test_dir
@@ -149,10 +151,11 @@ def test_all_project_files(self):
         for filepath in support.all_project_files():
             with open(filepath, "rb") as fp:
                 encoding = tokenize.detect_encoding(fp.readline)[0]
-                fp.seek(0)
+            self.assertTrue(encoding is not None,
+                            "can't detect encoding for %s" % filepath)
+            with open(filepath, "r") as fp:
                 source = fp.read()
-                if encoding:
-                    source = source.decode(encoding)
+                source = source.decode(encoding)
             tree = driver.parse_string(source)
             new = str(tree)
             if encoding:
@@ -199,10 +202,10 @@ def test_multiline_str_literals(self):
         self.validate(s)
 
 
-def diff(fn, result):
-    f = open("@", "wb")
+def diff(fn, result, encoding):
+    f = open("@", "w")
     try:
-        f.write(result)
+        f.write(result.encode(encoding))
     finally:
         f.close()
     try: