CLN/ENH: Add list versions of iterator methods.

py3compat now has lrange, lzip, lmap, lreduce and lfilter, which work
like the python builtins and produce lists (to make it more clear where
looking for iterators vs. containers).

Author: jtratner

bench/alignment.py

@@ -1,14 +1,14 @@
 # Setup
-from pandas.util.py3compat import range
+from pandas.util.py3compat import range, lrange
 import numpy as np
 import pandas
 import la
 N = 1000
 K = 50
 arr1 = np.random.randn(N, K)
 arr2 = np.random.randn(N, K)
-idx1 = list(range(N))
-idx2 = list(range(K))
+idx1 = lrange(N)
+idx2 = lrange(K)
 
 # pandas
 dma1 = pandas.DataFrame(arr1, idx1, idx2)

bench/bench_merge.py

@@ -1,14 +1,14 @@
 from pandas import *
 from pandas.util.testing import rands
-from pandas.util.py3compat import range
+from pandas.util.py3compat import range, lrange
 import random
 
 N = 10000
 ngroups = 10
 
 
 def get_test_data(ngroups=100, n=N):
-    unique_groups = list(range(ngroups))
+    unique_groups = lrange(ngroups)
     arr = np.asarray(np.tile(unique_groups, n / ngroups), dtype=object)
 
     if len(arr) < n:
@@ -66,7 +66,7 @@ def get_test_data(ngroups=100, n=N):
 
 
 # R results
-from pandas.util.py3compat import StringIO
+from pandas.util.py3compat import StringIO, lrange
 # many to one
 r_results = read_table(StringIO("""      base::merge   plyr data.table
 inner      0.2475 0.1183     0.1100
@@ -94,7 +94,7 @@ def get_test_data(ngroups=100, n=N):
 
 # many to many
 
-from pandas.util.py3compat import StringIO
+from pandas.util.py3compat import StringIO, lrange
 # many to one
 r_results = read_table(StringIO("""base::merge   plyr data.table
 inner      0.4610 0.1276     0.1269

bench/bench_merge_sqlite.py

@@ -4,8 +4,7 @@
 import time
 from pandas import DataFrame
 from pandas.util.testing import rands
-from pandas.util.py3compat import range
-from six.moves import zip
+from pandas.util.py3compat import range, zip
 import random
 
 N = 10000

bench/bench_take_indexing.py

@@ -6,7 +6,7 @@
 
 from pandas import DataFrame
 import timeit
-from six.moves import zip
+from pandas.util.py3compat import zip
 
 setup = """
 from pandas import Series

bench/bench_unique.py

@@ -2,7 +2,7 @@
 from pandas import *
 from pandas.util.testing import rands
 from pandas.util.py3compat import range
-from six.moves import zip
+from pandas.util.py3compat import zip
 import pandas._tseries as lib
 import numpy as np
 import matplotlib.pyplot as plt

bench/better_unique.py

@@ -1,7 +1,7 @@
 from __future__ import print_function
 from pandas import DataFrame
 from pandas.util.py3compat import range
-from six.moves import zip
+from pandas.util.py3compat import zip
 import timeit
 
 setup = """

bench/serialize.py

@@ -1,5 +1,5 @@
 from __future__ import print_function
-from pandas.util.py3compat import range
+from pandas.util.py3compat import range, lrange
 import time
 import os
 import numpy as np
@@ -22,7 +22,7 @@ def roundtrip_archive(N, iterations=10):
     # Create data
     arr = np.random.randn(N, N)
     lar = la.larry(arr)
-    dma = pandas.DataFrame(arr, list(range(N)), list(range(N)))
+    dma = pandas.DataFrame(arr, lrange(N), lrange(N))
 
     # filenames
     filename_numpy = '/Users/wesm/tmp/numpy.npz'

bench/test.py

@@ -3,7 +3,7 @@
 import itertools
 import collections
 import scipy.ndimage as ndi
-from six.moves import zip
+from pandas.util.py3compat import zip
 
 N = 10000
 

doc/sphinxext/comment_eater.py

@@ -1,4 +1,4 @@
-from six.moves import cStringIO
+from pandas.util.py3compat import cStringIO
 import compiler
 import inspect
 import textwrap

doc/sphinxext/compiler_unparse.py

@@ -12,7 +12,7 @@
 """
 
 import sys
-from six.moves import cStringIO as StringIO
+from pandas.util.py3compat import cStringIO as StringIO
 from compiler.ast import Const, Name, Tuple, Div, Mul, Sub, Add
 
 def unparse(ast, single_line_functions=False):

doc/sphinxext/ipython_directive.py

@@ -58,8 +58,8 @@
 #-----------------------------------------------------------------------------
 
 # Stdlib
-from pandas.util.py3compat import range
-from six.moves import map, cStringIO as StringIO
+from pandas.util.py3compat import range, lmap
+from pandas.util.py3compat import map, cStringIO as StringIO
 import ast
 import os
 import re
@@ -72,7 +72,7 @@
 from docutils import nodes
 from sphinx.util.compat import Directive
 import six
-from six.moves import zip
+from pandas.util.py3compat import zip
 
 matplotlib.use('Agg')
 
@@ -303,7 +303,7 @@ def process_input(self, data, input_prompt, lineno):
         def _remove_first_space_if_any(line):
             return line[1:] if line.startswith(' ') else line
 
-        input_lines = list(map(_remove_first_space_if_any, input.split('\n')))
+        input_lines = lmap(_remove_first_space_if_any, input.split('\n'))
 
         self.datacontent = data
 

doc/sphinxext/plot_directive.py

@@ -77,11 +77,11 @@
 
 from pandas.util.py3compat import range
 import sys, os, glob, shutil, imp, warnings, re, textwrap, traceback
-from six.moves import cStringIO as StringIO
+from pandas.util.py3compat import cStringIO as StringIO
 import sphinx
 
 import warnings
-from six.moves import map
+from pandas.util.py3compat import map
 warnings.warn("A plot_directive module is also available under "
               "matplotlib.sphinxext; expect this numpydoc.plot_directive "
               "module to be deprecated after relevant features have been "

examples/finance.py

@@ -3,6 +3,7 @@
 """
 
 from datetime import datetime
+from pandas.util.py3compat import zip
 
 import matplotlib.finance as fin
 import numpy as np
@@ -19,7 +20,7 @@
 
 def getQuotes(symbol, start, end):
     quotes = fin.quotes_historical_yahoo(symbol, start, end)
-    dates, open, close, high, low, volume = list(zip(*quotes))
+    dates, open, close, high, low, volume = zip(*quotes)
 
     data = {
         'open': open,

pandas/compat/scipy.py

@@ -2,7 +2,7 @@
 Shipping functions from SciPy to reduce dependency on having SciPy installed
 """
 
-from pandas.util.py3compat import range
+from pandas.util.py3compat import range, lrange
 import numpy as np
 
 
@@ -224,9 +224,9 @@ def percentileofscore(a, score, kind='rank'):
     if kind == 'rank':
         if not(np.any(a == score)):
             a = np.append(a, score)
-            a_len = np.array(list(range(len(a))))
+            a_len = np.array(lrange(len(a)))
         else:
-            a_len = np.array(list(range(len(a)))) + 1.0
+            a_len = np.array(lrange(len(a))) + 1.0
 
         a = np.sort(a)
         idx = [a == score]

pandas/core/common.py

@@ -16,7 +16,7 @@
 
 from pandas.util import py3compat
 from pandas.util.py3compat import StringIO, BytesIO, range, long
-from six.moves import zip, map
+from pandas.util.py3compat import zip, map
 import six
 
 

pandas/core/config.py

@@ -53,7 +53,7 @@
 from collections import namedtuple
 import warnings
 import six
-from six.moves import map
+from pandas.util.py3compat import map, lmap
 
 DeprecatedOption = namedtuple('DeprecatedOption', 'key msg rkey removal_ver')
 RegisteredOption = namedtuple(
@@ -746,7 +746,7 @@ def is_one_of_factory(legal_values):
     def inner(x):
         from pandas.core.common import pprint_thing as pp
         if not x in legal_values:
-            pp_values = list(map(pp, legal_values))
+            pp_values = lmap(pp, legal_values)
             raise ValueError("Value must be one of %s" % pp("|".join(pp_values)))
 
     return inner

pandas/core/format.py

@@ -1,13 +1,11 @@
 from __future__ import print_function
 # pylint: disable=W0141
 
-from pandas.util.py3compat import range
 from pandas.util import compat
 import sys
 import six
-from six.moves import map, zip, reduce
 
-from pandas.util.py3compat import StringIO
+from pandas.util.py3compat import StringIO, lzip, range, map, zip, reduce
 from pandas.core.common import adjoin, isnull, notnull
 from pandas.core.index import Index, MultiIndex, _ensure_index
 from pandas.util import py3compat
@@ -419,7 +417,7 @@ def is_numeric_dtype(dtype):
 
         if isinstance(self.columns, MultiIndex):
             fmt_columns = self.columns.format(sparsify=False, adjoin=False)
-            fmt_columns = list(zip(*fmt_columns))
+            fmt_columns = lzip(*fmt_columns)
             dtypes = self.frame.dtypes.values
             need_leadsp = dict(zip(fmt_columns, map(is_numeric_dtype, dtypes)))
             str_columns = list(zip(*[[' ' + y
@@ -718,7 +716,7 @@ def _write_hierarchical_rows(self, fmt_values, indent):
 
         idx_values = frame.index.format(sparsify=False, adjoin=False,
                                         names=False)
-        idx_values = list(zip(*idx_values))
+        idx_values = lzip(*idx_values)
 
         if self.fmt.sparsify:
 

pandas/core/frame.py

@@ -12,9 +12,7 @@
 # pylint: disable=E1101,E1103
 # pylint: disable=W0212,W0231,W0703,W0622
 
-from six.moves import zip
-from pandas.util.py3compat import StringIO
-from pandas.util.py3compat import range
+from pandas.util.py3compat import range, zip, lrange, lmap, lzip, StringIO
 from pandas.util import compat
 import operator
 import sys
@@ -59,7 +57,6 @@
 
 from pandas.core.config import get_option, set_option
 import six
-from six.moves import map
 
 #----------------------------------------------------------------------
 # Docstring templates
@@ -1151,7 +1148,7 @@ def to_records(self, index=True, convert_datetime64=True):
             else:
                 if isinstance(self.index, MultiIndex):
                     # array of tuples to numpy cols. copy copy copy
-                    ix_vals = list(map(np.array,zip(*self.index.values)))
+                    ix_vals = lmap(np.array,zip(*self.index.values))
                 else:
                     ix_vals = [self.index.values]
 
@@ -1166,10 +1163,10 @@ def to_records(self, index=True, convert_datetime64=True):
                         count += 1
             elif index_names[0] is None:
                 index_names = ['index']
-            names = index_names + list(map(str, self.columns))
+            names = index_names + lmap(str, self.columns)
         else:
             arrays = [self[c].values for c in self.columns]
-            names = list(map(str, self.columns))
+            names = lmap(str, self.columns)
 
         dtype = np.dtype([(x, v.dtype) for x, v in zip(names, arrays)])
         return np.rec.fromarrays(arrays, dtype=dtype, names=names)
@@ -1197,7 +1194,7 @@ def from_items(cls, items, columns=None, orient='columns'):
         -------
         frame : DataFrame
         """
-        keys, values = list(zip(*items))
+        keys, values = lzip(*items)
 
         if orient == 'columns':
             if columns is not None:
@@ -2911,7 +2908,7 @@ def _maybe_cast(values, labels=None):
 
             if not drop:
                 names = self.index.names
-                zipped = list(zip(self.index.levels, self.index.labels))
+                zipped = lzip(self.index.levels, self.index.labels)
 
                 multi_col = isinstance(self.columns, MultiIndex)
                 for i, (lev, lab) in reversed(list(enumerate(zipped))):
@@ -4536,7 +4533,7 @@ def _apply_broadcast(self, func, axis):
     def applymap(self, func):
         """
         Apply a function to a DataFrame that is intended to operate
-        elementwise, i.e. like doing list(map(func, series)) for each series in the
+        elementwise, i.e. like doing map(func, series) for each series in the
         DataFrame
 
         Parameters
@@ -4889,7 +4886,7 @@ def pretty_name(x):
                            series.min(), series.quantile(lb), series.median(),
                            series.quantile(ub), series.max()])
 
-        return self._constructor(list(map(list, zip(*destat))), index=destat_columns,
+        return self._constructor(lmap(list, zip(*destat)), index=destat_columns,
                                  columns=numdata.columns)
 
     #----------------------------------------------------------------------
@@ -5850,7 +5847,7 @@ def _to_arrays(data, columns, coerce_float=False, dtype=None):
         return arrays, columns
     else:
         # last ditch effort
-        data = list(map(tuple, data))
+        data = lmap(tuple, data)
         return _list_to_arrays(data, columns,
                                coerce_float=coerce_float,
                                dtype=dtype)
@@ -5924,7 +5921,7 @@ def _convert_object_array(content, columns, coerce_float=False, dtype=None):
 
 
 def _get_names_from_index(data):
-    index = list(range(len(data)))
+    index = lrange(len(data))
     has_some_name = any([s.name is not None for s in data])
     if not has_some_name:
         return index

pandas/core/generic.py

@@ -11,7 +11,7 @@
 from pandas.tseries.index import DatetimeIndex
 import pandas.core.common as com
 import six
-from six.moves import map, zip
+from pandas.util.py3compat import map, zip
 
 
 class PandasError(Exception):