use pandas-dev-flaker

Author: MarcoGorelli

.pre-commit-config.yaml

@@ -36,7 +36,10 @@ repos:
     rev: 3.9.0
     hooks:
     -   id: flake8
-        additional_dependencies: [flake8-comprehensions>=3.1.0, flake8-bugbear>=21.3.2]
+        additional_dependencies:
+            - flake8-comprehensions>=3.1.0
+            - flake8-bugbear>=21.3.2
+            - pandas-dev-flaker==0.1.2
     -   id: flake8
         name: flake8 (cython)
         types: [cython]
@@ -69,7 +72,11 @@ repos:
     rev: v1.2.2
     hooks:
     -   id: yesqa
-        additional_dependencies: [flake8==3.9.0]
+        additional_dependencies:
+            - flake8==3.9.0
+            - flake8-comprehensions>=3.1.0
+            - flake8-bugbear>=21.3.2
+            - pandas-dev-flaker==0.1.2
 -   repo: local
     hooks:
     -   id: flake8-rst
@@ -80,28 +87,6 @@ repos:
         types: [rst]
         args: [--filename=*.rst]
         additional_dependencies: [flake8-rst==0.7.0, flake8==3.7.9]
-    -   id: frame-or-series-union
-        name: Check for use of Union[Series, DataFrame] instead of FrameOrSeriesUnion alias
-        entry: Union\[.*(Series,.*DataFrame|DataFrame,.*Series).*\]
-        language: pygrep
-        types: [python]
-        exclude: ^pandas/_typing\.py$
-    -   id: inconsistent-namespace-usage
-        name: 'Check for inconsistent use of pandas namespace'
-        entry: python scripts/check_for_inconsistent_pandas_namespace.py
-        language: python
-        types: [python]
-    -   id: no-os-remove
-        name: Check code for instances of os.remove
-        entry: os\.remove
-        language: pygrep
-        types: [python]
-        files: ^pandas/tests/
-        exclude: |
-            (?x)^
-            pandas/tests/io/excel/test_writers\.py
-            |pandas/tests/io/pytables/common\.py
-            |pandas/tests/io/pytables/test_store\.py$
     -   id: unwanted-patterns
         name: Unwanted patterns
         language: pygrep
@@ -111,52 +96,10 @@ repos:
             \#\ type:\ (?!ignore)
             |\#\ type:\s?ignore(?!\[)
 
-            # foo._class__ instead of type(foo)
-            |\.__class__
-
-            # np.bool/np.object instead of np.bool_/np.object_
-            |np\.bool[^_8]
-            |np\.object[^_8]
-
-            # imports from pandas.core.common instead of `import pandas.core.common as com`
-            |from\ pandas\.core\.common\ import
-            |from\ pandas\.core\ import\ common
-
-            # imports from collections.abc instead of `from collections import abc`
-            |from\ collections\.abc\ import
-
-            # Numpy
-            |from\ numpy\ import\ random
-            |from\ numpy\.random\ import
-
             # Incorrect code-block / IPython directives
             |\.\.\ code-block\ ::
             |\.\.\ ipython\ ::
         types_or: [python, cython, rst]
-        exclude: ^doc/source/development/code_style\.rst  # contains examples of patterns to avoid
-    -   id: unwanted-patterns-in-tests
-        name: Unwanted patterns in tests
-        language: pygrep
-        entry: |
-            (?x)
-            # pytest.xfail instead of pytest.mark.xfail
-            pytest\.xfail
-
-            # imports from pandas._testing instead of `import pandas._testing as tm`
-            |from\ pandas\._testing\ import
-            |from\ pandas\ import\ _testing\ as\ tm
-
-            # No direct imports from conftest
-            |conftest\ import
-            |import\ conftest
-
-            # pandas.testing instead of tm
-            |pd\.testing\.
-
-            # pd.api.types instead of from pandas.api.types import ...
-            |(pd|pandas)\.api\.types\.
-        files: ^pandas/tests/
-        types_or: [python, cython, rst]
     -   id: pip-to-conda
         name: Generate pip dependency from conda
         description: This hook checks if the conda environment.yml and requirements-dev.txt are equal
@@ -178,35 +121,6 @@ repos:
         language: python
         types: [rst]
         files: ^doc/source/(development|reference)/
-    -   id: unwanted-patterns-bare-pytest-raises
-        name: Check for use of bare pytest raises
-        language: python
-        entry: python scripts/validate_unwanted_patterns.py --validation-type="bare_pytest_raises"
-        types: [python]
-        files: ^pandas/tests/
-        exclude: ^pandas/tests/extension/
-    -   id: unwanted-patterns-private-function-across-module
-        name: Check for use of private functions across modules
-        language: python
-        entry: python scripts/validate_unwanted_patterns.py --validation-type="private_function_across_module"
-        types: [python]
-        exclude: ^(asv_bench|pandas/tests|doc)/
-    -   id: unwanted-patterns-private-import-across-module
-        name: Check for import of private attributes across modules
-        language: python
-        entry: python scripts/validate_unwanted_patterns.py --validation-type="private_import_across_module"
-        types: [python]
-        exclude: ^(asv_bench|pandas/tests|doc)/
-    -   id: unwanted-patterns-strings-to-concatenate
-        name: Check for use of not concatenated strings
-        language: python
-        entry: python scripts/validate_unwanted_patterns.py --validation-type="strings_to_concatenate"
-        types_or: [python, cython]
-    -   id: unwanted-patterns-strings-with-wrong-placed-whitespace
-        name: Check for strings with wrong placed spaces
-        language: python
-        entry: python scripts/validate_unwanted_patterns.py --validation-type="strings_with_wrong_placed_whitespace"
-        types_or: [python, cython]
     -   id: use-pd_array-in-core
         name: Import pandas.array as pd_array in core
         language: python

asv_bench/benchmarks/gil.py

@@ -31,7 +31,7 @@
 except ImportError:
     from pandas import algos
 try:
-    from pandas._testing import test_parallel
+    from pandas._testing import test_parallel  # noqa: PDF014
 
     have_real_test_parallel = True
 except ImportError:

asv_bench/benchmarks/pandas_vb_common.py

@@ -70,7 +70,7 @@ class BaseIO:
     def remove(self, f):
         """Remove created files"""
         try:
-            os.remove(f)
+            os.remove(f)  # noqa: PDF008
         except OSError:
             # On Windows, attempting to remove a file that is in use
             # causes an exception to be raised

ci/code_checks.sh

@@ -64,27 +64,6 @@ fi
 ### PATTERNS ###
 if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
 
-    MSG='Check for use of exec' ; echo $MSG
-    invgrep -R --include="*.py*" -E "[^a-zA-Z0-9_]exec\(" pandas
-    RET=$(($RET + $?)) ; echo $MSG "DONE"
-
-    MSG='Check for pytest warns' ; echo $MSG
-    invgrep -r -E --include '*.py' 'pytest\.warns' pandas/tests/
-    RET=$(($RET + $?)) ; echo $MSG "DONE"
-
-    MSG='Check for pytest raises without context' ; echo $MSG
-    invgrep -r -E --include '*.py' "[[:space:]] pytest.raises" pandas/tests/
-    RET=$(($RET + $?)) ; echo $MSG "DONE"
-
-    MSG='Check for use of builtin filter function' ; echo $MSG
-    invgrep -R --include="*.py" -P '(?<!def)[\(\s]filter\(' pandas
-    RET=$(($RET + $?)) ; echo $MSG "DONE"
-
-    # Check for the following code in testing: `np.testing` and `np.array_equal`
-    MSG='Check for invalid testing' ; echo $MSG
-    invgrep -r -E --include '*.py' --exclude testing.py '(numpy|np)(\.testing|\.array_equal)' pandas/tests/
-    RET=$(($RET + $?)) ; echo $MSG "DONE"
-
     # Check for the following code in the extension array base tests: `tm.assert_frame_equal` and `tm.assert_series_equal`
     MSG='Check for invalid EA testing' ; echo $MSG
     invgrep -r -E --include '*.py' --exclude base.py 'tm.assert_(series|frame)_equal' pandas/tests/extension/base
@@ -98,15 +77,6 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
     invgrep -R --include="*.rst" -E "[a-zA-Z0-9]\`\`?[a-zA-Z0-9]" doc/source/
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
-    # Check for the following code in testing: `unittest.mock`, `mock.Mock()` or `mock.patch`
-    MSG='Check that unittest.mock is not used (pytest builtin monkeypatch fixture should be used instead)' ; echo $MSG
-    invgrep -r -E --include '*.py' '(unittest(\.| import )mock|mock\.Mock\(\)|mock\.patch)' pandas/tests/
-    RET=$(($RET + $?)) ; echo $MSG "DONE"
-
-    MSG='Check for use of {foo!r} instead of {repr(foo)}' ; echo $MSG
-    invgrep -R --include=*.{py,pyx} '!r}' pandas
-    RET=$(($RET + $?)) ; echo $MSG "DONE"
-    echo $MSG "DONE"
 fi
 
 ### CODE ###

environment.yml

@@ -117,3 +117,4 @@ dependencies:
   - pip:
     - git+https://github.com/pydata/pydata-sphinx-theme.git@master
     - numpydoc < 1.2  # 2021-02-09 1.2dev breaking CI
+    - pandas-dev-flaker==0.1.2

pandas/_testing/__init__.py

@@ -915,7 +915,7 @@ def external_error_raised(expected_exception: type[Exception]) -> ContextManager
     """
     import pytest
 
-    return pytest.raises(expected_exception, match=None)
+    return pytest.raises(expected_exception, match=None)  # noqa: PDF010
 
 
 cython_table = pd.core.common._cython_table.items()

pandas/core/window/ewm.py

@@ -21,7 +21,7 @@
 from pandas.core.dtypes.common import is_datetime64_ns_dtype
 from pandas.core.dtypes.missing import isna
 
-import pandas.core.common as common
+import pandas.core.common as com
 from pandas.core.util.numba_ import maybe_use_numba
 from pandas.core.window.common import zsqrt
 from pandas.core.window.doc import (
@@ -51,7 +51,7 @@ def get_center_of_mass(
     halflife: float | None,
     alpha: float | None,
 ) -> float:
-    valid_count = common.count_not_none(comass, span, halflife, alpha)
+    valid_count = com.count_not_none(comass, span, halflife, alpha)
     if valid_count > 1:
         raise ValueError("comass, span, halflife, and alpha are mutually exclusive")
 
@@ -279,7 +279,7 @@ def __init__(
             self._deltas = np.diff(_times) / _halflife
             # Halflife is no longer applicable when calculating COM
             # But allow COM to still be calculated if the user passes other decay args
-            if common.count_not_none(self.com, self.span, self.alpha) > 0:
+            if com.count_not_none(self.com, self.span, self.alpha) > 0:
                 self._com = get_center_of_mass(self.com, self.span, None, self.alpha)
             else:
                 self._com = 1.0

pandas/core/window/rolling.py

@@ -56,7 +56,7 @@
     DataError,
     SelectionMixin,
 )
-import pandas.core.common as common
+import pandas.core.common as com
 from pandas.core.indexes.api import (
     Index,
     MultiIndex,
@@ -643,7 +643,7 @@ def _apply_pairwise(
             )
 
             gb_pairs = (
-                common.maybe_make_list(pair) for pair in self._grouper.indices.keys()
+                com.maybe_make_list(pair) for pair in self._grouper.indices.keys()
             )
             groupby_codes = []
             groupby_levels = []

pandas/tests/api/test_api.py

@@ -275,7 +275,7 @@ class TestTesting(Base):
     ]
 
     def test_testing(self):
-        from pandas import testing
+        from pandas import testing  # noqa: PDF015
 
         self.check(testing, self.funcs)
 

pandas/tests/indexes/object/test_astype.py

@@ -1,5 +1,5 @@
 from pandas import Index
-import pandas.testing as tm
+import pandas._testing as tm
 
 
 def test_astype_str_from_bytes():

pandas/tests/io/pytables/common.py

@@ -16,7 +16,7 @@
 def safe_remove(path):
     if path is not None:
         try:
-            os.remove(path)
+            os.remove(path)  # noqa: PDF008
         except OSError:
             pass
 

pandas/tests/io/pytables/test_store.py

@@ -911,7 +911,7 @@ def do_copy(f, new_f=None, keys=None, propindexes=True, **kwargs):
                     os.close(fd)
                 except (OSError, ValueError):
                     pass
-                os.remove(new_f)
+                os.remove(new_f)  # noqa: PDF008
 
         # new table
         df = tm.makeDataFrame()

pandas/tests/tools/test_to_numeric.py

@@ -232,9 +232,7 @@ def test_type_check(errors):
     # see gh-11776
     df = DataFrame({"a": [1, -3.14, 7], "b": ["4", "5", "6"]})
     kwargs = {"errors": errors} if errors is not None else {}
-    error_ctx = pytest.raises(TypeError, match="1-d array")
-
-    with error_ctx:
+    with pytest.raises(TypeError, match="1-d array"):
         to_numeric(df, **kwargs)
 
 

requirements-dev.txt

@@ -80,3 +80,4 @@ tabulate>=0.8.3
 natsort
 git+https://github.com/pydata/pydata-sphinx-theme.git@master
 numpydoc < 1.2
+pandas-dev-flaker==0.1.2