Revert "STYLE use pandas-dev-flaker (#40906) (#50519)

* Revert "STYLE use pandas-dev-flaker (#40906)"

This reverts commit 1f27ed0.

* fixups

* fixup?

* make _compressors public

* use typing.Callable

* reduce diff

* further reduce

* one more

* fixup regex

* fixup regex

* add xfail to request

* remove else after xfail mark

* make copy_on_write public

Co-authored-by: MarcoGorelli <>

Author: MarcoGorelli

.pre-commit-config.yaml

@@ -70,12 +70,9 @@ repos:
     rev: 6.0.0
     hooks:
     -   id: flake8
-        # Need to patch os.remove rule in pandas-dev-flaker
-        exclude: ^ci/fix_wheels.py
         additional_dependencies: &flake8_dependencies
         - flake8==6.0.0
         - flake8-bugbear==22.7.1
-        - pandas-dev-flaker==0.5.0
 -   repo: https://github.com/pycqa/pylint
     rev: v2.15.9
     hooks:
@@ -183,6 +180,21 @@ repos:
         types: [rst]
         args: [--filename=*.rst]
         additional_dependencies: [flake8-rst==0.7.0, flake8==3.7.9]
+    -   id: inconsistent-namespace-usage
+        name: 'Check for inconsistent use of pandas namespace'
+        entry: python scripts/check_for_inconsistent_pandas_namespace.py
+        exclude: ^pandas/core/interchange/
+        language: python
+        types: [python]
+    -   id: no-os-remove
+        name: Check code for instances of os.remove
+        entry: os\.remove
+        language: pygrep
+        types: [python]
+        files: ^pandas/tests/
+        exclude: |
+            (?x)^
+            pandas/tests/io/pytables/test_store\.py$
     -   id: unwanted-patterns
         name: Unwanted patterns
         language: pygrep
@@ -192,6 +204,20 @@ repos:
             \#\ type:\ (?!ignore)
             |\#\ type:\s?ignore(?!\[)
 
+            # foo._class__ instead of type(foo)
+            |\.__class__
+
+            # np.bool/np.object instead of np.bool_/np.object_
+            |np\.bool[^_8`]
+            |np\.object[^_8`]
+
+            # imports from collections.abc instead of `from collections import abc`
+            |from\ collections\.abc\ import
+
+            # Numpy
+            |from\ numpy\ import\ random
+            |from\ numpy\.random\ import
+
             # Incorrect code-block / IPython directives
             |\.\.\ code-block\ ::
             |\.\.\ ipython\ ::
@@ -200,7 +226,17 @@ repos:
 
             # Check for deprecated messages without sphinx directive
             |(DEPRECATED|DEPRECATE|Deprecated)(:|,|\.)
+
+            # {foo!r} instead of {repr(foo)}
+            |!r}
+
+            # builtin filter function
+            |(?<!def)[\(\s]filter\(
+
+            # exec
+            |[^a-zA-Z0-9_]exec\(
         types_or: [python, cython, rst]
+        exclude: ^doc/source/development/code_style\.rst  # contains examples of patterns to avoid
     -   id: cython-casting
         name: Check Cython casting is `<type>obj`, not `<type> obj`
         language: pygrep
@@ -231,6 +267,58 @@ repos:
         files: ^pandas/tests/extension/base
         types: [python]
         exclude: ^pandas/tests/extension/base/base\.py
+    -   id: unwanted-patterns-in-tests
+        name: Unwanted patterns in tests
+        language: pygrep
+        entry: |
+            (?x)
+            # pytest.xfail instead of pytest.mark.xfail
+            pytest\.xfail
+
+            # imports from pandas._testing instead of `import pandas._testing as tm`
+            |from\ pandas\._testing\ import
+            |from\ pandas\ import\ _testing\ as\ tm
+
+            # No direct imports from conftest
+            |conftest\ import
+            |import\ conftest
+
+            # pandas.testing instead of tm
+            |pd\.testing\.
+
+            # pd.api.types instead of from pandas.api.types import ...
+            |(pd|pandas)\.api\.types\.
+
+            # np.testing, np.array_equal
+            |(numpy|np)(\.testing|\.array_equal)
+
+            # unittest.mock (use pytest builtin monkeypatch fixture instead)
+            |(unittest(\.| import )mock|mock\.Mock\(\)|mock\.patch)
+
+            # pytest raises without context
+            |\s\ pytest.raises
+
+            # pytest.warns (use tm.assert_produces_warning instead)
+            |pytest\.warns
+        files: ^pandas/tests/
+        types_or: [python, cython, rst]
+    -   id: unwanted-patterns-in-ea-tests
+        name: Unwanted patterns in EA tests
+        language: pygrep
+        entry: |
+            (?x)
+            tm.assert_(series|frame)_equal
+        files: ^pandas/tests/extension/base/
+        exclude: ^pandas/tests/extension/base/base\.py$
+        types_or: [python, cython, rst]
+    -   id: unwanted-patterns-in-cython
+        name: Unwanted patterns in Cython code
+        language: pygrep
+        entry: |
+            (?x)
+            # `<type>obj` as opposed to `<type> obj`
+            [a-zA-Z0-9*]>[ ]
+        types: [cython]
     -   id: pip-to-conda
         name: Generate pip dependency from conda
         language: python
@@ -251,6 +339,38 @@ repos:
         language: python
         types: [rst]
         files: ^doc/source/(development|reference)/
+    -   id: unwanted-patterns-bare-pytest-raises
+        name: Check for use of bare pytest raises
+        language: python
+        entry: python scripts/validate_unwanted_patterns.py --validation-type="bare_pytest_raises"
+        types: [python]
+        files: ^pandas/tests/
+        exclude: ^pandas/tests/extension/
+    -   id: unwanted-patterns-private-function-across-module
+        name: Check for use of private functions across modules
+        language: python
+        entry: python scripts/validate_unwanted_patterns.py --validation-type="private_function_across_module"
+        types: [python]
+        exclude: ^(asv_bench|pandas/tests|doc)/
+    -   id: unwanted-patterns-private-import-across-module
+        name: Check for import of private attributes across modules
+        language: python
+        entry: python scripts/validate_unwanted_patterns.py --validation-type="private_import_across_module"
+        types: [python]
+        exclude: |
+            (?x)
+            ^(asv_bench|pandas/tests|doc)/
+            |scripts/validate_min_versions_in_sync\.py$
+    -   id: unwanted-patterns-strings-to-concatenate
+        name: Check for use of not concatenated strings
+        language: python
+        entry: python scripts/validate_unwanted_patterns.py --validation-type="strings_to_concatenate"
+        types_or: [python, cython]
+    -   id: unwanted-patterns-strings-with-misplaced-whitespace
+        name: Check for strings with misplaced spaces
+        language: python
+        entry: python scripts/validate_unwanted_patterns.py --validation-type="strings_with_wrong_placed_whitespace"
+        types_or: [python, cython]
     -   id: use-pd_array-in-core
         name: Import pandas.array as pd_array in core
         language: python

asv_bench/benchmarks/pandas_vb_common.py

@@ -70,7 +70,7 @@ class BaseIO:
     def remove(self, f):
         """Remove created files"""
         try:
-            os.remove(f)  # noqa: PDF008
+            os.remove(f)
         except OSError:
             # On Windows, attempting to remove a file that is in use
             # causes an exception to be raised

doc/scripts/eval_performance.py

@@ -6,8 +6,7 @@
 from pandas import DataFrame
 
 setup_common = """from pandas import DataFrame
-from numpy.random import randn
-df = DataFrame(randn(%d, 3), columns=list('abc'))
+df = DataFrame(np.random.randn(%d, 3), columns=list('abc'))
 %s"""
 
 setup_with = "s = 'a + b * (c ** 2 + b ** 2 - a) / (a * c) ** 3'"

doc/source/development/contributing_codebase.rst

@@ -43,7 +43,7 @@ Pre-commit
 ----------
 
 Additionally, :ref:`Continuous Integration <contributing.ci>` will run code formatting checks
-like ``black``, ``flake8`` (including a `pandas-dev-flaker <https://github.com/pandas-dev/pandas-dev-flaker>`_ plugin),
+like ``black``, ``flake8``,
 ``isort``, and ``cpplint`` and more using `pre-commit hooks <https://pre-commit.com/>`_
 Any warnings from these checks will cause the :ref:`Continuous Integration <contributing.ci>` to fail; therefore,
 it is helpful to run the check yourself before submitting code. This

doc/source/whatsnew/v1.4.0.rst

@@ -320,7 +320,7 @@ Null-values are no longer coerced to NaN-value in value_counts and mode
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 :meth:`Series.value_counts` and :meth:`Series.mode` no longer coerce ``None``,
-``NaT`` and other null-values to a NaN-value for ``np.object``-dtype. This
+``NaT`` and other null-values to a NaN-value for ``np.object_``-dtype. This
 behavior is now consistent with ``unique``, ``isin`` and others
 (:issue:`42688`).
 

environment.yml

@@ -90,7 +90,6 @@ dependencies:
   - gitdb
   - natsort  # DataFrame.sort_values doctest
   - numpydoc
-  - pandas-dev-flaker=0.5.0
   - pydata-sphinx-theme<0.11
   - pytest-cython  # doctest
   - sphinx

pandas/__init__.py

@@ -135,7 +135,7 @@
 )
 
 from pandas import api, arrays, errors, io, plotting, tseries
-from pandas import testing  # noqa:PDF015
+from pandas import testing
 from pandas.util._print_versions import show_versions
 
 from pandas.io.api import (

pandas/_libs/lib.pyx

@@ -1482,7 +1482,7 @@ def infer_dtype(value: object, skipna: bool = True) -> str:
         return val
 
     if values.descr.type_num != NPY_OBJECT:
-        # i.e. values.dtype != np.object
+        # i.e. values.dtype != np.object_
         # This should not be reached
         values = values.astype(object)
 

pandas/_testing/__init__.py

@@ -886,7 +886,7 @@ def external_error_raised(expected_exception: type[Exception]) -> ContextManager
     """
     import pytest
 
-    return pytest.raises(expected_exception, match=None)  # noqa: PDF010
+    return pytest.raises(expected_exception, match=None)
 
 
 cython_table = pd.core.common._cython_table.items()

pandas/compat/__init__.py

@@ -14,14 +14,14 @@
 import sys
 
 from pandas._typing import F
-import pandas.compat._compressors
 from pandas.compat._constants import (
     IS64,
     PY39,
     PY310,
     PY311,
     PYPY,
 )
+import pandas.compat.compressors
 from pandas.compat.numpy import (
     is_numpy_dev,
     np_version_under1p21,
@@ -131,7 +131,7 @@ def is_ci_environment() -> bool:
     return os.environ.get("PANDAS_CI", "0") == "1"
 
 
-def get_lzma_file() -> type[pandas.compat._compressors.LZMAFile]:
+def get_lzma_file() -> type[pandas.compat.compressors.LZMAFile]:
     """
     Importing the `LZMAFile` class from the `lzma` module.
 
@@ -145,13 +145,13 @@ def get_lzma_file() -> type[pandas.compat._compressors.LZMAFile]:
     RuntimeError
         If the `lzma` module was not imported correctly, or didn't exist.
     """
-    if not pandas.compat._compressors.has_lzma:
+    if not pandas.compat.compressors.has_lzma:
         raise RuntimeError(
             "lzma module not available. "
             "A Python re-install with the proper dependencies, "
             "might be required to solve this issue."
         )
-    return pandas.compat._compressors.LZMAFile
+    return pandas.compat.compressors.LZMAFile
 
 
 __all__ = [

pandas/compat/_compressors.py pandas/compat/compressors.py

@@ -1,8 +1,10 @@
 from __future__ import annotations
 
-from collections.abc import Callable  # noqa: PDF001
 import re
-from typing import Union
+from typing import (
+    Callable,
+    Union,
+)
 
 import numpy as np
 

pandas/core/arrays/string_arrow.py

@@ -133,11 +133,11 @@
 from pandas.core import (
     algorithms as algos,
     arraylike,
+    common,
     indexing,
     nanops,
     sample,
 )
-from pandas.core import common  # noqa: PDF018
 from pandas.core.array_algos.replace import should_use_regex
 from pandas.core.arrays import ExtensionArray
 from pandas.core.base import PandasObject
@@ -158,7 +158,7 @@
     SingleArrayManager,
 )
 from pandas.core.internals.construction import mgr_to_mgr
-from pandas.core.internals.managers import _using_copy_on_write
+from pandas.core.internals.managers import using_copy_on_write
 from pandas.core.methods.describe import describe_ndframe
 from pandas.core.missing import (
     clean_fill_method,
@@ -10102,7 +10102,7 @@ def truncate(
         if isinstance(ax, MultiIndex):
             setattr(result, self._get_axis_name(axis), ax.truncate(before, after))
 
-        if copy or (copy is None and not _using_copy_on_write()):
+        if copy or (copy is None and not using_copy_on_write()):
             result = result.copy(deep=copy)
 
         return result

pandas/core/generic.py

@@ -7,8 +7,10 @@
 from pandas.core.interchange.dataframe_protocol import DataFrame as DataFrameXchg
 
 if TYPE_CHECKING:
-    import pandas as pd
-    from pandas import Index
+    from pandas import (
+        DataFrame,
+        Index,
+    )
 
 
 class PandasDataFrameXchg(DataFrameXchg):
@@ -21,7 +23,7 @@ class PandasDataFrameXchg(DataFrameXchg):
     """
 
     def __init__(
-        self, df: pd.DataFrame, nan_as_null: bool = False, allow_copy: bool = True
+        self, df: DataFrame, nan_as_null: bool = False, allow_copy: bool = True
     ) -> None:
         """
         Constructor - an instance of this (private) class is returned from