Skip to content

Commit f6da1f1

Browse files
datapythonistadatapythonista
authored
STYLE: Fixing and refactoring linting (#22863)
1 parent 574eb75 commit f6da1f1

17 files changed

+203
-322
lines changed

‎.pep8speaks.yml

+11-2
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,18 @@
33
scanner:
44
diff_only: True # If True, errors caused by only the patch are shown
55

6+
# Opened issue in pep8speaks, so we can directly use the config in setup.cfg
7+
# (and avoid having to duplicate it here):
8+
# https://github.com/OrkoHunter/pep8speaks/issues/95
9+
610
pycodestyle:
711
max-line-length: 79
8-
ignore: # Errors and warnings to ignore
12+
ignore:
13+
- W503, # line break before binary operator
914
- E402, # module level import not at top of file
15+
- E722, # do not use bare except
1016
- E731, # do not assign a lambda expression, use a def
11-
- W503 # line break before binary operator
17+
- E741, # ambiguous variable name 'l'
18+
- C406, # Unnecessary list literal - rewrite as a dict literal.
19+
- C408, # Unnecessary dict call - rewrite as a literal.
20+
- C409 # Unnecessary list passed to tuple() - rewrite as a tuple literal.

‎.travis.yml

+3-8
Original file line numberDiff line numberDiff line change
@@ -45,15 +45,14 @@ matrix:
4545
- language-pack-zh-hans
4646
- dist: trusty
4747
env:
48-
- JOB="2.7, lint" ENV_FILE="ci/travis-27.yaml" TEST_ARGS="--skip-slow" LINT=true
48+
- JOB="2.7" ENV_FILE="ci/travis-27.yaml" TEST_ARGS="--skip-slow"
4949
addons:
5050
apt:
5151
packages:
5252
- python-gtk2
5353
- dist: trusty
5454
env:
55-
- JOB="3.6, coverage" ENV_FILE="ci/travis-36.yaml" TEST_ARGS="--skip-slow --skip-network" PANDAS_TESTING_MODE="deprecate" COVERAGE=true DOCTEST=true
56-
55+
- JOB="3.6, lint, coverage" ENV_FILE="ci/travis-36.yaml" TEST_ARGS="--skip-slow --skip-network" PANDAS_TESTING_MODE="deprecate" COVERAGE=true LINT=true
5756
- dist: trusty
5857
env:
5958
- JOB="3.7, NumPy dev" ENV_FILE="ci/travis-37-numpydev.yaml" TEST_ARGS="--skip-slow --skip-network -W error" PANDAS_TESTING_MODE="deprecate"
@@ -109,11 +108,7 @@ script:
109108
- ci/run_build_docs.sh
110109
- ci/script_single.sh
111110
- ci/script_multi.sh
112-
- ci/lint.sh
113-
- ci/doctests.sh
114-
- echo "checking imports"
115-
- source activate pandas && python ci/check_imports.py
116-
- echo "script done"
111+
- ci/code_checks.sh
117112

118113
after_success:
119114
- ci/upload_coverage.sh

‎ci/check_imports.py

-37
This file was deleted.

‎ci/code_checks.sh

+145
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
#!/bin/bash
2+
#
3+
# Run checks related to code quality.
4+
#
5+
# This script is intended for both the CI and to check locally that code standards are
6+
# respected. We are currently linting (PEP-8 and similar), looking for patterns of
7+
# common mistakes (sphinx directives with missing blank lines, old style classes,
8+
# unwanted imports...), and we also run doctests here (currently some files only).
9+
# In the future we may want to add the validation of docstrings and other checks here.
10+
#
11+
# Usage:
12+
# $ ./ci/code_checks.sh # run all checks
13+
# $ ./ci/code_checks.sh lint # run linting only
14+
# $ ./ci/code_checks.sh patterns # check for patterns that should not exist
15+
# $ ./ci/code_checks.sh doctests # run doctests
16+
17+
echo "inside $0"
18+
[[ $LINT ]] || { echo "NOT Linting. To lint use: LINT=true $0 $1"; exit 0; }
19+
[[ -z "$1" || "$1" == "lint" || "$1" == "patterns" || "$1" == "doctests" ]] || { echo "Unkown command $1. Usage: $0 [lint|patterns|doctests]"; exit 9999; }
20+
21+
source activate pandas
22+
RET=0
23+
CHECK=$1
24+
25+
26+
### LINTING ###
27+
if [[ -z "$CHECK" || "$CHECK" == "lint" ]]; then
28+
29+
# `setup.cfg` contains the list of error codes that are being ignored in flake8
30+
31+
echo "flake8 --version"
32+
flake8 --version
33+
34+
# pandas/_libs/src is C code, so no need to search there.
35+
MSG='Linting .py code' ; echo $MSG
36+
flake8 .
37+
RET=$(($RET + $?)) ; echo $MSG "DONE"
38+
39+
MSG='Linting .pyx code' ; echo $MSG
40+
flake8 pandas --filename=*.pyx --select=E501,E302,E203,E111,E114,E221,E303,E128,E231,E126,E265,E305,E301,E127,E261,E271,E129,W291,E222,E241,E123,F403,C400,C401,C402,C403,C404,C405,C406,C407,C408,C409,C410,C411
41+
RET=$(($RET + $?)) ; echo $MSG "DONE"
42+
43+
MSG='Linting .pxd and .pxi.in' ; echo $MSG
44+
flake8 pandas/_libs --filename=*.pxi.in,*.pxd --select=E501,E302,E203,E111,E114,E221,E303,E231,E126,F403
45+
RET=$(($RET + $?)) ; echo $MSG "DONE"
46+
47+
# readability/casting: Warnings about C casting instead of C++ casting
48+
# runtime/int: Warnings about using C number types instead of C++ ones
49+
# build/include_subdir: Warnings about prefacing included header files with directory
50+
51+
# We don't lint all C files because we don't want to lint any that are built
52+
# from Cython files nor do we want to lint C files that we didn't modify for
53+
# this particular codebase (e.g. src/headers, src/klib, src/msgpack). However,
54+
# we can lint all header files since they aren't "generated" like C files are.
55+
MSG='Linting .c and .h' ; echo $MSG
56+
cpplint --quiet --extensions=c,h --headers=h --recursive --filter=-readability/casting,-runtime/int,-build/include_subdir pandas/_libs/src/*.h pandas/_libs/src/parser pandas/_libs/ujson pandas/_libs/tslibs/src/datetime
57+
RET=$(($RET + $?)) ; echo $MSG "DONE"
58+
59+
fi
60+
61+
### PATTERNS ###
62+
if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
63+
64+
# Check for imports from pandas.core.common instead of `import pandas.core.common as com`
65+
MSG='Check for non-standard imports' ; echo $MSG
66+
! grep -R --include="*.py*" -E "from pandas.core.common import " pandas
67+
RET=$(($RET + $?)) ; echo $MSG "DONE"
68+
69+
MSG='Check for pytest warns' ; echo $MSG
70+
! grep -r -E --include '*.py' 'pytest\.warns' pandas/tests/
71+
RET=$(($RET + $?)) ; echo $MSG "DONE"
72+
73+
# Check for the following code in testing: `np.testing` and `np.array_equal`
74+
MSG='Check for invalid testing' ; echo $MSG
75+
! grep -r -E --include '*.py' --exclude testing.py '(numpy|np)(\.testing|\.array_equal)' pandas/tests/
76+
RET=$(($RET + $?)) ; echo $MSG "DONE"
77+
78+
# Check for the following code in the extension array base tests: `tm.assert_frame_equal` and `tm.assert_series_equal`
79+
MSG='Check for invalid EA testing' ; echo $MSG
80+
! grep -r -E --include '*.py' --exclude base.py 'tm.assert_(series|frame)_equal' pandas/tests/extension/base
81+
RET=$(($RET + $?)) ; echo $MSG "DONE"
82+
83+
MSG='Check for deprecated messages without sphinx directive' ; echo $MSG
84+
! grep -R --include="*.py" --include="*.pyx" -E "(DEPRECATED|DEPRECATE|Deprecated)(:|,|\.)" pandas
85+
RET=$(($RET + $?)) ; echo $MSG "DONE"
86+
87+
MSG='Check for old-style classes' ; echo $MSG
88+
! grep -R --include="*.py" -E "class\s\S*[^)]:" pandas scripts
89+
RET=$(($RET + $?)) ; echo $MSG "DONE"
90+
91+
MSG='Check for backticks incorrectly rendering because of missing spaces' ; echo $MSG
92+
! grep -R --include="*.rst" -E "[a-zA-Z0-9]\`\`?[a-zA-Z0-9]" doc/source/
93+
RET=$(($RET + $?)) ; echo $MSG "DONE"
94+
95+
MSG='Check for incorrect sphinx directives' ; echo $MSG
96+
! grep -R --include="*.py" --include="*.pyx" --include="*.rst" -E "\.\. (autosummary|contents|currentmodule|deprecated|function|image|important|include|ipython|literalinclude|math|module|note|raw|seealso|toctree|versionadded|versionchanged|warning):[^:]" ./pandas ./doc/source
97+
RET=$(($RET + $?)) ; echo $MSG "DONE"
98+
99+
MSG='Check for modules that pandas should not import' ; echo $MSG
100+
python -c "
101+
import sys
102+
import pandas
103+
104+
blacklist = {'bs4', 'gcsfs', 'html5lib', 'ipython', 'jinja2' 'hypothesis',
105+
'lxml', 'numexpr', 'openpyxl', 'py', 'pytest', 's3fs', 'scipy',
106+
'tables', 'xlrd', 'xlsxwriter', 'xlwt'}
107+
mods = blacklist & set(m.split('.')[0] for m in sys.modules)
108+
if mods:
109+
sys.stderr.write('pandas should not import: {}\n'.format(', '.join(mods)))
110+
sys.exit(len(mods))
111+
"
112+
RET=$(($RET + $?)) ; echo $MSG "DONE"
113+
114+
fi
115+
116+
### DOCTESTS ###
117+
if [[ -z "$CHECK" || "$CHECK" == "doctests" ]]; then
118+
119+
MSG='Doctests frame.py' ; echo $MSG
120+
pytest --doctest-modules -v pandas/core/frame.py \
121+
-k"-axes -combine -itertuples -join -nlargest -nsmallest -nunique -pivot_table -quantile -query -reindex -reindex_axis -replace -round -set_index -stack -to_dict -to_stata"
122+
RET=$(($RET + $?)) ; echo $MSG "DONE"
123+
124+
MSG='Doctests series.py' ; echo $MSG
125+
pytest --doctest-modules -v pandas/core/series.py \
126+
-k"-nonzero -reindex -searchsorted -to_dict"
127+
RET=$(($RET + $?)) ; echo $MSG "DONE"
128+
129+
MSG='Doctests generic.py' ; echo $MSG
130+
pytest --doctest-modules -v pandas/core/generic.py \
131+
-k"-_set_axis_name -_xs -describe -droplevel -groupby -interpolate -pct_change -pipe -reindex -reindex_axis -resample -to_json -transpose -values -xs"
132+
RET=$(($RET + $?)) ; echo $MSG "DONE"
133+
134+
MSG='Doctests top-level reshaping functions' ; echo $MSG
135+
pytest --doctest-modules -v \
136+
pandas/core/reshape/concat.py \
137+
pandas/core/reshape/pivot.py \
138+
pandas/core/reshape/reshape.py \
139+
pandas/core/reshape/tile.py \
140+
-k"-crosstab -pivot_table -cut"
141+
RET=$(($RET + $?)) ; echo $MSG "DONE"
142+
143+
fi
144+
145+
exit $RET

‎ci/doctests.sh

-60
This file was deleted.

0 commit comments

Comments
 (0)