Merge pull request HypothesisWorks#1014 from HypothesisWorks/fast-builds

Skip running tests if we're sure there aren't test relevant changes

Author: alexwlchan

.travis.yml

@@ -67,7 +67,7 @@ env:
         - TASK=deploy
 
 script:
-    - make $TASK
+    - python scripts/run_travis_make_task.py
 
 matrix:
     fast_finish: true

circle.yml

@@ -1,6 +1,6 @@
 test:
   override:
-    - scripts/run-circle.sh
+    - scripts/run_circle.py
 
 machine:
   # Courtesy of https://pewpewthespells.com/blog/building_python_on_circleci.html

scripts/hypothesistooling.py

@@ -327,3 +327,76 @@ def update_for_pending_release():
         'commit', '-m',
         'Bump version to %s and update changelog\n\n[skip ci]' % (__version__,)
     )
+
+
+def could_affect_tests(path):
+    """Does this file have any effect on test results?"""
+    # RST files are the input to some tests -- in particular, the
+    # documentation build and doctests.  Both of those jobs are always run,
+    # so we can ignore their effect here.
+    #
+    # IPython notebooks aren't currently used in any tests.
+    if path.endswith(('.rst', '.ipynb')):
+        return False
+
+    # These files exist but have no effect on tests.
+    if path in ('CITATION', 'LICENSE.txt', ):
+        return False
+
+    # We default to marking a file "interesting" unless we know otherwise --
+    # it's better to run tests that could have been skipped than skip tests
+    # when they needed to be run.
+    return True
+
+
+def changed_files_from_master():
+    """Returns a list of files which have changed between a branch and
+    master."""
+    files = set()
+    command = ['git', 'diff', '--name-only', 'HEAD', 'master']
+    diff_output = subprocess.check_output(command).decode('ascii')
+    for line in diff_output.splitlines():
+        filepath = line.strip()
+        if filepath:
+            files.add(filepath)
+    return files
+
+
+def should_run_ci_task(task, is_pull_request):
+    """Given a task name, should we run this task?"""
+    if not is_pull_request:
+        print('We only skip tests if the job is a pull request.')
+        return True
+
+    # These tests are usually fast; we always run them rather than trying
+    # to keep up-to-date rules of exactly which changed files mean they
+    # should run.
+    if task in [
+        'check-pyup-yml',
+        'check-release-file',
+        'check-shellcheck',
+        'documentation',
+        'lint',
+    ]:
+        print('We always run the %s task.' % task)
+        return True
+
+    # The remaining tasks are all some sort of test of Hypothesis
+    # functionality.  Since it's better to run tests when we don't need to
+    # than skip tests when it was important, we remove any files which we
+    # know are safe to ignore, and run tests if there's anything left.
+    changed_files = changed_files_from_master()
+
+    interesting_changed_files = [
+        f for f in changed_files if could_affect_tests(f)
+    ]
+
+    if interesting_changed_files:
+        print(
+            'Changes to the following files mean we need to run tests: %s' %
+            ', '.join(interesting_changed_files)
+        )
+        return True
+    else:
+        print('There are no changes which would need a test run.')
+        return False

scripts/run-circle.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+
+# coding=utf-8
+#
+# This file is part of Hypothesis, which may be found at
+# https://github.com/HypothesisWorks/hypothesis-python
+#
+# Most of this work is copyright (C) 2013-2017 David R. MacIver
+# (david@drmaciver.com), but it contains contributions by others. See
+# CONTRIBUTING.rst for a full list of people who may hold copyright, and
+# consult the git log if you need to determine who owns an individual
+# contribution.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public License,
+# v. 2.0. If a copy of the MPL was not distributed with this file, You can
+# obtain one at http://mozilla.org/MPL/2.0/.
+#
+# END HEADER
+
+from __future__ import division, print_function, absolute_import
+
+import os
+import sys
+import subprocess
+
+from hypothesistooling import should_run_ci_task
+
+if __name__ == '__main__':
+
+    if (
+        os.environ['CIRCLE_BRANCH'] != 'master' and
+        os.environ['CI_PULL_REQUEST'] == ''
+    ):
+        print('We only run CI builds on the master branch or in pull requests')
+        sys.exit(0)
+
+    is_pull_request = (os.environ['CI_PULL_REQUEST'] != '')
+
+    for task in ['check-pypy', 'check-py36', 'check-py27']:
+        if should_run_ci_task(task=task, is_pull_request=is_pull_request):
+            subprocess.check_call(['make', task])

scripts/run_circle.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+
+# coding=utf-8
+#
+# This file is part of Hypothesis, which may be found at
+# https://github.com/HypothesisWorks/hypothesis-python
+#
+# Most of this work is copyright (C) 2013-2017 David R. MacIver
+# (david@drmaciver.com), but it contains contributions by others. See
+# CONTRIBUTING.rst for a full list of people who may hold copyright, and
+# consult the git log if you need to determine who owns an individual
+# contribution.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public License,
+# v. 2.0. If a copy of the MPL was not distributed with this file, You can
+# obtain one at http://mozilla.org/MPL/2.0/.
+#
+# END HEADER
+
+from __future__ import division, print_function, absolute_import
+
+import os
+import subprocess
+
+from hypothesistooling import should_run_ci_task
+
+if __name__ == '__main__':
+    is_pull_request = (os.environ.get('TRAVIS_EVENT_TYPE') == 'pull_request')
+    task = os.environ['TASK']
+
+    if should_run_ci_task(task=task, is_pull_request=is_pull_request):
+        subprocess.check_call(['make', task])