From 983f1b2a49247982b837f2b7cdfb9ae6f1c6cca8 Mon Sep 17 00:00:00 2001 From: Adrien CABARBAYE Date: Thu, 9 Apr 2020 00:55:37 +0100 Subject: [PATCH 1/3] Refactoring --- .circleci/config.yml | 116 --------- .codeclimate.yml | 32 +++ .gitattributes | 1 + .github/ISSUE_TEMPLATE.md | 23 ++ .github/PULL_REQUEST_TEMPLATE.md | 17 ++ .gitignore | 124 ++------- .pre-commit-config.yaml | 52 ++++ CHANGELOG.md | 15 +- CODE_OF_CONDUCT.md | 84 ++++++ CONTRIBUTING.md | 141 ++++++---- DEVELOPMENT.md | 140 ++++++++++ KNOWN_ISSUES.md | 3 + LICENSE | 28 -- MANIFEST.in | 3 + Pipfile | 29 ++- Pipfile.lock | 0 README.md | 165 ++++++++---- azure-pipelines/build-release.yml | 240 ++++++++++++++++++ .../steps/determine-current-branch.yml | 10 + .../steps/generate-documentation.yml | 12 + .../steps/generate-spdx-documents.yml | 13 + .../install-development-dependencies.yml | 16 ++ azure-pipelines/steps/override-checkout.yml | 4 + .../steps/publish-code-coverage-results.yml | 8 + azure-pipelines/steps/tag-and-release.yml | 13 + codecov.yml | 33 +++ docs/index.html | 15 ++ docs/news/pyproject.toml | 9 - docs/news/template.rst | 34 --- news/20200409.major | 1 + news/README.md | 3 + pyproject.toml | 85 +++++++ scripts/autoversion.toml | 7 - scripts/tag_and_release.py | 90 ------- setup.cfg | 76 ++++-- setup.py | 60 +++-- snippet/__init__.py | 6 + snippet/__main__.py | 8 + snippet/_internal/__init__.py | 5 + .../_internal}/file_wrangler.py | 37 +-- snippet/_internal/logs.py | 8 + snippet/_internal/util.py | 13 + snippet/_internal/wrapper.py | 23 ++ snippet/_version.py | 17 ++ snippet/api.py | 21 ++ snippet/cli.py | 51 ++++ snippet/config.py | 124 +++++++++ snippet/exceptions.py | 41 +++ snippet/snippet.py | 192 ++++++++++++++ snippet/workflow.py | 71 ++++++ src/snippet/__init__.py | 17 -- src/snippet/__main__.py | 3 - src/snippet/_version.py | 4 - src/snippet/cli.py | 30 --- src/snippet/config.py | 91 ------- src/snippet/exceptions.py | 22 -- src/snippet/logs.py | 3 - src/snippet/snippet.py | 70 ----- src/snippet/util.py | 2 - src/snippet/workflow.py | 58 ----- src/snippet/wrapper.py | 16 -- tests/__init__.py | 13 +- tests/samples/config_fixture.md | 2 +- tests/samples/example.java | 4 + tests/samples/example.py | 11 +- tests/samples/fixture.md | 2 +- tests/test_config.py | 84 +++--- tests/test_direct.py | 37 +-- tests/test_parser.py | 175 ++++++------- tests/test_support.py | 16 +- tests/test_true.py | 11 + tests/test_workflow.py | 54 ++-- 72 files changed, 1977 insertions(+), 1067 deletions(-) delete mode 100644 .circleci/config.yml create mode 100644 .codeclimate.yml create mode 100644 .gitattributes create mode 100644 .github/ISSUE_TEMPLATE.md create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 .pre-commit-config.yaml create mode 100644 CODE_OF_CONDUCT.md create mode 100644 DEVELOPMENT.md create mode 100644 KNOWN_ISSUES.md create mode 100644 MANIFEST.in delete mode 100644 Pipfile.lock create mode 100644 azure-pipelines/build-release.yml create mode 100644 azure-pipelines/steps/determine-current-branch.yml create mode 100644 azure-pipelines/steps/generate-documentation.yml create mode 100644 azure-pipelines/steps/generate-spdx-documents.yml create mode 100644 azure-pipelines/steps/install-development-dependencies.yml create mode 100644 azure-pipelines/steps/override-checkout.yml create mode 100644 azure-pipelines/steps/publish-code-coverage-results.yml create mode 100644 azure-pipelines/steps/tag-and-release.yml create mode 100644 codecov.yml create mode 100644 docs/index.html delete mode 100644 docs/news/pyproject.toml delete mode 100644 docs/news/template.rst create mode 100644 news/20200409.major create mode 100644 news/README.md create mode 100644 pyproject.toml delete mode 100644 scripts/autoversion.toml delete mode 100644 scripts/tag_and_release.py create mode 100644 snippet/__init__.py create mode 100644 snippet/__main__.py create mode 100644 snippet/_internal/__init__.py rename {src/snippet => snippet/_internal}/file_wrangler.py (50%) create mode 100644 snippet/_internal/logs.py create mode 100644 snippet/_internal/util.py create mode 100644 snippet/_internal/wrapper.py create mode 100644 snippet/_version.py create mode 100644 snippet/api.py create mode 100644 snippet/cli.py create mode 100644 snippet/config.py create mode 100644 snippet/exceptions.py create mode 100644 snippet/snippet.py create mode 100644 snippet/workflow.py delete mode 100644 src/snippet/__init__.py delete mode 100644 src/snippet/__main__.py delete mode 100644 src/snippet/_version.py delete mode 100644 src/snippet/cli.py delete mode 100644 src/snippet/config.py delete mode 100644 src/snippet/exceptions.py delete mode 100644 src/snippet/logs.py delete mode 100644 src/snippet/snippet.py delete mode 100644 src/snippet/util.py delete mode 100644 src/snippet/workflow.py delete mode 100644 src/snippet/wrapper.py create mode 100644 tests/test_true.py diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index d1549d9..0000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,116 +0,0 @@ -# Python CircleCI 2.0 configuration file -# -# Check https://circleci.com/docs/2.0/language-python/ for more details -# -version: 2 - -install_dependencies: &install_dependencies - run: - name: install dependencies - command: | - python3 -m venv venv - . venv/bin/activate - sudo pip install -e . - sudo pip install setuptools green flake8 pyautoversion towncrier pytest twine==1.11 -jobs: - build: - docker: - # specify the version you desire here - # use `-browsers` prefix for selenium tests, e.g. `3.6.1-browsers` - - image: circleci/python:3.6.1 - - # Specify service dependencies here if necessary - # CircleCI maintains a library of pre-built images - # documented at https://circleci.com/docs/2.0/circleci-images/ - # - image: circleci/postgres:9.4 - - working_directory: ~/repo - - steps: - - checkout - - # Download and cache dependencies - - restore_cache: - keys: - - v1-dependencies-{{ checksum "requirements.txt" }} - # fallback to using the latest cache if no exact match is found - - v1-dependencies- - - - <<: *install_dependencies - - - save_cache: - paths: - - ./venv - key: v1-dependencies-{{ checksum "requirements.txt" }} - - # run tests! - # this example uses Django's built-in test-runner - # other common Python testing frameworks include pytest and nose - # https://pytest.org - # https://nose.readthedocs.io - - run: - name: run tests - command: | - . venv/bin/activate - pytest --junitxml=test-reports/results.xml - green - - - run: - name: run static analysis - command: | - . venv/bin/activate - flake8 - - - store_artifacts: - path: test-reports - destination: test-reports - - - store_test_results: - path: test-reports - - release: - docker: - - image: circleci/python:3.7.0 - working_directory: ~/repo - steps: - - checkout - - restore_cache: - keys: - - v2-dependencies-{{ checksum "requirements.txt" }} - # fallback to using the latest cache if no exact match is found - - v2-dependencies- - - <<: *install_dependencies - - run: - name: autoversion - command: | - autoversion --config=scripts/autoversion.toml --release --news - - run: - name: Generate changelog - command: towncrier --yes --name="" --version=$(cd ../../ && python setup.py --version) - working_directory: docs/news - - run: - name: Tag and Release - command: | - sudo -E python scripts/tag_and_release.py - - store_artifacts: - path: CHANGELOG.md - - save_cache: - paths: - - ./venv - key: v2-dependencies-{{ checksum "requirements.txt" }} - -workflows: - version: 2 - snippet_workflow: - jobs: - - build - - release_gate: - type: approval - filters: - branches: - only: master - requires: - - build - - release: - requires: - - release_gate diff --git a/.codeclimate.yml b/.codeclimate.yml new file mode 100644 index 0000000..b8edca9 --- /dev/null +++ b/.codeclimate.yml @@ -0,0 +1,32 @@ +version: "2" +checks: + argument-count: + config: + threshold: 4 + complex-logic: + config: + threshold: 4 + file-lines: + config: + threshold: 250 + method-complexity: + config: + threshold: 5 + method-count: + config: + threshold: 20 + method-lines: + config: + threshold: 25 + nested-control-flow: + config: + threshold: 4 + return-statements: + config: + threshold: 4 + similar-code: + config: + threshold: # language-specific defaults. an override will affect all languages. + identical-code: + config: + threshold: # language-specific defaults. an override will affect all languages. diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..176a458 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* text=auto diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000..688391a --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,23 @@ +### Description + + + + + +### Issue request type + + + +- [ ] Enhancement +- [ ] Bug diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..e1c570c --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,17 @@ +### Description + + + + + +### Test Coverage + + + +- [ ] This change is covered by existing or additional automated tests. +- [ ] Manual testing has been performed (and evidence provided) as automated testing was not feasible. +- [ ] Additional tests are not required for this change (e.g. documentation update). diff --git a/.gitignore b/.gitignore index 8e798d5..d86a058 100644 --- a/.gitignore +++ b/.gitignore @@ -1,110 +1,30 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class +# Don't lock the versions of this library or installation gridlock will ensue +Pipfile.lock -# C extensions -*.so +# PyCharm +.idea/ -# Distribution / packaging -.Python -env/ -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -*.egg-info/ -.installed.cfg -*.egg +# macOS +.DS_Store -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt +# Python +*.pyc +__pycache__/ +*.egg-info/ -# Unit test / coverage reports -htmlcov/ -.tox/ +# Coverage.py .coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -.hypothesis/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# pyenv -.python-version - -# celery beat schedule file -celerybeat-schedule - -# SageMath parsed files -*.sage.py - -# dotenv -.env - -# virtualenv -.venv -venv/ -ENV/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ +coverage/ +junit/ +htmlcov/ -# pytest -*.pytest* +# Package +build/ +dist/ +release-dist/ -# editors -*.idea +# Temporary file used by CI +dev-requirements.txt -# test directory -tmp_test_dir +# Local docs output +local_docs/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..ab74be3 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,52 @@ +- repo: local + hooks: + - id: assertnews + name: news file + entry: assert-news -l + language: python + types: [file] + require_serial: true + verbose: true + always_run: true + pass_filenames: false + + - id: licensing + name: licensing + entry: license-files + language: python + types: [file] + require_serial: true + always_run: true + verbose: true + pass_filenames: false + + - id: black + name: black + entry: black + language: python + types: [python] + require_serial: true + + - id: flake8 + name: flake8 + entry: flake8 + language: python + types: [python] + require_serial: true + + - id: mypy + name: mypy + entry: mypy -p snippet + language: python + types: [python] + require_serial: true + pass_filenames: false + + - id: pytest + name: pytest + entry: pytest -vvv + language: python + types: [python] + pass_filenames: false + always_run: true + diff --git a/CHANGELOG.md b/CHANGELOG.md index 28c476a..c1b250d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,13 @@ # Changelog -> This file is autogenerated. -> Only edit this file directly to correct typos. -> See [CONTRIBUTING)[./CONTRIBUTING.md] for instructions on adding new entries. - -This news file contains a log of notable changes to `snippet`. Please see [code-snippet](https://pypi.org/project/code-snippet/#history>) for -a list of versions that have been released on PyPI. + + +This document contains a history of significant changes which have been released for `code-snippet`. Please note that +beta releases are not included in this history. For a full list of all releases, please see the +[PyPI Release History](https://pypi.org/project/code-snippet/#history). [//]: # (begin_release_notes) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..a632520 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,84 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience +* Focusing on what is best not just for us as individuals, but for the overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or + advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email + address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior and will take appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, and will communicate reasons for moderation decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces. Examples of representing our community include using an official e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at support@mbed.com. All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing clarity around the nature of the violation and an explanation of why the behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series of actions. + +**Consequence**: A warning with consequences for continued behavior. No interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, for a specified period of time. This includes avoiding interactions in community spaces as well as external channels like social media. Violating these terms may lead to a temporary or permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public communication with the community for a specified period of time. No public or private interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this period. Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community standards, including sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within the project community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.0, +available at https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. + +Community Impact Guidelines were inspired by [Mozilla's code of conduct enforcement ladder](https://github.com/mozilla/diversity). + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see the FAQ at +https://www.contributor-covenant.org/faq. Translations are available at https://www.contributor-covenant.org/translations. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b389840..7bac029 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,62 +1,101 @@ -# Contributing +# Contribution Guide -Snippet is an open source project from Arm Mbed. +Snippet is an open source project from Arm Mbed and we really appreciate your contributions to the tools. We are committed to +fostering a welcoming community, please see our Code of Conduct, which can be found here: -We really appreciate your contributions! You can contribute by letting us know -about any [issues](https://github.com/ARMmbed/snippet/issues) -you have found, or by creating a [pull request](https://github.com/ARMmbed/snippet/pulls) -with a bug fix or new feature you find necessary, important or just attractive. +- [Code of Conduct](./CODE_OF_CONDUCT.md) -## How to Contribute Code +There are several ways to contribute: -Please keep contributions small and independent. We would much rather have -multiple pull requests for each cool thing you've done rather than have them all -in the same one. This will help us review, give feedback and merge in your -changes. +- Raise an issue found via [GitHub Issues](https://github.com/ARMmbed/snippet/issues). +- Open an [pull request](https://github.com/ARMmbed/snippet/pulls) to: + - Provide a fix. + - Add an enhancement feature. + - Correct, update or add documentation. +- Answering community questions on the [Mbed Forum](https://forums.mbed.com/). -- Fork the repository. -- Make your change and write unit tests, please do match the existing coding - style. -- Write a [good commit message](http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html) - and be sure to mention the issue if contributing a bug fix. -- Push to your fork. -- Submit a pull request. +## How to Contribute Documentation or Code -We will review your proposal, give you feedback and merge your changes if we -feel your contribution is generally useful and meets our quality criteria. +Please keep contributions small and independent. We would much rather have multiple pull requests for each thing done +rather than have them all in the same one. This will help us review, give feedback and merge in the changes. The +normal process to make a change is as follows: + +1. Fork the repository. +2. Make your change and write unit tests, please try to match the existing documentation and coding style. +3. Add a news file describing the changes and add it in the `/news` directory, see the section _News Files_ below. +4. Write a [good commit message](http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html). +5. Push to your fork. +6. Submit a pull request. + +We will review the proposed change as soon as we can and, if needed, give feedback. Please bear in mind that the tools +for Mbed OS are complex and cover a large number of use cases. This means we may ask for changes not only to ensure +that the proposed change meets our quality criteria, but also to make sure the that the change is generally useful and +doesn't impact other uses cases or maintainability. + +### News Files + +News files serve a different purpose to commit messages, which are generally written to inform developers of the +project. News files will form part of the release notes so should be written to target the consumer of the package or +tool. + +- A news file should be added for each merge request to the directory `/news`. +- The text of the file should be a single line describing the change and/or impact to the user. +- The filename of the news file should take the form `.`, e.g, `20191231.feature` where: + - The number is either the issue number or, if no issue exists, the date in the form `YYYYMMDD`. + - The extension should indicate the type of change as described in the following table: + +| Change Type | Extension | Version Impact | +|-------------------------------------------------------------------------------------------------------------------------|------------|-----------------| +| Backwards compatibility breakages or significant changes denoting a shift direction. | `.major` | Major increment | +| New features and enhancements (non breaking). | `.feature` | Minor increment | +| Bug fixes or corrections (non breaking). | `.bugfix` | Patch increment | +| Documentation impacting the consumer of the package (not repo documentation, such as this file, for this use `.misc`). | `.doc` | N/A | +| Deprecation of functionality or interfaces (not actual removal, for this use `.major`). | `.removal` | None | +| Changes to the repository that do not impact functionality e.g. build scripts change. | `.misc` | None | + +### Commit Hooks + +We use [pre-commit](https://pre-commit.com/) to install and run commit hooks, mirroring the code checks we run in our CI +environment. + +The `pre-commit` tool allows developers to easily install git hook scripts which will run on every `git commit`. The +`.pre-commit-config.yaml` in our repository sets up commit hooks to run pytest, black, mypy and flake8. Those checks +must pass in our CI before a PR is merged. Using commit hooks ensures you can't commit code which violates our style +and maintainability requirements. + +To install the commit hooks for the repository, run `pipenv install --dev` then `pipenv shell`, in the `pipenv shell` +type `pre-commit install`, the checks will now run automatically every time you try to `git commit` to the repository. ## Merging the Pull Request -When merging the pull request we will give it a title which provides context to changes: -* `:: (#)` - -An emoji will be used to highlight what has occurred in the change: - -Emoji | GitHub Markdown | Topic(s) -------|-----------------|--------- -🏁 | `:checkered_flag:` | New release -🎁 | `:gift:` | Features / New good stuff -🔧 | `:wrench:` | Bug / Defect fixes -❌ | `:x:` | Removing features / Deprecation -🔒 | `:lock:` | Security -🚀 | `:rocket:` | Performance -💰 | `:moneybag:` | Technical debt -📖 | `:book:` | Documentation -🔃 | `:arrows_clockwise:` | Synchronising (normally between branches) -⭕️ | `:o:` | CircleCI / Build system - -Additional emojis which are more likely to be used in commits than in a merge: - -Emoji | GitHub Markdown | Topic(s) -------|-----------------|--------- -🌈 | `:rainbow:` | Linting and appearance fixes -📰 | `:newspaper:` | Newsfile (news snippet) -🚧 | `:construction:` | Work In Progress (WIP) -⬆️ | `:arrow_up:` |️ Upgrade dependency -⬇️ | `:arrow_down:` | Downgrade dependency - -For us to accept your code contributions, we will need you to agree to our [Mbed -Contributor Agreement](http://developer.mbed.org/contributor_agreement/) to give -us the necessary rights to use and distribute your contributions. +When merging the pull request we will normally squash merge the changes give it a title which provides context to +the changes: + +- ` (#)` + +An emoji is used to highlight what has occurred in the change. Commonly used emojis can be seen below, but for a full +list please see [Gitmoji](https://gitmoji.carloscuesta.me/): + +Emoji | Topic(s) +------|--------- +✨ | New features or enhancements. +🐛 | Bug / defect fixes. +🔒 | Fixing security issues. +⚡️ | Improving performance. +♻️ | Refactoring or addressing technical debt. +💥 | Breaking changes or removing functionality. +❗️ | Notice of deprecation. +📝 | Writing or updating documentation. +👷 | Adding to the CI or build system. +💚️ | Fixing CI or build system issues. +🚀 | Releasing or deploying. + +For more on the version number scheme please see the [ReadMe](./README.md). + +## One Last Thing... + +For us to accept your code contributions, we will need you to agree to our +[Mbed Contributor Agreement](https://os.mbed.com/contributor_agreement/) to give us the necessary rights to use and +distribute your contributions. Thank you :smiley: diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md new file mode 100644 index 0000000..091a1f9 --- /dev/null +++ b/DEVELOPMENT.md @@ -0,0 +1,140 @@ +# Development and Testing + +For development and testing purposes, it is essential to use a virtual environment. It is recommended that `pipenv` is used. + +## Setup Pipenv + +To start developing, install pip and pipenv on your system. Note the latter is done at user level to keep the system installation of python clean which is important on a Mac (at least): + +```bash +sudo easy_install pip +``` + +Install pipenv (the --user is important, do not use `sudo`) + +```bash +pip install --user pipenv +``` + +Check that pipenv is in the binary path + +```bash +pipenv --version +``` + +If not, find the user base binary directory + +```bash +python -m site --user-base +#~ /Users//Library/Python/3.7 +``` + +Append `bin` to the directory returned and add this to your path by updating `~/.profile`. For example you might add the following: + +```bash +export PATH=~/Library/Python/3.7/bin/:$PATH +``` + +## Setup Development Environment + +Clone GitHub repository + +```bash +git clone git@github.com:ARMmbed/snippet.git +``` + +Setup Pipenv to use Python 3 (Python 2 is not supported) and install package development dependencies: + +```bash +cd code-snippet/ +pipenv --three +pipenv install --dev +``` + +## Unit Tests, Code Formatting and Static Analysis + +Shell into virtual environment: + +```bash +pipenv shell +``` + +Run unit tests: + +```bash +pytest +``` +Note that other test runners can be used (e.g. [green](https://github.com/CleanCut/green)) +as long as they support test written using unittest.TestCase. + + +Run code formatter (it will format files in place): + +```bash +black . +``` + +Run static analysis (note that no output means all is well): + +```bash +flake8 +``` + +## Documenting code + +Inclusion of docstrings is needed in all areas of the code for Flake8 +checks in the CI to pass. + +We use [google-style](http://google.github.io/styleguide/pyguide.html#381-docstrings) +docstrings. + +To set up google-style docstring prompts in Pycharm, in the menu navigate to +Preferences > Tools > Python Integrated Tools and in the dropdown for docstring +format select 'Google'. + +For longer explanations, you can also include markdown. Markdown can also be +kept in separate files in the `docs/user_docs` folder and included in a docstring in the +relevant place using the [reST include](https://docutils.sourceforge.io/docs/ref/rst/directives.html#including-an-external-document-fragment) as follows: + +```python + .. include:: ../docs/user_docs/documentation.md +``` + +### Building docs locally + +You can do a preview build of the documentation locally by running: + +```bash +generate-docs +``` + +This will generate the docs and output them to `local_docs`. +This should only be a preview. Since documentation is automatically generated +by the CI you shouldn't commit any docs html files manually. + +### Viewing docs generated by the CI + +If you want to preview the docs generated by the CI you can view them in +the Azure pipeline artifacts directory for the build. + +Documentation only gets committed back to this repo to the `docs` +directory during a release and this is what gets published to Github pages. +Don't modify any of the files in this directory by hand. + +## Type hints + +Type hints should be used in the code wherever possible. Since the +documentation shows the function signatures with the type hints +there is no need to include additional type information in the docstrings. + + +## Code Climate + +Code Climate is integrated with our GitHub flow. Failing the configured rules will yield a pull request not mergeable. + +If you prefer to view the Code Climate report on your machine, prior to sending a pull request, you can use the [cli provided by Code Climate](https://docs.codeclimate.com/docs/command-line-interface). + +Plugins for various tools are also available: + - [Atom](https://docs.codeclimate.com/docs/code-climate-atom-package) + - [PyCharm](https://plugins.jetbrains.com/plugin/13306-code-cleaner-with-code-climate-cli) + - [Vim](https://docs.codeclimate.com/docs/vim-plugin) diff --git a/KNOWN_ISSUES.md b/KNOWN_ISSUES.md new file mode 100644 index 0000000..0a28d7c --- /dev/null +++ b/KNOWN_ISSUES.md @@ -0,0 +1,3 @@ +# Known Issues + +This project is a work-in-progress and we are not currently tracking issues. diff --git a/LICENSE b/LICENSE index d645695..dd5b3a5 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,3 @@ - Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -173,30 +172,3 @@ defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..18efb63 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,3 @@ +graft snippet +global-exclude *.py[cod] __pycache__ *.so +global-include *.spdx diff --git a/Pipfile b/Pipfile index f2ae490..9d399f0 100644 --- a/Pipfile +++ b/Pipfile @@ -1,18 +1,23 @@ [[source]] - -url = "https://pypi.python.org/simple" -verify_ssl = true name = "pypi" - +url = "https://pypi.org/simple" +verify_ssl = true [dev-packages] - -"flake8" = "*" -ipython = "*" +black = "*" coverage = "*" -green = "*" - - -[packages] +flake8 = "*" +flake8-docstrings = "*" +flake8-black = "*" +flake8-copyright = "*" +flake8-file-encoding = "*" +mypy = ">=0.500" +pytest = "*" +pytest-cov = "*" +wheel = "*" +code-snippet = {editable = true, path = "."} +mbed-tools-ci-scripts = "*" +pre-commit = "*" -"-e ." = "*" +[pipenv] +allow_prereleases = true diff --git a/Pipfile.lock b/Pipfile.lock deleted file mode 100644 index e69de29..0000000 diff --git a/README.md b/README.md index 0e8adda..5889c19 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,21 @@ -# snippet -[![CircleCI](https://circleci.com/gh/ARMmbed/snippet.svg?style=svg&circle-token=f8151197e9160de7877eda3ae049d0925e9b7ff3)](https://circleci.com/gh/ARMmbed/snippet) +# Snippet -A Python3 tool to extract code snippets from source files +![Package](https://img.shields.io/badge/Package-code--snippet-lightgrey) +[![Documentation](https://img.shields.io/badge/Documentation-GitHub_Pages-blue)](https://armmbed.github.io/code-snippet) +[![PyPI](https://img.shields.io/pypi/v/code-snippet)](https://pypi.org/project/code-snippet/) +[![PyPI - Status](https://img.shields.io/pypi/status/code-snippet)](https://pypi.org/project/code-snippet/) +[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/code-snippet)](https://pypi.org/project/code-snippet/) +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/ARMmbed/code-snippet/blob/master/LICENSE) -Essentially, `snippet` extracts marked sections of text from a given -set of input files and saves them elsewhere. +[![Build Status](https://dev.azure.com/mbed-tools/code-snippet/_apis/build/status/Build%20and%20Release?branchName=master&stageName=CI%20Checkpoint)](https://dev.azure.com/mbed-tools/code-snippet/_build/latest?definitionId=TODO_AZURE&branchName=master) +[![Test Coverage](https://codecov.io/gh/ARMmbed/code-snippet/branch/master/graph/badge.svg)](https://codecov.io/gh/ARMmbed/code-snippet) +[![Maintainability](https://api.codeclimate.com/v1/badges/2050e74c1c485109d357/maintainability)](https://codeclimate.com/github/ARMmbed/snippet/maintainability) + +## Overview + +A tool to extract code snippets from source files + +Essentially, snippet extracts marked sections of text from a given set of input files and saves them elsewhere. Features include: - Works on any text file, e.g. @@ -14,59 +25,121 @@ any coding language by reading from source files - Hides sections from output - Performs validation to help avoid snippets breaking as code changes -## Rationale -Code documentation usually needs a written example demonstrating use of some code. This example code can however become quite easily outdated as a project evolves or even contain its own errors. -One solution is to write the examples as tests which can be run within the test system of choice. This ensures that the code of the examples is always valid and working. `snippet` can then be used to extract the relevant and informative part of the test and put it in a form which can then be rendered by the documentation system, providing fully tested code examples. +## Rationale +Code documentation usually needs a written example demonstrating use of some code. This example code can however become quite easily outdated as a project evolves or even contain its own errors. +One solution is to write the examples as tests which can be run within the test system of choice. This ensures that the code of the examples is always valid and working. `snippet` can then be used to extract the relevant and informative part of the test and put it in a form which can then be rendered by the documentation system, providing fully tested code examples. + +## Releases + +For release notes and a history of changes of all **production** releases, please see the following: + +- [Changelog](https://github.com/ARMmbed/snippet/blob/master/CHANGELOG.md) + +For a the list of all available versions please, please see the: + +- [PyPI Release History](https://pypi.org/project/code-snippet/#history) + +## Versioning + +The version scheme used follows [PEP440](https://www.python.org/dev/peps/pep-0440/) and +[Semantic Versioning](https://semver.org/). For production quality releases the version will look as follows: + +- `..` + +Beta releases are used to give early access to new functionality, for testing and to get feedback on experimental +features. As such these releases may not be stable and should not be used for production. Additionally any interfaces +introduced in a beta release may be removed or changed without notice. For **beta** releases the version will look as +follows: + +- `..-beta.` + +## Installation -## Getting started -### Prerequisites -- `snippet` requires Python 3 +It is recommended that a virtual environment such as [Pipenv](https://github.com/pypa/pipenv/blob/master/README.md) is +used for all installations to avoid Python dependency conflicts. + +To install the most recent production quality release use: -### Installation ``` pip install code-snippet ``` -### Configuration -Place a config file in the [toml format](https://github.com/toml-lang/toml) -in your project directory (e.g. `snippet.toml`). Any value defined in [the config object](https://github.com/ARMmbed/snippet/blob/master/src/snippet/config.py#L8) -can be overridden. -As an example, basic configuration typically includes input and output directories: +To install a specific release: ``` -[snippet] -input_glob = 'tests/unit/*.py' -output_dir = 'docs/examples' +pip install code-snippet== ``` -### Run -Run the following command in your project, using the Python interpreter you installed `snippet` to: +## Usage +### Configuration +Place a config file in the [toml format](https://github.com/toml-lang/toml) +in your project directory (e.g. `snippet.toml`). Any value defined in [the config object](https://github.com/ARMmbed/snippet/blob/master/src/snippet/config.py#L8) +can be overridden. + +As an example, basic configuration typically includes input and output directories: + +``` +[snippet] +input_glob = 'tests/unit/*.py' +output_dir = 'docs/examples' +``` +### Run +Run the following command in your project: + +``` +snippet +``` + +Alternatively, run snippet from anywhere and specify a working directory and config file: +``` +snippet path/to/root --config=path/to/config.toml +``` +Config files can be specified as glob patterns, defaulting to `*.toml`, and can +be set multiple times. Multiple files will be loaded in the order specified +and discovered. Settings loaded last will take precedence. -``` -python -m snippet -``` +For more information about how to use the tool, please have a look at the [Usage page](./USAGE.md) +The full CLI options are: +``` +> snippet --help +usage: __main__.py [-h] [--config CONFIG] [-v] [dir] + +positional arguments: + dir path to project root, used by any relative paths in loaded + configs [cwd] + +optional arguments: + -h, --help show this help message and exit + --config CONFIG paths (or globs) to config files + -v, --verbosity increase output verbosity +``` -Alternatively, run snippet from anywhere and specify a working directory and config file: -``` -python -m snippet path/to/root --config=path/to/config.toml -``` -Config files can be specified as glob patterns, defaulting to `*.toml`, and can -be set multiple times. Multiple files will be loaded in the order specified -and discovered. Settings loaded last will take precedence. +Interface definition and usage documentation (for developers of tooling) is available for the most recent +production release here: -### Usage -For more information about how to use the tool, please have a look at the [Usage page](./USAGE.md) -The full CLI options are: -``` -> python -m snippet --help -usage: __main__.py [-h] [--config CONFIG] [-v] [dir] +- [GitHub Pages](https://armmbed.github.io/snippet) -positional arguments: - dir path to project root, used by any relative paths in loaded - configs [cwd] +## Project Structure -optional arguments: - -h, --help show this help message and exit - --config CONFIG paths (or globs) to config files - -v, --verbosity increase output verbosity -``` +The follow described the major aspects of the project structure: + +- `azure-pipelines/` - CI configuration files for Azure Pipelines. +- `docs/` - Interface definition and usage documentation. +- `examples/` - Usage examples. +- `snippet/` - Python source files. +- `news/` - Collection of news files for unreleased changes. +- `tests/` - Unit and integration tests. + +## Getting Help + +- For interface definition and usage documentation, please see [GitHub Pages](https://armmbed.github.io/snippet). +- For a list of known issues and possible work arounds, please see [Known Issues](KNOWN_ISSUES.md). +- To raise a defect or enhancement please use [GitHub Issues](https://github.com/ARMmbed/snippet/issues). +- To ask a question please use the [Mbed Forum](https://forums.mbed.com/). + +## Contributing + +- Snippet is an open source project and we are committed to fostering a welcoming community, please see our + [Code of Conduct](https://github.com/ARMmbed/snippet/blob/master/CODE_OF_CONDUCT.md) for more information. +- For ways to contribute to the project, please see the [Contributions Guidelines](https://github.com/ARMmbed/snippet/blob/master/CONTRIBUTING.md) +- For a technical introduction into developing this package, please see the [Development Guide](https://github.com/ARMmbed/snippet/blob/master/DEVELOPMENT.md) diff --git a/azure-pipelines/build-release.yml b/azure-pipelines/build-release.yml new file mode 100644 index 0000000..579df96 --- /dev/null +++ b/azure-pipelines/build-release.yml @@ -0,0 +1,240 @@ +# Azure Pipeline for Build and Release a Python package. +# +# This pipeline performs multiple actions to ensure the quality of the package: +# - Performs static analysis and runs test on multiple Python versions and host platforms. +# - Uploads test coverage to Code Climate +# - Generates reference documentation +# - Optionally releases the software on PyPI + +# Rebuild after commits to master for GA releases, beta for pre-release and release branches for patches. +trigger: + - master + - beta + - releases/* + +# Trigger on a PR to master, beta or releases branches (this includes PRs from forks but secrets are not exposed). +pr: + - master + - beta + - releases/* + +stages: + - stage: AnalyseTest + displayName: 'Analyse and Test' + jobs: + - job: Test + strategy: + maxParallel: 10 + matrix: + Linux_Py_3_6: + python.version: '3.6' + vmImageName: ubuntu-latest + uploadCoverage: "false" + + Linux_Py_3_7: + python.version: '3.7' + vmImageName: ubuntu-latest + uploadCoverage: "true" + + Windows_Py_3_6: + python.version: '3.6' + vmImageName: windows-latest + uploadCoverage: "false" + + Windows_Py_3_7: + python.version: '3.7' + vmImageName: windows-latest + uploadCoverage: "false" + + macOS_Py_3_6: + python.version: '3.6' + vmImageName: macOS-latest + uploadCoverage: "false" + + macOS_Py_3_7: + python.version: '3.7' + vmImageName: macOS-latest + uploadCoverage: "false" + pool: + vmImage: $(vmImageName) + + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '$(python.version)' + displayName: 'Use Python $(python.version)' + + - template: steps/install-development-dependencies.yml + + # Static analysis is different for Python versions so should be run in each environment. + - script: | + flake8 + displayName: 'Static Analysis - general (flake8)' + + - script: | + mypy -p snippet + displayName: 'Static Analysis - type checks (mypy)' + + - script: | + pip install pytest-azurepipelines + displayName: 'Add pytest plugin to upload test results to Azure UI' + + - script: | + pytest + displayName: 'Run unit tests (pytest)' + + - template: steps/publish-code-coverage-results.yml + + - stage: AssertNews + displayName: 'Checks news files' + dependsOn: [] + jobs: + - job: News + displayName: 'Assert news files' + pool: + vmImage: 'ubuntu-latest' + + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.7' + inputs: + versionSpec: '3.7' + + - template: steps/determine-current-branch.yml + + - template: steps/install-development-dependencies.yml + + - script: | + assert-news -b $(current_branch) + displayName: 'Run check' + + - stage: DocBuild + displayName: 'Build Documentation' + dependsOn: [] + jobs: + - job: Docs + displayName: 'Build Documentation' + pool: + vmImage: 'ubuntu-latest' + + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.7' + inputs: + versionSpec: '3.7' + + - template: steps/install-development-dependencies.yml + + - template: steps/generate-documentation.yml + + - publish: $(temp_docs_path) + artifact: Documentation + displayName: 'Publish documentation' + + - stage: Licensing + displayName: 'Report licences in use (SPDX)' + dependsOn: [] + jobs: + - job: Licensing + displayName: 'Generate SPDX documents' + pool: + vmImage: 'ubuntu-latest' + + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.7' + inputs: + versionSpec: '3.7' + + - template: steps/install-development-dependencies.yml + + - template: steps/generate-spdx-documents.yml + + - publish: $(temp_spdx_reports_path) + artifact: SPDX + displayName: 'Publish SPDX reports' + + # Collect test and build stages together before the release stages to provide a pass/fail point for the status badge. + - stage: CiCheckpoint + displayName: 'CI Checkpoint' + dependsOn: + - AnalyseTest + - AssertNews + - DocBuild + - Licensing + jobs: + - job: ChecksPassing + displayName: 'Checks Passing' + # A dummy job is required due to a bug in Azure which runs the previous job if nothing is defined. + steps: + - bash: echo "All prerequisite stages have passed and the package should be suitable for release." + + - stage: BetaRelease + displayName: 'Beta Release' + # Only allow beta releases if the tests pass and we are on the beta branch. + dependsOn: + - CiCheckpoint + condition: and(succeeded(), eq(variables['build.sourceBranch'], 'refs/heads/beta')) + jobs: + - deployment: PyPIBetaRelease + displayName: 'PyPI Beta Release' + # The following environment has a manual approval step to gate releases. + # This can only be created and configured within the Environment section of Pipelines. + # The release can be approved from the Azure pipeline run. + environment: 'PyPI Release' + strategy: + runOnce: + deploy: + pool: + vmImage: 'ubuntu-latest' + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.7' + + - template: steps/determine-current-branch.yml + + - template: steps/override-checkout.yml + + - template: steps/install-development-dependencies.yml + + - template: steps/tag-and-release.yml + parameters: + message: 'Beta Release' + releaseType: 'beta' + currentBranch: $(current_branch) + + - stage: ProductionReleasePyPI + displayName: 'Production Release' + # Only allow production releases if the tests pass and we are on the master branch. + dependsOn: + - CiCheckpoint + condition: and(succeeded(), eq(variables['build.sourceBranch'], 'refs/heads/master')) + jobs: + - deployment: PyPIProductionRelease + displayName: 'PyPI Production Release' + # The following environment has a manual approval step to gate releases. + # This can only be created and configured within the Environment section of Pipelines. + # The release can be approved from the Azure pipeline run. + environment: 'PyPI Release' + strategy: + runOnce: + deploy: + pool: + vmImage: 'ubuntu-latest' + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.7' + + - template: steps/determine-current-branch.yml + + - template: steps/override-checkout.yml + + - template: steps/install-development-dependencies.yml + + - template: steps/tag-and-release.yml + parameters: + message: 'Production Release' + releaseType: 'release' + currentBranch: $(current_branch) diff --git a/azure-pipelines/steps/determine-current-branch.yml b/azure-pipelines/steps/determine-current-branch.yml new file mode 100644 index 0000000..4566d2c --- /dev/null +++ b/azure-pipelines/steps/determine-current-branch.yml @@ -0,0 +1,10 @@ +steps: + - bash: | + if [[ -z "$(System.PullRequest.SourceBranch)" ]]; then + branch_name="$(Build.SourceBranchName)" + else + branch_name="$(System.PullRequest.SourceBranch)" + fi + echo "Determined branch name: $branch_name" + echo "##vso[task.setvariable variable=current_branch]$branch_name" + displayName: "Set current_branch variable" diff --git a/azure-pipelines/steps/generate-documentation.yml b/azure-pipelines/steps/generate-documentation.yml new file mode 100644 index 0000000..21990aa --- /dev/null +++ b/azure-pipelines/steps/generate-documentation.yml @@ -0,0 +1,12 @@ +steps: + - bash: | + echo "##vso[task.setvariable variable=temp_docs_path]`get-config --key DOCUMENTATION_PRODUCTION_OUTPUT_PATH`" + displayName: 'Set variable for temporary docs dir' + + - script: | + generate-docs --output_dir $(temp_docs_path) + displayName: 'Generate documentation' + + - script: | + license-files + displayName: 'Add copyright/licence notice.' \ No newline at end of file diff --git a/azure-pipelines/steps/generate-spdx-documents.yml b/azure-pipelines/steps/generate-spdx-documents.yml new file mode 100644 index 0000000..9e3f08d --- /dev/null +++ b/azure-pipelines/steps/generate-spdx-documents.yml @@ -0,0 +1,13 @@ +steps: + - bash: | + echo "##vso[task.setvariable variable=temp_spdx_reports_path]licensing" + displayName: 'Set variable for temporary SPDX reports dir' + + - bash: | + mkdir -p $(temp_spdx_reports_path) + generate-spdx --output-dir $(temp_spdx_reports_path) + displayName: 'Generate SPDX documents' + + - script: | + license-files + displayName: 'Add copyright/licence notice.' \ No newline at end of file diff --git a/azure-pipelines/steps/install-development-dependencies.yml b/azure-pipelines/steps/install-development-dependencies.yml new file mode 100644 index 0000000..261f727 --- /dev/null +++ b/azure-pipelines/steps/install-development-dependencies.yml @@ -0,0 +1,16 @@ +steps: + # Note + # The below code generates a pip requirements file from the pipenv development requirements (also obtaining the + # normal dependencies from setup.py) and then installs via pip. As a virtual machine is already being used, pipenv + # is superfluous and eliminating pipenv in CI reduces overhead and reduce complexity, while retaining a single + # location for development dependencies. + # This code also forces the system to install latest tools as the ones present on the CI system may be too old + # for the process to go through properly. + - script: | + python -m pip install --upgrade pip wheel setuptools + pip install pipenv + python -m pipenv lock --dev -r > dev-requirements.txt + pip install -r dev-requirements.txt + pip install pytest-azurepipelines + pip list + displayName: 'Install development dependencies' diff --git a/azure-pipelines/steps/override-checkout.yml b/azure-pipelines/steps/override-checkout.yml new file mode 100644 index 0000000..d2cc88b --- /dev/null +++ b/azure-pipelines/steps/override-checkout.yml @@ -0,0 +1,4 @@ +steps: + - checkout: self + submodules: recursive + persistCredentials: true diff --git a/azure-pipelines/steps/publish-code-coverage-results.yml b/azure-pipelines/steps/publish-code-coverage-results.yml new file mode 100644 index 0000000..967103d --- /dev/null +++ b/azure-pipelines/steps/publish-code-coverage-results.yml @@ -0,0 +1,8 @@ +steps: + # This script only runs for one set of unit tests (to avoid duplicate uploads) and will only run when the variable + # CODECOV_TOKEN is available, which means this step is disabled for PRs from forked repositories. + # Use "bash" rather than "script" to allow upload from all host platforms including Windows. + - bash: | + bash <(curl -s https://codecov.io/bash) -Z -t $(CODECOV_TOKEN) + condition: and(succeeded(), eq(variables['uploadCoverage'], 'true'), ne(variables['CODECOV_TOKEN'], '')) + displayName: 'Upload test coverage to codecov.io' diff --git a/azure-pipelines/steps/tag-and-release.yml b/azure-pipelines/steps/tag-and-release.yml new file mode 100644 index 0000000..b2a506a --- /dev/null +++ b/azure-pipelines/steps/tag-and-release.yml @@ -0,0 +1,13 @@ +parameters: + message: 'Release' + releaseType: 'development' + currentBranch: '' + +steps: + - bash: tag-and-release -b ${{ parameters.currentBranch }} --release-type=${{ parameters.releaseType }} -vv + displayName: ${{ parameters.message }} + env: + GIT_TOKEN: $(GIT_TOKEN) + TWINE_USERNAME: $(TWINE_USERNAME) + TWINE_PASSWORD: $(TWINE_PASSWORD) + IGNORE_PYPI_TEST_UPLOAD: $(IGNORE_PYPI_TEST_UPLOAD) diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 0000000..cbd8a55 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,33 @@ +codecov: + require_ci_to_pass: yes + +coverage: + precision: 2 + round: down + range: "0...100" + status: + project: + default: + # basic + target: auto + threshold: 2% + base: auto + # advanced + branches: + - master + if_not_found: success + if_ci_failed: error + informational: true + only_pulls: false +parsers: + gcov: + branch_detection: + conditional: yes + loop: yes + method: no + macro: no + +comment: + layout: "reach,diff,flags,tree" + behavior: default + require_changes: no diff --git a/docs/index.html b/docs/index.html new file mode 100644 index 0000000..57fe639 --- /dev/null +++ b/docs/index.html @@ -0,0 +1,15 @@ + + + + + + snippet API documentation + + +

snippet API documentation

+

This is a placeholder for the API documentation, the documentation will be automatically generated when the package is released at version 1.0.0.

+ + diff --git a/docs/news/pyproject.toml b/docs/news/pyproject.toml deleted file mode 100644 index f694f2c..0000000 --- a/docs/news/pyproject.toml +++ /dev/null @@ -1,9 +0,0 @@ -[tool.towncrier] -directory = '.' -filename = '../../CHANGELOG.md' -package = 'snippet' -template = 'template.rst' -title_format = '{version} ({project_date})' -start_string = """ -[//]: # (begin_release_notes) -""" diff --git a/docs/news/template.rst b/docs/news/template.rst deleted file mode 100644 index c6c7df0..0000000 --- a/docs/news/template.rst +++ /dev/null @@ -1,34 +0,0 @@ -{# This is a custom template for towncrier md #} - -{% for section in sections %} -{% set underline = underlines[0] %} -{% if section %} -## {{section}} - -{% endif %} -{% if sections[section] %} -{% for category, val in definitions.items() if category in sections[section] %} -### {{ definitions[category]['name'] }} - -{% if definitions[category]['showcontent'] %} -{% for text, values in sections[section][category]|dictsort(by='value') %} -- {{ text }} ({{ values|sort|join(', ') }}) - -{% endfor %} -{% else %} -- {{ sections[section][category]['']|sort|join(', ') }} - -{% endif %} -{% if sections[section][category]|length == 0 %} - -No significant changes. - -{% else %} -{% endif %} -{% endfor %} -{% else %} - -No significant changes. - -{% endif %} -{% endfor %} diff --git a/news/20200409.major b/news/20200409.major new file mode 100644 index 0000000..7a54f2b --- /dev/null +++ b/news/20200409.major @@ -0,0 +1 @@ +Refactored thoroughly. \ No newline at end of file diff --git a/news/README.md b/news/README.md new file mode 100644 index 0000000..a5309c5 --- /dev/null +++ b/news/README.md @@ -0,0 +1,3 @@ +# News directory + +This directory comprises information about all the changes that happened since the last release. A file should be added to this directory for each PR. On release of the package the content of the file becomes part of the change log and this directory is reset. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..387f092 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,85 @@ +# +# Copyright (C) 2020 Arm Mbed. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +[ProjectConfig] +# Only path variables can and must contain 'DIR', 'PATH' or 'ROOT' in their name as +# these tokens are used to identify path variables from other variable types. +PROJECT_ROOT = "." +PROJECT_NAME = "Snippet" +PROJECT_UUID = "14544a30-0a1d-4edb-8e40-7ac66d741ba6" +PACKAGE_NAME = "code-snippet" +NEWS_DIR = "news/" +SOURCE_DIR = "snippet" +RELEASE_BRANCH_PATTERN = "^release.*$" +MODULE_TO_DOCUMENT = "snippet" +DOCUMENTATION_DEFAULT_OUTPUT_PATH = "local_docs" +DOCUMENTATION_PRODUCTION_OUTPUT_PATH = "docs" +VERSION_FILE_PATH = "snippet/_version.py" +CHANGELOG_FILE_PATH = "CHANGELOG.md" + +[AutoVersionConfig] +CONFIG_NAME = "DEFAULT" +PRERELEASE_TOKEN = "beta" +BUILD_TOKEN = "dev" +TAG_TEMPLATE = "release/{version}" +targets = [ "snippet/_version.py",] + +[AutoVersionConfig.key_aliases] +__version__ = "VERSION_KEY" +MAJOR = "major" +MINOR = "minor" +PATCH = "patch" +COMMIT = "COMMIT" + +[AutoVersionConfig.trigger_patterns] +major = "news/*.major" +minor = "news/*.feature" +patch = "news/*.bugfix" + +[tool.towncrier] +directory = "news" +filename = "CHANGELOG.md" +package = "snippet" +title_format = "{version} ({project_date})" +start_string = """ +[//]: # (begin_release_notes) +""" + +[[tool.towncrier.type]] +directory = "major" +name = "Major changes" +showcontent = true + +[[tool.towncrier.type]] +directory = "feature" +name = "Features" +showcontent = true + +[[tool.towncrier.type]] +directory = "bugfix" +name = "Bugfixes" +showcontent = true + +[[tool.towncrier.type]] +directory = "doc" +name = "Improved Documentation" +showcontent = true + +[[tool.towncrier.type]] +directory = "removal" +name = "Deprecations and Removals" +showcontent = true + +[[tool.towncrier.type]] +directory = "misc" +name = "Misc" +showcontent = false + +[tool.black] +line-length = 120 + +[spdx] +CreatorWebsite = "spdx.org" +PathToSpdx = "spdx/spdxdocs" +UUID = "6a1d2a3e-5f54-4521-b5fd-3fda5a4f879c" diff --git a/scripts/autoversion.toml b/scripts/autoversion.toml deleted file mode 100644 index e86fd5f..0000000 --- a/scripts/autoversion.toml +++ /dev/null @@ -1,7 +0,0 @@ -[AutoVersionConfig] -CONFIG_NAME = "SNIPPET" -targets = [ "src/snippet/_version.py"] - -[AutoVersionConfig.key_aliases] -__version__ = "VERSION_KEY" -COMMIT = "COMMIT" diff --git a/scripts/tag_and_release.py b/scripts/tag_and_release.py deleted file mode 100644 index c5a162a..0000000 --- a/scripts/tag_and_release.py +++ /dev/null @@ -1,90 +0,0 @@ -# -------------------------------------------------------------------------- -# Snippet -# (C) COPYRIGHT 2018 Arm Limited -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -------------------------------------------------------------------------- -"""Part of the CI process""" - -import os -import subprocess - - -def git_url_ssh_to_https(url): - """Convert a git url - - url will look like - https://github.com/ARMmbed/snippet.git - or - git@github.com:ARMmbed/snippet.git - we want: - https://${GITHUB_TOKEN}@github.com/ARMmbed/snippet.git - """ - path = url.split('github.com', 1)[1][1:].strip() - new = 'https://{GITHUB_TOKEN}@github.com/%s' % path - print('rewriting git url to: %s' % new) - return new.format(GITHUB_TOKEN=os.getenv('GITHUB_TOKEN')) - -VERSION_FILE='src/snippet/_version.py' - -def main(): - """Tags the current repository - - and commits changes to news files - """ - # see: - # https://packaging.python.org/tutorials/distributing-packages/#uploading-your-project-to-pypi - twine_repo = os.getenv('TWINE_REPOSITORY_URL') or os.getenv('TWINE_REPOSITORY') - print('tagging and releasing to %s as %s' % ( - twine_repo, - os.getenv('TWINE_USERNAME') - )) - - if not twine_repo: - raise Exception('cannot release to implicit pypi repository. explicitly set the repo/url.') - - version = subprocess.check_output([ 'python', 'setup.py', '--version']).decode().strip() - if 'dev' in version: - raise Exception('cannot release unversioned project: %s' % version) - - print('Preparing environment') - subprocess.check_call(['git', 'config', '--global', 'user.name', 'monty-bot']) - subprocess.check_call(['git', 'config', '--global', 'user.email', 'monty-bot@arm.com']) - url = subprocess.check_output(['git', 'remote', 'get-url', 'origin']) - branch_name = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']) - new_url = git_url_ssh_to_https(url.decode()) - subprocess.check_call(['git', 'remote', 'set-url', 'origin', new_url]) - branch_spec = 'origin/%s' % branch_name.decode('utf-8').strip() - subprocess.check_call(['git', 'branch', '--set-upstream-to', branch_spec]) - print('Generating a release package') - subprocess.check_call( - [ 'python', 'setup.py', 'clean', '--all', 'bdist_wheel', '--dist-dir', 'release-dist']) - print('Uploading to PyPI') - subprocess.check_call(['python', '-m', 'twine', 'upload', 'release-dist/*']) - print('Committing the changelog & version') - subprocess.check_call(['git', 'add', VERSION_FILE]) - subprocess.check_call(['git', 'add', 'CHANGELOG.md']) - subprocess.check_call(['git', 'add', 'docs/news/*']) - message = ':checkered_flag: :newspaper: Releasing version %s\n[skip ci]' % version - subprocess.check_call(['git', 'commit', '-m', message]) - print('Tagging the project') - subprocess.check_call(['git', 'tag', '-a', version, '-m', 'Release %s' % version]) - print('Pushing changes back to GitHub') - subprocess.check_call(['git', 'push', '--follow-tags']) - print('Marking this commit as latest') - subprocess.check_call(['git', 'tag', '-f', 'latest']) - subprocess.check_call(['git', 'push', '-f', '--tags']) - print('Done.') - -if __name__ == '__main__': - main() diff --git a/setup.cfg b/setup.cfg index b080d7d..aa450ef 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,25 +1,57 @@ +[metadata] +license_file = LICENSE + +[mypy] +# flake8-mypy expects the two following for sensible formatting +show_column_numbers = True +show_error_context = False + +# do not follow imports (except for ones found in typeshed) +follow_imports = skip + +# since we're ignoring imports, writing .mypy_cache doesn't make any sense +cache_dir = /dev/null + + +ignore_missing_imports = True +disallow_untyped_calls = True +warn_return_any = True +strict_optional = True +warn_no_return = True +warn_redundant_casts = True +warn_unused_ignores = True +disallow_untyped_defs = True +check_untyped_defs = True + [flake8] -exclude = __init__.py +exclude = + .git, + __pycache__, + config, + docs, + news, + dist +ignore = + # H301: one import per line + H301, + # H306: imports not in alphabetical order + H306, + # W503: line break before binary operator (this is no longer PEP8 compliant) + W503, +per-file-ignores = + # Top level init file improves user experience by short cutting imports, + # this is not used in the package online by calling clients + # F401: imported but unused + snippet/__init__.py:F401 + # Don't require docstrings in tests. + # We evaluate the need for them on case by case basis. + test_*.py:D1 + tests/**.py:D1 max-line-length = 120 -statistics = True -verbose = 2 -filename = - src - tests - -[coverage:run] -branch = true -parallel = true -omit = - **/tests/* -[coverage:report] -[coverage:html] -title = snippet -directory = report +docstring-convention = google +copyright-check = True +select = E,F,W,C,B,H,D -[green] -verbose = 2 -logging = False -processes = 1 -termcolor = True -run-coverage = True +[tool:pytest] +addopts = --cov snippet --cov-report xml:coverage/coverage.xml --cov-report=html +junit_family = xunit2 diff --git a/setup.py b/setup.py index 70a0046..1807c1a 100644 --- a/setup.py +++ b/setup.py @@ -1,38 +1,58 @@ +# +# Copyright (C) 2020 Arm Mbed. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +"""Package definition for PyPI.""" import os -from setuptools import find_packages from setuptools import setup -NAME = 'code-snippet' +PROJECT_SLUG = "code-snippet" +SOURCE_DIR = "snippet" __version__ = None repository_dir = os.path.dirname(__file__) -# single source for project version information without side effects -with open(os.path.join(repository_dir, 'src', 'snippet', '_version.py')) as fh: +# Read package version, this will set the variable `__version__` to the current version. +with open(os.path.join(repository_dir, SOURCE_DIR, "_version.py"), encoding="utf8") as fh: exec(fh.read()) -with open(os.path.join(repository_dir, 'README.md')) as fh: +# Use readme needed as long description in PyPI +with open(os.path.join(repository_dir, "README.md"), encoding="utf8") as fh: long_description = fh.read() -with open(os.path.join(repository_dir, 'requirements.txt')) as fh: - requirements = fh.readlines() - setup( - classifiers=( - 'Intended Audience :: Developers', - ), - author="David Hyman, Arm Mbed", + author="David Hyman, Mbed team", author_email="support@mbed.com", - description="Code snippet extraction", + classifiers=[ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Topic :: Software Development :: Build Tools", + "Topic :: Documentation", + "Topic :: Software Development :: Documentation", + ], + description="Code snippet extraction for documentation", + keywords="documentation-generator documentation-tool snippet-generator snippet project-management documentation", include_package_data=True, - install_requires=requirements, - license='Apache 2.0', + install_requires=["python-dotenv", "toml", "pystache", "mbed-tools-lib"], + license="Apache 2.0", + long_description_content_type="text/markdown", long_description=long_description, - name=NAME, - package_dir={'': 'src'}, - packages=find_packages('src'), - python_requires='>3.4', - url="https://github.com/ARMmbed/snippet", + name=PROJECT_SLUG, + packages=[SOURCE_DIR], + python_requires=">=3.6,<4", + url=f"https://github.com/ARMmbed/snippet", version=__version__, + entry_points=dict( + console_scripts=[ + f"snippet = {SOURCE_DIR}.cli:main", + f"code-snippet = {SOURCE_DIR}.cli:main", + f"code_snippet = {SOURCE_DIR}.cli:main", + ] + ), ) diff --git a/snippet/__init__.py b/snippet/__init__.py new file mode 100644 index 0000000..2f46f27 --- /dev/null +++ b/snippet/__init__.py @@ -0,0 +1,6 @@ +# +# Copyright (C) 2020 Arm Mbed. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +"""Snippet provides a way to extract code snippet so that they can be inserted in documentation.""" +from snippet._version import __version__ diff --git a/snippet/__main__.py b/snippet/__main__.py new file mode 100644 index 0000000..6276e8c --- /dev/null +++ b/snippet/__main__.py @@ -0,0 +1,8 @@ +# +# Copyright (C) 2020 Arm Mbed. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +"""Entrypoint for development purposes.""" +from snippet.cli import main + +main() diff --git a/snippet/_internal/__init__.py b/snippet/_internal/__init__.py new file mode 100644 index 0000000..ac3a3aa --- /dev/null +++ b/snippet/_internal/__init__.py @@ -0,0 +1,5 @@ +# +# Copyright (C) 2020 Arm Mbed. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +"""Code not to be accessed by external applications.""" diff --git a/src/snippet/file_wrangler.py b/snippet/_internal/file_wrangler.py similarity index 50% rename from src/snippet/file_wrangler.py rename to snippet/_internal/file_wrangler.py index d7df312..2dbadcd 100644 --- a/src/snippet/file_wrangler.py +++ b/snippet/_internal/file_wrangler.py @@ -1,3 +1,9 @@ +# +# Copyright (C) 2020 Arm Mbed. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +"""Utilities regarding file handling.""" + import os import glob @@ -6,53 +12,52 @@ import random from snippet.config import Config -from snippet.logs import logger +from snippet._internal.logs import LOGGER -def write_example(config: Config, example_name, example_block): - """Writes example to file""" +def write_example(config: Config, example_name: str, example_block: str) -> None: + """Writes example to file.""" output = pystache.render( config.output_template, name=example_name, code=example_block, comment_prefix=config.comment_prefix, comment_suffix=config.comment_suffix, - language_name=config.language_name + language_name=config.language_name, ) output_file_name = pystache.render( - config.output_file_name_template, - name=example_name.strip().replace(' ', '_').lower() + config.output_file_name_template, name=example_name.strip().replace(" ", "_").lower() ) if not os.path.exists(config.output_dir): - logger.info('creating output directory %s', config.output_dir) + LOGGER.info("creating output directory %s", config.output_dir) os.makedirs(config.output_dir) output_file = os.path.join(config.output_dir, output_file_name) - logger.info('writing %r to %s', example_name, output_file) + LOGGER.info("writing %r to %s", example_name, output_file) for i in range(1, config.write_attempts + 1): # we run a retry loop as there may be contention on the output file in # a multi-process environment try: - with open(output_file, 'a' if config.output_append else 'w') as fh: + with open(output_file, "a" if config.output_append else "w", encoding="utf8") as fh: fh.write(output) break except IOError as err: time.sleep(i * 0.5 + 0.1 * random.randint(0, 5)) - logger.info('write failed (%s) retrying attempt: %s', err, i) + LOGGER.info("write failed (%s) retrying attempt: %s", err, i) else: - raise IOError('could not write output file after %s attempts' % config.write_attempts) + raise IOError("could not write output file after %s attempts" % config.write_attempts) -def load_file_lines(path): - """Loads file into memory""" - with open(path, 'r', encoding='utf8') as fh: +def load_file_lines(path: str) -> list: + """Loads file into memory.""" + with open(path, "r", encoding="utf8") as fh: lines = fh.readlines() return lines -def find_files(config: Config): - """Finds input file paths, according to the config""" +def find_files(config: Config) -> list: + """Finds input file paths, according to the config.""" files = [] for glob_pattern in config.input_glob: files.extend(glob.glob(glob_pattern, recursive=True)) diff --git a/snippet/_internal/logs.py b/snippet/_internal/logs.py new file mode 100644 index 0000000..71b3efa --- /dev/null +++ b/snippet/_internal/logs.py @@ -0,0 +1,8 @@ +# +# Copyright (C) 2020 Arm Mbed. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +"""Logger definition.""" +import logging + +LOGGER = logging.getLogger(__name__) diff --git a/snippet/_internal/util.py b/snippet/_internal/util.py new file mode 100644 index 0000000..ee42303 --- /dev/null +++ b/snippet/_internal/util.py @@ -0,0 +1,13 @@ +# +# Copyright (C) 2020 Arm Mbed. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +"""Utilities.""" +from typing import Any + + +def ensure_list(item: Any) -> list: + """Ensures an item is a list.""" + if item: + return item if isinstance(item, list) else [item] + return list() diff --git a/snippet/_internal/wrapper.py b/snippet/_internal/wrapper.py new file mode 100644 index 0000000..bfdbd30 --- /dev/null +++ b/snippet/_internal/wrapper.py @@ -0,0 +1,23 @@ +# +# Copyright (C) 2020 Arm Mbed. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +"""Decorator.""" +import traceback +from snippet.config import Config +from typing import Callable, Any, List + + +def wrap(config: Config, failures: List[Any], identifier: str, nullary_function: Callable, default: Any = None) -> Any: + """Executes a function (`nullary_function`) with no arguments. + + to pass arguments, use partials + stores any exceptions in `failures` + """ + try: + return nullary_function() + except Exception: + if config.stop_on_first_failure: + raise + failures.append((identifier, traceback.format_exc())) + return default diff --git a/snippet/_version.py b/snippet/_version.py new file mode 100644 index 0000000..abfbe0a --- /dev/null +++ b/snippet/_version.py @@ -0,0 +1,17 @@ +# +# Copyright (C) 2020 Arm Mbed. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +"""The version number is based on semantic versioning. + +References +- https://semver.org/ +- https://www.python.org/dev/peps/pep-0440/ + +This file is autogenerated, do not modify by hand. +""" +__version__ = "1.1.0" +COMMIT = "5d5a5cbd12ef49909d78c2247a2f988efb9c4894" +MAJOR = 1 +MINOR = 1 +PATCH = 0 diff --git a/snippet/api.py b/snippet/api.py new file mode 100644 index 0000000..b6c0189 --- /dev/null +++ b/snippet/api.py @@ -0,0 +1,21 @@ +# +# Copyright (C) 2020 Arm Mbed. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +"""Code Snippet APIs.""" +import textwrap + +from snippet import workflow, config +from snippet._internal.logs import LOGGER + + +def extract_code_snippets(config: config.Config) -> None: + """Extracts code snippets according to configuration.""" + LOGGER.debug("project directory is %r", config.project_root) + examples, paths, failures = workflow.run(config) + + if failures: + LOGGER.error( + "failures:\n%s", textwrap.indent("\n".join(f"{name}: {exc}" for name, exc in failures), prefix=" ") + ) + raise Exception(f"There were %s failures!" % len(failures)) diff --git a/snippet/cli.py b/snippet/cli.py new file mode 100644 index 0000000..6febb1d --- /dev/null +++ b/snippet/cli.py @@ -0,0 +1,51 @@ +# +# Copyright (C) 2020 Arm Mbed. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +"""CLI definition.""" +import argparse + +import os +import sys +import dotenv + +from mbed_tools_lib.logging import set_log_level, MbedToolsHandler + +from snippet import config +from snippet.api import extract_code_snippets +from snippet._internal.logs import LOGGER + + +def main() -> int: + """Script CLI.""" + parser = argparse.ArgumentParser() + parser.add_argument("--config", type=str, action="append", help="paths (or globs) to config files") + parser.add_argument( + "dir", + nargs="?", + default=os.getcwd(), + help="path to project root, used by any relative paths in loaded configs [cwd]", + ) + parser.add_argument( + "-v", + "--verbose", + action="count", + default=0, + help="Set the verbosity level, enter multiple times to increase verbosity.", + ) + parser.add_argument( + "-t", "--traceback", action="store_true", default=True, help="Show a traceback when an error is raised." + ) + args = parser.parse_args() + set_log_level(args.verbose) + dotenv.load_dotenv(dotenv.find_dotenv(usecwd=True, raise_error_if_not_found=False)) + # Use the context manager to ensure tools exceptions (expected behaviour) are shown as messages to the user, + # but all other exceptions (unexpected behaviour) are shown as errors. + with MbedToolsHandler(LOGGER, args.traceback): + extract_code_snippets(config.get_config(config_paths=args.config, project_root=args.dir,)) + return 0 + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/snippet/config.py b/snippet/config.py new file mode 100644 index 0000000..b7cb22e --- /dev/null +++ b/snippet/config.py @@ -0,0 +1,124 @@ +# +# Copyright (C) 2020 Arm Mbed. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +"""Definition of the configuration of snippet.""" +import glob +import logging +import os +from pathlib import Path + +import toml + +from snippet._internal.logs import LOGGER +from snippet._internal.util import ensure_list +from typing import Optional, List + + +class EnvironmentVariables: + """Environment variables used by snippet.""" + + @property + def SNIPPET_CONFIG_PATH(self) -> Optional[str]: + """Path to the configuration file for snippet.""" + return os.getenv("SNIPPET_CONFIG_PATH") + + +DEFAULT_PROJECT_ROOT_PATH = "." + + +class Config: + """Definition of snippet's configuration.""" + + # IO + project_root = DEFAULT_PROJECT_ROOT_PATH # the project root used for relative IO paths (set by commandline) + input_glob = ["tests/example/*.py"] + output_append = True # if the output file exists, append to it + output_dir = "." + output_file_name_template = "{{name}}.md" # a mustache template for the output file name + write_attempts = 3 # number of retries when writing output files + + # Language and style + language_name = "python" + comment_prefix = "# " + comment_suffix = "" + # a mustache template for each file (triple braces important for code literals, no escaping) + output_template = "```{{language_name}}\n{{comment_prefix}}example: {{{name}}}{{comment_suffix}}\n{{{code}}}\n```\n" + + # Logger + log_level = logging.INFO + + # Code block indicators + start_flag = "an example" + end_flag = "end of example" + + # Hidden block indicators + cloak_flag = "cloak" + uncloak_flag = "uncloak" + + # Validation and formatting logic + drop_lines: List[str] = list() # drop lines containing these phrases + replacements = {"self.": ""} # straightforward replacements + fail_on_contains = ["assert"] # fail if these strings are found in code blocks + auto_dedent = True # keep code left-aligned with the start flag + fail_on_dedent = True # fail if code is dedented before reaching the end flag + stop_on_first_failure = False # fail early + + +def get_config(config_paths: Optional[list] = None, **options: dict) -> Config: + """Gets Snippet's configuration.""" + config = Config() + config_paths = _determine_config_paths(config_paths, options) + new_options = _load_configs(config_paths) + + # passed keyword args override other parameters + new_options.update(options) + + # update the config object + for k, v in new_options.items(): + setattr(config, k, v) + + return config + + +def _load_configs(config_paths: list) -> dict: + """Loads all the config files.""" + new_options = {} + for toml_file in _find_configs(glob_patterns=config_paths): + LOGGER.debug("trying config from %s", toml_file) + with open(toml_file, encoding="utf8") as f: + try: + config_file_contents = toml.load(f) + except toml.TomlDecodeError as e: + LOGGER.debug("failed to load %s: %s", toml_file, e) + continue + snippet_config = config_file_contents.get("snippet") + if snippet_config: + LOGGER.info("loading config from %s", toml_file) + new_options.update(snippet_config) + return new_options + + +def _find_configs(glob_patterns: list) -> list: + """Finds all the different configuration files on the file system.""" + configs = [] + for glob_pattern in glob_patterns: + configs.extend(glob.glob(glob_pattern, recursive=True)) + return configs + + +def _config_paths_from_env() -> list: + """Gets configuration paths from environment.""" + return ensure_list(EnvironmentVariables().SNIPPET_CONFIG_PATH) + + +def _determine_config_paths(config_paths: Optional[list], options: dict) -> list: + """Determines the paths to all configuration files.""" + project_root = Path(options.get("project_root", DEFAULT_PROJECT_ROOT_PATH)).absolute() + + config_paths = config_paths or list() + config_paths.extend(_config_paths_from_env()) + # fallback option - search the project directory + if len(config_paths) == 0: + config_paths.append(str((project_root.joinpath("**", "*.toml")))) + return config_paths diff --git a/snippet/exceptions.py b/snippet/exceptions.py new file mode 100644 index 0000000..244ed25 --- /dev/null +++ b/snippet/exceptions.py @@ -0,0 +1,41 @@ +# +# Copyright (C) 2020 Arm Mbed. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +"""Snippet exceptions.""" + + +class SnippetError(Exception): + """Generic error.""" + + pass + + +class DuplicateName(SnippetError): + """Found duplicated names.""" + + pass + + +class ValidationFailure(SnippetError): + """Failed snippet validation.""" + + pass + + +class TagMismatch(SnippetError): + """Tags mismatch in snippet.""" + + pass + + +class StartEndMismatch(TagMismatch): + """Snippet format problem.""" + + pass + + +class CloakMismatch(TagMismatch): + """Invalid cloaking in snippet.""" + + pass diff --git a/snippet/snippet.py b/snippet/snippet.py new file mode 100644 index 0000000..f213534 --- /dev/null +++ b/snippet/snippet.py @@ -0,0 +1,192 @@ +# +# Copyright (C) 2020 Arm Mbed. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +"""Text snippet extractor.""" +from typing import List, Optional + +from snippet import exceptions +from snippet.config import Config + + +class Example: + """An example.""" + + def __init__(self, path: str, line_num: int, example_name: str, line: str) -> None: + """Initialiser.""" + self._key = (path, line_num, example_name) + self._strip = len(line) - len(line.lstrip()) + self._text: List[str] = list() + self._cloaking = False + + def add_line(self, line: str) -> None: + """Adds a line.""" + if self._cloaking: + return + self._text.append(line) + + def cloak(self, line_num: int) -> None: + """Starts cloaking.""" + if self._cloaking: + raise exceptions.CloakMismatch(f"Already cloaked at {self.debug_id} ({line_num})") + self._cloaking = True + + def uncloak(self, line_num: int) -> None: + """Stops cloaking.""" + if not self._cloaking: + raise exceptions.CloakMismatch(f"Already uncloaked at {self.debug_id} ({line_num})") + self._cloaking = False + + @property + def is_cloaking(self) -> bool: + """States whether it's in cloaking mode.""" + return self._cloaking + + @property + def is_empty(self) -> bool: + """States whether the example is empty or not.""" + return len(self._text) == 0 + + @property + def text(self) -> List[str]: + """Gets example text.""" + return self._text + + @property + def strip_number(self) -> int: + """Gets the example strip number.""" + return self._strip + + @property + def key(self) -> tuple: + """Gets the example key.""" + return self._key + + @property + def debug_id(self) -> str: + """Gets some debug information about the example.""" + return str(self.key) + + +class Examples: + """All the examples in a file.""" + + def __init__(self) -> None: + """Initialiser.""" + self._examples: List[Example] = list() + self._current_example: Optional[Example] = None + + def set_current(self, example: Example, line_num: int) -> None: + """Sets current example.""" + if self._current_example: + raise exceptions.StartEndMismatch(f"Already capturing at {self._current_example.debug_id} ({line_num})") + self._current_example = example + + def store_current(self, line_num: int) -> None: + """Stores current example.""" + if not self._current_example: + raise exceptions.StartEndMismatch(f"Not yet capturing at {line_num}") + if self._current_example.is_cloaking: + raise exceptions.CloakMismatch( + f"End of example reached whilst still cloaked {self._current_example.debug_id} ({line_num})" + ) + if not self._current_example.is_empty: + self._examples.append(self._current_example) + self._current_example = None + + def cloak(self, line_num: int) -> None: + """Start cloaking.""" + if self._current_example: + self._current_example.cloak(line_num) + + def uncloak(self, line_num: int) -> None: + """Stops cloaking.""" + if self._current_example: + self._current_example.uncloak(line_num) + + def end(self, line_num: int) -> None: + """Ends.""" + if self._current_example: + raise exceptions.StartEndMismatch( + f"EOF reached whilst still capturing {self._current_example.debug_id} ({line_num})" + ) + + def add_line(self, line: str) -> None: + """Adds a line.""" + if self._current_example: + self._current_example.add_line(line) + + def validate_dedent(self, line: str, line_num: int) -> None: + """Validates dedent.""" + if not self._current_example: + return + if any(line[: self._current_example.strip_number].lstrip()): + raise exceptions.ValidationFailure( + f"Unexpected dedent whilst capturing {self._current_example.debug_id} ({line_num})" + ) + + def validate_line(self, fail_on_contains: List[str], line: str, line_num: int) -> None: + """Validates line.""" + for trigger in fail_on_contains: + if trigger in line: + debug_info = self._current_example.debug_id if self._current_example else "" + raise exceptions.ValidationFailure(f"Unexpected phrase {repr(trigger)} at {debug_info} ({line_num})") + + def clean_line(self, line: str) -> Optional[str]: + """Cleans a line.""" + if not line: + return None + if not self._current_example: + return line + start = self._current_example.strip_number + return line[start:].rstrip() + + @property + def all(self) -> list: + """Gets all the examples.""" + return self._examples + + +def extract_snippets_from_text(config: Config, lines: list, path: str) -> dict: + """Finds snippets in lines of text.""" + examples = Examples() + line_index = 0 + for line_num, line in enumerate(lines): + line_index = line_num + if config.start_flag in line: + # start capturing code from the next line + examples.set_current( + Example(path=path, line_num=line_num, example_name=line.rsplit(":")[-1].strip(), line=line), line_num + ) + continue + + if config.end_flag in line: + # stop capturing, and discard empty blocks + examples.store_current(line_num) + continue + + if config.uncloak_flag in line: + examples.uncloak(line_num) + continue + + if config.cloak_flag in line: + examples.cloak(line_num) + continue + + # whilst capturing, append code lines to the current block + if config.fail_on_dedent: + examples.validate_dedent(line, line_num) + clean_line = examples.clean_line(line) + if not clean_line: + continue + if any(match in clean_line for match in config.drop_lines): + continue + for r_before, r_after in config.replacements.items(): + clean_line = clean_line.replace(r_before, r_after) + examples.validate_line(config.fail_on_contains, clean_line, line_num) + + # add this line of code to the example block + examples.add_line(clean_line) + + examples.end(line_index) + return {example.key: example.text for example in examples.all} diff --git a/snippet/workflow.py b/snippet/workflow.py new file mode 100644 index 0000000..6860419 --- /dev/null +++ b/snippet/workflow.py @@ -0,0 +1,71 @@ +# +# Copyright (C) 2020 Arm Mbed. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +"""Definition of the full workflow.""" +import textwrap +from functools import partial +from pathlib import Path +from typing import Tuple, Any, List, Dict + +from snippet import exceptions +from snippet._internal import file_wrangler +from snippet._internal.logs import LOGGER +from snippet._internal.util import ensure_list +from snippet._internal.wrapper import wrap +from snippet.config import Config +from snippet.snippet import extract_snippets_from_text + + +def run(config: Config) -> Tuple[dict, list, list]: + """Retrieves all the code snippets according to configuration.""" + failures: List[Any] = list() + + _set_config(config) + examples, paths = _find_all_code_examples(config, failures) + + _check_for_duplicates(examples) + + for (path, line_num, example_name), code_lines in examples.items(): + example_block = "\n".join(code_lines) + LOGGER.info("example: %r", example_name) + LOGGER.debug("example code: %s", example_block) + + wrap(config, failures, path, partial(file_wrangler.write_example, config, example_name, example_block)) + + return examples, paths, failures + + +def _check_for_duplicates(examples: dict) -> None: + unique_example_names: Dict[str, Any] = dict() + for (path, line_num, example_name), code_lines in examples.items(): + existing = unique_example_names.get(example_name) + if existing: + raise exceptions.DuplicateName("Example with duplicate name %s %s matches %s" % (path, line_num, existing)) + else: + unique_example_names[example_name] = (path, line_num, example_name) + + +def _set_config(config: Config) -> None: + # validate and set IO directories that are relative to project root + config.input_glob = [ + str(Path(config.project_root).joinpath(str(pattern)).absolute()) for pattern in ensure_list(config.input_glob) + ] + config.output_dir = str(Path(config.project_root).joinpath(config.output_dir).absolute()) + + +def _find_all_code_examples(config: Config, failures: List[Any]) -> Tuple[dict, list]: + paths = file_wrangler.find_files(config) + LOGGER.debug("files to parse:\n%s", textwrap.indent("\n".join(paths), prefix=" ")) + examples = dict() + + for path in paths: + # load the file + lines = wrap(config, failures, path, partial(file_wrangler.load_file_lines, path), []) + + # extract snippets + new_examples = wrap(config, failures, path, partial(extract_snippets_from_text, config, lines, path), {}) + + # store the new examples for analysis + examples.update(new_examples) + return examples, paths diff --git a/src/snippet/__init__.py b/src/snippet/__init__.py deleted file mode 100644 index eec8024..0000000 --- a/src/snippet/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -import logging -import textwrap - -from snippet.config import Config -from snippet.workflow import run - - -def main(config: Config): - if config.log_level: - logging.basicConfig(level=config.log_level) - logger = logging.getLogger(__name__) - logger.debug('project directory is %r', config.project_root) - examples, paths, failures = run(config) - - if failures: - logger.error('failures:\n%s', textwrap.indent('\n'.join(f'{name}: {exc}' for name, exc in failures), prefix=' ')) - raise Exception(f'There were %s failures!' % len(failures)) diff --git a/src/snippet/__main__.py b/src/snippet/__main__.py deleted file mode 100644 index 03d523a..0000000 --- a/src/snippet/__main__.py +++ /dev/null @@ -1,3 +0,0 @@ -from snippet.cli import run_from_cli - -run_from_cli() diff --git a/src/snippet/_version.py b/src/snippet/_version.py deleted file mode 100644 index 26d3a89..0000000 --- a/src/snippet/_version.py +++ /dev/null @@ -1,4 +0,0 @@ -# This project's release version -__version__ = '1.1.0' -# This project's release commit hash -COMMIT = "5d5a5cbd12ef49909d78c2247a2f988efb9c4894" diff --git a/src/snippet/cli.py b/src/snippet/cli.py deleted file mode 100644 index 42a7c2f..0000000 --- a/src/snippet/cli.py +++ /dev/null @@ -1,30 +0,0 @@ -import argparse -import os -import logging - -import snippet - - -def get_cli_opts(): - parser = argparse.ArgumentParser() - parser.add_argument('--config', type=str, action='append', - help='paths (or globs) to config files') - parser.add_argument('dir', nargs='?', default=os.getcwd(), - help='path to project root, used by any relative paths in loaded configs [cwd]') - parser.add_argument('-v', '--verbosity', action='count', default=0, - help='increase output verbosity') - return parser - - -def run_from_cli(): - args = get_cli_opts().parse_args() - log_level = logging.WARNING - 10 * args.verbosity - logging.basicConfig(level=log_level) - snippet.main(snippet.config.get_config( - config_paths=args.config, - project_root=args.dir, - )) - - -if __name__ == '__main__': - run_from_cli() diff --git a/src/snippet/config.py b/src/snippet/config.py deleted file mode 100644 index 419c09f..0000000 --- a/src/snippet/config.py +++ /dev/null @@ -1,91 +0,0 @@ -import logging -import glob -import os - -import toml - -from snippet.logs import logger - - -class Config: - # IO - project_root = '.' # the project root used for relative IO paths (set by commandline) - input_glob = 'tests/example/*.py' - output_append = True # if the output file exists, append to it - output_dir = '.' - output_file_name_template = '{{name}}.md' # a mustache template for the output file name - write_attempts = 3 # number of retries when writing output files - - # Language and style - language_name = 'python' - comment_prefix = '# ' - comment_suffix = '' - # a mustache template for each file (triple braces important for code literals, no escaping) - output_template = '```{{language_name}}\n{{comment_prefix}}example: {{{name}}}{{comment_suffix}}\n{{{code}}}\n```\n' - - # Logger - log_level = logging.INFO - - # Code block indicators - start_flag = 'an example' - end_flag = 'end of example' - - # Hidden block indicators - cloak_flag = 'cloak' - uncloak_flag = 'uncloak' - - # Validation and formatting logic - drop_lines = [] # drop lines containing these phrases - replacements = {'self.': ''} # straightforward replacements - fail_on_contains = ['assert'] # fail if these strings are found in code blocks - auto_dedent = True # keep code left-aligned with the start flag - fail_on_dedent = True # fail if code is dedented before reaching the end flag - stop_on_first_failure = False # fail early - - -def find_configs(glob_patterns): - configs = [] - for glob_pattern in glob_patterns: - configs.extend(glob.glob(glob_pattern, recursive=True)) - return configs - - -def config_paths_from_env(): - env_var = os.environ.get('SNIPPET_CONFIG_PATH') - return list(env_var) if env_var else [] - - -def get_config(config_paths=None, **options): - config = Config() - project_root = os.path.abspath(options.get('project_root', config.project_root)) - - new_options = {} - - config_paths = config_paths or [] - config_paths.extend(config_paths_from_env()) - - # fallback option - search the project directory - if not config_paths: - config_paths.append(os.path.join(project_root, '**', '*.toml')) - - for toml_file in find_configs(glob_patterns=config_paths): - logger.debug('trying config from %s', toml_file) - with open(toml_file) as f: - try: - config_file_contents = toml.load(f) - except toml.TomlDecodeError as e: - logger.debug('failed to load %s: %s', toml_file, e) - continue - snippet_config = config_file_contents.get('snippet') - if snippet_config: - logger.info('loading config from %s', toml_file) - new_options.update(snippet_config) - - # passed keyword args override other parameters - new_options.update(options) - - # update the config object - for k, v in new_options.items(): - setattr(config, k, v) - - return config diff --git a/src/snippet/exceptions.py b/src/snippet/exceptions.py deleted file mode 100644 index f398e21..0000000 --- a/src/snippet/exceptions.py +++ /dev/null @@ -1,22 +0,0 @@ -class SnippetError(Exception): - pass - - -class DuplicateName(SnippetError): - pass - - -class ValidationFailure(SnippetError): - pass - - -class TagMismatch(SnippetError): - pass - - -class StartEndMismatch(TagMismatch): - pass - - -class CloakMismatch(TagMismatch): - pass diff --git a/src/snippet/logs.py b/src/snippet/logs.py deleted file mode 100644 index eea436a..0000000 --- a/src/snippet/logs.py +++ /dev/null @@ -1,3 +0,0 @@ -import logging - -logger = logging.getLogger(__name__) diff --git a/src/snippet/snippet.py b/src/snippet/snippet.py deleted file mode 100644 index 4020048..0000000 --- a/src/snippet/snippet.py +++ /dev/null @@ -1,70 +0,0 @@ -from snippet.config import Config -from snippet import exceptions - - -def extract_snippets(config: Config, lines, path): - """Finds snippets in lines of text""" - current_key = None - current_block = None - current_strip = None - capture = False - cloak = False - examples = {} - - for line_num, line in enumerate(lines): - - if config.start_flag in line: - # start capturing code from the next line - example_name = line.rsplit(':')[-1].strip() - current_key = (path, line_num, example_name) - current_block = [] - examples[current_key] = current_block - current_strip = len(line) - len(line.lstrip()) - if capture: - raise exceptions.StartEndMismatch(f'Already capturing at {current_key}') - capture = True - continue - - current_debug_key = f'{current_key} ({line_num})' - - if config.end_flag in line: - # stop capturing, and discard empty blocks - if not capture: - raise exceptions.StartEndMismatch(f'Not yet capturing at {current_debug_key}') - capture = False - if not current_block: - examples.pop(current_key) - continue - - if config.uncloak_flag in line: - if not cloak: - raise exceptions.CloakMismatch(f'Already uncloaked at {current_debug_key}') - cloak = False - continue - - if capture and not cloak: - if config.cloak_flag in line: - cloak = True - continue - - # whilst capturing, append code lines to the current block - if config.fail_on_dedent and any(line[:current_strip].lstrip()): - raise exceptions.ValidationFailure(f'Unexpected dedent whilst capturing {current_debug_key}') - clean_line = line[current_strip:].rstrip() - if any(match in clean_line for match in config.drop_lines): - continue - for r_before, r_after in config.replacements.items(): - clean_line = clean_line.replace(r_before, r_after) - for trigger in config.fail_on_contains: - if trigger in clean_line: - raise exceptions.ValidationFailure(f'Unexpected phrase {repr(trigger)} at {current_debug_key}') - # add this line of code to the example block - current_block.append(clean_line) - - if capture: - raise exceptions.StartEndMismatch(f'EOF reached whilst still capturing {current_debug_key}') - - if cloak: - raise exceptions.CloakMismatch(f'EOF reached whilst still cloaked {current_debug_key}') - - return examples diff --git a/src/snippet/util.py b/src/snippet/util.py deleted file mode 100644 index 7b7717b..0000000 --- a/src/snippet/util.py +++ /dev/null @@ -1,2 +0,0 @@ -def ensure_list(item): - return item if isinstance(item, list) else [item] diff --git a/src/snippet/workflow.py b/src/snippet/workflow.py deleted file mode 100644 index 6fd43f5..0000000 --- a/src/snippet/workflow.py +++ /dev/null @@ -1,58 +0,0 @@ -import textwrap -import os -from functools import partial - -from snippet import file_wrangler -from snippet.config import Config -from snippet.snippet import extract_snippets -from snippet.wrapper import wrap -from snippet import exceptions -from snippet.logs import logger -from snippet.util import ensure_list - - -def run(config: Config): - examples = {} - failures = [] - - # validate and set IO directories that are relative to project root - config.input_glob = [ - os.path.abspath(os.path.join(config.project_root, pattern)) for pattern in ensure_list(config.input_glob) - ] - config.output_dir = os.path.abspath(os.path.join(config.project_root, config.output_dir)) - - paths = file_wrangler.find_files(config) - logger.debug('files to parse:\n%s', textwrap.indent('\n'.join(paths), prefix=' ')) - - for path in paths: - # load the file - lines = wrap(config, failures, path, partial( - file_wrangler.load_file_lines, path - ), []) - - # extract snippets - new_examples = wrap(config, failures, path, partial( - extract_snippets, config, lines, path - ), {}) - - # store the new examples for analysis - examples.update(new_examples) - - unique_example_names = dict() - for (path, line_num, example_name), code_lines in examples.items(): - existing = unique_example_names.get(example_name) - if existing: - raise exceptions.DuplicateName('Example with duplicate name %s %s matches %s' % (path, line_num, existing)) - else: - unique_example_names[example_name] = (path, line_num, example_name) - - for (path, line_num, example_name), code_lines in examples.items(): - example_block = '\n'.join(code_lines) - logger.info('example: %r', example_name) - logger.debug('example code: %s', example_block) - - wrap(config, failures, path, partial( - file_wrangler.write_example, config, example_name, example_block - )) - - return examples, paths, failures diff --git a/src/snippet/wrapper.py b/src/snippet/wrapper.py deleted file mode 100644 index e9737de..0000000 --- a/src/snippet/wrapper.py +++ /dev/null @@ -1,16 +0,0 @@ -import traceback - - -def wrap(config, failures, identifier, nullary_function, default=None): - """executes a function (`nullary_function`) with no arguments - - to pass arguments, use partials - stores any exceptions in `failures` - """ - try: - return nullary_function() - except Exception as e: - if config.stop_on_first_failure: - raise - failures.append((identifier, traceback.format_exc())) - return default diff --git a/tests/__init__.py b/tests/__init__.py index 2d4aa87..7d0571f 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,4 +1,9 @@ -import os -tmp_test_dirname = 'tmp_test_dir' -tmp_test_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), tmp_test_dirname)) -sample_input_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'samples')) +# +# Copyright (C) 2020 Arm Mbed. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +from pathlib import Path + +tmp_test_dirname = "tmp_test_dir" +tmp_test_dir = Path(__file__).parent.joinpath(tmp_test_dirname).absolute() +sample_input_dir = Path(__file__).parent.joinpath("samples").absolute() diff --git a/tests/samples/config_fixture.md b/tests/samples/config_fixture.md index da4bd39..7b5dadc 100644 --- a/tests/samples/config_fixture.md +++ b/tests/samples/config_fixture.md @@ -1,6 +1,6 @@ ```python # example: this config is itself an example -input_glob = 'does not match anything' +input_glob = "does not match anything" stop_on_first_failure = true ``` diff --git a/tests/samples/example.java b/tests/samples/example.java index 658725f..00d4087 100644 --- a/tests/samples/example.java +++ b/tests/samples/example.java @@ -1,3 +1,7 @@ +/* + * Copyright (C) 2020 Arm Mbed. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ import static org.junit.Assert.fail; import java.util.Arrays; diff --git a/tests/samples/example.py b/tests/samples/example.py index 3afe5a0..43ff9b6 100644 --- a/tests/samples/example.py +++ b/tests/samples/example.py @@ -1,11 +1,16 @@ -# this is a python module +# +# Copyright (C) 2020 Arm Mbed. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +"""this is a python module.""" class A: - """A class""" + """A class.""" + def my_example(self): # an example: number 1 - print('x: 5') + print("x: 5") for number in range(5): print(number) # end of example diff --git a/tests/samples/fixture.md b/tests/samples/fixture.md index f0f1de1..af65ced 100644 --- a/tests/samples/fixture.md +++ b/tests/samples/fixture.md @@ -1,6 +1,6 @@ ```python # example: number 1 -print('x: 5') +print("x: 5") for number in range(5): print(number) ``` diff --git a/tests/test_config.py b/tests/test_config.py index 2639d1c..92a1836 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,3 +1,7 @@ +# +# Copyright (C) 2020 Arm Mbed. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# import os import shutil import filecmp @@ -6,40 +10,47 @@ import textwrap import unittest -import snippet +from snippet import config as snippet_config from tests import tmp_test_dir from tests import sample_input_dir class Test(unittest.TestCase): - @classmethod def setUpClass(cls): # use a plain directory not-really-in-tmp to avoid cross-process perms issues in windows os.makedirs(tmp_test_dir) - cls.tmp_fp = os.path.join(tmp_test_dir, 'config.toml') - with open(cls.tmp_fp, 'w') as fh: - fh.write(textwrap.dedent(""" + cls.tmp_fp = os.path.join(tmp_test_dir, "config.toml") + with open(cls.tmp_fp, "w", encoding="utf8") as fh: + fh.write( + textwrap.dedent( + """ [snippet] # an example: this config is itself an example - input_glob = 'does not match anything' + input_glob = "does not match anything" stop_on_first_failure = true - end_flag = 'custom value' + end_flag = "custom value" - foo = 'bar' - fizz = 'buzz' - """).lstrip()) + foo = "bar" + fizz = "buzz" + """ + ).lstrip() + ) - cls.tmp_fp_2 = os.path.join(tmp_test_dir, 'config2.toml') - with open(cls.tmp_fp_2, 'w') as fh: - fh.write(textwrap.dedent(""" + cls.tmp_fp_2 = os.path.join(tmp_test_dir, "config2.toml") + with open(cls.tmp_fp_2, "w", encoding="utf8") as fh: + fh.write( + textwrap.dedent( + """ [snippet] input_glob = 'config.toml' foo = 'baz' - """).lstrip()) + """ + ).lstrip() + ) @classmethod def tearDownClass(cls): @@ -47,32 +58,43 @@ def tearDownClass(cls): def test_config_from_file(self): # explicitly load config from a file - config = snippet.config.get_config(config_paths=[self.tmp_fp]) - self.assertEqual(config.end_flag, 'custom value') - self.assertEqual(config.foo, 'bar') - self.assertEqual(config.fizz, 'buzz') + config = snippet_config.get_config(config_paths=[self.tmp_fp]) + self.assertEqual(config.end_flag, "custom value") + self.assertEqual(config.foo, "bar") + self.assertEqual(config.fizz, "buzz") def test_config_from_multi_globs(self): # explicitly load from two files - config = snippet.config.get_config(config_paths=[self.tmp_fp, self.tmp_fp_2]) - self.assertEqual(config.foo, 'baz') - self.assertEqual(config.fizz, 'buzz') + config = snippet_config.get_config(config_paths=[self.tmp_fp, self.tmp_fp_2]) + self.assertEqual(config.foo, "baz") + self.assertEqual(config.fizz, "buzz") def test_config_from_cli(self): # load config when run as a module subprocess.check_call( - [sys.executable, '-m', 'snippet', tmp_test_dir, '--config', self.tmp_fp, '--config', self.tmp_fp_2], - stderr=subprocess.STDOUT + [ + sys.executable, + "-m", + "snippet", + str(tmp_test_dir), + "--config", + str(self.tmp_fp), + "--config", + str(self.tmp_fp_2), + ], + stderr=subprocess.STDOUT, ) - self.assertTrue(filecmp.cmp( - os.path.join(tmp_test_dir, 'this_config_is_itself_an_example.md'), - os.path.join(sample_input_dir, 'config_fixture.md'), - shallow=False, - )) + self.assertTrue( + filecmp.cmp( + os.path.join(tmp_test_dir, "this_config_is_itself_an_example.md"), + os.path.join(sample_input_dir, "config_fixture.md"), + shallow=False, + ) + ) def test_auto_config(self): # load config, without explicitly setting the config path - config = snippet.config.get_config() - self.assertEqual(config.end_flag, 'custom value') - self.assertEqual(config.fizz, 'buzz') + config = snippet_config.get_config() + self.assertEqual(config.end_flag, "custom value") + self.assertEqual(config.fizz, "buzz") diff --git a/tests/test_direct.py b/tests/test_direct.py index c01a0e7..1addff4 100644 --- a/tests/test_direct.py +++ b/tests/test_direct.py @@ -1,6 +1,10 @@ +# +# Copyright (C) 2020 Arm Mbed. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# import unittest -import snippet -import os +from snippet import config as snippet_config, api +from pathlib import Path import shutil import filecmp @@ -9,29 +13,30 @@ class Test(unittest.TestCase): - def tearDown(self): shutil.rmtree(tmp_test_dir) def test_run(self): # writing two different languages sequentially to the same file - config = snippet.Config() + config = snippet_config.Config() config.output_dir = tmp_test_dir config.output_append = True # only detect the python file - config.input_glob = os.path.join(sample_input_dir, 'example.py') - snippet.main(config=config) + config.input_glob = Path(sample_input_dir).joinpath("example.py") + api.extract_code_snippets(config=config) # only detect the java file - config.input_glob = os.path.join(sample_input_dir, 'example.java') - config.language_name = 'java' - config.comment_prefix = '// ' - snippet.main(config=config) - - self.assertTrue(filecmp.cmp( - os.path.join(tmp_test_dir, 'number_1.md'), - os.path.join(sample_input_dir, 'fixture.md'), - shallow=False, - )) + config.input_glob = Path(sample_input_dir).joinpath("example.java") + config.language_name = "java" + config.comment_prefix = "// " + api.extract_code_snippets(config=config) + + self.assertTrue( + filecmp.cmp( + Path(tmp_test_dir).joinpath("number_1.md"), + Path(sample_input_dir).joinpath("fixture.md"), + shallow=False, + ) + ) diff --git a/tests/test_parser.py b/tests/test_parser.py index 0bd6825..79fa44e 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -1,17 +1,21 @@ +# +# Copyright (C) 2020 Arm Mbed. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# import unittest -from snippet.snippet import extract_snippets +from snippet.snippet import extract_snippets_from_text from snippet.config import Config from snippet import exceptions -start = f'# this is {Config.start_flag}: ' -newline = '\n' -A = 'items = my_api().list_items()\n' -B = 'for item in items:\n' -C = ' print(item.name)\n' -stop = f'# {Config.end_flag}\n' -cloak = f'# {Config.cloak_flag}\n' -uncloak = f'# {Config.uncloak_flag}\n' +start = f"# this is {Config.start_flag}: " +newline = "\n" +A = "items = my_api().list_items()\n" +B = "for item in items:\n" +C = " print(item.name)\n" +stop = f"# {Config.end_flag}\n" +cloak = f"# {Config.cloak_flag}\n" +uncloak = f"# {Config.uncloak_flag}\n" # however the output is combined, it should match this sample_output = """items = my_api().list_items()\nfor item in items:\n print(item.name)""" @@ -22,144 +26,120 @@ class Test(unittest.TestCase): def go(self, config, sequence): # shorthand to get the code equivalent post-parsing - text = ''.join(sequence) - result = extract_snippets(config, text.splitlines(), None) - return ['\n'.join(block) for k, block in result.items()] + text = "".join(sequence) + print(text) + result = extract_snippets_from_text(config, text.splitlines(), "dummy_path") + print(result) + return ["\n".join(block) for k, block in result.items()] def go_exact(self, config, sequence): # *really* shorthand, assumes the result matches the sample exactly - self.assertEqual( - sample_output, - self.go( - config, - sequence - )[0] - ) + self.assertEqual(sample_output, self.go(config, sequence)[0]) # print('test sequence:\n', ''.join(sequence)) def test_empty(self): - self.assertEqual(self.go(Config(), [start, 'test', newline, stop]), []) + self.assertEqual(self.go(Config(), [start, "test", newline, stop]), []) def test_plain(self): - self.go_exact(Config(), [start, 'test', newline, A, B, C, stop]) + self.go_exact(Config(), [start, "test", newline, A, B, C, stop]) def test_indent(self): # left pad the sequence by two spaces, to check result is dedented to depth of start - sequence = [f' {x}' for x in [start + 'test' + newline, A, B, C]] + sequence = [f" {x}" for x in [start + "test" + newline, A, B, C]] sequence.append(stop) - self.go_exact( - Config(), - sequence - ) + self.go_exact(Config(), sequence) def test_indent_tabs(self): # left pad the sequence by two tabs, to check result is dedented to depth of start - sequence = [f'\t\t{x}' for x in [start + 'test' + newline, A, B, C]] + sequence = [f"\t\t{x}" for x in [start + "test" + newline, A, B, C]] sequence.append(stop) - self.go_exact( - Config(), - sequence - ) + self.go_exact(Config(), sequence) def test_indent_with_blank_line(self): # left pad the sequence and then have some blank lines - sequence = [f' {x}' for x in [start + 'test' + newline, A, B, C, stop]] - sequence.insert(2, '') - self.go_exact( - Config(), - sequence - ) + sequence = [f" {x}" for x in [start + "test" + newline, A, B, C, stop]] + sequence.insert(2, "") + self.go_exact(Config(), sequence) def test_trigger_phrase(self): with self.assertRaises(exceptions.ValidationFailure): - self.go_exact(Config(), [start, 'test', newline, A, B, C, 'assert\n', stop]) + self.go_exact(Config(), [start, "test", newline, A, B, C, "assert\n", stop]) def test_dedent_code(self): with self.assertRaises(exceptions.ValidationFailure): - sequence = [f' {x}' for x in [start + 'test' + newline, A, B, C, stop]] + sequence = [f" {x}" for x in [start + "test" + newline, A, B, C, stop]] sequence[-2] = sequence[-2].lstrip() - self.go_exact( - Config(), - sequence - ) + self.go_exact(Config(), sequence) def test_unstarted(self): with self.assertRaises(exceptions.StartEndMismatch): - self.go_exact( - Config(), - [A, B, C, stop] - ) + self.go_exact(Config(), [A, B, C, stop]) def test_unfinished(self): with self.assertRaises(exceptions.StartEndMismatch): - self.go_exact( - Config(), - [start, 'test', newline, A, B, C] - ) + self.go_exact(Config(), [start, "test", newline, A, B, C]) def test_double_start(self): with self.assertRaises(exceptions.StartEndMismatch): - self.go_exact( - Config(), - [start, 'test', newline, A, start, 'test again', newline, B, C, stop] - ) + self.go_exact(Config(), [start, "test", newline, A, start, "test again", newline, B, C, stop]) def test_double_stop(self): with self.assertRaises(exceptions.StartEndMismatch): - self.go_exact( - Config(), - [start, 'test', newline, A, stop, B, C, stop] - ) + self.go_exact(Config(), [start, "test", newline, A, stop, B, C, stop]) def test_prefix(self): - self.go_exact( - Config(), - ['some other stuff', start, 'test', newline, A, B, C, stop, 'other stuff'] - ) + self.go_exact(Config(), ["some other stuff", start, "test", newline, A, B, C, stop, "other stuff"]) def test_cloak(self): self.go_exact( Config(), - ['some other stuff\n', start, 'test', newline, - A, - cloak, - 'ignore this stuff\n', - uncloak, - B, C, - stop, 'other stuff'] + [ + "some other stuff\n", + start, + "test", + newline, + A, + cloak, + "ignore this stuff\n", + uncloak, + B, + C, + stop, + "other stuff", + ], ) def test_cloak_unstarted(self): with self.assertRaises(exceptions.CloakMismatch): - self.go_exact( - Config(), - ['some other stuff', start, 'test', newline, A, uncloak, B, C, stop, 'other stuff'] - ) + self.go_exact(Config(), ["some other stuff", start, "test", newline, A, uncloak, B, C, stop, "other stuff"]) def test_cloak_unfinished(self): with self.assertRaises(exceptions.CloakMismatch): - self.go_exact( - Config(), - ['some other stuff', start, 'test', newline, A, cloak, B, C, stop, 'other stuff'] - ) + self.go_exact(Config(), ["some other stuff", start, "test", newline, A, cloak, B, C, stop, "other stuff"]) def test_multiple(self): sequence = [ - 'some other stuff', - start, 'test 1 ', newline, + "some other stuff", + start, + "test 1 ", + newline, A, cloak, - 'something to hide?', + "something to hide?", uncloak, - B, C, + B, + C, stop, - 'other stuff', - 'more other stuff', - start, ' test 2 ', newline, + "other stuff", + "more other stuff", + start, + " test 2 ", + newline, A, - B, C, + B, + C, stop, - 'more stuff' + "more stuff", ] for i, parsed in enumerate(self.go(Config(), sequence)): with self.subTest(attempt=i): @@ -167,25 +147,12 @@ def test_multiple(self): def test_drop_lines(self): config = Config() - config.drop_lines = ['# ignore me'] + config.drop_lines = ["# ignore me"] self.go_exact( - config, - [start, 'test', newline, - A, - '# ignore me, this comment is unhelpful\n', - B, C, - stop, 'other stuff'] + config, [start, "test", newline, A, "# ignore me, this comment is unhelpful\n", B, C, stop, "other stuff"] ) def test_replacements(self): config = Config() - config.replacements = {'self.': ''} - self.go_exact( - config, - [start, 'test', newline, - 'self.' + A, - B, - C, - stop] - ) - + config.replacements = {"self.": ""} + self.go_exact(config, [start, "test", newline, "self." + A, B, C, stop]) diff --git a/tests/test_support.py b/tests/test_support.py index de10bfe..bde980e 100644 --- a/tests/test_support.py +++ b/tests/test_support.py @@ -1,27 +1,32 @@ +# +# Copyright (C) 2020 Arm Mbed. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# import unittest -from snippet.wrapper import wrap +from snippet._internal.wrapper import wrap from snippet.config import Config class Test(unittest.TestCase): def test_wrap_passthrough(self): - counter = {'a': 0} + counter = {"a": 0} def x(): - counter['a'] += 1 + counter["a"] += 1 return counter f = [] result = wrap(Config(), f, None, x) self.assertEqual(result, counter) - self.assertEqual(counter['a'], 1) + self.assertEqual(counter["a"], 1) self.assertEqual(f, []) def test_wrap_raises(self): def x(): raise NotImplementedError() + config = Config() config.stop_on_first_failure = True f = [] @@ -31,6 +36,7 @@ def x(): def test_wrap_wraps(self): def x(): raise NotImplementedError() + config = Config() config.stop_on_first_failure = False f = [] @@ -40,4 +46,4 @@ def x(): self.assertIs(result, default) self.assertEqual(len(f), 1) self.assertIsNone(f[0][0]) - self.assertIn('Traceback', f[0][1]) + self.assertIn("Traceback", f[0][1]) diff --git a/tests/test_true.py b/tests/test_true.py new file mode 100644 index 0000000..738311d --- /dev/null +++ b/tests/test_true.py @@ -0,0 +1,11 @@ +# +# Copyright (C) 2020 Arm Mbed. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +from snippet import __version__ +from unittest import TestCase + + +class TestPackage(TestCase): + def test_version(self): + self.assertIsNotNone(__version__) diff --git a/tests/test_workflow.py b/tests/test_workflow.py index a97acde..a876a38 100644 --- a/tests/test_workflow.py +++ b/tests/test_workflow.py @@ -1,29 +1,30 @@ +# +# Copyright (C) 2020 Arm Mbed. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# import tempfile -import os import unittest +from pathlib import Path from snippet import exceptions from snippet import workflow from snippet.config import Config - from tests import test_parser as P class Test(unittest.TestCase): - example_name = 'this snippet' + example_name = "this snippet" expect_examples = 1 - text = ''.join([ - 'blah blah\n', P.start, example_name, P.newline, P.A, P.B, P.C, P.stop, '# rhubarb\n' - ]) + text = "".join(["blah blah\n", P.start, example_name, P.newline, P.A, P.B, P.C, P.stop, "# rhubarb\n"]) def setUp(self): self.tmpdir = tempfile.TemporaryDirectory() - self.tmp_fp = os.path.join(self.tmpdir.name, 'sample.txt') - with open(self.tmp_fp, 'w') as fh: - fh.write('\n') + self.tmp_fp = str(Path(self.tmpdir.name).joinpath("sample.txt")) + with open(self.tmp_fp, "w", encoding="utf8") as fh: + fh.write("\n") def test_read(self): - with open(self.tmp_fp, 'w') as fh: + with open(self.tmp_fp, "w", encoding="utf8") as fh: fh.write(self.text) config = Config() @@ -33,35 +34,36 @@ def test_read(self): examples, paths, failures = workflow.run(config) - with self.subTest(part='found the file'): + with self.subTest(part="found the file"): self.assertEqual([self.tmp_fp], paths) - with self.subTest(part='no failures'): + with self.subTest(part="no failures"): self.assertEqual([], failures) - with self.subTest(part='one example extracted'): + with self.subTest(part="one example extracted"): self.assertEqual(len(examples), self.expect_examples) for k, v in examples.items(): - with self.subTest(part='example matches'): - self.assertEqual('\n'.join(v), P.sample_output) + with self.subTest(part="example matches"): + self.assertEqual("\n".join(v), P.sample_output) self.assertIn(self.example_name, k[-1]) + def tearDown(self): + if self.tmpdir: + self.tmpdir.cleanup() + self.tmpdir = None + class TestMultipleExtract(Test): # two examples in one file expect_examples = 2 - text = ''.join([ - 'blah blah\n', P.start, Test.example_name, P.newline, P.A, P.B, P.C, P.stop, '# rhubarb\n' - ]) - text = text + '\n' + text.replace(Test.example_name, 'this snippet 2') + text = "".join(["blah blah\n", P.start, Test.example_name, P.newline, P.A, P.B, P.C, P.stop, "# rhubarb\n"]) + text = text + "\n" + text.replace(Test.example_name, "this snippet 2") class TestDuplicateNames(Test): - text = ''.join([ - 'blah blah\n', P.start, Test.example_name, P.newline, P.A, P.B, P.C, P.stop, '# rhubarb\n' - ]) - text = text + '\n' + text + text = "".join(["blah blah\n", P.start, Test.example_name, P.newline, P.A, P.B, P.C, P.stop, "# rhubarb\n"]) + text = text + "\n" + text def test_read(self): with self.assertRaises(exceptions.DuplicateName): @@ -70,7 +72,5 @@ def test_read(self): class TestNoExamplesInFile(Test): expect_examples = 0 - text = ''.join([ - 'blah blah\n', Test.example_name, P.newline, P.A, P.B, P.C, '# rhubarb\n' - ]) - text = text + '\n' + text + text = "".join(["blah blah\n", Test.example_name, P.newline, P.A, P.B, P.C, "# rhubarb\n"]) + text = text + "\n" + text From 5cd37c1a91d502d06e1c4616dc620fe58d22d230 Mon Sep 17 00:00:00 2001 From: Adrien CABARBAYE Date: Thu, 9 Apr 2020 01:24:39 +0100 Subject: [PATCH 2/3] fix --- tests/test_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_config.py b/tests/test_config.py index 92a1836..b4b57c3 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -45,9 +45,9 @@ def setUpClass(cls): textwrap.dedent( """ [snippet] - input_glob = 'config.toml' + input_glob = "config.toml" - foo = 'baz' + foo = "baz" """ ).lstrip() ) From 3848f2beb90c4d14dc2132ac3f0aa39ad14e0310 Mon Sep 17 00:00:00 2001 From: Adrien CABARBAYE Date: Thu, 9 Apr 2020 08:21:10 +0100 Subject: [PATCH 3/3] Fixed failing tests --- snippet/snippet.py | 6 +----- tests/test_parser.py | 33 ++++++++++++++++++++++++++++++--- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/snippet/snippet.py b/snippet/snippet.py index f213534..34bd9af 100644 --- a/snippet/snippet.py +++ b/snippet/snippet.py @@ -132,10 +132,8 @@ def validate_line(self, fail_on_contains: List[str], line: str, line_num: int) - debug_info = self._current_example.debug_id if self._current_example else "" raise exceptions.ValidationFailure(f"Unexpected phrase {repr(trigger)} at {debug_info} ({line_num})") - def clean_line(self, line: str) -> Optional[str]: + def clean_line(self, line: str) -> str: """Cleans a line.""" - if not line: - return None if not self._current_example: return line start = self._current_example.strip_number @@ -177,8 +175,6 @@ def extract_snippets_from_text(config: Config, lines: list, path: str) -> dict: if config.fail_on_dedent: examples.validate_dedent(line, line_num) clean_line = examples.clean_line(line) - if not clean_line: - continue if any(match in clean_line for match in config.drop_lines): continue for r_before, r_after in config.replacements.items(): diff --git a/tests/test_parser.py b/tests/test_parser.py index 79fa44e..8d3807e 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -7,7 +7,6 @@ from snippet.config import Config from snippet import exceptions - start = f"# this is {Config.start_flag}: " newline = "\n" A = "items = my_api().list_items()\n" @@ -27,9 +26,7 @@ class Test(unittest.TestCase): def go(self, config, sequence): # shorthand to get the code equivalent post-parsing text = "".join(sequence) - print(text) result = extract_snippets_from_text(config, text.splitlines(), "dummy_path") - print(result) return ["\n".join(block) for k, block in result.items()] def go_exact(self, config, sequence): @@ -40,6 +37,36 @@ def go_exact(self, config, sequence): def test_empty(self): self.assertEqual(self.go(Config(), [start, "test", newline, stop]), []) + def test_with_empty_lines(self): + self.assertEqual( + self.go( + Config(), + f""" + {start} + + // Listening to device state changes for 2 minutes. + Thread.sleep(120000); + + // Stopping the Wobble. + wibbler.stop(); + + {stop} + """, + ), + [ + "".join( + [ + "\n", + "// Listening to device state changes for 2 minutes.\n", + "Thread.sleep(120000);\n", + "\n", + "// Stopping the Wobble.\n", + "wibbler.stop();\n", + ] + ) + ], + ) + def test_plain(self): self.go_exact(Config(), [start, "test", newline, A, B, C, stop])