From 19494d9ae0d41c34a077ca900770fbbad48daf2a Mon Sep 17 00:00:00 2001 From: Luca Cipriani Date: Tue, 16 Jul 2019 15:46:06 +0200 Subject: [PATCH 1/8] Added integration test --- .gitignore | 4 ++- test/README.md | 15 ++++++++++ test/pytest.ini | 7 +++++ test/requirements.txt | 14 +++++++++ test/test_main.py | 67 +++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 test/README.md create mode 100644 test/pytest.ini create mode 100644 test/requirements.txt create mode 100644 test/test_main.py diff --git a/.gitignore b/.gitignore index 199c1fe75e4..f5a8089e7a7 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,6 @@ /arduino-cli.yaml /wiki .idea -coverage_*.txt \ No newline at end of file +coverage_*.txt +__pycache__ +venv diff --git a/test/README.md b/test/README.md new file mode 100644 index 00000000000..576dc51458f --- /dev/null +++ b/test/README.md @@ -0,0 +1,15 @@ +## Integration tests + +This dir contains integration tests, the aim is to test the Command Line Interface and its output +from a pure user point of view. + +### Installation + + cd test + virtualenv --python=python3 venv + source venv/bin/activate + pip install -r requirements.txt + +### Running tests + + pytest \ No newline at end of file diff --git a/test/pytest.ini b/test/pytest.ini new file mode 100644 index 00000000000..4d77846a98b --- /dev/null +++ b/test/pytest.ini @@ -0,0 +1,7 @@ +[pytest] +filterwarnings = + error + ignore::DeprecationWarning + ignore::ResourceWarning + +addopts = -s --verbose \ No newline at end of file diff --git a/test/requirements.txt b/test/requirements.txt new file mode 100644 index 00000000000..cb0bb320dbd --- /dev/null +++ b/test/requirements.txt @@ -0,0 +1,14 @@ +atomicwrites==1.3.0 +attrs==19.1.0 +importlib-metadata==0.18 +invoke==1.2.0 +more-itertools==7.1.0 +packaging==19.0 +pep8==1.7.1 +pluggy==0.12.0 +py==1.8.0 +pyparsing==2.4.0 +pytest==5.0.1 +six==1.12.0 +wcwidth==0.1.7 +zipp==0.5.2 \ No newline at end of file diff --git a/test/test_main.py b/test/test_main.py new file mode 100644 index 00000000000..c47af202de0 --- /dev/null +++ b/test/test_main.py @@ -0,0 +1,67 @@ +from invoke import run, Responder +import os +import json + +this_test_path = os.path.dirname(os.path.realpath(__file__)) +# Calculate absolute path of the CLI +cli_path = os.path.join(this_test_path, '..', 'arduino-cli') + +# Useful reference: +# http://docs.pyinvoke.org/en/1.2/api/runners.html#invoke.runners.Result + + +def cli_line(*args): + # Accept a list of arguments cli_line('lib list --format json') + # Return a full command line string e.g. 'arduino-cli help --format json' + cli_full_line = ' '.join([cli_path, ' '.join(str(arg) for arg in args)]) + # print(cli_full_line) + return cli_full_line + + +def run_command(*args): + result = run(cli_line(*args), echo=False, hide='out') + return result + + +def test_command_help(): + result = run_command('help') + assert result.ok + assert result.stderr == '' + assert 'Usage' in result.stdout + # result.out + + +def test_command_lib_list(): + result = run_command('lib list') + assert result.stderr == '' + result = run_command('lib list', '--format json') + assert '{}' == result.stdout + + +def test_command_lib_install(): + libs = ['\"AzureIoTProtocol_MQTT\"', '\"CMMC MQTT Connector\"', '\"WiFiNINA\"'] + # Should be safe to run install multiple times + result_1 = run_command('lib install {}'.format(' '.join(libs))) + assert result_1.ok + result_2 = run_command('lib install {}'.format(' '.join(libs))) + assert result_2.ok + + +def test_command_lib_remove(): + libs = ['\"AzureIoTProtocol_MQTT\"', '\"CMMC MQTT Connector\"', '\"WiFiNINA\"'] + result = run_command('lib uninstall {}'.format(' '.join(libs))) + + +def test_command_board_list(): + result = run_command('board list --format json') + # check is a valid json and contains a list of ports + ports = json.loads(result.stdout).get('ports') + assert isinstance(ports, list) + for port in ports: + assert 'protocol' in port + assert 'protocol_label' in port + + +def test_command_board_listall(): + result = run_command('board listall') + assert ['Board', 'Name', 'FQBN'] == result.stdout.splitlines()[0].strip().split() From a9fb4cdc9dac47d27b4326c3e1941a9e254665f2 Mon Sep 17 00:00:00 2001 From: Luca Cipriani Date: Wed, 17 Jul 2019 11:09:58 +0200 Subject: [PATCH 2/8] Added command lib search testing --- test/conftest.py | 21 +++++++++++++++++++++ test/pytest.ini | 2 +- test/test_main.py | 32 +++++++++++++++++++++++++++++--- 3 files changed, 51 insertions(+), 4 deletions(-) create mode 100644 test/conftest.py diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 00000000000..cc4ec91c7f5 --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,21 @@ +import pytest + + +def pytest_addoption(parser): + parser.addoption( + "--runslow", action="store_true", default=False, help="run slow tests" + ) + + +def pytest_configure(config): + config.addinivalue_line("markers", "slow: mark test as slow to run") + + +def pytest_collection_modifyitems(config, items): + if config.getoption("--runslow"): + # --runslow given in cli: do not skip slow tests + return + skip_slow = pytest.mark.skip(reason="need --runslow option to run") + for item in items: + if "slow" in item.keywords: + item.add_marker(skip_slow) \ No newline at end of file diff --git a/test/pytest.ini b/test/pytest.ini index 4d77846a98b..90432408e87 100644 --- a/test/pytest.ini +++ b/test/pytest.ini @@ -4,4 +4,4 @@ filterwarnings = ignore::DeprecationWarning ignore::ResourceWarning -addopts = -s --verbose \ No newline at end of file +addopts = -s --verbose --runslow \ No newline at end of file diff --git a/test/test_main.py b/test/test_main.py index c47af202de0..d9ccc4f7437 100644 --- a/test/test_main.py +++ b/test/test_main.py @@ -1,6 +1,7 @@ from invoke import run, Responder import os import json +import pytest this_test_path = os.path.dirname(os.path.realpath(__file__)) # Calculate absolute path of the CLI @@ -14,12 +15,11 @@ def cli_line(*args): # Accept a list of arguments cli_line('lib list --format json') # Return a full command line string e.g. 'arduino-cli help --format json' cli_full_line = ' '.join([cli_path, ' '.join(str(arg) for arg in args)]) - # print(cli_full_line) return cli_full_line def run_command(*args): - result = run(cli_line(*args), echo=False, hide='out') + result = run(cli_line(*args), echo=False, hide=True) return result @@ -28,7 +28,6 @@ def test_command_help(): assert result.ok assert result.stderr == '' assert 'Usage' in result.stdout - # result.out def test_command_lib_list(): @@ -50,6 +49,24 @@ def test_command_lib_install(): def test_command_lib_remove(): libs = ['\"AzureIoTProtocol_MQTT\"', '\"CMMC MQTT Connector\"', '\"WiFiNINA\"'] result = run_command('lib uninstall {}'.format(' '.join(libs))) + assert result.ok + +@pytest.mark.slow +def test_command_lib_search(): + result = run_command('lib search') + out_lines = result.stdout.splitlines() + libs = [] + # Create an array with just the name of the vars + for line in out_lines: + if 'Name: ' in line: + libs.append(line.split()[1].strip('\"')) + number_of_libs = len(libs) + # It would be strange to have less than 2000 Arduino Libs published + assert number_of_libs > 2000 + result = run_command('lib search --format json') + libs_found_from_json = json.loads(result.stdout) + number_of_libs_from_json = len(libs_found_from_json.get('libraries')) + assert number_of_libs == number_of_libs_from_json def test_command_board_list(): @@ -65,3 +82,12 @@ def test_command_board_list(): def test_command_board_listall(): result = run_command('board listall') assert ['Board', 'Name', 'FQBN'] == result.stdout.splitlines()[0].strip().split() + +def test_command_version(): + result = run_command('version --format json') + parsed_out = json.loads(result.stdout) + + assert parsed_out.get('command', False) == 'arduino-cli' + assert parsed_out.get('version', False) + assert parsed_out.get('commit', False) + assert parsed_out.get('build_date', False) From fccce601e207632d6cc314e649c8f027a8b0e9a5 Mon Sep 17 00:00:00 2001 From: Luca Cipriani Date: Wed, 17 Jul 2019 14:31:43 +0200 Subject: [PATCH 3/8] Added update-index test and semver testinf for versioning --- test/pytest.ini | 2 +- test/test_main.py | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/test/pytest.ini b/test/pytest.ini index 90432408e87..4d77846a98b 100644 --- a/test/pytest.ini +++ b/test/pytest.ini @@ -4,4 +4,4 @@ filterwarnings = ignore::DeprecationWarning ignore::ResourceWarning -addopts = -s --verbose --runslow \ No newline at end of file +addopts = -s --verbose \ No newline at end of file diff --git a/test/test_main.py b/test/test_main.py index d9ccc4f7437..d9e38875c1e 100644 --- a/test/test_main.py +++ b/test/test_main.py @@ -2,6 +2,7 @@ import os import json import pytest +import semver this_test_path = os.path.dirname(os.path.realpath(__file__)) # Calculate absolute path of the CLI @@ -36,6 +37,9 @@ def test_command_lib_list(): result = run_command('lib list', '--format json') assert '{}' == result.stdout +# def test_command_lib_download(): +# result = run_command('lib download') + def test_command_lib_install(): libs = ['\"AzureIoTProtocol_MQTT\"', '\"CMMC MQTT Connector\"', '\"WiFiNINA\"'] @@ -45,6 +49,9 @@ def test_command_lib_install(): result_2 = run_command('lib install {}'.format(' '.join(libs))) assert result_2.ok +def test_command_lib_update_index(): + result = run_command('lib update-index') + assert 'Updating index: library_index.json downloaded' == result.stdout.splitlines()[-1].strip() def test_command_lib_remove(): libs = ['\"AzureIoTProtocol_MQTT\"', '\"CMMC MQTT Connector\"', '\"WiFiNINA\"'] @@ -83,11 +90,13 @@ def test_command_board_listall(): result = run_command('board listall') assert ['Board', 'Name', 'FQBN'] == result.stdout.splitlines()[0].strip().split() + def test_command_version(): result = run_command('version --format json') parsed_out = json.loads(result.stdout) assert parsed_out.get('command', False) == 'arduino-cli' - assert parsed_out.get('version', False) - assert parsed_out.get('commit', False) + assert isinstance(semver.parse(parsed_out.get('version', False)), dict) + assert isinstance(parsed_out.get('commit', False), str) assert parsed_out.get('build_date', False) + From 95d575c4fe03e12c77ad00a8fe48342e2e2e12c5 Mon Sep 17 00:00:00 2001 From: Luca Cipriani Date: Wed, 17 Jul 2019 17:08:30 +0200 Subject: [PATCH 4/8] Improved error reporting, update test of version command --- test/pytest.ini | 2 +- test/test_main.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/test/pytest.ini b/test/pytest.ini index 4d77846a98b..acd4fa1729b 100644 --- a/test/pytest.ini +++ b/test/pytest.ini @@ -4,4 +4,4 @@ filterwarnings = ignore::DeprecationWarning ignore::ResourceWarning -addopts = -s --verbose \ No newline at end of file +addopts = -s --verbose --tb=short \ No newline at end of file diff --git a/test/test_main.py b/test/test_main.py index d9e38875c1e..6d0a3e9fde8 100644 --- a/test/test_main.py +++ b/test/test_main.py @@ -3,6 +3,7 @@ import json import pytest import semver +from datetime import datetime this_test_path = os.path.dirname(os.path.realpath(__file__)) # Calculate absolute path of the CLI @@ -95,8 +96,8 @@ def test_command_version(): result = run_command('version --format json') parsed_out = json.loads(result.stdout) - assert parsed_out.get('command', False) == 'arduino-cli' - assert isinstance(semver.parse(parsed_out.get('version', False)), dict) - assert isinstance(parsed_out.get('commit', False), str) - assert parsed_out.get('build_date', False) + assert parsed_out.get('Application', False) == 'arduino-cli' + assert isinstance(semver.parse(parsed_out.get('VersionString', False)), dict) + assert isinstance(parsed_out.get('Commit', False), str) + assert datetime.strptime(parsed_out.get('BuildDate')[:-2], '%Y-%m-%dT%H:%M:%S.%f') From 2dd934c687d2c66b7e2abc90271824b90da0f4ed Mon Sep 17 00:00:00 2001 From: Luca Cipriani Date: Thu, 18 Jul 2019 14:34:20 +0200 Subject: [PATCH 5/8] Avoid UnexpectedExit error so the output is more understandable, added result.ok check everywhere --- test/test_main.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/test/test_main.py b/test/test_main.py index 6d0a3e9fde8..ad9bd03b7bd 100644 --- a/test/test_main.py +++ b/test/test_main.py @@ -1,4 +1,4 @@ -from invoke import run, Responder +from invoke import run, Responder, exceptions import os import json import pytest @@ -21,7 +21,7 @@ def cli_line(*args): def run_command(*args): - result = run(cli_line(*args), echo=False, hide=True) + result = run(cli_line(*args), echo=False, hide=True, warn=True) return result @@ -34,6 +34,7 @@ def test_command_help(): def test_command_lib_list(): result = run_command('lib list') + assert result.ok assert result.stderr == '' result = run_command('lib list', '--format json') assert '{}' == result.stdout @@ -52,6 +53,7 @@ def test_command_lib_install(): def test_command_lib_update_index(): result = run_command('lib update-index') + assert result.ok assert 'Updating index: library_index.json downloaded' == result.stdout.splitlines()[-1].strip() def test_command_lib_remove(): @@ -62,6 +64,7 @@ def test_command_lib_remove(): @pytest.mark.slow def test_command_lib_search(): result = run_command('lib search') + assert result.ok out_lines = result.stdout.splitlines() libs = [] # Create an array with just the name of the vars @@ -72,6 +75,7 @@ def test_command_lib_search(): # It would be strange to have less than 2000 Arduino Libs published assert number_of_libs > 2000 result = run_command('lib search --format json') + assert result.ok libs_found_from_json = json.loads(result.stdout) number_of_libs_from_json = len(libs_found_from_json.get('libraries')) assert number_of_libs == number_of_libs_from_json @@ -79,6 +83,7 @@ def test_command_lib_search(): def test_command_board_list(): result = run_command('board list --format json') + assert result.ok # check is a valid json and contains a list of ports ports = json.loads(result.stdout).get('ports') assert isinstance(ports, list) @@ -89,15 +94,16 @@ def test_command_board_list(): def test_command_board_listall(): result = run_command('board listall') + assert result.ok assert ['Board', 'Name', 'FQBN'] == result.stdout.splitlines()[0].strip().split() def test_command_version(): result = run_command('version --format json') + assert result.ok parsed_out = json.loads(result.stdout) assert parsed_out.get('Application', False) == 'arduino-cli' assert isinstance(semver.parse(parsed_out.get('VersionString', False)), dict) assert isinstance(parsed_out.get('Commit', False), str) assert datetime.strptime(parsed_out.get('BuildDate')[:-2], '%Y-%m-%dT%H:%M:%S.%f') - From 005ee714e5580ec579a2fa1d671e42c7337db05c Mon Sep 17 00:00:00 2001 From: Luca Cipriani Date: Thu, 18 Jul 2019 14:45:16 +0200 Subject: [PATCH 6/8] Update requirements, removed unused comments --- .gitignore | 3 ++- test/requirements.txt | 10 +++++++++- test/test_main.py | 3 --- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index f5a8089e7a7..a3924579657 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ /debug /arduino-cli /main -/.vscode/settings.json +/.vscode/ /cmd/formatter/debug.test /arduino-cli.yaml /wiki @@ -9,3 +9,4 @@ coverage_*.txt __pycache__ venv +.pytest_cache diff --git a/test/requirements.txt b/test/requirements.txt index cb0bb320dbd..435c582dadc 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,14 +1,22 @@ +astroid==2.2.5 atomicwrites==1.3.0 attrs==19.1.0 importlib-metadata==0.18 invoke==1.2.0 +isort==4.3.21 +lazy-object-proxy==1.4.1 +mccabe==0.6.1 more-itertools==7.1.0 packaging==19.0 pep8==1.7.1 pluggy==0.12.0 py==1.8.0 +pylint==2.3.1 pyparsing==2.4.0 pytest==5.0.1 +semver==2.8.1 six==1.12.0 +typed-ast==1.4.0 wcwidth==0.1.7 -zipp==0.5.2 \ No newline at end of file +wrapt==1.11.2 +zipp==0.5.2 diff --git a/test/test_main.py b/test/test_main.py index ad9bd03b7bd..bf59bdd4e9f 100644 --- a/test/test_main.py +++ b/test/test_main.py @@ -39,9 +39,6 @@ def test_command_lib_list(): result = run_command('lib list', '--format json') assert '{}' == result.stdout -# def test_command_lib_download(): -# result = run_command('lib download') - def test_command_lib_install(): libs = ['\"AzureIoTProtocol_MQTT\"', '\"CMMC MQTT Connector\"', '\"WiFiNINA\"'] From 6e7338a410e6cf31edfcd2e3ea0733775608f000 Mon Sep 17 00:00:00 2001 From: Luca Cipriani Date: Thu, 18 Jul 2019 14:54:46 +0200 Subject: [PATCH 7/8] moved from custom markers to pytest.ini config for slow files --- test/conftest.py | 21 --------------------- test/pytest.ini | 3 +++ 2 files changed, 3 insertions(+), 21 deletions(-) delete mode 100644 test/conftest.py diff --git a/test/conftest.py b/test/conftest.py deleted file mode 100644 index cc4ec91c7f5..00000000000 --- a/test/conftest.py +++ /dev/null @@ -1,21 +0,0 @@ -import pytest - - -def pytest_addoption(parser): - parser.addoption( - "--runslow", action="store_true", default=False, help="run slow tests" - ) - - -def pytest_configure(config): - config.addinivalue_line("markers", "slow: mark test as slow to run") - - -def pytest_collection_modifyitems(config, items): - if config.getoption("--runslow"): - # --runslow given in cli: do not skip slow tests - return - skip_slow = pytest.mark.skip(reason="need --runslow option to run") - for item in items: - if "slow" in item.keywords: - item.add_marker(skip_slow) \ No newline at end of file diff --git a/test/pytest.ini b/test/pytest.ini index acd4fa1729b..bde27d789f4 100644 --- a/test/pytest.ini +++ b/test/pytest.ini @@ -4,4 +4,7 @@ filterwarnings = ignore::DeprecationWarning ignore::ResourceWarning +markers = + slow: marks tests as slow (deselect with '-m "not slow"') + addopts = -s --verbose --tb=short \ No newline at end of file From 1e4264e7a7b92b8f9e06d8daa284a6f718883234 Mon Sep 17 00:00:00 2001 From: Luca Cipriani Date: Thu, 18 Jul 2019 16:02:12 +0200 Subject: [PATCH 8/8] Testing with real lib names --- test/test_main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_main.py b/test/test_main.py index bf59bdd4e9f..5d03b6a9c35 100644 --- a/test/test_main.py +++ b/test/test_main.py @@ -69,8 +69,8 @@ def test_command_lib_search(): if 'Name: ' in line: libs.append(line.split()[1].strip('\"')) number_of_libs = len(libs) - # It would be strange to have less than 2000 Arduino Libs published - assert number_of_libs > 2000 + assert sorted(libs) == libs + assert ['WiFi101', 'WiFi101OTA'] == [lib for lib in libs if 'WiFi101' in lib] result = run_command('lib search --format json') assert result.ok libs_found_from_json = json.loads(result.stdout)