From 3e6b1a10d020165d2d519331ca31f89c8f9454dc Mon Sep 17 00:00:00 2001 From: Joseph Klix Date: Mon, 4 Apr 2022 11:26:44 -0700 Subject: [PATCH 01/66] rename ISSUE_TEMPLATE folder --- .github/{ISSUE_TEMPLATES => ISSUE_TEMPLATE}/bug-report.yml | 0 .github/{ISSUE_TEMPLATES => ISSUE_TEMPLATE}/config.yml | 0 .github/{ISSUE_TEMPLATES => ISSUE_TEMPLATE}/documentation.yml | 0 .github/{ISSUE_TEMPLATES => ISSUE_TEMPLATE}/feature-request.yml | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename .github/{ISSUE_TEMPLATES => ISSUE_TEMPLATE}/bug-report.yml (100%) rename .github/{ISSUE_TEMPLATES => ISSUE_TEMPLATE}/config.yml (100%) rename .github/{ISSUE_TEMPLATES => ISSUE_TEMPLATE}/documentation.yml (100%) rename .github/{ISSUE_TEMPLATES => ISSUE_TEMPLATE}/feature-request.yml (100%) diff --git a/.github/ISSUE_TEMPLATES/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml similarity index 100% rename from .github/ISSUE_TEMPLATES/bug-report.yml rename to .github/ISSUE_TEMPLATE/bug-report.yml diff --git a/.github/ISSUE_TEMPLATES/config.yml b/.github/ISSUE_TEMPLATE/config.yml similarity index 100% rename from .github/ISSUE_TEMPLATES/config.yml rename to .github/ISSUE_TEMPLATE/config.yml diff --git a/.github/ISSUE_TEMPLATES/documentation.yml b/.github/ISSUE_TEMPLATE/documentation.yml similarity index 100% rename from .github/ISSUE_TEMPLATES/documentation.yml rename to .github/ISSUE_TEMPLATE/documentation.yml diff --git a/.github/ISSUE_TEMPLATES/feature-request.yml b/.github/ISSUE_TEMPLATE/feature-request.yml similarity index 100% rename from .github/ISSUE_TEMPLATES/feature-request.yml rename to .github/ISSUE_TEMPLATE/feature-request.yml From 10b897139f9fa70451984b109c8ef15829671737 Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Mon, 4 Apr 2022 16:22:46 -0700 Subject: [PATCH 02/66] add integration test --- test-integration/.gitignore | 0 test-integration/Credentials/.gitignore | 0 .../Credentials/certificate.pem.crt | 20 ++ .../Credentials/privateKey.pem.key | 28 ++ test-integration/Credentials/rootCA.crt | 20 ++ test-integration/IntegrationTests/.gitignore | 0 ...estAsyncAPIGeneralNotificationCallbacks.py | 159 +++++++++ ...IntegrationTestAutoReconnectResubscribe.py | 200 ++++++++++++ .../IntegrationTestClientReusability.py | 128 ++++++++ ...nTestConfigurablePublishMessageQueueing.py | 303 ++++++++++++++++++ .../IntegrationTestDiscovery.py | 215 +++++++++++++ .../IntegrationTestJobsClient.py | 184 +++++++++++ .../IntegrationTestMQTTConnection.py | 173 ++++++++++ ...tOfflineQueueingForSubscribeUnsubscribe.py | 211 ++++++++++++ .../IntegrationTestProgressiveBackoff.py | 285 ++++++++++++++++ .../IntegrationTests/IntegrationTestShadow.py | 246 ++++++++++++++ .../TestToolLibrary/MQTTClientManager.py | 146 +++++++++ .../TestToolLibrary/SDKPackage/__init__.py | 1 + .../TestToolLibrary/__init__.py | 0 .../TestToolLibrary/checkInManager.py | 18 ++ .../TestToolLibrary/simpleThreadManager.py | 110 +++++++ .../IntegrationTests/TestToolLibrary/skip.py | 110 +++++++ test-integration/Tools/.gitignore | 0 test-integration/Tools/getSDKZIP.sh | 47 +++ test-integration/Tools/retrieve-key.py | 59 ++++ test-integration/run/.gitignore | 0 test-integration/run/run.sh | 202 ++++++++++++ 27 files changed, 2865 insertions(+) create mode 100644 test-integration/.gitignore create mode 100644 test-integration/Credentials/.gitignore create mode 100644 test-integration/Credentials/certificate.pem.crt create mode 100644 test-integration/Credentials/privateKey.pem.key create mode 100644 test-integration/Credentials/rootCA.crt create mode 100644 test-integration/IntegrationTests/.gitignore create mode 100644 test-integration/IntegrationTests/IntegrationTestAsyncAPIGeneralNotificationCallbacks.py create mode 100644 test-integration/IntegrationTests/IntegrationTestAutoReconnectResubscribe.py create mode 100644 test-integration/IntegrationTests/IntegrationTestClientReusability.py create mode 100644 test-integration/IntegrationTests/IntegrationTestConfigurablePublishMessageQueueing.py create mode 100644 test-integration/IntegrationTests/IntegrationTestDiscovery.py create mode 100644 test-integration/IntegrationTests/IntegrationTestJobsClient.py create mode 100644 test-integration/IntegrationTests/IntegrationTestMQTTConnection.py create mode 100644 test-integration/IntegrationTests/IntegrationTestOfflineQueueingForSubscribeUnsubscribe.py create mode 100644 test-integration/IntegrationTests/IntegrationTestProgressiveBackoff.py create mode 100644 test-integration/IntegrationTests/IntegrationTestShadow.py create mode 100644 test-integration/IntegrationTests/TestToolLibrary/MQTTClientManager.py create mode 100644 test-integration/IntegrationTests/TestToolLibrary/SDKPackage/__init__.py create mode 100644 test-integration/IntegrationTests/TestToolLibrary/__init__.py create mode 100644 test-integration/IntegrationTests/TestToolLibrary/checkInManager.py create mode 100644 test-integration/IntegrationTests/TestToolLibrary/simpleThreadManager.py create mode 100644 test-integration/IntegrationTests/TestToolLibrary/skip.py create mode 100644 test-integration/Tools/.gitignore create mode 100644 test-integration/Tools/getSDKZIP.sh create mode 100644 test-integration/Tools/retrieve-key.py create mode 100644 test-integration/run/.gitignore create mode 100644 test-integration/run/run.sh diff --git a/test-integration/.gitignore b/test-integration/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/test-integration/Credentials/.gitignore b/test-integration/Credentials/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/test-integration/Credentials/certificate.pem.crt b/test-integration/Credentials/certificate.pem.crt new file mode 100644 index 0000000..aeb69b2 --- /dev/null +++ b/test-integration/Credentials/certificate.pem.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDWTCCAkGgAwIBAgIUCulzDAbSLt4/M7FiCvxE59ZxeWowDQYJKoZIhvcNAQEL +BQAwTTFLMEkGA1UECwxCQW1hem9uIFdlYiBTZXJ2aWNlcyBPPUFtYXpvbi5jb20g +SW5jLiBMPVNlYXR0bGUgU1Q9V2FzaGluZ3RvbiBDPVVTMB4XDTE2MDkxNzIxNDI1 +MVoXDTQ5MTIzMTIzNTk1OVowHjEcMBoGA1UEAwwTQVdTIElvVCBDZXJ0aWZpY2F0 +ZTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAIqu7JpefK48aLZbaswK +nNY/X20yzD5fa/duH8YZc6G0XskR3jKxtMQa/xz4qShFRyi/XEDWFeVBGe6zsOjo +7Qbf/Q6HzGOxfFbJNd+xo9t/6dxSX7nOFAa3o6EDH4ID6vQGg5S3kmDQWTNfq32Y +NuLfY4d9e8zHXVU/RCTUgbE+FPg7Fli5BoPWUufowsp5ZKPQNrIBwMrDuqY6PRFa +tOolybJi5MiQyaNWHAwoiozoZaJxuMTk+f/a658UYSJmkWLVhAAB1BwRqpJOErUx +dOylGCgcYAcgDJYL7CnJlWlsX/ydGHqu9qah192lpxeiv96oTnDiHbsRfsvjnIAI +XNcCAwEAAaNgMF4wHwYDVR0jBBgwFoAUQh4sEAJgaN2krLfjtGNaPSHeTcQwHQYD +VR0OBBYEFBUVF2WWXjLEdXCq9BSJghTbJzGpMAwGA1UdEwEB/wQCMAAwDgYDVR0P +AQH/BAQDAgeAMA0GCSqGSIb3DQEBCwUAA4IBAQB43Nf+StCdSbQMLFywxnyfcp3u +dZHCrtN45LAUmcJ64LAcr6vAWfdQzAInuv1eSpFl/IXW/k3gjijm6XEC2yCL3efB +6ckJ0vczkX6UrqSOtbltZM8fRjBM2rrRGuaL4M56J4XjEchVJdZ1UXKFtxkeWR2H +NTNQzqH4pEHCwraJA1odAFj1qmh7L2bAQFmC8rBi7y4bRaYPs6v3esxP9a93AZnn +g07camolyUlAMt9sFFqkWxGFST+MvZGN4LG8iytHhBwvp8CdwFXBIRPnJhlIz6uh +/a4p1NgUyFAvE07EhyESRQ9Uf+Og286sOfZPDhmw22VJzAkf3vz7Q8jY1EW0 +-----END CERTIFICATE----- diff --git a/test-integration/Credentials/privateKey.pem.key b/test-integration/Credentials/privateKey.pem.key new file mode 100644 index 0000000..3bbb2c4 --- /dev/null +++ b/test-integration/Credentials/privateKey.pem.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCKruyaXnyuPGi2 +W2rMCpzWP19tMsw+X2v3bh/GGXOhtF7JEd4ysbTEGv8c+KkoRUcov1xA1hXlQRnu +s7Do6O0G3/0Oh8xjsXxWyTXfsaPbf+ncUl+5zhQGt6OhAx+CA+r0BoOUt5Jg0Fkz +X6t9mDbi32OHfXvMx11VP0Qk1IGxPhT4OxZYuQaD1lLn6MLKeWSj0DayAcDKw7qm +Oj0RWrTqJcmyYuTIkMmjVhwMKIqM6GWicbjE5Pn/2uufFGEiZpFi1YQAAdQcEaqS +ThK1MXTspRgoHGAHIAyWC+wpyZVpbF/8nRh6rvamodfdpacXor/eqE5w4h27EX7L +45yACFzXAgMBAAECggEAFWtS67ywMRDvc0rHQeBZvNVo3aq81N1UBZEioywHKfB7 +uz5hPR8pNiidQlI9fhTFGswDaZgnJZNad/2U00Q2Z5tssDPVI4ikFZtaEMSU9C/b +ZMXsNvxXJ3pxcUrUMPZ98VaYN/3qQ72qKT8bc3hw8bVi3+hHmKcqCSKOWcnghglA +OUyCFZiEWnxPRQ8VUkLWawmr1c/1fBCJT2HD9PnLiYv9FwqCU2gUuhR8yzkl0/xF +pxtgr6tUdflCD2MD4jjhICTFKuD75q1ZbkqHwCYSXw6NqayR9k2zID1h/E47FRA0 +6R50nt5c8jsXERo31KGCECEHXt3S9sIsU8HrruSO4QKBgQDMliNUIa/INc5ggo6H +9B15oxZB1pVJSw4ZREuznGJCy6XjjEtayiHhP+nqQQ2Ks5SxA1JXqPQOaoeKTV6i +fitkMMipt5hA3ZHdg6M1ElBveT2zba89TB5kxEJtBxkwK1Az1L7qtJCuw/gJqL27 +mLUyngAvKFRRslgz91GxCERSfwKBgQCtiPj4Ijs4dmEhdDcDt/RogIwOQ0aMIta6 +OIcclXNzTJe1G6GldmfSojdDQHWRGCX83qDtZkwrIg3t0nd+4DQCoSIDhEx4ZKhd +od1Yr+8R94DEk7l5dULfDroDNx56tSZKKx8IXBJZEY1/MPqCxIv1tyUTGHhNtsAx +2BrCumCZqQKBgQCxOY45H0VrJlE1AWP/GdU+vaxWNFD2QPJhqOv7F4l3W3rE949/ +goJ+4iL8LoZQlOhFvx7hmRZyNo5bnFJSaQGltSze+JAIAOiO/62uF8NeDaUJfgbE +DuB1Yh443GFRfPPpMm6AWxLKkjCYDXjuvYaZ5o06TLFeZCRMP/6gYQzueQKBgDY7 +RRs+j2VQ0YAD8qFt3qq96YcXjTeEN7jQq6DKL99Lg2ms7fJos3+HBGA3B8bFVjHV +XVNzkcA1D0dQa9mxtW9Q7fFNahyB0IAacrMhoEPsJkjIpcEIXVKaJpWUpyjP7dxl +53fUVkORkbe7Fb7CL78lcieqkQYwY5XwJETQvBmpAoGAMcC/2oWca/+An8CxKxpR +i5qyKsqner21+PbxISSTJPgfDdzkQ9CwdJnSSZW8bT9eeE/D2kqo43rRLpTo/DOm +18EEr6M3ED1sgzdOEYiCsKEVEr/MZNOv6155KkSN5mJknKWE+TOvu3Zr8k8Ejjim +Iouc5aXZwdb4ERaY7jNsTm8= +-----END PRIVATE KEY----- diff --git a/test-integration/Credentials/rootCA.crt b/test-integration/Credentials/rootCA.crt new file mode 100644 index 0000000..a6f3e92 --- /dev/null +++ b/test-integration/Credentials/rootCA.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDQTCCAimgAwIBAgITBmyfz5m/jAo54vB4ikPmljZbyjANBgkqhkiG9w0BAQsF +ADA5MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6 +b24gUm9vdCBDQSAxMB4XDTE1MDUyNjAwMDAwMFoXDTM4MDExNzAwMDAwMFowOTEL +MAkGA1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEZMBcGA1UEAxMQQW1hem9uIFJv +b3QgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALJ4gHHKeNXj +ca9HgFB0fW7Y14h29Jlo91ghYPl0hAEvrAIthtOgQ3pOsqTQNroBvo3bSMgHFzZM +9O6II8c+6zf1tRn4SWiw3te5djgdYZ6k/oI2peVKVuRF4fn9tBb6dNqcmzU5L/qw +IFAGbHrQgLKm+a/sRxmPUDgH3KKHOVj4utWp+UhnMJbulHheb4mjUcAwhmahRWa6 +VOujw5H5SNz/0egwLX0tdHA114gk957EWW67c4cX8jJGKLhD+rcdqsq08p8kDi1L +93FcXmn/6pUCyziKrlA4b9v7LWIbxcceVOF34GfID5yHI9Y/QCB/IIDEgEw+OyQm +jgSubJrIqg0CAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMC +AYYwHQYDVR0OBBYEFIQYzIU07LwMlJQuCFmcx7IQTgoIMA0GCSqGSIb3DQEBCwUA +A4IBAQCY8jdaQZChGsV2USggNiMOruYou6r4lK5IpDB/G/wkjUu0yKGX9rbxenDI +U5PMCCjjmCXPI6T53iHTfIUJrU6adTrCC2qJeHZERxhlbI1Bjjt/msv0tadQ1wUs +N+gDS63pYaACbvXy8MWy7Vu33PqUXHeeE6V/Uq2V8viTO96LXFvKWlJbYK8U90vv +o/ufQJVtMVT8QtPHRh8jrdkPSHCa2XV4cdFyQzR1bldZwgJcJmApzyMZFo6IQ6XU +5MsI+yMRQ+hDKXJioaldXgjUkK642M4UwtBV8ob2xJNDd2ZhwLnoQdeXeGADbkpy +rqXRfboQnoZsG4q5WTP468SQvvG5 +-----END CERTIFICATE----- diff --git a/test-integration/IntegrationTests/.gitignore b/test-integration/IntegrationTests/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/test-integration/IntegrationTests/IntegrationTestAsyncAPIGeneralNotificationCallbacks.py b/test-integration/IntegrationTests/IntegrationTestAsyncAPIGeneralNotificationCallbacks.py new file mode 100644 index 0000000..409700e --- /dev/null +++ b/test-integration/IntegrationTests/IntegrationTestAsyncAPIGeneralNotificationCallbacks.py @@ -0,0 +1,159 @@ +# This integration test verifies the functionality of asynchronous API for plain MQTT operations, as well as general +# notification callbacks. There are 2 phases for this test: +# a) Testing async APIs + onMessage general notification callback +# b) Testing onOnline, onOffline notification callbacks +# To achieve test goal a) and b), the client will follow the routine described below: +# 1. Client does async connect to AWS IoT and captures the CONNACK event and onOnline callback event in the record +# 2. Client does async subscribe to a topic and captures the SUBACK event in the record +# 3. Client does several async publish (QoS1) to the same topic and captures the PUBACK event in the record +# 4. Since client subscribes and publishes to the same topic, onMessage callback should be triggered. We capture these +# events as well in the record. +# 5. Client does async disconnect. This would trigger the offline callback and disconnect event callback. We capture +# them in the record. +# We should be able to receive all ACKs for all operations and corresponding general notification callback triggering +# events. + + +import random +import string +import time +import sys +sys.path.insert(0, "./configuration/IntegrationTests/TestToolLibrary") +sys.path.insert(0, "./configuration/IntegrationTests/TestToolLibrary/SDKPackage") + +from TestToolLibrary.checkInManager import checkInManager +from TestToolLibrary.MQTTClientManager import MQTTClientManager +from TestToolLibrary.skip import skip_when_match +from TestToolLibrary.skip import ModeIsALPN +from TestToolLibrary.skip import Python2VersionLowerThan +from TestToolLibrary.skip import Python3VersionLowerThan + + +TOPIC = "topic/test/async_cb/" +MESSAGE_PREFIX = "MagicMessage-" +NUMBER_OF_PUBLISHES = 3 +HOST = "ajje7lpljulm4-ats.iot.us-east-1.amazonaws.com" +ROOT_CA = "./configuration/Credentials/rootCA.crt" +CERT = "./configuration/Credentials/certificate.pem.crt" +KEY = "./configuration/Credentials/privateKey.pem.key" +CLIENT_ID = "PySdkIntegTest_AsyncAPI_Callbacks" + +KEY_ON_ONLINE = "OnOnline" +KEY_ON_OFFLINE = "OnOffline" +KEY_ON_MESSAGE = "OnMessage" +KEY_CONNACK = "Connack" +KEY_DISCONNECT = "Disconnect" +KEY_PUBACK = "Puback" +KEY_SUBACK = "Suback" +KEY_UNSUBACK = "Unsuback" + + +class CallbackManager(object): + + def __init__(self): + self.callback_invocation_record = { + KEY_ON_ONLINE : 0, + KEY_ON_OFFLINE : 0, + KEY_ON_MESSAGE : 0, + KEY_CONNACK : 0, + KEY_DISCONNECT : 0, + KEY_PUBACK : 0, + KEY_SUBACK : 0, + KEY_UNSUBACK : 0 + } + + def on_online(self): + print("OMG, I am online!") + self.callback_invocation_record[KEY_ON_ONLINE] += 1 + + def on_offline(self): + print("OMG, I am offline!") + self.callback_invocation_record[KEY_ON_OFFLINE] += 1 + + def on_message(self, message): + print("OMG, I got a message!") + self.callback_invocation_record[KEY_ON_MESSAGE] += 1 + + def connack(self, mid, data): + print("OMG, I got a connack!") + self.callback_invocation_record[KEY_CONNACK] += 1 + + def disconnect(self, mid, data): + print("OMG, I got a disconnect!") + self.callback_invocation_record[KEY_DISCONNECT] += 1 + + def puback(self, mid): + print("OMG, I got a puback!") + self.callback_invocation_record[KEY_PUBACK] += 1 + + def suback(self, mid, data): + print("OMG, I got a suback!") + self.callback_invocation_record[KEY_SUBACK] += 1 + + def unsuback(self, mid): + print("OMG, I got an unsuback!") + self.callback_invocation_record[KEY_UNSUBACK] += 1 + + +def get_random_string(length): + return "".join(random.choice(string.ascii_lowercase) for i in range(length)) + + +############################################################################ +# Main # +# Check inputs +my_check_in_manager = checkInManager(1) +my_check_in_manager.verify(sys.argv) +mode = my_check_in_manager.mode + +skip_when_match(ModeIsALPN(mode).And( + Python2VersionLowerThan((2, 7, 10)).Or(Python3VersionLowerThan((3, 5, 0))) +), "This test is not applicable for mode %s and Python verison %s. Skipping..." % (mode, sys.version_info[:3])) + +# Performing +############ +print("Connecting...") +callback_manager = CallbackManager() +sdk_mqtt_client = MQTTClientManager()\ + .create_nonconnected_mqtt_client(mode, CLIENT_ID, HOST, (ROOT_CA, CERT, KEY), callback_manager) +sdk_mqtt_client.connectAsync(keepAliveIntervalSecond=1, ackCallback=callback_manager.connack) # Add callback +print("Wait some time to make sure we are connected...") +time.sleep(10) # 10 sec + +topic = TOPIC + get_random_string(4) +print("Subscribing to topic: " + topic) +sdk_mqtt_client.subscribeAsync(topic, 1, ackCallback=callback_manager.suback, messageCallback=None) +print("Wait some time to make sure we are subscribed...") +time.sleep(3) # 3 sec + +print("Publishing...") +for i in range(NUMBER_OF_PUBLISHES): + sdk_mqtt_client.publishAsync(topic, MESSAGE_PREFIX + str(i), 1, ackCallback=callback_manager.puback) + time.sleep(1) +print("Wait sometime to make sure we finished with publishing...") +time.sleep(2) + +print("Unsubscribing...") +sdk_mqtt_client.unsubscribeAsync(topic, ackCallback=callback_manager.unsuback) +print("Wait sometime to make sure we finished with unsubscribing...") +time.sleep(2) + +print("Disconnecting...") +sdk_mqtt_client.disconnectAsync(ackCallback=callback_manager.disconnect) + +print("Wait sometime to let the test result sync...") +time.sleep(3) + +print("Verifying...") +try: + assert callback_manager.callback_invocation_record[KEY_ON_ONLINE] == 1 + assert callback_manager.callback_invocation_record[KEY_CONNACK] == 1 + assert callback_manager.callback_invocation_record[KEY_SUBACK] == 1 + assert callback_manager.callback_invocation_record[KEY_PUBACK] == NUMBER_OF_PUBLISHES + assert callback_manager.callback_invocation_record[KEY_ON_MESSAGE] == NUMBER_OF_PUBLISHES + assert callback_manager.callback_invocation_record[KEY_UNSUBACK] == 1 + assert callback_manager.callback_invocation_record[KEY_DISCONNECT] == 1 + assert callback_manager.callback_invocation_record[KEY_ON_OFFLINE] == 1 +except BaseException as e: + print("Failed! %s" % e.message) +print("Pass!") diff --git a/test-integration/IntegrationTests/IntegrationTestAutoReconnectResubscribe.py b/test-integration/IntegrationTests/IntegrationTestAutoReconnectResubscribe.py new file mode 100644 index 0000000..8292075 --- /dev/null +++ b/test-integration/IntegrationTests/IntegrationTestAutoReconnectResubscribe.py @@ -0,0 +1,200 @@ +# This integration test verifies the functionality in the Python core of Yun/Python SDK +# for auto-reconnect and auto-resubscribe. +# It starts two threads using two different connections to AWS IoT: +# Thread A publishes 10 messages to topicB first, then quiet for a while, and finally +# publishes another 10 messages to topicB. +# Thread B subscribes to topicB and waits to receive messages. Once it receives the first +# 10 messages. It simulates a network error, disconnecting from the broker. In a short time, +# it should automatically reconnect and resubscribe to the previous topic and be able to +# receive the next 10 messages from thread A. +# Because of auto-reconnect/resubscribe, thread B should be able to receive all of the +# messages from topicB published by thread A without calling subscribe again in user code +# explicitly. + + +import random +import string +import sys +import time +sys.path.insert(0, "./configuration/IntegrationTests/TestToolLibrary") +sys.path.insert(0, "./configuration/IntegrationTests/TestToolLibrary/SDKPackage") + +import TestToolLibrary.checkInManager as checkInManager +import TestToolLibrary.MQTTClientManager as MQTTClientManager +from TestToolLibrary import simpleThreadManager +from TestToolLibrary.SDKPackage.AWSIoTPythonSDK.exception.AWSIoTExceptions import publishError +from TestToolLibrary.SDKPackage.AWSIoTPythonSDK.exception.AWSIoTExceptions import subscribeError +from TestToolLibrary.SDKPackage.AWSIoTPythonSDK.exception.AWSIoTExceptions import subscribeTimeoutException +from TestToolLibrary.skip import skip_when_match +from TestToolLibrary.skip import ModeIsALPN +from TestToolLibrary.skip import Python2VersionLowerThan +from TestToolLibrary.skip import Python3VersionLowerThan + + +# Callback unit +class callbackUnit: + def __init__(self): + self._internalSet = set() + + # Callback fro clientSub + def messageCallback(self, client, userdata, message): + print("Received a new message: " + str(message.payload)) + self._internalSet.add(message.payload.decode('utf-8')) + + def getInternalSet(self): + return self._internalSet + + +# Simulate a network error +def manualNetworkError(srcPyMQTTCore): + # Ensure we close the socket + if srcPyMQTTCore._internal_async_client._paho_client._sock: + srcPyMQTTCore._internal_async_client._paho_client._sock.close() + srcPyMQTTCore._internal_async_client._paho_client._sock = None + if srcPyMQTTCore._internal_async_client._paho_client._ssl: + srcPyMQTTCore._internal_async_client._paho_client._ssl.close() + srcPyMQTTCore._internal_async_client._paho_client._ssl = None + # Fake that we have detected the disconnection + srcPyMQTTCore._internal_async_client._paho_client.on_disconnect(None, None, 0) + + +# runFunctionUnit +class runFunctionUnit(): + def __init__(self): + self._messagesPublished = set() + self._topicB = "topicB/" + "".join(random.choice(string.ascii_lowercase) for i in range(4)) + + # ThreadA runtime function: + # 1. Publish 10 messages to topicB. + # 2. Take a nap: 20 sec + # 3. Publish another 10 messages to topicB. + def threadARuntime(self, pyCoreClient): + time.sleep(3) # Ensure a valid subscription + messageCount = 0 + # First 10 messages + while messageCount < 10: + try: + pyCoreClient.publish(self._topicB, str(messageCount), 1, False) + self._messagesPublished.add(str(messageCount)) + except publishError: + print("Publish error!") + except Exception as e: + print("Unknown exception!") + print("Type: " + str(type(e))) + print("Message: " + str(e.message)) + messageCount += 1 + time.sleep(0.5) # TPS = 2 + # Take a nap + time.sleep(20) + # Second 10 messages + while messageCount < 20: + try: + pyCoreClient.publish(self._topicB, str(messageCount), 1, False) + self._messagesPublished.add(str(messageCount)) + except publishError: + print("Publish Error!") + except Exception as e: + print("Unknown exception!") + print("Type: " + str(type(e))) + print("Message: " + str(e.message)) + messageCount += 1 + time.sleep(0.5) + print("Publish thread terminated.") + + # ThreadB runtime function: + # 1. Subscribe to topicB + # 2. Wait for a while + # 3. Create network blocking, triggering auto-reconnect and auto-resubscribe + # 4. On connect, wait for another while + def threadBRuntime(self, pyCoreClient, callback): + try: + # Subscribe to topicB + pyCoreClient.subscribe(self._topicB, 1, callback) + except subscribeTimeoutException: + print("Subscribe timeout!") + except subscribeError: + print("Subscribe error!") + except Exception as e: + print("Unknown exception!") + print("Type: " + str(type(e))) + print("Message: " + str(e.message)) + # Wait to get the first 10 messages from thread A + time.sleep(10) + # Block the network for 3 sec + print("Block the network for 3 sec...") + blockingTimeTenMs = 300 + while blockingTimeTenMs != 0: + manualNetworkError(pyCoreClient) + blockingTimeTenMs -= 1 + time.sleep(0.01) + print("Leave it to the main thread to keep waiting...") + + +############################################################################ +# Main # +# Check inputs +myCheckInManager = checkInManager.checkInManager(1) +myCheckInManager.verify(sys.argv) + +host = "ajje7lpljulm4-ats.iot.us-east-1.amazonaws.com" +rootCA = "./configuration/Credentials/rootCA.crt" +certificate = "./configuration/Credentials/certificate.pem.crt" +privateKey = "./configuration/Credentials/privateKey.pem.key" +mode = myCheckInManager.mode + +skip_when_match(ModeIsALPN(mode).And( + Python2VersionLowerThan((2, 7, 10)).Or(Python3VersionLowerThan((3, 5, 0))) +), "This test is not applicable for mode %s and Python verison %s. Skipping..." % (mode, sys.version_info[:3])) + +# Init Python core and connect +myMQTTClientManager = MQTTClientManager.MQTTClientManager() +clientPub = myMQTTClientManager.create_connected_mqtt_core("integrationTestMQTT_ClientPub", host, rootCA, + certificate, privateKey, mode=mode) +clientSub = myMQTTClientManager.create_connected_mqtt_core("integrationTestMQTT_ClientSub", host, rootCA, + certificate, privateKey, mode=mode) + +if clientPub is None or clientSub is None: + print("Clients not init!") + exit(4) + +print("Two clients are connected!") + +# Configurations +################ +# Callback unit +subCallbackUnit = callbackUnit() +# Threads +mySimpleThreadManager = simpleThreadManager.simpleThreadManager() +myRunFunctionUnit = runFunctionUnit() +publishThreadID = mySimpleThreadManager.createOneTimeThread(myRunFunctionUnit.threadARuntime, [clientPub]) +subscribeThreadID = mySimpleThreadManager.createOneTimeThread(myRunFunctionUnit.threadBRuntime, + [clientSub, subCallbackUnit.messageCallback]) + +# Performing +############ +mySimpleThreadManager.startThreadWithID(subscribeThreadID) +mySimpleThreadManager.startThreadWithID(publishThreadID) +mySimpleThreadManager.joinOneTimeThreadWithID(subscribeThreadID) +mySimpleThreadManager.joinOneTimeThreadWithID(publishThreadID) +time.sleep(3) # Just in case messages arrive slowly + +# Verifying +########### +# Length +print("Check if the length of the two sets are equal...") +print("Received from subscription: " + str(len(subCallbackUnit.getInternalSet()))) +print("Sent through publishes: " + str(len(myRunFunctionUnit._messagesPublished))) +if len(myRunFunctionUnit._messagesPublished) != len(subCallbackUnit.getInternalSet()): + print("Number of messages not equal!") + exit(4) +# Content +print("Check if the content if the two sets are equivalent...") +if myRunFunctionUnit._messagesPublished != subCallbackUnit.getInternalSet(): + print("Sent through publishes:") + print(myRunFunctionUnit._messagesPublished) + print("Received from subscription:") + print(subCallbackUnit.getInternalSet()) + print("Set content not equal!") + exit(4) +else: + print("Yes!") diff --git a/test-integration/IntegrationTests/IntegrationTestClientReusability.py b/test-integration/IntegrationTests/IntegrationTestClientReusability.py new file mode 100644 index 0000000..2cd1d26 --- /dev/null +++ b/test-integration/IntegrationTests/IntegrationTestClientReusability.py @@ -0,0 +1,128 @@ +# This integration test verifies the re-usability of SDK MQTT client. +# By saying re-usability, we mean that users should be able to reuse +# the same SDK MQTT client object to connect and invoke other APIs +# after a disconnect API call has been invoked on that client object. +# This test contains 2 clients living 2 separate threads: +# 1. Thread publish: In this thread, a MQTT client will do the following +# in a loop: +# a. Connect to AWS IoT +# b. Publish several messages to a dedicated topic +# c. Disconnect from AWS IoT +# d. Sleep for a while +# 2. Thread subscribe: In this thread, a MQTT client will do nothing +# other than subscribing to a dedicated topic and counting the incoming +# messages. +# Assuming the client is reusable, the subscriber should be able to +# receive all the messages published by the publisher from the same +# client object in different connect sessions. + + +import uuid +import time +import sys +sys.path.insert(0, "./configuration/IntegrationTests/TestToolLibrary") +sys.path.insert(0, "./configuration/IntegrationTests/TestToolLibrary/SDKPackage") + +from threading import Event +from TestToolLibrary.checkInManager import checkInManager +from TestToolLibrary.simpleThreadManager import simpleThreadManager +from TestToolLibrary.MQTTClientManager import MQTTClientManager +from TestToolLibrary.skip import skip_when_match +from TestToolLibrary.skip import ModeIsALPN +from TestToolLibrary.skip import Python2VersionLowerThan +from TestToolLibrary.skip import Python3VersionLowerThan + + +TOPIC = "topic/" + str(uuid.uuid1()) +CLIENT_ID_PUB = "publisher" + str(uuid.uuid1()) +CLIENT_ID_SUB = "subscriber" + str(uuid.uuid1()) +MESSAGE_PREFIX = "Message-" +NUMBER_OF_MESSAGES_PER_LOOP = 3 +NUMBER_OF_LOOPS = 3 +SUB_WAIT_TIME_OUT_SEC = 20 +HOST = "ajje7lpljulm4-ats.iot.us-east-1.amazonaws.com" +ROOT_CA = "./configuration/Credentials/rootCA.crt" +CERT = "./configuration/Credentials/certificate.pem.crt" +KEY = "./configuration/Credentials/privateKey.pem.key" + + +class ClientTwins(object): + + def __init__(self, client_pub, client_sub): + self._client_pub = client_pub + self._client_sub = client_sub + self._message_publish_set = set() + self._message_receive_set = set() + self._publish_done = Event() + + def run_publisher(self, *params): + self._publish_done.clear() + time.sleep(3) + for i in range(NUMBER_OF_LOOPS): + self._single_publish_loop(i) + time.sleep(2) + self._publish_done.set() + + def _single_publish_loop(self, iteration_count): + print("In loop %d: " % iteration_count) + self._client_pub.connect() + print("Publisher connected!") + for i in range(NUMBER_OF_MESSAGES_PER_LOOP): + message = MESSAGE_PREFIX + str(iteration_count) + "_" + str(i) + self._client_pub.publish(TOPIC, message, 1) + print("Publisher published %s to topic %s" % (message, TOPIC)) + self._message_publish_set.add(message.encode("utf-8")) + time.sleep(1) + self._client_pub.disconnect() + print("Publisher disconnected!\n\n") + + def run_subscriber(self, *params): + self._client_sub.connect() + self._client_sub.subscribe(TOPIC, 1, self._callback) + self._publish_done.wait(20) + self._client_sub.disconnect() + + def _callback(self, client, user_data, message): + self._message_receive_set.add(message.payload) + print("Subscriber received %s from topic %s" % (message.payload, message.topic)) + + def verify(self): + assert len(self._message_receive_set) != 0 + assert len(self._message_publish_set) != 0 + assert self._message_publish_set == self._message_receive_set + + +############################################################################ +# Main # +my_check_in_manager = checkInManager(1) +my_check_in_manager.verify(sys.argv) +mode = my_check_in_manager.mode + +skip_when_match(ModeIsALPN(mode).And( + Python2VersionLowerThan((2, 7, 10)).Or(Python3VersionLowerThan((3, 5, 0))) +), "This test is not applicable for mode %s and Python verison %s. Skipping..." % (mode, sys.version_info[:3])) + +simple_thread_manager = simpleThreadManager() + +client_pub = MQTTClientManager().create_nonconnected_mqtt_client(mode, CLIENT_ID_PUB, HOST, (ROOT_CA, CERT, KEY)) +print("Client publisher initialized.") +client_sub = MQTTClientManager().create_nonconnected_mqtt_client(mode, CLIENT_ID_SUB, HOST, (ROOT_CA, CERT, KEY)) +print("Client subscriber initialized.") +client_twins = ClientTwins(client_pub, client_sub) +print("Client twins initialized.") + +publisher_thread_id = simple_thread_manager.createOneTimeThread(client_twins.run_publisher, []) +subscriber_thread_id = simple_thread_manager.createOneTimeThread(client_twins.run_subscriber, []) +simple_thread_manager.startThreadWithID(subscriber_thread_id) +print("Started subscriber thread.") +simple_thread_manager.startThreadWithID(publisher_thread_id) +print("Started publisher thread.") + +print("Main thread starts waiting.") +simple_thread_manager.joinOneTimeThreadWithID(publisher_thread_id) +simple_thread_manager.joinOneTimeThreadWithID(subscriber_thread_id) +print("Main thread waiting is done!") + +print("Verifying...") +client_twins.verify() +print("Pass!") diff --git a/test-integration/IntegrationTests/IntegrationTestConfigurablePublishMessageQueueing.py b/test-integration/IntegrationTests/IntegrationTestConfigurablePublishMessageQueueing.py new file mode 100644 index 0000000..f213bcb --- /dev/null +++ b/test-integration/IntegrationTests/IntegrationTestConfigurablePublishMessageQueueing.py @@ -0,0 +1,303 @@ +# This integration test verifies the functionality in the Python core of Yun SDK +# for configurable offline publish message queueing. +# For each offline publish queue to be tested, it starts two threads using +# different connections to AWS IoT: +# Thread A subscribes to TopicOnly and wait to receive messages published to +# TopicOnly from ThreadB. +# Thread B publishes to TopicOnly with a manual network error which triggers the +# offline publish message queueing. According to different configurations, the +# internal queue should keep as many publish requests as configured and then +# republish them once the connection is back. +# * After the network is down but before the client gets the notification of being +# * disconnected, QoS0 messages in between this "blind-window" will be lost. However, +# * once the client gets the notification, it should start queueing messages up to +# * its queue size limit. +# * Therefore, all published messages are QoS0, we are verifying the total amount. +# * Configuration to be tested: +# 1. Limited queueing section, limited response (in-flight) section, drop oldest +# 2. Limited queueing section, limited response (in-flight) section, drop newest + + +import threading +import sys +import time +import random +import string +sys.path.insert(0, "./configuration/IntegrationTests/TestToolLibrary") +sys.path.insert(0, "./configuration/IntegrationTests/TestToolLibrary/SDKPackage") + +import TestToolLibrary.checkInManager as checkInManager +import TestToolLibrary.MQTTClientManager as MQTTClientManager +from TestToolLibrary.SDKPackage.AWSIoTPythonSDK.core.util.enums import DropBehaviorTypes +from TestToolLibrary.SDKPackage.AWSIoTPythonSDK.exception.AWSIoTExceptions import publishError +from TestToolLibrary.SDKPackage.AWSIoTPythonSDK.exception.AWSIoTExceptions import publishQueueFullException +from TestToolLibrary.SDKPackage.AWSIoTPythonSDK.exception.AWSIoTExceptions import subscribeError +from TestToolLibrary.SDKPackage.AWSIoTPythonSDK.exception.AWSIoTExceptions import subscribeTimeoutException +from TestToolLibrary.skip import skip_when_match +from TestToolLibrary.skip import ModeIsALPN +from TestToolLibrary.skip import Python2VersionLowerThan +from TestToolLibrary.skip import Python3VersionLowerThan + + +# Class that implements the publishing thread: Thread A, with network failure +# This thread will publish 3 messages first, and then keep publishing +# with a network failure, and then publish another set of 3 messages +# once the connection is resumed. +# * TPS = 1 +class threadPub: + def __init__(self, pyCoreClient, numberOfOfflinePublish, srcTopic): + self._publishMessagePool = list() + self._pyCoreClient = pyCoreClient + self._numberOfOfflinePublish = numberOfOfflinePublish + self._topic = srcTopic + + # Simulate a network error + def _manualNetworkError(self): + # Ensure we close the socket + if self._pyCoreClient._internal_async_client._paho_client._sock: + self._pyCoreClient._internal_async_client._paho_client._sock.close() + self._pyCoreClient._internal_async_client._paho_client._sock = None + if self._pyCoreClient._internal_async_client._paho_client._ssl: + self._pyCoreClient._internal_async_client._paho_client._ssl.close() + self._pyCoreClient._internal_async_client._paho_client._ssl = None + # Fake that we have detected the disconnection + self._pyCoreClient._internal_async_client._paho_client.on_disconnect(None, None, 0) + + def _runtime(self): + messageCount = 0 + # Publish 3 messages + print("Thread A: Publish 3 messages.") + step1PublishCount = 3 + while step1PublishCount != 0: + currentMessage = str(messageCount) + self._publishMessagePool.append(int(currentMessage)) + try: + self._pyCoreClient.publish(self._topic, currentMessage, 0, False) + print("Thread A: Published a message: " + str(currentMessage)) + step1PublishCount -= 1 + messageCount += 1 + except publishError: + print("Publish Error!") + except publishQueueFullException: + print("Internal Publish Queue is FULL!") + except Exception as e: + print("Unknown exception!") + print("Type: " + str(type(e))) + print("Message: " + str(e.message)) + time.sleep(1) + # Network Failure, publish #numberOfOfflinePublish# messages + # Scanning rate = 100 TPS + print( + "Thread A: Simulate an network error. Keep publishing for " + str(self._numberOfOfflinePublish) + " messages.") + step2LoopCount = self._numberOfOfflinePublish * 100 + while step2LoopCount != 0: + self._manualNetworkError() + if step2LoopCount % 100 == 0: + currentMessage = str(messageCount) + self._publishMessagePool.append(int(currentMessage)) + try: + self._pyCoreClient.publish(self._topic, currentMessage, 0, False) + print("Thread A: Published a message: " + str(currentMessage)) + except publishError: + print("Publish Error!") + except Exception as e: + print("Unknown exception!") + print("Type: " + str(type(e))) + print("Message: " + str(e.message)) + messageCount += 1 + step2LoopCount -= 1 + time.sleep(0.01) + # Reconnecting + reconnectTiming = 0 # count per 0.01 seconds + while reconnectTiming <= 1000: + if reconnectTiming % 100 == 0: + print("Thread A: Counting reconnect time: " + str(reconnectTiming / 100) + "seconds.") + reconnectTiming += 1 + time.sleep(0.01) + print("Thread A: Reconnected!") + # Publish another set of 3 messages + print("Thread A: Publish 3 messages again.") + step3PublishCount = 3 + while step3PublishCount != 0: + currentMessage = str(messageCount) + self._publishMessagePool.append(int(currentMessage)) + try: + self._pyCoreClient.publish(self._topic, currentMessage, 0, False) + print("Thread A: Published a message: " + str(currentMessage)) + step3PublishCount -= 1 + messageCount += 1 + except publishError: + print("Publish Error!") + except Exception as e: + print("Unknown exception!") + print("Type: " + str(type(e))) + print("Message: " + str(e.message)) + time.sleep(1) + # Wrap up: Sleep for extra 5 seconds + time.sleep(5) + + def startThreadAndGo(self): + threadHandler = threading.Thread(target=self._runtime) + threadHandler.start() + return threadHandler + + def getPublishMessagePool(self): + return self._publishMessagePool + + +# Class that implements the subscribing thread: Thread B. +# Basically this thread does nothing but subscribes to TopicOnly and keeps receiving messages. +class threadSub: + def __init__(self, pyCoreClient, srcTopic): + self._keepRunning = True + self._pyCoreClient = pyCoreClient + self._subscribeMessagePool = list() + self._topic = srcTopic + + def _messageCallback(self, client, userdata, message): + print("Thread B: Received a new message from topic: " + str(message.topic)) + print("Thread B: Payload is: " + str(message.payload)) + self._subscribeMessagePool.append(int(message.payload)) + + def _runtime(self): + # Subscribe to self._topic + try: + self._pyCoreClient.subscribe(self._topic, 1, self._messageCallback) + except subscribeTimeoutException: + print("Subscribe timeout!") + except subscribeError: + print("Subscribe error!") + except Exception as e: + print("Unknown exception!") + print("Type: " + str(type(e))) + print("Message: " + str(e.message)) + time.sleep(2.2) + print("Thread B: Subscribed to self._topic.") + print("Thread B: Now wait for Thread A.") + # Scanning rate is 100 TPS + while self._keepRunning: + time.sleep(0.01) + + def startThreadAndGo(self): + threadHandler = threading.Thread(target=self._runtime) + threadHandler.start() + return threadHandler + + def stopRunning(self): + self._keepRunning = False + + def getSubscribeMessagePool(self): + return self._subscribeMessagePool + + +# Generate answer for this integration test using queue configuration +def generateAnswer(data, queueingSize, srcMode): + dataInWork = sorted(data) + dataHead = dataInWork[:3] + dataTail = dataInWork[-3:] + dataRet = dataHead + dataInWork = dataInWork[3:] + dataInWork = dataInWork[:-3] + if srcMode == 0: # DROP_OLDEST + dataInWork = dataInWork[(-1 * queueingSize):] + dataRet.extend(dataInWork) + dataRet.extend(dataTail) + return sorted(dataRet) + elif srcMode == 1: # DROP_NEWEST + dataInWork = dataInWork[:queueingSize] + dataRet.extend(dataInWork) + dataRet.extend(dataTail) + return sorted(dataRet) + else: + print("Unsupported drop behavior!") + raise ValueError + + +# Create thread object, load in pyCoreClient and perform the set of integration tests +def performConfigurableOfflinePublishQueueTest(clientPub, clientSub): + print("Test DROP_NEWEST....") + clientPub[0].configure_offline_requests_queue(10, DropBehaviorTypes.DROP_NEWEST) # dropNewest + clientSub[0].configure_offline_requests_queue(10, DropBehaviorTypes.DROP_NEWEST) # dropNewest + # Create Topics + TopicOnly = "TopicOnly/" + "".join(random.choice(string.ascii_lowercase) for i in range(4)) + # Create thread object + threadPubObject = threadPub(clientPub[0], 15, TopicOnly) # Configure to publish 15 messages during network outage + threadSubObject = threadSub(clientSub[0], TopicOnly) + threadSubHandler = threadSubObject.startThreadAndGo() + time.sleep(3) + threadPubHandler = threadPubObject.startThreadAndGo() + threadPubHandler.join() + threadSubObject.stopRunning() + threadSubHandler.join() + # Verify result + print("Verify DROP_NEWEST:") + answer = generateAnswer(threadPubObject.getPublishMessagePool(), 10, 1) + print("ANSWER:") + print(answer) + print("ACTUAL:") + print(threadSubObject.getSubscribeMessagePool()) + # We are doing QoS0 publish. We cannot guarantee when the drop will happen since we cannot guarantee a fixed time out + # of disconnect detection. However, once offline requests queue starts involving, it should queue up to its limit, + # thus the total number of received messages after draining should always match. + if len(threadSubObject.getSubscribeMessagePool()) == len(answer): + print("Passed.") + else: + print("Verify DROP_NEWEST failed!!!") + return False + time.sleep(5) + print("Test DROP_OLDEST....") + clientPub[0].configure_offline_requests_queue(10, DropBehaviorTypes.DROP_OLDEST) # dropOldest + clientSub[0].configure_offline_requests_queue(10, DropBehaviorTypes.DROP_OLDEST) # dropOldest + # Create thread object + threadPubObject = threadPub(clientPub[0], 15, TopicOnly) # Configure to publish 15 messages during network outage + threadSubObject = threadSub(clientSub[0], TopicOnly) + threadSubHandler = threadSubObject.startThreadAndGo() + time.sleep(3) + threadPubHandler = threadPubObject.startThreadAndGo() + threadPubHandler.join() + threadSubObject.stopRunning() + threadSubHandler.join() + # Verify result + print("Verify DROP_OLDEST:") + answer = generateAnswer(threadPubObject.getPublishMessagePool(), 10, 0) + print(answer) + print("ACTUAL:") + print(threadSubObject.getSubscribeMessagePool()) + if len(threadSubObject.getSubscribeMessagePool()) == len(answer): + print("Passed.") + else: + print("Verify DROP_OLDEST failed!!!") + return False + return True + + +# Check inputs +myCheckInManager = checkInManager.checkInManager(1) +myCheckInManager.verify(sys.argv) + +host = "ajje7lpljulm4-ats.iot.us-east-1.amazonaws.com" +rootCA = "./configuration/Credentials/rootCA.crt" +certificate = "./configuration/Credentials/certificate.pem.crt" +privateKey = "./configuration/Credentials/privateKey.pem.key" +mode = myCheckInManager.mode + +skip_when_match(ModeIsALPN(mode).And( + Python2VersionLowerThan((2, 7, 10)).Or(Python3VersionLowerThan((3, 5, 0))) +), "This test is not applicable for mode %s and Python verison %s. Skipping..." % (mode, sys.version_info[:3])) + +# Init Python core and connect +myMQTTClientManager = MQTTClientManager.MQTTClientManager() +clientPub = myMQTTClientManager.create_connected_mqtt_core("integrationTestMQTT_ClientPub", host, rootCA, + certificate, privateKey, mode=mode) +clientSub = myMQTTClientManager.create_connected_mqtt_core("integrationTestMQTT_ClientSub", host, rootCA, + certificate, privateKey, mode=mode) + +if clientPub is None or clientSub is None: + exit(4) + +print("Two clients are connected!") + +# Functionality test +if not performConfigurableOfflinePublishQueueTest([clientPub], [clientSub]): + print("The above Drop behavior broken!") + exit(4) diff --git a/test-integration/IntegrationTests/IntegrationTestDiscovery.py b/test-integration/IntegrationTests/IntegrationTestDiscovery.py new file mode 100644 index 0000000..9127eed --- /dev/null +++ b/test-integration/IntegrationTests/IntegrationTestDiscovery.py @@ -0,0 +1,215 @@ +import sys +sys.path.insert(0, "./configuration/IntegrationTests/TestToolLibrary") +sys.path.insert(0, "./configuration/IntegrationTests/TestToolLibrary/SDKPackage") + +from TestToolLibrary.SDKPackage.AWSIoTPythonSDK.core.greengrass.discovery.providers import DiscoveryInfoProvider +from TestToolLibrary.checkInManager import checkInManager +from TestToolLibrary.skip import skip_when_match +from TestToolLibrary.skip import ModeIsWebSocket + + +HOST = "arc9d2oott9lj-ats.iot.us-east-1.amazonaws.com" # 003261610643 +PORT = 8443 +CA = "./configuration/Credentials/rootCA.crt" +CERT = "./configuration/Credentials/certificate_drs.pem.crt" +KEY = "./configuration/Credentials/privateKey_drs.pem.key" +TIME_OUT_SEC = 30 +# This is a pre-generated test data from DRS integration tests +ID_PREFIX = "Id-" +GGC_ARN = "arn:aws:iot:us-east-1:003261610643:thing/DRS_GGC_0kegiNGA_0" +GGC_PORT_NUMBER_BASE = 8080 +GGC_HOST_ADDRESS_PREFIX = "192.168.101." +METADATA_PREFIX = "Description-" +GROUP_ID = "627bf63d-ae64-4f58-a18c-80a44fcf4088" +THING_NAME = "DRS_GGAD_0kegiNGA_0" +EXPECTED_CA_CONTENT = "-----BEGIN CERTIFICATE-----\n" \ + "MIIEFTCCAv2gAwIBAgIVAPZfc4GMLZPmXbnoaZm6jRDqDs4+MA0GCSqGSIb3DQEB\n" \ + "CwUAMIGoMQswCQYDVQQGEwJVUzEYMBYGA1UECgwPQW1hem9uLmNvbSBJbmMuMRww\n" \ + "GgYDVQQLDBNBbWF6b24gV2ViIFNlcnZpY2VzMRMwEQYDVQQIDApXYXNoaW5ndG9u\n" \ + "MRAwDgYDVQQHDAdTZWF0dGxlMTowOAYDVQQDDDEwMDMyNjE2MTA2NDM6NjI3YmY2\n" \ + "M2QtYWU2NC00ZjU4LWExOGMtODBhNDRmY2Y0MDg4MCAXDTE3MDUyNTE4NDI1OVoY\n" \ + "DzIwOTcwNTI1MTg0MjU4WjCBqDELMAkGA1UEBhMCVVMxGDAWBgNVBAoMD0FtYXpv\n" \ + "bi5jb20gSW5jLjEcMBoGA1UECwwTQW1hem9uIFdlYiBTZXJ2aWNlczETMBEGA1UE\n" \ + "CAwKV2FzaGluZ3RvbjEQMA4GA1UEBwwHU2VhdHRsZTE6MDgGA1UEAwwxMDAzMjYx\n" \ + "NjEwNjQzOjYyN2JmNjNkLWFlNjQtNGY1OC1hMThjLTgwYTQ0ZmNmNDA4ODCCASIw\n" \ + "DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKEWtZtKyJUg2VUwZkbkVtltrfam\n" \ + "s9LMIdKNA3Wz4zSLhZjKHiTSkQmpZwKle5ziYs6Q5hfeT8WC0FNAVv1JhnwsuGfT\n" \ + "sG0UO5dSn7wqXOJigKC1CaSGqeFpKB0/a3wR1L6pCGVbLZ86/sPCEPHHJDieQ+Ps\n" \ + "RnOcUGb4CuIBnI2N+lafWNa4F4KRSVJCEeZ6u4iWVVdIEcDLKlakY45jtVvQqwnz\n" \ + "3leFsN7PTLEkVq5u1PXSbT5DWv6p+5NoDnGAT7j7Wbr2yJw7DtpBOL6oWkAdbFAQ\n" \ + "2097e8mIxNYE9xAzRlb5wEr6jZl/8K60v9P83OapMeuOg4JS8FGulHXbDg0CAwEA\n" \ + "AaMyMDAwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU21ELaPCH9Oh001OS0JMv\n" \ + "n8hU8dYwDQYJKoZIhvcNAQELBQADggEBABW66eH/+/v9Nq5jtJzflfrqAfBOzWLj\n" \ + "UTEv6szkYzV5Crr8vnu2P5OlyA0NdiKGiAm0AgoDkf+n9HU3Hc0zm3G/QaAO2UmN\n" \ + "9MwtIp29BSRf+gd1bX/WZTtl5I5xl290BDfr5o08I6TOf0A4P8IAkGwku5j0IQjM\n" \ + "ns2HH5UVki155dtmWDEGX6q35KABbsmv3tO1+geJVYnd1QkHzR5IXA12gxlMw9GJ\n" \ + "+cOw+rwJJ2ZcXo3HFoXBcsPqPOa1SO3vTl3XWQ+jX3vyDsxh/VGoJ4epsjwmJ+dW\n" \ + "sHJoqsa3ZPDW0LcEuYgdzYWRhumGwH9fJJUx0GS4Tdg4ud+6jpuyflU=\n" \ + "-----END CERTIFICATE-----\n" +# The expected response from DRS should be: +''' +{ + "GGGroups": [ + { + "GGGroupId": "627bf63d-ae64-4f58-a18c-80a44fcf4088", + "Cores": [ + { + "thingArn": "arn:aws:iot:us-east-1:003261610643:thing\/DRS_GGC_0kegiNGA_0", + "Connectivity": [ + { + "Id": "Id-0", + "HostAddress": "192.168.101.0", + "PortNumber": 8080, + "Metadata": "Description-0" + }, + { + "Id": "Id-1", + "HostAddress": "192.168.101.1", + "PortNumber": 8081, + "Metadata": "Description-1" + }, + { + "Id": "Id-2", + "HostAddress": "192.168.101.2", + "PortNumber": 8082, + "Metadata": "Description-2" + } + ] + } + ], + "CAs": [ + "-----BEGIN CERTIFICATE-----\n + MIIEFTCCAv2gAwIBAgIVAPZfc4GMLZPmXbnoaZm6jRDqDs4+MA0GCSqGSIb3DQEB\n + CwUAMIGoMQswCQYDVQQGEwJVUzEYMBYGA1UECgwPQW1hem9uLmNvbSBJbmMuMRww\n + GgYDVQQLDBNBbWF6b24gV2ViIFNlcnZpY2VzMRMwEQYDVQQIDApXYXNoaW5ndG9u\n + MRAwDgYDVQQHDAdTZWF0dGxlMTowOAYDVQQDDDEwMDMyNjE2MTA2NDM6NjI3YmY2\n + M2QtYWU2NC00ZjU4LWExOGMtODBhNDRmY2Y0MDg4MCAXDTE3MDUyNTE4NDI1OVoY\n + DzIwOTcwNTI1MTg0MjU4WjCBqDELMAkGA1UEBhMCVVMxGDAWBgNVBAoMD0FtYXpv\n + bi5jb20gSW5jLjEcMBoGA1UECwwTQW1hem9uIFdlYiBTZXJ2aWNlczETMBEGA1UE\n + CAwKV2FzaGluZ3RvbjEQMA4GA1UEBwwHU2VhdHRsZTE6MDgGA1UEAwwxMDAzMjYx\n + NjEwNjQzOjYyN2JmNjNkLWFlNjQtNGY1OC1hMThjLTgwYTQ0ZmNmNDA4ODCCASIw\n + DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKEWtZtKyJUg2VUwZkbkVtltrfam\n + s9LMIdKNA3Wz4zSLhZjKHiTSkQmpZwKle5ziYs6Q5hfeT8WC0FNAVv1JhnwsuGfT\n + sG0UO5dSn7wqXOJigKC1CaSGqeFpKB0\/a3wR1L6pCGVbLZ86\/sPCEPHHJDieQ+Ps\n + RnOcUGb4CuIBnI2N+lafWNa4F4KRSVJCEeZ6u4iWVVdIEcDLKlakY45jtVvQqwnz\n + 3leFsN7PTLEkVq5u1PXSbT5DWv6p+5NoDnGAT7j7Wbr2yJw7DtpBOL6oWkAdbFAQ\n + 2097e8mIxNYE9xAzRlb5wEr6jZl\/8K60v9P83OapMeuOg4JS8FGulHXbDg0CAwEA\n + AaMyMDAwDwYDVR0TAQH\/BAUwAwEB\/zAdBgNVHQ4EFgQU21ELaPCH9Oh001OS0JMv\n + n8hU8dYwDQYJKoZIhvcNAQELBQADggEBABW66eH\/+\/v9Nq5jtJzflfrqAfBOzWLj\n + UTEv6szkYzV5Crr8vnu2P5OlyA0NdiKGiAm0AgoDkf+n9HU3Hc0zm3G\/QaAO2UmN\n + 9MwtIp29BSRf+gd1bX\/WZTtl5I5xl290BDfr5o08I6TOf0A4P8IAkGwku5j0IQjM\n + ns2HH5UVki155dtmWDEGX6q35KABbsmv3tO1+geJVYnd1QkHzR5IXA12gxlMw9GJ\n + +cOw+rwJJ2ZcXo3HFoXBcsPqPOa1SO3vTl3XWQ+jX3vyDsxh\/VGoJ4epsjwmJ+dW\n + sHJoqsa3ZPDW0LcEuYgdzYWRhumGwH9fJJUx0GS4Tdg4ud+6jpuyflU=\n + -----END CERTIFICATE-----\n" + ] + } + ] +} +''' + + +def create_discovery_info_provider(): + discovery_info_provider = DiscoveryInfoProvider() + discovery_info_provider.configureEndpoint(HOST, PORT) + discovery_info_provider.configureCredentials(CA, CERT, KEY) + discovery_info_provider.configureTimeout(TIME_OUT_SEC) + return discovery_info_provider + + +def perform_integ_test_discovery(): + discovery_info_provider = create_discovery_info_provider() + return discovery_info_provider.discover(THING_NAME) + + +def _verify_connectivity_info(actual_connectivity_info): + info_id = actual_connectivity_info.id + sequence_number_string = info_id[-1:] + assert actual_connectivity_info.host == GGC_HOST_ADDRESS_PREFIX + sequence_number_string + assert actual_connectivity_info.port == GGC_PORT_NUMBER_BASE + int(sequence_number_string) + assert actual_connectivity_info.metadata == METADATA_PREFIX + sequence_number_string + + +def _verify_connectivity_info_list(actual_connectivity_info_list): + for actual_connectivity_info in actual_connectivity_info_list: + _verify_connectivity_info(actual_connectivity_info) + + +def _verify_ggc_info(actual_ggc_info): + assert actual_ggc_info.coreThingArn == GGC_ARN + assert actual_ggc_info.groupId == GROUP_ID + _verify_connectivity_info_list(actual_ggc_info.connectivityInfoList) + + +def _verify_ca_list(ca_list): + assert len(ca_list) == 1 + try: + group_id, ca = ca_list[0] + assert group_id == GROUP_ID + assert ca == EXPECTED_CA_CONTENT + except: + assert ca_list[0] == EXPECTED_CA_CONTENT + + +def verify_all_cores(discovery_info): + print("Verifying \"getAllCores\"...") + ggc_info_list = discovery_info.getAllCores() + assert len(ggc_info_list) == 1 + _verify_ggc_info(ggc_info_list[0]) + print("Pass!") + + +def verify_all_cas(discovery_info): + print("Verifying \"getAllCas\"...") + ca_list = discovery_info.getAllCas() + _verify_ca_list(ca_list) + print("Pass!") + + +def verify_all_groups(discovery_info): + print("Verifying \"getAllGroups\"...") + group_list = discovery_info.getAllGroups() + assert len(group_list) == 1 + group_info = group_list[0] + _verify_ca_list(group_info.caList) + _verify_ggc_info(group_info.coreConnectivityInfoList[0]) + print("Pass!") + + +def verify_group_object(discovery_info): + print("Verifying \"toObjectAtGroupLevel\"...") + group_info_object = discovery_info.toObjectAtGroupLevel() + _verify_connectivity_info(group_info_object + .get(GROUP_ID) + .getCoreConnectivityInfo(GGC_ARN) + .getConnectivityInfo(ID_PREFIX + "0")) + _verify_connectivity_info(group_info_object + .get(GROUP_ID) + .getCoreConnectivityInfo(GGC_ARN) + .getConnectivityInfo(ID_PREFIX + "1")) + _verify_connectivity_info(group_info_object + .get(GROUP_ID) + .getCoreConnectivityInfo(GGC_ARN) + .getConnectivityInfo(ID_PREFIX + "2")) + print("Pass!") + + +############################################################################ +# Main # +my_check_in_manager = checkInManager(1) +my_check_in_manager.verify(sys.argv) +mode = my_check_in_manager.mode + +skip_when_match(ModeIsWebSocket(mode), "This test is not applicable for mode: %s. Skipping..." % mode) + +# GG Discovery only applies mutual auth with cert +try: + discovery_info = perform_integ_test_discovery() + + verify_all_cores(discovery_info) + verify_all_cas(discovery_info) + verify_all_groups(discovery_info) + verify_group_object(discovery_info) +except BaseException as e: + print("Failed! " + e.message) + exit(4) diff --git a/test-integration/IntegrationTests/IntegrationTestJobsClient.py b/test-integration/IntegrationTests/IntegrationTestJobsClient.py new file mode 100644 index 0000000..e76ba59 --- /dev/null +++ b/test-integration/IntegrationTests/IntegrationTestJobsClient.py @@ -0,0 +1,184 @@ +# This integration test verifies the jobs client functionality in the +# Python SDK. +# It performs a number of basic operations without expecting an actual job or +# job execution to be present. The callbacks associated with these actions +# are written to accept and pass server responses given when no jobs or job +# executions exist. +# Finally, the tester pumps through all jobs queued for the given thing +# doing a basic echo of the job document and updating the job execution +# to SUCCEEDED + +import random +import string +import time +import sys +sys.path.insert(0, "./configuration/IntegrationTests/TestToolLibrary") +sys.path.insert(0, "./configuration/IntegrationTests/TestToolLibrary/SDKPackage") + +from TestToolLibrary import simpleThreadManager +import TestToolLibrary.checkInManager as checkInManager +import TestToolLibrary.MQTTClientManager as MQTTClientManager +from TestToolLibrary.SDKPackage.AWSIoTPythonSDK.exception.AWSIoTExceptions import publishError +from TestToolLibrary.SDKPackage.AWSIoTPythonSDK.exception.AWSIoTExceptions import subscribeError +from TestToolLibrary.SDKPackage.AWSIoTPythonSDK.exception.AWSIoTExceptions import subscribeTimeoutException +from TestToolLibrary.skip import skip_when_match +from TestToolLibrary.skip import ModeIsALPN +from TestToolLibrary.skip import Python2VersionLowerThan +from TestToolLibrary.skip import Python3VersionLowerThan + +from AWSIoTPythonSDK.MQTTLib import AWSIoTMQTTThingJobsClient +from AWSIoTPythonSDK.core.jobs.thingJobManager import jobExecutionTopicType +from AWSIoTPythonSDK.core.jobs.thingJobManager import jobExecutionTopicReplyType +from AWSIoTPythonSDK.core.jobs.thingJobManager import jobExecutionStatus + +import threading +import datetime +import argparse +import json + +IOT_JOBS_MQTT_RESPONSE_WAIT_SECONDS = 5 + +class JobsMessageProcessor(object): + def __init__(self, awsIoTMQTTThingJobsClient, clientToken): + #keep track of this to correlate request/responses + self.clientToken = clientToken + self.awsIoTMQTTThingJobsClient = awsIoTMQTTThingJobsClient + + def _setupCallbacks(self): + print('Creating test subscriptions...') + assert True == self.awsIoTMQTTThingJobsClient.createJobSubscription(self.getPendingJobAcceptedCallback, jobExecutionTopicType.JOB_GET_PENDING_TOPIC, jobExecutionTopicReplyType.JOB_ACCEPTED_REPLY_TYPE) + assert True == self.awsIoTMQTTThingJobsClient.createJobSubscription(self.getPendingJobRejectedCallback, jobExecutionTopicType.JOB_GET_PENDING_TOPIC, jobExecutionTopicReplyType.JOB_REJECTED_REPLY_TYPE) + assert True == self.awsIoTMQTTThingJobsClient.createJobSubscription(self.describeJobExecAcceptedCallback, jobExecutionTopicType.JOB_DESCRIBE_TOPIC, jobExecutionTopicReplyType.JOB_ACCEPTED_REPLY_TYPE, '+') + assert True == self.awsIoTMQTTThingJobsClient.createJobSubscription(self.describeJobExecRejectedCallback, jobExecutionTopicType.JOB_DESCRIBE_TOPIC, jobExecutionTopicReplyType.JOB_REJECTED_REPLY_TYPE, '+') + assert True == self.awsIoTMQTTThingJobsClient.createJobSubscription(self.startNextPendingJobAcceptedCallback, jobExecutionTopicType.JOB_START_NEXT_TOPIC, jobExecutionTopicReplyType.JOB_ACCEPTED_REPLY_TYPE) + assert True == self.awsIoTMQTTThingJobsClient.createJobSubscription(self.startNextPendingJobRejectedCallback, jobExecutionTopicType.JOB_START_NEXT_TOPIC, jobExecutionTopicReplyType.JOB_REJECTED_REPLY_TYPE) + assert True == self.awsIoTMQTTThingJobsClient.createJobSubscription(self.updateJobAcceptedCallback, jobExecutionTopicType.JOB_UPDATE_TOPIC, jobExecutionTopicReplyType.JOB_ACCEPTED_REPLY_TYPE, '+') + assert True == self.awsIoTMQTTThingJobsClient.createJobSubscription(self.updateJobRejectedCallback, jobExecutionTopicType.JOB_UPDATE_TOPIC, jobExecutionTopicReplyType.JOB_REJECTED_REPLY_TYPE, '+') + + def getPendingJobAcceptedCallback(self, client, userdata, message): + self.testResult = (True, 'GetPending accepted callback invoked!') + self.waitEvent.set() + + def getPendingJobRejectedCallback(self, client, userdata, message): + self.testResult = (False, 'GetPending rejection callback invoked!') + self.waitEvent.set() + + def describeJobExecAcceptedCallback(self, client, userdata, message): + self.testResult = (True, 'DescribeJobExecution accepted callback invoked!') + self.waitEvent.set() + + def describeJobExecRejectedCallback(self, client, userdata, message): + self.testResult = (False, 'DescribeJobExecution rejected callback invoked!') + self.waitEvent.set() + + def startNextPendingJobAcceptedCallback(self, client, userdata, message): + self.testResult = (True, 'StartNextPendingJob accepted callback invoked!') + payload = json.loads(message.payload.decode('utf-8')) + if 'execution' not in payload: + self.done = True + else: + print('Found job! Document: ' + payload['execution']['jobDocument']) + threading.Thread(target=self.awsIoTMQTTThingJobsClient.sendJobsUpdate(payload['execution']['jobId'], jobExecutionStatus.JOB_EXECUTION_SUCCEEDED)).start() + self.waitEvent.set() + + def startNextPendingJobRejectedCallback(self, client, userdata, message): + self.testResult = (False, 'StartNextPendingJob rejected callback invoked!') + self.waitEvent.set() + + def updateJobAcceptedCallback(self, client, userdata, message): + self.testResult = (True, 'UpdateJob accepted callback invoked!') + self.waitEvent.set() + + def updateJobRejectedCallback(self, client, userdata, message): + #rejection is still a successful test because job IDs may or may not exist, and could exist in unknown state + self.testResult = (True, 'UpdateJob rejected callback invoked!') + self.waitEvent.set() + + def executeJob(self, execution): + print('Executing job ID, version, number: {}, {}, {}'.format(execution['jobId'], execution['versionNumber'], execution['executionNumber'])) + print('With jobDocument: ' + json.dumps(execution['jobDocument'])) + + def runTests(self): + print('Running jobs tests...') + ##create subscriptions + self._setupCallbacks() + + #make publish calls + self._init_test_wait() + self._test_send_response_confirm(self.awsIoTMQTTThingJobsClient.sendJobsDescribe('$next')) + + self._init_test_wait() + self._test_send_response_confirm(self.awsIoTMQTTThingJobsClient.sendJobsUpdate('junkJobId', jobExecutionStatus.JOB_EXECUTION_SUCCEEDED)) + + self._init_test_wait() + self._test_send_response_confirm(self.awsIoTMQTTThingJobsClient.sendJobsQuery(jobExecutionTopicType.JOB_GET_PENDING_TOPIC)) + + self._init_test_wait() + self._test_send_response_confirm(self.awsIoTMQTTThingJobsClient.sendJobsStartNext()) + + self.processAllJobs() + + def processAllJobs(self): + #process all enqueued jobs + print('Processing all jobs found in queue for thing...') + self.done = False + while not self.done: + self._attemptStartNextJob() + time.sleep(5) + + def _attemptStartNextJob(self): + statusDetails = {'StartedBy': 'ClientToken: {} on {}'.format(self.clientToken, datetime.datetime.now().isoformat())} + threading.Thread(target=self.awsIoTMQTTThingJobsClient.sendJobsStartNext, kwargs = {'statusDetails': statusDetails}).start() + + def _init_test_wait(self): + self.testResult = (False, 'Callback not invoked') + self.waitEvent = threading.Event() + + def _test_send_response_confirm(self, sendResult): + if not sendResult: + print('Failed to send jobs message') + exit(4) + else: + #wait 25 seconds for expected callback response to happen + if not self.waitEvent.wait(IOT_JOBS_MQTT_RESPONSE_WAIT_SECONDS): + print('Did not receive expected callback within %d second timeout' % IOT_JOBS_MQTT_RESPONSE_WAIT_SECONDS) + exit(4) + elif not self.testResult[0]: + print('Callback result has failed the test with message: ' + self.testResult[1]) + exit(4) + else: + print('Recieved expected result: ' + self.testResult[1]) + + +############################################################################ +# Main # +# Check inputs +myCheckInManager = checkInManager.checkInManager(1) +myCheckInManager.verify(sys.argv) + +host = "ajje7lpljulm4-ats.iot.us-east-1.amazonaws.com" +rootCA = "./configuration/Credentials/rootCA.crt" +certificate = "./configuration/Credentials/certificate.pem.crt" +privateKey = "./configuration/Credentials/privateKey.pem.key" +mode = myCheckInManager.mode + +skip_when_match(ModeIsALPN(mode).And( + Python2VersionLowerThan((2, 7, 10)).Or(Python3VersionLowerThan((3, 5, 0))) +), "This test is not applicable for mode %s and Python verison %s. Skipping..." % (mode, sys.version_info[:3])) + +# Init Python core and connect +myMQTTClientManager = MQTTClientManager.MQTTClientManager() +client = myMQTTClientManager.create_connected_mqtt_client(mode, "integrationTestJobs_Client", host, (rootCA, certificate, privateKey)) + +clientId = 'AWSPythonkSDKTestThingClient' +thingName = 'AWSPythonkSDKTestThing' +jobsClient = AWSIoTMQTTThingJobsClient(clientId, thingName, QoS=1, awsIoTMQTTClient=client) + +print('Connecting to MQTT server and setting up callbacks...') +jobsMsgProc = JobsMessageProcessor(jobsClient, clientId) +print('Starting jobs tests...') +jobsMsgProc.runTests() +print('Done running jobs tests') + +#can call this on the jobsClient, or myAWSIoTMQTTClient directly +jobsClient.disconnect() diff --git a/test-integration/IntegrationTests/IntegrationTestMQTTConnection.py b/test-integration/IntegrationTests/IntegrationTestMQTTConnection.py new file mode 100644 index 0000000..77c95ce --- /dev/null +++ b/test-integration/IntegrationTests/IntegrationTestMQTTConnection.py @@ -0,0 +1,173 @@ +# This integration test verifies the functionality in the Python core of IoT Yun/Python SDK +# for basic MQTT connection. +# It starts two threads using two different connections to AWS IoT: +# Thread A: publish to "deviceSDK/PyIntegrationTest/Topic", X messages, QoS1, TPS=50 +# Thread B: subscribe to "deviceSDK/PyIntegrationTest/Topic", QoS1 +# Thread B will be started first with extra delay to ensure a valid subscription +# Then thread A will be started. +# Verify send/receive messages are equivalent + + +import random +import string +import time +import sys +sys.path.insert(0, "./configuration/IntegrationTests/TestToolLibrary") +sys.path.insert(0, "./configuration/IntegrationTests/TestToolLibrary/SDKPackage") + +from TestToolLibrary import simpleThreadManager +import TestToolLibrary.checkInManager as checkInManager +import TestToolLibrary.MQTTClientManager as MQTTClientManager +from TestToolLibrary.SDKPackage.AWSIoTPythonSDK.exception.AWSIoTExceptions import publishError +from TestToolLibrary.SDKPackage.AWSIoTPythonSDK.exception.AWSIoTExceptions import subscribeError +from TestToolLibrary.SDKPackage.AWSIoTPythonSDK.exception.AWSIoTExceptions import subscribeTimeoutException +from TestToolLibrary.skip import skip_when_match +from TestToolLibrary.skip import ModeIsALPN +from TestToolLibrary.skip import Python2VersionLowerThan +from TestToolLibrary.skip import Python3VersionLowerThan + + +API_TYPE_SYNC = "sync" +API_TYPE_ASYNC = "async" + + +# Callback unit for subscribe +class callbackUnit: + def __init__(self, srcSet, apiType): + self._internalSet = srcSet + self._apiType = apiType + + # Callback for clientSub + def messageCallback(self, client, userdata, message): + print(self._apiType + ": Received a new message: " + str(message.payload)) + self._internalSet.add(message.payload.decode('utf-8')) + + def getInternalSet(self): + return self._internalSet + + +# Run function unit +class runFunctionUnit: + def __init__(self, apiType): + self._messagesPublished = set() + self._apiType = apiType + + # Run function for publish thread (one time) + def threadPublish(self, pyCoreClient, numberOfTotalMessages, topic, TPS): + # One time thread + time.sleep(3) # Extra waiting time for valid subscription + messagesLeftToBePublished = numberOfTotalMessages + while messagesLeftToBePublished != 0: + try: + currentMessage = str(messagesLeftToBePublished) + self._performPublish(pyCoreClient, topic, 1, currentMessage) + self._messagesPublished.add(currentMessage) + except publishError: + print("Publish Error for message: " + currentMessage) + except Exception as e: + print("Unknown exception: " + str(type(e)) + " " + str(e.message)) + messagesLeftToBePublished -= 1 + time.sleep(1 / float(TPS)) + print("End of publish thread.") + + def _performPublish(self, pyCoreClient, topic, qos, payload): + if self._apiType == API_TYPE_SYNC: + pyCoreClient.publish(topic, payload, qos, False) + if self._apiType == API_TYPE_ASYNC: + pyCoreClient.publish_async(topic, payload, qos, False, None) # TODO: See if we can also check PUBACKs + + +############################################################################ +# Main # +# Check inputs +myCheckInManager = checkInManager.checkInManager(2) +myCheckInManager.verify(sys.argv) + +host = "ajje7lpljulm4-ats.iot.us-east-1.amazonaws.com" +rootCA = "./configuration/Credentials/rootCA.crt" +certificate = "./configuration/Credentials/certificate.pem.crt" +privateKey = "./configuration/Credentials/privateKey.pem.key" +mode = myCheckInManager.mode + +skip_when_match(ModeIsALPN(mode).And( + Python2VersionLowerThan((2, 7, 10)).Or(Python3VersionLowerThan((3, 5, 0))) +), "This test is not applicable for mode %s and Python verison %s. Skipping..." % (mode, sys.version_info[:3])) + +# Init Python core and connect +myMQTTClientManager = MQTTClientManager.MQTTClientManager() +clientPub = myMQTTClientManager.create_connected_mqtt_core("integrationTestMQTT_ClientPub", host, rootCA, + certificate, privateKey, mode=mode) +clientSub = myMQTTClientManager.create_connected_mqtt_core("integrationTestMQTT_ClientSub", host, rootCA, + certificate, privateKey, mode=mode) + +if clientPub is None or clientSub is None: + exit(4) + +print("Two clients are connected!") + +# Configurations +################ +# Data/Data pool +TPS = 20 +numberOfTotalMessagesAsync = myCheckInManager.customParameter +numberOfTotalMessagesSync = numberOfTotalMessagesAsync / 10 +subSetAsync = set() +subSetSync = set() +subCallbackUnitAsync = callbackUnit(subSetAsync, API_TYPE_ASYNC) +subCallbackUnitSync = callbackUnit(subSetSync, API_TYPE_SYNC) +syncTopic = "YunSDK/PyIntegrationTest/Topic/sync" + "".join(random.choice(string.ascii_lowercase) for i in range(4)) +print(syncTopic) +asyncTopic = "YunSDK/PyIntegrationTest/Topic/async" + "".join(random.choice(string.ascii_lowercase) for j in range(4)) +# clientSub +try: + clientSub.subscribe(asyncTopic, 1, subCallbackUnitAsync.messageCallback) + clientSub.subscribe(syncTopic, 1, subCallbackUnitSync.messageCallback) + time.sleep(3) +except subscribeTimeoutException: + print("Subscribe timeout!") +except subscribeError: + print("Subscribe error!") +except Exception as e: + print("Unknown exception!") + print("Type: " + str(type(e))) + print("Message: " + str(e.message)) +# Threads +mySimpleThreadManager = simpleThreadManager.simpleThreadManager() +myRunFunctionUnitSyncPub = runFunctionUnit(API_TYPE_SYNC) +myRunFunctionUnitAsyncPub = runFunctionUnit(API_TYPE_ASYNC) +publishSyncThreadID = mySimpleThreadManager.createOneTimeThread(myRunFunctionUnitSyncPub.threadPublish, + [clientPub, numberOfTotalMessagesSync, syncTopic, TPS]) +publishAsyncThreadID = mySimpleThreadManager.createOneTimeThread(myRunFunctionUnitAsyncPub.threadPublish, + [clientPub, numberOfTotalMessagesAsync, asyncTopic, TPS]) + +# Performing +############ +mySimpleThreadManager.startThreadWithID(publishSyncThreadID) +mySimpleThreadManager.startThreadWithID(publishAsyncThreadID) +mySimpleThreadManager.joinOneTimeThreadWithID(publishSyncThreadID) +mySimpleThreadManager.joinOneTimeThreadWithID(publishAsyncThreadID) +time.sleep(numberOfTotalMessagesAsync / float(TPS) * 0.5) + +# Verifying +########### +# Length +print("Check if the length of the two sets are equal...") +print("Received from subscription (sync pub): " + str(len(subCallbackUnitSync.getInternalSet()))) +print("Received from subscription (async pub): " + str(len(subCallbackUnitAsync.getInternalSet()))) +print("Sent through sync publishes: " + str(len(myRunFunctionUnitSyncPub._messagesPublished))) +print("Sent through async publishes: " + str(len(myRunFunctionUnitAsyncPub._messagesPublished))) +if len(myRunFunctionUnitSyncPub._messagesPublished) != len(subCallbackUnitSync.getInternalSet()): + print("[Sync pub] Number of messages not equal!") + exit(4) +if len(myRunFunctionUnitAsyncPub._messagesPublished) != len(subCallbackUnitAsync.getInternalSet()): + print("[Asyn pub] Number of messages not equal!") + exit(4) +# Content +print("Check if the content if the two sets are equivalent...") +if myRunFunctionUnitSyncPub._messagesPublished != subCallbackUnitSync.getInternalSet(): + print("[Sync pub] Set content not equal!") + exit(4) +elif myRunFunctionUnitAsyncPub._messagesPublished != subCallbackUnitAsync.getInternalSet(): + print("[Async pub] Set content not equal!") +else: + print("Yes!") diff --git a/test-integration/IntegrationTests/IntegrationTestOfflineQueueingForSubscribeUnsubscribe.py b/test-integration/IntegrationTests/IntegrationTestOfflineQueueingForSubscribeUnsubscribe.py new file mode 100644 index 0000000..307e116 --- /dev/null +++ b/test-integration/IntegrationTests/IntegrationTestOfflineQueueingForSubscribeUnsubscribe.py @@ -0,0 +1,211 @@ +# This integration test verifies the functionality off queueing up subscribe/unsubscribe requests submitted by the +# client when it is offline, and drain them out when the client is reconnected. The test contains 2 clients running in +# 2 different threads: +# +# In thread A, client_sub_unsub follows the below workflow: +# 1. Client connects to AWS IoT. +# 2. Client subscribes to "topic_A". +# 3. Experience a simulated network error which brings the client offline. +# 4. While offline, client subscribes to "topic_B' and unsubscribes from "topic_A". +# 5. Client reconnects, comes back online and drains out all offline queued requests. +# 6. Client stays and receives messages published in another thread. +# +# In thread B, client_pub follows the below workflow: +# 1. Client in thread B connects to AWS IoT. +# 2. After client in thread A connects and subscribes to "topic_A", client in thread B publishes messages to "topic_A". +# 3. Client in thread B keeps sleeping until client in thread A goes back online and reaches to a stable state (draining done). +# 4. Client in thread B then publishes messages to "topic_A" and "topic_B". +# +# Since client in thread A does a unsubscribe to "topic_A", it should never receive messages published to "topic_A" after +# it reconnects and gets stable. It should have the messages from "topic_A" published at the very beginning. +# Since client in thread A does a subscribe to "topic_B", it should receive messages published to "topic_B" after it +# reconnects and gets stable. + + +import random +import string +import time +from threading import Event +from threading import Thread +import sys +sys.path.insert(0, "./configuration/IntegrationTests/TestToolLibrary") +sys.path.insert(0, "./configuration/IntegrationTests/TestToolLibrary/SDKPackage") + +from TestToolLibrary.SDKPackage.AWSIoTPythonSDK.core.protocol.internal.clients import ClientStatus +from TestToolLibrary.checkInManager import checkInManager +from TestToolLibrary.MQTTClientManager import MQTTClientManager +from TestToolLibrary.skip import skip_when_match +from TestToolLibrary.skip import ModeIsALPN +from TestToolLibrary.skip import Python2VersionLowerThan +from TestToolLibrary.skip import Python3VersionLowerThan + + +TOPIC_A = "topic/test/offline_sub_unsub/a" +TOPIC_B = "topic/test/offline_sub_unsub/b" +MESSAGE_PREFIX = "MagicMessage-" +NUMBER_OF_PUBLISHES = 3 +HOST = "ajje7lpljulm4-ats.iot.us-east-1.amazonaws.com" +ROOT_CA = "./configuration/Credentials/rootCA.crt" +CERT = "./configuration/Credentials/certificate.pem.crt" +KEY = "./configuration/Credentials/privateKey.pem.key" +CLIENT_PUB_ID = "PySdkIntegTest_OfflineSubUnsub_pub" +CLIENT_SUB_UNSUB_ID = "PySdkIntegTest_OfflineSubUnsub_subunsub" +KEEP_ALIVE_SEC = 1 +EVENT_WAIT_TIME_OUT_SEC = 5 + + +def get_random_string(length): + return "".join(random.choice(string.ascii_lowercase) for i in range(length)) + + +class DualClientRunner(object): + + def __init__(self, mode): + self._publish_end_flag = Event() + self._stable_flag = Event() + self._received_messages_topic_a = list() + self._received_messages_topic_b = list() + self.__mode = mode + self._client_pub = self._create_connected_client(CLIENT_PUB_ID) + print("Created connected client pub.") + self._client_sub_unsub = self._create_connected_client(CLIENT_SUB_UNSUB_ID) + print("Created connected client sub/unsub.") + self._client_sub_unsub.subscribe(TOPIC_A, 1, self._collect_sub_messages) + print("Client sub/unsub subscribed to topic: %s" % TOPIC_A) + time.sleep(2) # Make sure the subscription is valid + + def _create_connected_client(self, id_prefix): + return MQTTClientManager().create_connected_mqtt_client(self.__mode, id_prefix, HOST, (ROOT_CA, CERT, KEY)) + + def start(self): + thread_client_sub_unsub = Thread(target=self._thread_client_sub_unsub_runtime) + thread_client_pub = Thread(target=self._thread_client_pub_runtime) + thread_client_sub_unsub.start() + thread_client_pub.start() + thread_client_pub.join() + thread_client_sub_unsub.join() + + def _thread_client_sub_unsub_runtime(self): + print("Start client sub/unsub runtime thread...") + print("Client sub/unsub waits on the 1st round of publishes to end...") + if not self._publish_end_flag.wait(EVENT_WAIT_TIME_OUT_SEC): + raise RuntimeError("Timed out in waiting for the publishes to topic: %s" % TOPIC_A) + print("Client sub/unsub gets notified.") + self._publish_end_flag.clear() + + print("Client sub/unsub now goes offline...") + self._go_offline_and_send_requests() + + # Wait until the connection is stable and then notify + print("Client sub/unsub waits on a stable connection...") + self._wait_until_stable_connection() + + print("Client sub/unsub waits on the 2nd round of publishes to end...") + if not self._publish_end_flag.wait(EVENT_WAIT_TIME_OUT_SEC): + raise RuntimeError("Timed out in waiting for the publishes to topic: %s" % TOPIC_B) + print("Client sub/unsub gets notified.") + self._publish_end_flag.clear() + + print("Client sub/unsub runtime thread ends.") + + def _wait_until_stable_connection(self): + reconnect_timing = 0 + while self._client_sub_unsub._mqtt_core._client_status.get_status() != ClientStatus.STABLE: + time.sleep(0.01) + reconnect_timing += 1 + if reconnect_timing % 100 == 0: + print("Client sub/unsub: Counting reconnect time: " + str(reconnect_timing / 100) + " seconds.") + print("Client sub/unsub: Counting reconnect time result: " + str(float(reconnect_timing) / 100) + " seconds.") + self._stable_flag.set() + + def _collect_sub_messages(self, client, userdata, message): + if message.topic == TOPIC_A: + print("Client sub/unsub: Got a message from %s" % TOPIC_A) + self._received_messages_topic_a.append(message.payload) + if message.topic == TOPIC_B: + print("Client sub/unsub: Got a message from %s" % TOPIC_B) + self._received_messages_topic_b.append(message.payload) + + def _go_offline_and_send_requests(self): + do_once = True + loop_count = EVENT_WAIT_TIME_OUT_SEC * 100 + while loop_count != 0: + self._manual_network_error() + if loop_count % 100 == 0: + print("Client sub/unsub: Offline time down count: %d sec" % (loop_count / 100)) + if do_once and (loop_count / 100) <= (EVENT_WAIT_TIME_OUT_SEC / 2): + print("Client sub/unsub: Performing offline sub/unsub...") + self._client_sub_unsub.subscribe(TOPIC_B, 1, self._collect_sub_messages) + self._client_sub_unsub.unsubscribe(TOPIC_A) + print("Client sub/unsub: Done with offline sub/unsub.") + do_once = False + loop_count -= 1 + time.sleep(0.01) + + def _manual_network_error(self): + # Ensure we close the socket + if self._client_sub_unsub._mqtt_core._internal_async_client._paho_client._sock: + self._client_sub_unsub._mqtt_core._internal_async_client._paho_client._sock.close() + self._client_sub_unsub._mqtt_core._internal_async_client._paho_client._sock = None + if self._client_sub_unsub._mqtt_core._internal_async_client._paho_client._ssl: + self._client_sub_unsub._mqtt_core._internal_async_client._paho_client._ssl.close() + self._client_sub_unsub._mqtt_core._internal_async_client._paho_client._ssl = None + # Fake that we have detected the disconnection + self._client_sub_unsub._mqtt_core._internal_async_client._paho_client.on_disconnect(None, None, 0) + + def _thread_client_pub_runtime(self): + print("Start client pub runtime thread...") + print("Client pub: 1st round of publishes") + for i in range(NUMBER_OF_PUBLISHES): + self._client_pub.publish(TOPIC_A, MESSAGE_PREFIX + str(i), 1) + print("Client pub: Published a message") + time.sleep(0.5) + time.sleep(1) + print("Client pub: Publishes done. Notifying...") + self._publish_end_flag.set() + + print("Client pub waits on client sub/unsub to be stable...") + time.sleep(1) + if not self._stable_flag.wait(EVENT_WAIT_TIME_OUT_SEC * 3): # We wait longer for the reconnect/stabilization + raise RuntimeError("Timed out in waiting for client_sub_unsub to be stable") + self._stable_flag.clear() + + print("Client pub: 2nd round of publishes") + for j in range(NUMBER_OF_PUBLISHES): + self._client_pub.publish(TOPIC_B, MESSAGE_PREFIX + str(j), 1) + print("Client pub: Published a message to %s" % TOPIC_B) + self._client_pub.publish(TOPIC_A, MESSAGE_PREFIX + str(j) + "-dup", 1) + print("Client pub: Published a message to %s" % TOPIC_A) + time.sleep(0.5) + time.sleep(1) + print("Client pub: Publishes done. Notifying...") + self._publish_end_flag.set() + + print("Client pub runtime thread ends.") + + def verify(self): + print("Verifying...") + assert len(self._received_messages_topic_a) == NUMBER_OF_PUBLISHES # We should only receive the first round + assert len(self._received_messages_topic_b) == NUMBER_OF_PUBLISHES # We should only receive the second round + print("Pass!") + + +############################################################################ +# Main # +# Check inputs +my_check_in_manager = checkInManager(1) +my_check_in_manager.verify(sys.argv) +mode = my_check_in_manager.mode + +skip_when_match(ModeIsALPN(mode).And( + Python2VersionLowerThan((2, 7, 10)).Or(Python3VersionLowerThan((3, 5, 0))) +), "This test is not applicable for mode %s and Python verison %s. Skipping..." % (mode, sys.version_info[:3])) + +# Performing +############ +dual_client_runner = DualClientRunner(mode) +dual_client_runner.start() + +# Verifying +########### +dual_client_runner.verify() diff --git a/test-integration/IntegrationTests/IntegrationTestProgressiveBackoff.py b/test-integration/IntegrationTests/IntegrationTestProgressiveBackoff.py new file mode 100644 index 0000000..f21b9ba --- /dev/null +++ b/test-integration/IntegrationTests/IntegrationTestProgressiveBackoff.py @@ -0,0 +1,285 @@ +# This integration test verifies the functionality in the Python core of Yun/Python SDK +# for progressive backoff logic in auto-reconnect. +# It starts two threads using two different connections to AWS IoT: +# Thread B subscribes to "coolTopic" and waits for incoming messages. Network +# failure will happen occasionally in thread B with a variant interval (connected +# period), simulating stable/unstable connection so as to test the reset logic of +# backoff timing. Once thread B is back online, an internal flag will be set to +# notify the other thread, Thread A, to start publishing to the same topic. +# Thread A will publish a set of messages (a fixed number of messages) to "coolTopic" +# using QoS1 and does nothing in the rest of the time. It will only start publishing +# when it gets ready notification from thread B. No network failure will happen in +# thread A. +# Because thread A is always online and only publishes when thread B is back online, +# all messages published to "coolTopic" should be received by thread B. In meantime, +# thread B should have an increasing amount of backoff waiting period until the +# connected period reaches the length of time for a stable connection. After that, +# the backoff waiting period should be reset. +# The following things will be verified to pass the test: +# 1. All messages are received. +# 2. Backoff waiting period increases as configured before the thread reaches to a +# stable connection. +# 3. Backoff waiting period does not exceed the maximum allowed time. +# 4. Backoff waiting period is reset after the thread reaches to a stable connection. + + +import string +import random +import time +import threading +import sys +sys.path.insert(0, "./configuration/IntegrationTests/TestToolLibrary") +sys.path.insert(0, "./configuration/IntegrationTests/TestToolLibrary/SDKPackage") + +from TestToolLibrary import simpleThreadManager +import TestToolLibrary.checkInManager as checkInManager +import TestToolLibrary.MQTTClientManager as MQTTClientManager +from TestToolLibrary.SDKPackage.AWSIoTPythonSDK.exception.AWSIoTExceptions import publishError +from TestToolLibrary.SDKPackage.AWSIoTPythonSDK.exception.AWSIoTExceptions import subscribeError +from TestToolLibrary.SDKPackage.AWSIoTPythonSDK.exception.AWSIoTExceptions import subscribeTimeoutException +from TestToolLibrary.SDKPackage.AWSIoTPythonSDK.core.protocol.internal.clients import ClientStatus +from TestToolLibrary.skip import skip_when_match +from TestToolLibrary.skip import ModeIsALPN +from TestToolLibrary.skip import Python2VersionLowerThan +from TestToolLibrary.skip import Python3VersionLowerThan + + +# Class that implements all the related threads in the test in a controllable manner +class threadPool: + def __init__(self, srcTotalNumberOfNetworkFailure, clientPub, clientSub): + self._threadBReadyFlag = 0 # 0-Not connected, 1-Connected+Subscribed, -1-ShouldExit + self._threadBReadyFlagMutex = threading.Lock() + self._targetedTopic = "coolTopic" + "".join(random.choice(string.ascii_lowercase) for i in range(4)) + self._publishMessagePool = set() + self._receiveMessagePool = set() + self._roundOfNetworkFailure = 1 + self._totalNumberOfNetworkFailure = srcTotalNumberOfNetworkFailure + self._clientPub = clientPub + self._clientSub = clientSub + self._pubCount = 0 + self._reconnectTimeRecord = list() + self._connectedTimeRecord = list() + + # Message callback for collecting incoming messages from the subscribed topic + def _messageCallback(self, client, userdata, message): + print("Thread B: Received new message: " + str(message.payload)) + self._receiveMessagePool.add(message.payload.decode('utf-8')) + + # The one that publishes + def threadARuntime(self): + exitNow = False + while not exitNow: + self._threadBReadyFlagMutex.acquire() + # Thread A is still reconnecting, WAIT! + if self._threadBReadyFlag == 0: + pass + # Thread A is connected and subscribed, PUBLISH! + elif self._threadBReadyFlag == 1: + self._publish3Messages() + self._threadBReadyFlag = 0 # Reset the readyFlag + # Thread A has finished all rounds of network failure/reconnect, EXIT! + else: + exitNow = True + self._threadBReadyFlagMutex.release() + time.sleep(0.01) # 0.01 sec scanning + + # Publish a set of messages: 3 + def _publish3Messages(self): + loopCount = 3 + while loopCount != 0: + try: + currentMessage = "Message" + str(self._pubCount) + self._clientPub.publish(self._targetedTopic, currentMessage, 1, False) + print("Thread A: Published new message: " + str(currentMessage)) + self._publishMessagePool.add(currentMessage) + self._pubCount += 1 + loopCount -= 1 + except publishError: + print("Publish error!") + except Exception as e: + print("Unknown exception!") + print("Type: " + str(type(e))) + print("Message: " + str(e.message)) + time.sleep(0.5) + + # The one that subscribes and has network failures + def threadBRuntime(self): + # Subscribe to the topic + try: + self._clientSub.subscribe(self._targetedTopic, 1, self._messageCallback) + except subscribeTimeoutException: + print("Subscribe timeout!") + except subscribeError: + print("Subscribe error!") + except Exception as e: + print("Unknown exception!") + print("Type: " + str(type(e))) + print("Message: " + str(e.message)) + print("Thread B: Subscribe request sent. Staring waiting for subscription processing...") + time.sleep(3) + print("Thread B: Done waiting.") + self._threadBReadyFlagMutex.acquire() + self._threadBReadyFlag = 1 + self._threadBReadyFlagMutex.release() + # Start looping with network failure + connectedPeriodSecond = 3 + while self._roundOfNetworkFailure <= self._totalNumberOfNetworkFailure: + self._connectedTimeRecord.append(connectedPeriodSecond) + # Wait for connectedPeriodSecond + print("Thread B: Connected time: " + str(connectedPeriodSecond) + " seconds.") + print("Thread B: Stable time: 60 seconds.") + time.sleep(connectedPeriodSecond) + print("Thread B: Network failure. Round: " + str(self._roundOfNetworkFailure) + ". 0.5 seconds.") + print("Thread B: Backoff time for this round should be: " + str( + self._clientSub._internal_async_client._paho_client._backoffCore._currentBackoffTimeSecond) + " second(s).") + # Set the readyFlag + self._threadBReadyFlagMutex.acquire() + self._threadBReadyFlag = 0 + self._threadBReadyFlagMutex.release() + # Now lose connection for 0.5 seconds, preventing multiple reconnect attempts + loseConnectionLoopCount = 50 + while loseConnectionLoopCount != 0: + self._manualNetworkError() + loseConnectionLoopCount -= 1 + time.sleep(0.01) + # Wait until the connection/subscription is recovered + reconnectTiming = 0 + while self._clientSub._client_status.get_status() != ClientStatus.STABLE: + time.sleep(0.01) + reconnectTiming += 1 + if reconnectTiming % 100 == 0: + print("Thread B: Counting reconnect time: " + str(reconnectTiming / 100) + " seconds.") + print("Thread B: Counting reconnect time result: " + str(float(reconnectTiming) / 100) + " seconds.") + self._reconnectTimeRecord.append(reconnectTiming / 100) + + time.sleep(3) # For valid subscription + + # Update thread B status + self._threadBReadyFlagMutex.acquire() + self._threadBReadyFlag = 1 + self._threadBReadyFlagMutex.release() + + # Update connectedPeriodSecond + connectedPeriodSecond += (2 ** (self._roundOfNetworkFailure - 1)) + # Update roundOfNetworkFailure + self._roundOfNetworkFailure += 1 + + # Notify thread A shouldExit + self._threadBReadyFlagMutex.acquire() + self._threadBReadyFlag = -1 + self._threadBReadyFlagMutex.release() + + # Simulate a network error + def _manualNetworkError(self): + # Only the subscriber needs the network error + if self._clientSub._internal_async_client._paho_client._sock: + self._clientSub._internal_async_client._paho_client._sock.close() + self._clientSub._internal_async_client._paho_client._sock = None + if self._clientSub._internal_async_client._paho_client._ssl: + self._clientSub._internal_async_client._paho_client._ssl.close() + self._clientSub._internal_async_client._paho_client._ssl = None + # Fake that we have detected the disconnection + self._clientSub._internal_async_client._paho_client.on_disconnect(None, None, 0) + + def getReconnectTimeRecord(self): + return self._reconnectTimeRecord + + def getConnectedTimeRecord(self): + return self._connectedTimeRecord + + +# Generate the correct backoff timing to compare the test result with +def generateCorrectAnswer(baseTime, maximumTime, stableTime, connectedTimeRecord): + answer = list() + currentTime = baseTime + nextTime = baseTime + for i in range(0, len(connectedTimeRecord)): + if connectedTimeRecord[i] >= stableTime or i == 0: + currentTime = baseTime + else: + currentTime = min(currentTime * 2, maximumTime) + answer.append(currentTime) + return answer + + +# Verify backoff time +# Corresponding element should have no diff or a bias greater than 1.5 +def verifyBackoffTime(answerList, resultList): + result = True + for i in range(0, len(answerList)): + if abs(answerList[i] - resultList[i]) > 1.5: + result = False + break + return result + + +############################################################################ +# Main # +# Check inputs +myCheckInManager = checkInManager.checkInManager(2) +myCheckInManager.verify(sys.argv) + +#host via describe-endpoint on this OdinMS: com.amazonaws.iot.device.sdk.credentials.testing.websocket +host = "ajje7lpljulm4-ats.iot.us-east-1.amazonaws.com" +rootCA = "./configuration/Credentials/rootCA.crt" +certificate = "./configuration/Credentials/certificate.pem.crt" +privateKey = "./configuration/Credentials/privateKey.pem.key" +mode = myCheckInManager.mode + +skip_when_match(ModeIsALPN(mode).And( + Python2VersionLowerThan((2, 7, 10)).Or(Python3VersionLowerThan((3, 5, 0))) +), "This test is not applicable for mode %s and Python verison %s. Skipping..." % (mode, sys.version_info[:3])) + +# Init Python core and connect +myMQTTClientManager = MQTTClientManager.MQTTClientManager() +clientPub = myMQTTClientManager.create_connected_mqtt_core("integrationTestMQTT_ClientPub", host, rootCA, + certificate, privateKey, mode=mode) +clientSub = myMQTTClientManager.create_connected_mqtt_core("integrationTestMQTT_ClientSub", host, rootCA, + certificate, privateKey, mode=mode) + +if clientPub is None or clientSub is None: + exit(4) + +# Extra configuration for clients +clientPub.configure_reconnect_back_off(1, 16, 60) +clientSub.configure_reconnect_back_off(1, 16, 60) + +print("Two clients are connected!") + +# Configurations +################ +# Custom parameters +NumberOfNetworkFailure = myCheckInManager.customParameter +# ThreadPool object +threadPoolObject = threadPool(NumberOfNetworkFailure, clientPub, clientSub) +# Threads +mySimpleThreadManager = simpleThreadManager.simpleThreadManager() +threadAID = mySimpleThreadManager.createOneTimeThread(threadPoolObject.threadARuntime, []) +threadBID = mySimpleThreadManager.createOneTimeThread(threadPoolObject.threadBRuntime, []) + +# Performing +############ +mySimpleThreadManager.startThreadWithID(threadBID) +mySimpleThreadManager.startThreadWithID(threadAID) +mySimpleThreadManager.joinOneTimeThreadWithID(threadBID) +mySimpleThreadManager.joinOneTimeThreadWithID(threadAID) + +# Verifying +########### +print("Verify that all messages are received...") +if threadPoolObject._publishMessagePool == threadPoolObject._receiveMessagePool: + print("Passed. Recv/Pub: " + str(len(threadPoolObject._receiveMessagePool)) + "/" + str( + len(threadPoolObject._publishMessagePool))) +else: + print("Not all messages are received!") + exit(4) +print("Verify reconnect backoff time record...") +print("ConnectedTimeRecord: " + str(threadPoolObject.getConnectedTimeRecord())) +print("ReconnectTimeRecord: " + str(threadPoolObject.getReconnectTimeRecord())) +print("Answer: " + str(generateCorrectAnswer(1, 16, 60, threadPoolObject.getConnectedTimeRecord()))) +if verifyBackoffTime(generateCorrectAnswer(1, 16, 60, threadPoolObject.getConnectedTimeRecord()), + threadPoolObject.getReconnectTimeRecord()): + print("Passed.") +else: + print("Backoff time does not match theoretical value!") + exit(4) diff --git a/test-integration/IntegrationTests/IntegrationTestShadow.py b/test-integration/IntegrationTests/IntegrationTestShadow.py new file mode 100644 index 0000000..6197df3 --- /dev/null +++ b/test-integration/IntegrationTests/IntegrationTestShadow.py @@ -0,0 +1,246 @@ +# This integration test verifies the functionality in the Python core of Yun/Python SDK +# for IoT shadow operations: shadowUpdate and delta. +# 1. The test generates a X-byte-long random sting and breaks it into a random +# number of chunks, with a fixed length variation from 1 byte to 10 bytes. +# 2. Two threads are created to do shadowUpdate and delta on the same device +# shadow. The update thread updates the desired state with an increasing sequence +# number and a chunk. It is terminated when there are no more chunks to be sent. +# 3. The delta thread listens on delta topic and receives the changes in device +# shadow JSON document. It parses out the sequence number and the chunk, then pack +# them into a dictionary with sequence number as the key and the chunk as the value. +# 4. To verify the result of the test, the random string is re-assembled for both +# the update thread and the delta thread to see if they are equal. +# 5. Since shadow operations are all QoS0 (Pub/Sub), it is still a valid case when +# the re-assembled strings are not equal. Then we need to make sure that the number +# of the missing chunks does not exceed 10% of the total number of chunk transmission +# that succeeds. + +import time +import random +import string +import json +import sys +sys.path.insert(0, "./configuration/IntegrationTests/TestToolLibrary") +sys.path.insert(0, "./configuration/IntegrationTests/TestToolLibrary/SDKPackage") + +from TestToolLibrary import simpleThreadManager +import TestToolLibrary.checkInManager as checkInManager +import TestToolLibrary.MQTTClientManager as MQTTClientManager +from TestToolLibrary.SDKPackage.AWSIoTPythonSDK.core.shadow.deviceShadow import deviceShadow +from TestToolLibrary.SDKPackage.AWSIoTPythonSDK.core.shadow.shadowManager import shadowManager +from TestToolLibrary.SDKPackage.AWSIoTPythonSDK.exception.AWSIoTExceptions import publishError +from TestToolLibrary.SDKPackage.AWSIoTPythonSDK.exception.AWSIoTExceptions import subscribeError +from TestToolLibrary.SDKPackage.AWSIoTPythonSDK.exception.AWSIoTExceptions import subscribeTimeoutException +from TestToolLibrary.skip import skip_when_match +from TestToolLibrary.skip import ModeIsALPN +from TestToolLibrary.skip import Python2VersionLowerThan +from TestToolLibrary.skip import Python3VersionLowerThan + + +# Global configuration +TPS = 1 # Update speed, Spectre does not tolerate high TPS shadow operations... + + +# Class that manages the generation and chopping of the random string +class GibberishBox: + def __init__(self, length): + self._content = self._generateGibberish(length) + + def getGibberish(self): + return self._content + + # Random string generator: lower/upper case letter + digits + def _generateGibberish(self, length): + s = string.ascii_lowercase + string.digits + string.ascii_uppercase + return ''.join(random.sample(s * length, length)) + + # Spit out the gibberish chunk by chunk (1-10 bytes) + def gibberishSpitter(self): + randomLength = random.randrange(1, 11) + ret = None + if self._content is not None: + ret = self._content[0:randomLength] + self._content = self._content[randomLength:] + return ret + + +# Class that manages the callback function and record of chunks for re-assembling +class callbackContainer: + def __init__(self): + self._internalDictionary = dict() + + def getInternalDictionary(self): + return self._internalDictionary + + def testCallback(self, payload, type, token): + print("Type: " + type) + print(payload) + print("&&&&&&&&&&&&&&&&&&&&") + # This is the shadow delta callback, so the token should be None + if type == "accepted": + JsonDict = json.loads(payload) + try: + sequenceNumber = int(JsonDict['state']['desired']['sequenceNumber']) + gibberishChunk = JsonDict['state']['desired']['gibberishChunk'] + self._internalDictionary[sequenceNumber] = gibberishChunk + except KeyError as e: + print(e.message) + print("No such key!") + else: + JsonDict = json.loads(payload) + try: + sequenceNumber = int(JsonDict['state']['sequenceNumber']) + gibberishChunk = JsonDict['state']['gibberishChunk'] + self._internalDictionary[sequenceNumber] = gibberishChunk + except KeyError as e: + print(e.message) + print("No such key!") + + +# Thread runtime function +def threadShadowUpdate(deviceShadow, callback, TPS, gibberishBox, maxNumMessage): + time.sleep(2) + chunkSequence = 0 + while True: + currentChunk = gibberishBox.gibberishSpitter() + if currentChunk != "": + outboundJsonDict = dict() + outboundJsonDict["state"] = dict() + outboundJsonDict["state"]["desired"] = dict() + outboundJsonDict["state"]["desired"]["sequenceNumber"] = chunkSequence + outboundJsonDict["state"]["desired"]["gibberishChunk"] = currentChunk + outboundJSON = json.dumps(outboundJsonDict) + chunkSequence += 1 + try: + deviceShadow.shadowUpdate(outboundJSON, callback, 5) + except publishError: + print("Publish error!") + except subscribeTimeoutException: + print("Subscribe timeout!") + except subscribeError: + print("Subscribe error!") + except Exception as e: + print("Unknown exception!") + print("Type: " + str(type(e))) + print("Message: " + str(e.message)) + time.sleep(1 / TPS) + else: + break + print("Update thread completed.") + + +# Re-assemble gibberish +def reAssembleGibberish(srcDict, maxNumMessage): + ret = "" + for i in range(0, maxNumMessage): + try: + ret += srcDict[i] + except KeyError: + pass + return ret + + +# RandomShadowNameSuffix +def randomString(lengthOfString): + return "".join(random.choice(string.ascii_lowercase) for i in range(lengthOfString)) + + +############################################################################ +# Main # +# Check inputs +myCheckInManager = checkInManager.checkInManager(2) +myCheckInManager.verify(sys.argv) + +host = "ajje7lpljulm4-ats.iot.us-east-1.amazonaws.com" +rootCA = "./configuration/Credentials/rootCA.crt" +certificate = "./configuration/Credentials/certificate.pem.crt" +privateKey = "./configuration/Credentials/privateKey.pem.key" +mode = myCheckInManager.mode + +skip_when_match(ModeIsALPN(mode).And( + Python2VersionLowerThan((2, 7, 10)).Or(Python3VersionLowerThan((3, 5, 0))) +), "This test is not applicable for mode %s and Python verison %s. Skipping..." % (mode, sys.version_info[:3])) + +# Init Python core and connect +myMQTTClientManager = MQTTClientManager.MQTTClientManager() +clientPub = myMQTTClientManager.create_connected_mqtt_core("integrationTestMQTT_ClientPub", host, rootCA, + certificate, privateKey, mode=mode) +clientSub = myMQTTClientManager.create_connected_mqtt_core("integrationTestMQTT_ClientSub", host, rootCA, + certificate, privateKey, mode=mode) + +if clientPub is None or clientSub is None: + exit(4) + +print("Two clients are connected!") + +# Configurations +################ +# Data +gibberishLength = myCheckInManager.customParameter +# Init device shadow instance +shadowManager1 = shadowManager(clientPub) +shadowManager2 = shadowManager(clientSub) +shadowName = "GibberChunk" + randomString(5) +deviceShadow1 = deviceShadow(shadowName, True, shadowManager1) +deviceShadow2 = deviceShadow(shadowName, True, shadowManager2) +print("Two device shadow instances are created!") + +# Callbacks +callbackHome_Update = callbackContainer() +callbackHome_Delta = callbackContainer() + +# Listen on delta topic +try: + deviceShadow2.shadowRegisterDeltaCallback(callbackHome_Delta.testCallback) +except subscribeError: + print("Subscribe error!") +except subscribeTimeoutException: + print("Subscribe timeout!") +except Exception as e: + print("Unknown exception!") + print("Type: " + str(type(e))) + print("Message: " + str(e.message)) + +# Init gibberishBox +cipher = GibberishBox(gibberishLength) +gibberish = cipher.getGibberish() +print("Random string: " + gibberish) + +# Threads +mySimpleThreadManager = simpleThreadManager.simpleThreadManager() +updateThreadID = mySimpleThreadManager.createOneTimeThread(threadShadowUpdate, + [deviceShadow1, callbackHome_Update.testCallback, TPS, + cipher, gibberishLength]) + +# Performing +############ +# Functionality test +mySimpleThreadManager.startThreadWithID(updateThreadID) +mySimpleThreadManager.joinOneTimeThreadWithID(updateThreadID) +time.sleep(10) # Just in case + +# Now check the gibberish +gibberishUpdateResult = reAssembleGibberish(callbackHome_Update.getInternalDictionary(), gibberishLength) +gibberishDeltaResult = reAssembleGibberish(callbackHome_Delta.getInternalDictionary(), gibberishLength) +print("Update:") +print(gibberishUpdateResult) +print("Delta:") +print(gibberishDeltaResult) +print("Origin:") +print(gibberish) + +if gibberishUpdateResult != gibberishDeltaResult: + # Since shadow operations are on QoS0 (Pub/Sub), there could be a chance + # where incoming messages are missing on the subscribed side + # A ratio of 95% must be guaranteed to pass this test + dictUpdate = callbackHome_Update.getInternalDictionary() + dictDelta = callbackHome_Delta.getInternalDictionary() + maxBaseNumber = max(len(dictUpdate), len(dictDelta)) + diff = float(abs(len(dictUpdate) - len(dictDelta))) / maxBaseNumber + print("Update/Delta string not equal, missing rate is: " + str(diff * 100) + "%.") + # Largest chunk is 10 bytes, total length is X bytes. + # Minimum number of chunks is X/10 + # Maximum missing rate = 10% + if diff > 0.1: + print("Missing rate too high!") + exit(4) diff --git a/test-integration/IntegrationTests/TestToolLibrary/MQTTClientManager.py b/test-integration/IntegrationTests/TestToolLibrary/MQTTClientManager.py new file mode 100644 index 0000000..9cdf3d5 --- /dev/null +++ b/test-integration/IntegrationTests/TestToolLibrary/MQTTClientManager.py @@ -0,0 +1,146 @@ +import random +import string +import traceback +from ssl import SSLError + +import TestToolLibrary.SDKPackage.AWSIoTPythonSDK.core.protocol.paho.client as paho +from TestToolLibrary.SDKPackage.AWSIoTPythonSDK.MQTTLib import AWSIoTMQTTClient +from TestToolLibrary.SDKPackage.AWSIoTPythonSDK.core.util.providers import CertificateCredentialsProvider +# from TestToolLibrary.SDKPackage.AWSIoTPythonSDK.core.util.providers import CiphersProvider +from TestToolLibrary.SDKPackage.AWSIoTPythonSDK.core.util.enums import DropBehaviorTypes +from TestToolLibrary.SDKPackage.AWSIoTPythonSDK.core.util.providers import EndpointProvider +from TestToolLibrary.SDKPackage.AWSIoTPythonSDK.core.protocol.mqtt_core import MqttCore +from TestToolLibrary.SDKPackage.AWSIoTPythonSDK.exception.AWSIoTExceptions import connectError +from TestToolLibrary.SDKPackage.AWSIoTPythonSDK.exception.AWSIoTExceptions import connectTimeoutException + + +CERT_MUTUAL_AUTH = "MutualAuth" +WEBSOCKET = 'Websocket' +CERT_ALPN = "ALPN" + + +# Class that manages the creation, configuration and connection of MQTT Client +class MQTTClientManager: + + def create_connected_mqtt_client(self, mode, client_id, host, credentials_data, callbacks=None): + client = self.create_nonconnected_mqtt_client(mode, client_id, host, credentials_data, callbacks) + return self._connect_client(client) + + def create_nonconnected_mqtt_client(self, mode, client_id, host, credentials_data, callbacks=None): + if mode == CERT_MUTUAL_AUTH: + sdk_mqtt_client = self._create_nonconnected_mqtt_client_with_cert(client_id, host, 8883, credentials_data) + elif mode == WEBSOCKET: + root_ca, certificate, private_key = credentials_data + sdk_mqtt_client = AWSIoTMQTTClient(clientID=client_id + "_" + self._random_string(3), useWebsocket=True) + sdk_mqtt_client.configureEndpoint(host, 443) + sdk_mqtt_client.configureCredentials(CAFilePath=root_ca) + elif mode == CERT_ALPN: + sdk_mqtt_client = self._create_nonconnected_mqtt_client_with_cert(client_id, host, 443, credentials_data) + else: + raise RuntimeError("Test mode: " + str(mode) + " not supported!") + + sdk_mqtt_client.configureConnectDisconnectTimeout(10) + sdk_mqtt_client.configureMQTTOperationTimeout(5) + + if callbacks is not None: + sdk_mqtt_client.onOnline = callbacks.on_online + sdk_mqtt_client.onOffline = callbacks.on_offline + sdk_mqtt_client.onMessage = callbacks.on_message + + return sdk_mqtt_client + + def _create_nonconnected_mqtt_client_with_cert(self, client_id, host, port, credentials_data): + root_ca, certificate, private_key = credentials_data + sdk_mqtt_client = AWSIoTMQTTClient(clientID=client_id + "_" + self._random_string(3)) + sdk_mqtt_client.configureEndpoint(host, port) + sdk_mqtt_client.configureCredentials(CAFilePath=root_ca, KeyPath=private_key, CertificatePath=certificate) + + return sdk_mqtt_client + + def create_connected_mqtt_core(self, client_id, host, root_ca, certificate, private_key, mode): + client = self.create_nonconnected_mqtt_core(client_id, host, root_ca, certificate, private_key, mode) + return self._connect_client(client) + + def create_nonconnected_mqtt_core(self, client_id, host, root_ca, certificate, private_key, mode): + client = None + protocol = None + port = None + is_websocket = False + is_alpn = False + + if mode == CERT_MUTUAL_AUTH: + protocol = paho.MQTTv311 + port = 8883 + elif mode == WEBSOCKET: + protocol = paho.MQTTv31 + port = 443 + is_websocket = True + elif mode == CERT_ALPN: + protocol = paho.MQTTv311 + port = 443 + is_alpn = True + else: + print("Error in creating the client") + + if protocol is None or port is None: + print("Not enough input parameters") + return client # client is None is the necessary params are not there + + try: + client = MqttCore(client_id + "_" + self._random_string(3), True, protocol, is_websocket) + + endpoint_provider = EndpointProvider() + endpoint_provider.set_host(host) + endpoint_provider.set_port(port) + + # Once is_websocket is True, certificate_credentials_provider will NOT be used + # by the client even if it is configured + certificate_credentials_provider = CertificateCredentialsProvider() + certificate_credentials_provider.set_ca_path(root_ca) + certificate_credentials_provider.set_cert_path(certificate) + certificate_credentials_provider.set_key_path(private_key) + + # cipher_provider = CiphersProvider() + # cipher_provider.set_ciphers(None) + + client.configure_endpoint(endpoint_provider) + # client.configure_cert_credentials(certificate_credentials_provider, cipher_provider) + client.configure_cert_credentials(certificate_credentials_provider) + client.configure_connect_disconnect_timeout_sec(10) + client.configure_operation_timeout_sec(5) + client.configure_offline_requests_queue(10, DropBehaviorTypes.DROP_NEWEST) + + if is_alpn: + client.configure_alpn_protocols() + except Exception as e: + print("Unknown exception in creating the client: " + str(e)) + finally: + return client + + def _random_string(self, length): + return "".join(random.choice(string.ascii_lowercase) for i in range(length)) + + def _connect_client(self, client): + if client is None: + return client + + try: + client.connect(1) + except connectTimeoutException as e: + print("Connect timeout: " + str(e)) + return None + except connectError as e: + print("Connect error:" + str(e)) + return None + except SSLError as e: + print("Connect SSL error: " + str(e)) + return None + except IOError as e: + print("Credentials not found: " + str(e)) + return None + except Exception as e: + print("Unknown exception in connect: ") + traceback.print_exc() + return None + + return client diff --git a/test-integration/IntegrationTests/TestToolLibrary/SDKPackage/__init__.py b/test-integration/IntegrationTests/TestToolLibrary/SDKPackage/__init__.py new file mode 100644 index 0000000..1ad354e --- /dev/null +++ b/test-integration/IntegrationTests/TestToolLibrary/SDKPackage/__init__.py @@ -0,0 +1 @@ +__version__ = "1.4.9" diff --git a/test-integration/IntegrationTests/TestToolLibrary/__init__.py b/test-integration/IntegrationTests/TestToolLibrary/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test-integration/IntegrationTests/TestToolLibrary/checkInManager.py b/test-integration/IntegrationTests/TestToolLibrary/checkInManager.py new file mode 100644 index 0000000..2faaa02 --- /dev/null +++ b/test-integration/IntegrationTests/TestToolLibrary/checkInManager.py @@ -0,0 +1,18 @@ +# Class that is responsible for input/dependency verification +import sys + + +class checkInManager: + + def __init__(self, numberOfInputParameters): + self._numberOfInputParameters = numberOfInputParameters + self.mode = None + self.customParameter = None + + def verify(self, args): + # Check if we got the correct command line params + if len(args) != self._numberOfInputParameters + 1: + exit(4) + self.mode = str(args[1]) + if self._numberOfInputParameters + 1 > 2: + self.customParameter = int(args[2]) diff --git a/test-integration/IntegrationTests/TestToolLibrary/simpleThreadManager.py b/test-integration/IntegrationTests/TestToolLibrary/simpleThreadManager.py new file mode 100644 index 0000000..023f8e7 --- /dev/null +++ b/test-integration/IntegrationTests/TestToolLibrary/simpleThreadManager.py @@ -0,0 +1,110 @@ +# Library for controllable threads. Should be able to: +# 1. Create different threads +# 2. Terminate certain thread +# 3. Join certain thread + + +import time +import threading + + +# Describe the type, property and control flag for a certain thread +class _threadControlUnit: + # Constants for different thread type + _THREAD_TYPE_ONETIME = 0 + _THREAD_TYPE_LOOP = 1 + + def __init__(self, threadID, threadType, runFunction, runParameters, scanningSpeedSecond=0.01): + if threadID is None or threadType is None or runFunction is None or runParameters is None: + raise ValueError("None input detected.") + if threadType != self._THREAD_TYPE_ONETIME and threadType != self._THREAD_TYPE_LOOP: + raise ValueError("Thread type not supported.") + self.threadID = threadID + self.threadType = threadType + self.runFunction = runFunction + self.runParameters = runParameters + self.threadObject = None # Holds the real thread object + # Now configure control flag, only meaning for loop thread type + if self.threadType == self._THREAD_TYPE_LOOP: + self.stopSign = False # Enabled infinite loop by default + self.scanningSpeedSecond = scanningSpeedSecond + else: + self.stopSign = None # No control flag for one time thread + self.scanningSpeedSecond = -1 + + def _oneTimeRunFunction(self): + self.runFunction(*self.runParameters) + + def _loopRunFunction(self): + while not self.stopSign: + self.runFunction(*self.runParameters) # There should be no manual delay in this function + time.sleep(self.scanningSpeedSecond) + + def _stopMe(self): + self.stopSign = True + + def _setThreadObject(self, threadObject): + self.threadObject = threadObject + + def _getThreadObject(self): + return self.threadObject + + +# Class that manages all threadControlUnit +# Used in a single thread +class simpleThreadManager: + def __init__(self): + self._internalCount = 0 + self._controlCenter = dict() + + def createOneTimeThread(self, runFunction, runParameters): + returnID = self._internalCount + self._controlCenter[self._internalCount] = _threadControlUnit(self._internalCount, + _threadControlUnit._THREAD_TYPE_ONETIME, + runFunction, runParameters) + self._internalCount += 1 + return returnID + + def createLoopThread(self, runFunction, runParameters, scanningSpeedSecond): + returnID = self._internalCount + self._controlCenter[self._internalCount] = _threadControlUnit(self._internalCount, + _threadControlUnit._THREAD_TYPE_LOOP, runFunction, + runParameters, scanningSpeedSecond) + self._internalCount += 1 + return returnID + + def stopLoopThreadWithID(self, threadID): + threadToStop = self._controlCenter.get(threadID) + if threadToStop is None: + raise ValueError("No such threadID.") + else: + if threadToStop.threadType == _threadControlUnit._THREAD_TYPE_LOOP: + threadToStop._stopMe() + time.sleep(3 * threadToStop.scanningSpeedSecond) + else: + raise TypeError("Error! Try to stop a one time thread.") + + def startThreadWithID(self, threadID): + threadToStart = self._controlCenter.get(threadID) + if threadToStart is None: + raise ValueError("No such threadID.") + else: + currentThreadType = threadToStart.threadType + newThreadObject = None + if currentThreadType == _threadControlUnit._THREAD_TYPE_LOOP: + newThreadObject = threading.Thread(target=threadToStart._loopRunFunction) + else: # One time thread + newThreadObject = threading.Thread(target=threadToStart._oneTimeRunFunction) + newThreadObject.start() + threadToStart._setThreadObject(newThreadObject) + + def joinOneTimeThreadWithID(self, threadID): + threadToJoin = self._controlCenter.get(threadID) + if threadToJoin is None: + raise ValueError("No such threadID.") + else: + if threadToJoin.threadType == _threadControlUnit._THREAD_TYPE_ONETIME: + currentThreadObject = threadToJoin._getThreadObject() + currentThreadObject.join() + else: + raise TypeError("Error! Try to join a loop thread.") diff --git a/test-integration/IntegrationTests/TestToolLibrary/skip.py b/test-integration/IntegrationTests/TestToolLibrary/skip.py new file mode 100644 index 0000000..4d6e5ca --- /dev/null +++ b/test-integration/IntegrationTests/TestToolLibrary/skip.py @@ -0,0 +1,110 @@ +import sys +from TestToolLibrary.MQTTClientManager import CERT_ALPN +from TestToolLibrary.MQTTClientManager import WEBSOCKET + +# This module manages the skip policy validation for each test + + +def skip_when_match(policy, message): + if policy.validate(): + print(message) + exit(0) # Exit the Python interpreter + + +class Policy(object): + + AND = "and" + OR = "or" + + def __init__(self): + self._relations = [] + + # Use caps to avoid collision with Python built-in and/or keywords + def And(self, policy): + self._relations.append((self.AND, policy)) + return self + + def Or(self, policy): + self._relations.append((self.OR, policy)) + return self + + def validate(self): + result = self.validate_impl() + + for element in self._relations: + operand, policy = element + if operand == self.AND: + result = result and policy.validate() + elif operand == self.OR: + result = result or policy.validate() + else: + raise RuntimeError("Unrecognized operand: " + str(operand)) + + return result + + def validate_impl(self): + raise RuntimeError("Not implemented") + + +class PythonVersion(Policy): + + HIGHER = "higher" + LOWER = "lower" + EQUALS = "equals" + + def __init__(self, actual_version, expected_version, operand): + Policy.__init__(self) + self._actual_version = actual_version + self._expected_version = expected_version + self._operand = operand + + def validate_impl(self): + if self._operand == self.LOWER: + return self._actual_version < self._expected_version + elif self._operand == self.HIGHER: + return self._actual_version > self._expected_version + elif self._operand == self.EQUALS: + return self._actual_version == self._expected_version + else: + raise RuntimeError("Unsupported operand: " + self._operand) + + +class Python2VersionLowerThan(PythonVersion): + + def __init__(self, version): + PythonVersion.__init__(self, sys.version_info[:3], version, PythonVersion.LOWER) + + def validate_impl(self): + return sys.version_info[0] == 2 and PythonVersion.validate_impl(self) + + +class Python3VersionLowerThan(PythonVersion): + + def __init__(self, version): + PythonVersion.__init__(self, sys.version_info[:3], version, PythonVersion.LOWER) + + def validate_impl(self): + return sys.version_info[0] == 3 and PythonVersion.validate_impl(self) + + +class ModeIs(Policy): + + def __init__(self, actual_mode, expected_mode): + Policy.__init__(self) + self._actual_mode = actual_mode + self._expected_mode = expected_mode + + def validate_impl(self): + return self._actual_mode == self._expected_mode + + +class ModeIsALPN(ModeIs): + + def __init__(self, actual_mode): + ModeIs.__init__(self, actual_mode=actual_mode, expected_mode=CERT_ALPN) + + +class ModeIsWebSocket(ModeIs): + + def __init__(self, actual_mode): + ModeIs.__init__(self, actual_mode=actual_mode, expected_mode=WEBSOCKET) diff --git a/test-integration/Tools/.gitignore b/test-integration/Tools/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/test-integration/Tools/getSDKZIP.sh b/test-integration/Tools/getSDKZIP.sh new file mode 100644 index 0000000..7cb939f --- /dev/null +++ b/test-integration/Tools/getSDKZIP.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# +# This script checks which environment this +# is, brazil or ToD Worker and trying to find +# ZIP package for the latest build of SDK in +# certain directory. +# Will print out the location of ZIP package +# and return code specifying environment type: +# 0 - Brazil +# 1 - ToD +# -1 - Not supported +# Otherwise print nothing. The environment is +# not supported. + +# Define usage +USAGE="usage: getSDKZIP.sh " +# if input args not correct, echo usage +if [ $# -ne 1 ]; then + echo ${USAGE} +else +# Find SDK ZIP package + ZIPLocation="" + if [ "$1"x == "IoTYunSDK"x ]; then + ZIPLocation=$(find ../IotSdkArduino/build/zipPackage -name AWS-IoT-Arduino-Yun-SDK-latest.zip) + if [ -n "$ZIPLocation" ]; then + echo ${ZIPLocation} + exit 0 + fi + ZIPLocation=$(find ./test-runtime/ -name AWS-IoT-Arduino-Yun-SDK-latest.zip) + if [ -n "$ZIPLocation" ]; then + echo ${ZIPLocation} + exit 1 + fi + elif [ "$1"x == "IoTPySDK"x ]; then + if [ -d "../../src/IotSdkPython/AWSIoTPythonSDK/" ]; then + echo "../../src/IotSdkPython/AWSIoTPythonSDK/" + exit 2 + fi + if [ -d "./test-runtime/aws-iot-device-sdk-python/AWSIoTPythonSDK" ]; then + echo "./test-runtime/aws-iot-device-sdk-python/AWSIoTPythonSDK" + exit 3 + fi + else + exit -1 + fi + exit -1 +fi diff --git a/test-integration/Tools/retrieve-key.py b/test-integration/Tools/retrieve-key.py new file mode 100644 index 0000000..fd6336a --- /dev/null +++ b/test-integration/Tools/retrieve-key.py @@ -0,0 +1,59 @@ + +import boto3 +import base64 +import sys +from botocore.exceptions import ClientError + +def main(): + secret_name = sys.argv[1] + region_name = "us-east-1" + + # Create a Secrets Manager client + session = boto3.session.Session() + client = session.client( + service_name='secretsmanager', + region_name=region_name + ) + # In this sample we only handle the specific exceptions for the 'GetSecretValue' API. + # See https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_GetSecretValue.html + # We rethrow the exception by default. + + try: + get_secret_value_response = client.get_secret_value( + SecretId=secret_name + ) + + except ClientError as e: + if e.response['Error']['Code'] == 'DecryptionFailureException': + # Secrets Manager can't decrypt the protected secret text using the provided KMS key. + # Deal with the exception here, and/or rethrow at your discretion. + raise e + elif e.response['Error']['Code'] == 'InternalServiceErrorException': + # An error occurred on the server side. + # Deal with the exception here, and/or rethrow at your discretion. + raise e + elif e.response['Error']['Code'] == 'InvalidParameterException': + # You provided an invalid value for a parameter. + # Deal with the exception here, and/or rethrow at your discretion. + raise e + elif e.response['Error']['Code'] == 'InvalidRequestException': + # You provided a parameter value that is not valid for the current state of the resource. + # Deal with the exception here, and/or rethrow at your discretion. + raise e + elif e.response['Error']['Code'] == 'ResourceNotFoundException': + # We can't find the resource that you asked for. + # Deal with the exception here, and/or rethrow at your discretion. + raise e + print(e) + else: + # Decrypts secret using the associated KMS key. + # Depending on whether the secret is a string or binary, one of these fields will be populated. + if 'SecretString' in get_secret_value_response: + secret = get_secret_value_response['SecretString'] + else: + secret = base64.b64decode(get_secret_value_response['SecretBinary']) + print(secret) + + +if __name__ == '__main__': + sys.exit(main()) # next section explains the use of sys.exit \ No newline at end of file diff --git a/test-integration/run/.gitignore b/test-integration/run/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/test-integration/run/run.sh b/test-integration/run/run.sh new file mode 100644 index 0000000..cbc3ce4 --- /dev/null +++ b/test-integration/run/run.sh @@ -0,0 +1,202 @@ +#!/bin/bash +# +# This script manages the start of integration +# tests for Python core in AWS IoT Arduino Yun +# SDK. The tests should be able to run both in +# Brazil and ToD Worker environment. +# The script will perform the following tasks: +# 1. Retrieve credentials as needed from Odin +# 2. Obtain ZIP package and unzip it locally +# 3. Obtain Python executable +# 4. Start the integration tests and check results +# 5. Report any status returned. +# To start the tests as TodWorker: +# > run.sh MutualAuth 1000 100 7 +# or +# > run.sh Websocket 1000 100 7 +# or +# > run.sh ALPN 1000 100 7 +# +# To start the tests from desktop: +# > run.sh MutualAuthT 1000 100 7 +# or +# > run.sh WebsocketT 1000 100 7 +# or +# > run.sh ALPNT 1000 100 7 +# +# 1000 MQTT messages, 100 bytes of random string +# in length and 7 rounds of network failure for +# progressive backoff. +# Test mode (MutualAuth/Websocket) must be +# specified. +# Scale number must also be specified (see usage) + +# Define const +USAGE="usage: run.sh " +OdinSetForMutualAuth_Desktop="com.amazon.aws.iot.sdk.desktop.certificate" +OdinSetForWebsocket_Desktop="com.amazon.aws.iot.sdk.desktop.websockets" +OdinSetForMutualAuth_TodWorker="com.amazonaws.iot.device.sdk.credentials.testing.certificates.us-east-1" +AWSMutualAuth_TodWorker_private_key="arn:aws:secretsmanager:us-east-1:123124136734:secret:V1IotSdkIntegrationTestPrivateKey-vNUQU8" +AWSMutualAuth_TodWorker_certificate="arn:aws:secretsmanager:us-east-1:123124136734:secret:V1IotSdkIntegrationTestCertificate-vTRwjE" + +OdinSetForGGDiscovery_TodWorker="com.amazonaws.iot.device.sdk.credentials.testing.gg.discovery.us-east-1" +OdinSetForWebsocket_TodWorker="com.amazonaws.iot.device.sdk.credentials.testing.websocket" +RetrieveOdinMaterial="./configuration/Tools/retrieve-material.sh" +RetrieveAWSKeys="./configuration/Tools/retrieve-key.py" +CREDENTIAL_DIR="./configuration/Credentials/" +TEST_DIR="./configuration/IntegrationTests/" +CA_CERT_URL="https://www.amazontrust.com/repository/AmazonRootCA1.pem" +CA_CERT_PATH=${CREDENTIAL_DIR}rootCA.crt +ACCESS_KEY_ID_ARN="arn:aws:secretsmanager:us-east-1:123124136734:secret:V1IotSdkIntegrationTestWebsocketAccessKeyId-1YdB9z" +ACCESS_KEY_ARN="arn:aws:secretsmanager:us-east-1:123124136734:secret:V1IotSdkIntegrationTestWebsocketSecretAccessKey-MKTSaV" + + + +# If input args not correct, echo usage +if [ $# -ne 5 ]; then + echo ${USAGE} +else +# Description + echo "[STEP] Start run.sh" + echo "***************************************************" + echo "About to start integration tests for $1..." + echo "Test Mode: $2" +# Determine the Python versions need to test for this SDK + pythonExecutableArray=() + if [ "$1"x == "IoTYunSDK"x ]; then + pythonExecutableArray[0]="2" + elif [ "$1"x == "IoTPySDK"x ]; then + pythonExecutableArray[0]="3" + else + echo "Cannot determine Python executables for this unrecognized SDK!" + exit 1 + fi +# Retrieve credentials as needed from Odin + TestMode="" + echo "[STEP] Retrieve credentials from Odin" + echo "***************************************************" + if [ "$2"x == "MutualAuth"x -o "$2"x == "MutualAuthT"x ]; then + OdinSetName=${OdinSetForMutualAuth_TodWorker} + AWSSetName_privatekey=${AWSMutualAuth_TodWorker_private_key} + AWSSetName_certificate=${AWSMutualAuth_TodWorker_certificate} + OdinSetDRSName=${OdinSetForGGDiscovery_TodWorker} + TestMode="MutualAuth" + if [ "$2"x == "MutualAuthT"x ]; then + OdinSetName=${OdinSetForMutualAuth_Desktop} + fi + OdinMaterialTypeCertificate="Certificate" + OdinMaterialTypeKey="PrivateKey" + python ${RetrieveAWSKeys} ${AWSSetName_certificate} > ${CREDENTIAL_DIR}certificate.pem.crt + python ${RetrieveAWSKeys} ${AWSSetName_privatekey} > ${CREDENTIAL_DIR}privateKey.pem.key + curl -s "${CA_CERT_URL}" > ${CA_CERT_PATH} + echo -e "URL retrieved certificate data:\n$(cat ${CA_CERT_PATH})\n" + #${RetrieveOdinMaterial} ${OdinSetDRSName} ${OdinMaterialTypeCertificate} > ${CREDENTIAL_DIR}certificate_drs.pem.crt + #${RetrieveOdinMaterial} ${OdinSetDRSName} ${OdinMaterialTypeKey} > ${CREDENTIAL_DIR}privateKey_drs.pem.key + elif [ "$2"x == "Websocket"x -o "$2"x == "WebsocketT"x ]; then + OdinSetName=${OdinSetForWebsocket_TodWorker} + TestMode="Websocket" + if [ "$2"x == "WebsocketT"x ]; then + OdinSetName=${OdinSetForWebsocket_Desktop} + fi + OdinMaterialTypeID="Principal" + OdinMaterialTypeSecret="Credential" + python ${RetrieveAWSKeys} ${ACCESS_KEY_ID_ARN} > KEY_ID + python ${RetrieveAWSKeys} ${ACCESS_KEY_ARN} > SECRET_KEY + #SECRET_KEY=$(${RetrieveOdinMaterial} ${OdinSetName} ${OdinMaterialTypeSecret}) + export AWS_ACCESS_KEY_ID=${KEY_ID} + export AWS_SECRET_ACCESS_KEY=${SECRET_KEY} + curl -s "${CA_CERT_URL}" > ${CA_CERT_PATH} + echo -e "URL retrieved certificate data:\n$(cat ${CA_CERT_PATH})\n" + elif [ "$2"x == "ALPN"x -o "$2"x == "ALPNT"x ]; then + OdinSetName=${OdinSetForMutualAuth_TodWorker} + OdinSetDRSName=${OdinSetForGGDiscovery_TodWorker} + TestMode="ALPN" + if [ "$2"x == "ALPNT"x ]; then + OdinSetName=${OdinSetForMutualAuth_Desktop} + fi + OdinMaterialTypeCertificate="Certificate" + OdinMaterialTypeKey="PrivateKey" + ${RetrieveOdinMaterial} ${OdinSetName} ${OdinMaterialTypeCertificate} > ${CREDENTIAL_DIR}certificate.pem.crt + ${RetrieveOdinMaterial} ${OdinSetName} ${OdinMaterialTypeKey} > ${CREDENTIAL_DIR}privateKey.pem.key + curl -s "${CA_CERT_URL}" > ${CA_CERT_PATH} + echo -e "URL retrieved certificate data:\n$(cat ${CA_CERT_PATH})\n" + #${RetrieveOdinMaterial} ${OdinSetDRSName} ${OdinMaterialTypeCertificate} > ${CREDENTIAL_DIR}certificate_drs.pem.crt + #${RetrieveOdinMaterial} ${OdinSetDRSName} ${OdinMaterialTypeKey} > ${CREDENTIAL_DIR}privateKey_drs.pem.key + else + echo "Mode not supported" + exit 1 + fi +# Obtain ZIP package and unzip it locally + echo ${TestMode} + echo "[STEP] Obtain ZIP package" + echo "***************************************************" + ZIPLocation="C:/Iot/workspace/aws-iot-device-sdk-python/AWSIoTPythonSDK" + if [ $? -eq "-1" ]; then + echo "Cannot find SDK ZIP package" + exit 2 + fi + if [ "$1"x == "IoTYunSDK"x ]; then + unzip -q ${ZIPLocation} -d ./configuration/IntegrationTests/TestToolLibrary/SDKPackage/ + elif [ "$1"x == "IoTPySDK"x ]; then + cp -R ${ZIPLocation} ./configuration/IntegrationTests/TestToolLibrary/SDKPackage/ + else + echo "Error in getSDKZIP" + exit 2 + fi +# Obtain Python executable + for iii in "${pythonExecutableArray[@]}" + do + echo "[STEP] Obtain Python executable" + echo "***************************************************" + echo "Python version ${iii}.x..." + PYTHON=$(./configuration/Tools/getPython.sh ${iii}) + if [ $? -eq "-1" ]; then + echo "Cannot find Python executable" + exit 3 + fi + # Start integration test + # Loop through the tes directory and execute all available integration tests + echo "[STEP] Start integration tests" + echo "***************************************************" + for file in `ls ${TEST_DIR}` + do + # if [ ${file}x == "IntegrationTestMQTTConnection.py"x ]; then + if [ ${file##*.}x == "py"x ]; then + echo "[SUB] Running test: ${file}..." + + Scale=10 + case "$file" in + "IntegrationTestMQTTConnection.py") Scale=$3 + ;; + "IntegrationTestShadow.py") Scale=$4 + ;; + "IntegrationTestAutoReconnectResubscribe.py") Scale="" + ;; + "IntegrationTestProgressiveBackoff.py") Scale=$5 + ;; + "IntegrationTestConfigurablePublishMessageQueueing.py") Scale="" + ;; + "IntegrationTestDiscovery.py") Scale="" + ;; + "IntegrationTestAsyncAPIGeneralNotificationCallbacks.py") Scale="" + ;; + "IntegrationTestOfflineQueueingForSubscribeUnsubscribe.py") Scale="" + ;; + "IntegrationTestClientReusability.py") Scale="" + ;; + "IntegrationTestJobsClient.py") Scale="" + esac + + python3 ${TEST_DIR}${file} ${TestMode} ${Scale} + currentTestStatus=$? + echo "[SUB] Test: ${file} completed. Exiting with status: ${currentTestStatus}" + if [ ${currentTestStatus} -ne 0 ]; then + echo "!!!!!!!!!!!!!Test: ${file} in Python version ${iii}.x failed.!!!!!!!!!!!!!" + exit ${currentTestStatus} + fi + echo "" + fi + done + echo "All integration tests passed for Python version ${iii}.x." + done +fi From e3a15efa967b086f04fc31f82aec2700470dfe2e Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Mon, 4 Apr 2022 16:28:08 -0700 Subject: [PATCH 03/66] add a test CI for integration test --- .github/workflows/ci.yml | 18 ++++++++++++++++++ ...TestAsyncAPIGeneralNotificationCallbacks.py | 10 +++++----- .../IntegrationTestAutoReconnectResubscribe.py | 10 +++++----- .../IntegrationTestClientReusability.py | 10 +++++----- ...onTestConfigurablePublishMessageQueueing.py | 10 +++++----- .../IntegrationTestDiscovery.py | 10 +++++----- .../IntegrationTestJobsClient.py | 10 +++++----- .../IntegrationTestMQTTConnection.py | 10 +++++----- ...stOfflineQueueingForSubscribeUnsubscribe.py | 10 +++++----- .../IntegrationTestProgressiveBackoff.py | 10 +++++----- .../IntegrationTests/IntegrationTestShadow.py | 10 +++++----- test-integration/run/run.sh | 14 +++++++------- 12 files changed, 75 insertions(+), 57 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 157eb33..02d70d6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,3 +31,21 @@ jobs: pip install mock python3 -m pytest test + integration-tests: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + test-type: [MutualAuth] + #[MutualAuth, Websocket, ALPN] + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: '3.6' + - name: Integration tests + run: | + pip install pytest + pip install mock + pip install boto3 + .\test-integration\run\run.sh IoTPySDK ${{ matrix.test-type }} 1000 100 7 \ No newline at end of file diff --git a/test-integration/IntegrationTests/IntegrationTestAsyncAPIGeneralNotificationCallbacks.py b/test-integration/IntegrationTests/IntegrationTestAsyncAPIGeneralNotificationCallbacks.py index 409700e..b69fdcd 100644 --- a/test-integration/IntegrationTests/IntegrationTestAsyncAPIGeneralNotificationCallbacks.py +++ b/test-integration/IntegrationTests/IntegrationTestAsyncAPIGeneralNotificationCallbacks.py @@ -18,8 +18,8 @@ import string import time import sys -sys.path.insert(0, "./configuration/IntegrationTests/TestToolLibrary") -sys.path.insert(0, "./configuration/IntegrationTests/TestToolLibrary/SDKPackage") +sys.path.insert(0, "./test-integration/IntegrationTests/TestToolLibrary") +sys.path.insert(0, "./test-integration/IntegrationTests/TestToolLibrary/SDKPackage") from TestToolLibrary.checkInManager import checkInManager from TestToolLibrary.MQTTClientManager import MQTTClientManager @@ -33,9 +33,9 @@ MESSAGE_PREFIX = "MagicMessage-" NUMBER_OF_PUBLISHES = 3 HOST = "ajje7lpljulm4-ats.iot.us-east-1.amazonaws.com" -ROOT_CA = "./configuration/Credentials/rootCA.crt" -CERT = "./configuration/Credentials/certificate.pem.crt" -KEY = "./configuration/Credentials/privateKey.pem.key" +ROOT_CA = "./test-integration/Credentials/rootCA.crt" +CERT = "./test-integration/Credentials/certificate.pem.crt" +KEY = "./test-integration/Credentials/privateKey.pem.key" CLIENT_ID = "PySdkIntegTest_AsyncAPI_Callbacks" KEY_ON_ONLINE = "OnOnline" diff --git a/test-integration/IntegrationTests/IntegrationTestAutoReconnectResubscribe.py b/test-integration/IntegrationTests/IntegrationTestAutoReconnectResubscribe.py index 8292075..1ee251a 100644 --- a/test-integration/IntegrationTests/IntegrationTestAutoReconnectResubscribe.py +++ b/test-integration/IntegrationTests/IntegrationTestAutoReconnectResubscribe.py @@ -16,8 +16,8 @@ import string import sys import time -sys.path.insert(0, "./configuration/IntegrationTests/TestToolLibrary") -sys.path.insert(0, "./configuration/IntegrationTests/TestToolLibrary/SDKPackage") +sys.path.insert(0, "./test-integration/IntegrationTests/TestToolLibrary") +sys.path.insert(0, "./test-integration/IntegrationTests/TestToolLibrary/SDKPackage") import TestToolLibrary.checkInManager as checkInManager import TestToolLibrary.MQTTClientManager as MQTTClientManager @@ -137,9 +137,9 @@ def threadBRuntime(self, pyCoreClient, callback): myCheckInManager.verify(sys.argv) host = "ajje7lpljulm4-ats.iot.us-east-1.amazonaws.com" -rootCA = "./configuration/Credentials/rootCA.crt" -certificate = "./configuration/Credentials/certificate.pem.crt" -privateKey = "./configuration/Credentials/privateKey.pem.key" +rootCA = "./test-integration/Credentials/rootCA.crt" +certificate = "./test-integration/Credentials/certificate.pem.crt" +privateKey = "./test-integration/Credentials/privateKey.pem.key" mode = myCheckInManager.mode skip_when_match(ModeIsALPN(mode).And( diff --git a/test-integration/IntegrationTests/IntegrationTestClientReusability.py b/test-integration/IntegrationTests/IntegrationTestClientReusability.py index 2cd1d26..f747e19 100644 --- a/test-integration/IntegrationTests/IntegrationTestClientReusability.py +++ b/test-integration/IntegrationTests/IntegrationTestClientReusability.py @@ -20,8 +20,8 @@ import uuid import time import sys -sys.path.insert(0, "./configuration/IntegrationTests/TestToolLibrary") -sys.path.insert(0, "./configuration/IntegrationTests/TestToolLibrary/SDKPackage") +sys.path.insert(0, "./test-integration/IntegrationTests/TestToolLibrary") +sys.path.insert(0, "./test-integration/IntegrationTests/TestToolLibrary/SDKPackage") from threading import Event from TestToolLibrary.checkInManager import checkInManager @@ -41,9 +41,9 @@ NUMBER_OF_LOOPS = 3 SUB_WAIT_TIME_OUT_SEC = 20 HOST = "ajje7lpljulm4-ats.iot.us-east-1.amazonaws.com" -ROOT_CA = "./configuration/Credentials/rootCA.crt" -CERT = "./configuration/Credentials/certificate.pem.crt" -KEY = "./configuration/Credentials/privateKey.pem.key" +ROOT_CA = "./test-integration/Credentials/rootCA.crt" +CERT = "./test-integration/Credentials/certificate.pem.crt" +KEY = "./test-integration/Credentials/privateKey.pem.key" class ClientTwins(object): diff --git a/test-integration/IntegrationTests/IntegrationTestConfigurablePublishMessageQueueing.py b/test-integration/IntegrationTests/IntegrationTestConfigurablePublishMessageQueueing.py index f213bcb..32f4514 100644 --- a/test-integration/IntegrationTests/IntegrationTestConfigurablePublishMessageQueueing.py +++ b/test-integration/IntegrationTests/IntegrationTestConfigurablePublishMessageQueueing.py @@ -23,8 +23,8 @@ import time import random import string -sys.path.insert(0, "./configuration/IntegrationTests/TestToolLibrary") -sys.path.insert(0, "./configuration/IntegrationTests/TestToolLibrary/SDKPackage") +sys.path.insert(0, "./test-integration/IntegrationTests/TestToolLibrary") +sys.path.insert(0, "./test-integration/IntegrationTests/TestToolLibrary/SDKPackage") import TestToolLibrary.checkInManager as checkInManager import TestToolLibrary.MQTTClientManager as MQTTClientManager @@ -276,9 +276,9 @@ def performConfigurableOfflinePublishQueueTest(clientPub, clientSub): myCheckInManager.verify(sys.argv) host = "ajje7lpljulm4-ats.iot.us-east-1.amazonaws.com" -rootCA = "./configuration/Credentials/rootCA.crt" -certificate = "./configuration/Credentials/certificate.pem.crt" -privateKey = "./configuration/Credentials/privateKey.pem.key" +rootCA = "./test-integration/Credentials/rootCA.crt" +certificate = "./test-integration/Credentials/certificate.pem.crt" +privateKey = "./test-integration/Credentials/privateKey.pem.key" mode = myCheckInManager.mode skip_when_match(ModeIsALPN(mode).And( diff --git a/test-integration/IntegrationTests/IntegrationTestDiscovery.py b/test-integration/IntegrationTests/IntegrationTestDiscovery.py index 9127eed..8f23aa9 100644 --- a/test-integration/IntegrationTests/IntegrationTestDiscovery.py +++ b/test-integration/IntegrationTests/IntegrationTestDiscovery.py @@ -1,6 +1,6 @@ import sys -sys.path.insert(0, "./configuration/IntegrationTests/TestToolLibrary") -sys.path.insert(0, "./configuration/IntegrationTests/TestToolLibrary/SDKPackage") +sys.path.insert(0, "./test-integration/IntegrationTests/TestToolLibrary") +sys.path.insert(0, "./test-integration/IntegrationTests/TestToolLibrary/SDKPackage") from TestToolLibrary.SDKPackage.AWSIoTPythonSDK.core.greengrass.discovery.providers import DiscoveryInfoProvider from TestToolLibrary.checkInManager import checkInManager @@ -10,9 +10,9 @@ HOST = "arc9d2oott9lj-ats.iot.us-east-1.amazonaws.com" # 003261610643 PORT = 8443 -CA = "./configuration/Credentials/rootCA.crt" -CERT = "./configuration/Credentials/certificate_drs.pem.crt" -KEY = "./configuration/Credentials/privateKey_drs.pem.key" +CA = "./test-integration/Credentials/rootCA.crt" +CERT = "./test-integration/Credentials/certificate_drs.pem.crt" +KEY = "./test-integration/Credentials/privateKey_drs.pem.key" TIME_OUT_SEC = 30 # This is a pre-generated test data from DRS integration tests ID_PREFIX = "Id-" diff --git a/test-integration/IntegrationTests/IntegrationTestJobsClient.py b/test-integration/IntegrationTests/IntegrationTestJobsClient.py index e76ba59..9eb5690 100644 --- a/test-integration/IntegrationTests/IntegrationTestJobsClient.py +++ b/test-integration/IntegrationTests/IntegrationTestJobsClient.py @@ -12,8 +12,8 @@ import string import time import sys -sys.path.insert(0, "./configuration/IntegrationTests/TestToolLibrary") -sys.path.insert(0, "./configuration/IntegrationTests/TestToolLibrary/SDKPackage") +sys.path.insert(0, "./test-integration/IntegrationTests/TestToolLibrary") +sys.path.insert(0, "./test-integration/IntegrationTests/TestToolLibrary/SDKPackage") from TestToolLibrary import simpleThreadManager import TestToolLibrary.checkInManager as checkInManager @@ -157,9 +157,9 @@ def _test_send_response_confirm(self, sendResult): myCheckInManager.verify(sys.argv) host = "ajje7lpljulm4-ats.iot.us-east-1.amazonaws.com" -rootCA = "./configuration/Credentials/rootCA.crt" -certificate = "./configuration/Credentials/certificate.pem.crt" -privateKey = "./configuration/Credentials/privateKey.pem.key" +rootCA = "./test-integration/Credentials/rootCA.crt" +certificate = "./test-integration/Credentials/certificate.pem.crt" +privateKey = "./test-integration/Credentials/privateKey.pem.key" mode = myCheckInManager.mode skip_when_match(ModeIsALPN(mode).And( diff --git a/test-integration/IntegrationTests/IntegrationTestMQTTConnection.py b/test-integration/IntegrationTests/IntegrationTestMQTTConnection.py index 77c95ce..9deb10e 100644 --- a/test-integration/IntegrationTests/IntegrationTestMQTTConnection.py +++ b/test-integration/IntegrationTests/IntegrationTestMQTTConnection.py @@ -12,8 +12,8 @@ import string import time import sys -sys.path.insert(0, "./configuration/IntegrationTests/TestToolLibrary") -sys.path.insert(0, "./configuration/IntegrationTests/TestToolLibrary/SDKPackage") +sys.path.insert(0, "./test-integration/IntegrationTests/TestToolLibrary") +sys.path.insert(0, "./test-integration/IntegrationTests/TestToolLibrary/SDKPackage") from TestToolLibrary import simpleThreadManager import TestToolLibrary.checkInManager as checkInManager @@ -84,9 +84,9 @@ def _performPublish(self, pyCoreClient, topic, qos, payload): myCheckInManager.verify(sys.argv) host = "ajje7lpljulm4-ats.iot.us-east-1.amazonaws.com" -rootCA = "./configuration/Credentials/rootCA.crt" -certificate = "./configuration/Credentials/certificate.pem.crt" -privateKey = "./configuration/Credentials/privateKey.pem.key" +rootCA = "./test-integration/Credentials/rootCA.crt" +certificate = "./test-integration/Credentials/certificate.pem.crt" +privateKey = "./test-integration/Credentials/privateKey.pem.key" mode = myCheckInManager.mode skip_when_match(ModeIsALPN(mode).And( diff --git a/test-integration/IntegrationTests/IntegrationTestOfflineQueueingForSubscribeUnsubscribe.py b/test-integration/IntegrationTests/IntegrationTestOfflineQueueingForSubscribeUnsubscribe.py index 307e116..67cdfb2 100644 --- a/test-integration/IntegrationTests/IntegrationTestOfflineQueueingForSubscribeUnsubscribe.py +++ b/test-integration/IntegrationTests/IntegrationTestOfflineQueueingForSubscribeUnsubscribe.py @@ -28,8 +28,8 @@ from threading import Event from threading import Thread import sys -sys.path.insert(0, "./configuration/IntegrationTests/TestToolLibrary") -sys.path.insert(0, "./configuration/IntegrationTests/TestToolLibrary/SDKPackage") +sys.path.insert(0, "./test-integration/IntegrationTests/TestToolLibrary") +sys.path.insert(0, "./test-integration/IntegrationTests/TestToolLibrary/SDKPackage") from TestToolLibrary.SDKPackage.AWSIoTPythonSDK.core.protocol.internal.clients import ClientStatus from TestToolLibrary.checkInManager import checkInManager @@ -45,9 +45,9 @@ MESSAGE_PREFIX = "MagicMessage-" NUMBER_OF_PUBLISHES = 3 HOST = "ajje7lpljulm4-ats.iot.us-east-1.amazonaws.com" -ROOT_CA = "./configuration/Credentials/rootCA.crt" -CERT = "./configuration/Credentials/certificate.pem.crt" -KEY = "./configuration/Credentials/privateKey.pem.key" +ROOT_CA = "./test-integration/Credentials/rootCA.crt" +CERT = "./test-integration/Credentials/certificate.pem.crt" +KEY = "./test-integration/Credentials/privateKey.pem.key" CLIENT_PUB_ID = "PySdkIntegTest_OfflineSubUnsub_pub" CLIENT_SUB_UNSUB_ID = "PySdkIntegTest_OfflineSubUnsub_subunsub" KEEP_ALIVE_SEC = 1 diff --git a/test-integration/IntegrationTests/IntegrationTestProgressiveBackoff.py b/test-integration/IntegrationTests/IntegrationTestProgressiveBackoff.py index f21b9ba..7fb7139 100644 --- a/test-integration/IntegrationTests/IntegrationTestProgressiveBackoff.py +++ b/test-integration/IntegrationTests/IntegrationTestProgressiveBackoff.py @@ -28,8 +28,8 @@ import time import threading import sys -sys.path.insert(0, "./configuration/IntegrationTests/TestToolLibrary") -sys.path.insert(0, "./configuration/IntegrationTests/TestToolLibrary/SDKPackage") +sys.path.insert(0, "./test-integration/IntegrationTests/TestToolLibrary") +sys.path.insert(0, "./test-integration/IntegrationTests/TestToolLibrary/SDKPackage") from TestToolLibrary import simpleThreadManager import TestToolLibrary.checkInManager as checkInManager @@ -221,9 +221,9 @@ def verifyBackoffTime(answerList, resultList): #host via describe-endpoint on this OdinMS: com.amazonaws.iot.device.sdk.credentials.testing.websocket host = "ajje7lpljulm4-ats.iot.us-east-1.amazonaws.com" -rootCA = "./configuration/Credentials/rootCA.crt" -certificate = "./configuration/Credentials/certificate.pem.crt" -privateKey = "./configuration/Credentials/privateKey.pem.key" +rootCA = "./test-integration/Credentials/rootCA.crt" +certificate = "./test-integration/Credentials/certificate.pem.crt" +privateKey = "./test-integration/Credentials/privateKey.pem.key" mode = myCheckInManager.mode skip_when_match(ModeIsALPN(mode).And( diff --git a/test-integration/IntegrationTests/IntegrationTestShadow.py b/test-integration/IntegrationTests/IntegrationTestShadow.py index 6197df3..0b1885d 100644 --- a/test-integration/IntegrationTests/IntegrationTestShadow.py +++ b/test-integration/IntegrationTests/IntegrationTestShadow.py @@ -20,8 +20,8 @@ import string import json import sys -sys.path.insert(0, "./configuration/IntegrationTests/TestToolLibrary") -sys.path.insert(0, "./configuration/IntegrationTests/TestToolLibrary/SDKPackage") +sys.path.insert(0, "./test-integration/IntegrationTests/TestToolLibrary") +sys.path.insert(0, "./test-integration/IntegrationTests/TestToolLibrary/SDKPackage") from TestToolLibrary import simpleThreadManager import TestToolLibrary.checkInManager as checkInManager @@ -152,9 +152,9 @@ def randomString(lengthOfString): myCheckInManager.verify(sys.argv) host = "ajje7lpljulm4-ats.iot.us-east-1.amazonaws.com" -rootCA = "./configuration/Credentials/rootCA.crt" -certificate = "./configuration/Credentials/certificate.pem.crt" -privateKey = "./configuration/Credentials/privateKey.pem.key" +rootCA = "./test-integration/Credentials/rootCA.crt" +certificate = "./test-integration/Credentials/certificate.pem.crt" +privateKey = "./test-integration/Credentials/privateKey.pem.key" mode = myCheckInManager.mode skip_when_match(ModeIsALPN(mode).And( diff --git a/test-integration/run/run.sh b/test-integration/run/run.sh index cbc3ce4..9c66f8e 100644 --- a/test-integration/run/run.sh +++ b/test-integration/run/run.sh @@ -41,10 +41,10 @@ AWSMutualAuth_TodWorker_certificate="arn:aws:secretsmanager:us-east-1:1231241367 OdinSetForGGDiscovery_TodWorker="com.amazonaws.iot.device.sdk.credentials.testing.gg.discovery.us-east-1" OdinSetForWebsocket_TodWorker="com.amazonaws.iot.device.sdk.credentials.testing.websocket" -RetrieveOdinMaterial="./configuration/Tools/retrieve-material.sh" -RetrieveAWSKeys="./configuration/Tools/retrieve-key.py" -CREDENTIAL_DIR="./configuration/Credentials/" -TEST_DIR="./configuration/IntegrationTests/" +RetrieveOdinMaterial="./test-integration/Tools/retrieve-material.sh" +RetrieveAWSKeys="./test-integration/Tools/retrieve-key.py" +CREDENTIAL_DIR="./test-integration/Credentials/" +TEST_DIR="./test-integration/IntegrationTests/" CA_CERT_URL="https://www.amazontrust.com/repository/AmazonRootCA1.pem" CA_CERT_PATH=${CREDENTIAL_DIR}rootCA.crt ACCESS_KEY_ID_ARN="arn:aws:secretsmanager:us-east-1:123124136734:secret:V1IotSdkIntegrationTestWebsocketAccessKeyId-1YdB9z" @@ -136,9 +136,9 @@ else exit 2 fi if [ "$1"x == "IoTYunSDK"x ]; then - unzip -q ${ZIPLocation} -d ./configuration/IntegrationTests/TestToolLibrary/SDKPackage/ + unzip -q ${ZIPLocation} -d ./test-integration/IntegrationTests/TestToolLibrary/SDKPackage/ elif [ "$1"x == "IoTPySDK"x ]; then - cp -R ${ZIPLocation} ./configuration/IntegrationTests/TestToolLibrary/SDKPackage/ + cp -R ${ZIPLocation} ./test-integration/IntegrationTests/TestToolLibrary/SDKPackage/ else echo "Error in getSDKZIP" exit 2 @@ -149,7 +149,7 @@ else echo "[STEP] Obtain Python executable" echo "***************************************************" echo "Python version ${iii}.x..." - PYTHON=$(./configuration/Tools/getPython.sh ${iii}) + PYTHON=$(./test-integration/Tools/getPython.sh ${iii}) if [ $? -eq "-1" ]; then echo "Cannot find Python executable" exit 3 From 972dd5ff57ae598510ed602d32053e2546534154 Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Mon, 4 Apr 2022 16:29:54 -0700 Subject: [PATCH 04/66] update CI command --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 02d70d6..aae4ed9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,4 +48,4 @@ jobs: pip install pytest pip install mock pip install boto3 - .\test-integration\run\run.sh IoTPySDK ${{ matrix.test-type }} 1000 100 7 \ No newline at end of file + ./test-integration/run/run.sh IoTPySDK ${{ matrix.test-type }} 1000 100 7 \ No newline at end of file From a43eda363a9a10c684c6bb910c71e33ac51f9cda Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Mon, 4 Apr 2022 16:33:39 -0700 Subject: [PATCH 05/66] update run.sh access permission --- test-integration/run/run.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 test-integration/run/run.sh diff --git a/test-integration/run/run.sh b/test-integration/run/run.sh old mode 100644 new mode 100755 From ba571fc1335625516ea4115adfe3ca65b2fbd88e Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Mon, 4 Apr 2022 16:43:24 -0700 Subject: [PATCH 06/66] fix pySDK path --- test-integration/run/run.sh | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/test-integration/run/run.sh b/test-integration/run/run.sh index 9c66f8e..00a46d7 100755 --- a/test-integration/run/run.sh +++ b/test-integration/run/run.sh @@ -130,7 +130,7 @@ else echo ${TestMode} echo "[STEP] Obtain ZIP package" echo "***************************************************" - ZIPLocation="C:/Iot/workspace/aws-iot-device-sdk-python/AWSIoTPythonSDK" + ZIPLocation="../AWSIoTPythonSDK" if [ $? -eq "-1" ]; then echo "Cannot find SDK ZIP package" exit 2 @@ -146,17 +146,6 @@ else # Obtain Python executable for iii in "${pythonExecutableArray[@]}" do - echo "[STEP] Obtain Python executable" - echo "***************************************************" - echo "Python version ${iii}.x..." - PYTHON=$(./test-integration/Tools/getPython.sh ${iii}) - if [ $? -eq "-1" ]; then - echo "Cannot find Python executable" - exit 3 - fi - # Start integration test - # Loop through the tes directory and execute all available integration tests - echo "[STEP] Start integration tests" echo "***************************************************" for file in `ls ${TEST_DIR}` do From 15a9e8443cc5a3a28dda36c0b4a6c0cfa440fae5 Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Mon, 4 Apr 2022 16:43:51 -0700 Subject: [PATCH 07/66] fix pySDK path --- test-integration/run/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-integration/run/run.sh b/test-integration/run/run.sh index 00a46d7..bafdc8f 100755 --- a/test-integration/run/run.sh +++ b/test-integration/run/run.sh @@ -130,7 +130,7 @@ else echo ${TestMode} echo "[STEP] Obtain ZIP package" echo "***************************************************" - ZIPLocation="../AWSIoTPythonSDK" + ZIPLocation="./AWSIoTPythonSDK" if [ $? -eq "-1" ]; then echo "Cannot find SDK ZIP package" exit 2 From 2a1c3b943fedc96551f993a590081bd396b5e9a5 Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Mon, 4 Apr 2022 16:48:24 -0700 Subject: [PATCH 08/66] update test case using cipher_provider --- .../TestToolLibrary/MQTTClientManager.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/test-integration/IntegrationTests/TestToolLibrary/MQTTClientManager.py b/test-integration/IntegrationTests/TestToolLibrary/MQTTClientManager.py index 9cdf3d5..05c671a 100644 --- a/test-integration/IntegrationTests/TestToolLibrary/MQTTClientManager.py +++ b/test-integration/IntegrationTests/TestToolLibrary/MQTTClientManager.py @@ -6,7 +6,7 @@ import TestToolLibrary.SDKPackage.AWSIoTPythonSDK.core.protocol.paho.client as paho from TestToolLibrary.SDKPackage.AWSIoTPythonSDK.MQTTLib import AWSIoTMQTTClient from TestToolLibrary.SDKPackage.AWSIoTPythonSDK.core.util.providers import CertificateCredentialsProvider -# from TestToolLibrary.SDKPackage.AWSIoTPythonSDK.core.util.providers import CiphersProvider +from TestToolLibrary.SDKPackage.AWSIoTPythonSDK.core.util.providers import CiphersProvider from TestToolLibrary.SDKPackage.AWSIoTPythonSDK.core.util.enums import DropBehaviorTypes from TestToolLibrary.SDKPackage.AWSIoTPythonSDK.core.util.providers import EndpointProvider from TestToolLibrary.SDKPackage.AWSIoTPythonSDK.core.protocol.mqtt_core import MqttCore @@ -100,12 +100,11 @@ def create_nonconnected_mqtt_core(self, client_id, host, root_ca, certificate, p certificate_credentials_provider.set_cert_path(certificate) certificate_credentials_provider.set_key_path(private_key) - # cipher_provider = CiphersProvider() - # cipher_provider.set_ciphers(None) + cipher_provider = CiphersProvider() + cipher_provider.set_ciphers(None) client.configure_endpoint(endpoint_provider) - # client.configure_cert_credentials(certificate_credentials_provider, cipher_provider) - client.configure_cert_credentials(certificate_credentials_provider) + client.configure_cert_credentials(certificate_credentials_provider, cipher_provider) client.configure_connect_disconnect_timeout_sec(10) client.configure_operation_timeout_sec(5) client.configure_offline_requests_queue(10, DropBehaviorTypes.DROP_NEWEST) From def51baa65184b6b035dedc692334a0a7a1074b0 Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Tue, 5 Apr 2022 14:21:29 -0700 Subject: [PATCH 09/66] update run scripts and ci to use AWS keys --- .github/workflows/ci.yml | 5 +- .../Credentials/certificate.pem.crt | 20 -- .../Credentials/privateKey.pem.key | 28 --- test-integration/Credentials/rootCA.crt | 20 -- test-integration/run/run.sh | 183 ++++++++---------- 5 files changed, 88 insertions(+), 168 deletions(-) delete mode 100644 test-integration/Credentials/certificate.pem.crt delete mode 100644 test-integration/Credentials/privateKey.pem.key delete mode 100644 test-integration/Credentials/rootCA.crt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aae4ed9..064a5f7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,13 +36,14 @@ jobs: strategy: fail-fast: false matrix: - test-type: [MutualAuth] + test-type: [ MutualAuth, MutualAuthT , Websocket, ALPN, ALPNT] + python-version: [ '2.x', '3.x' ] #[MutualAuth, Websocket, ALPN] steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: '3.6' + python-version: ${{ matrix.python-version }} - name: Integration tests run: | pip install pytest diff --git a/test-integration/Credentials/certificate.pem.crt b/test-integration/Credentials/certificate.pem.crt deleted file mode 100644 index aeb69b2..0000000 --- a/test-integration/Credentials/certificate.pem.crt +++ /dev/null @@ -1,20 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDWTCCAkGgAwIBAgIUCulzDAbSLt4/M7FiCvxE59ZxeWowDQYJKoZIhvcNAQEL -BQAwTTFLMEkGA1UECwxCQW1hem9uIFdlYiBTZXJ2aWNlcyBPPUFtYXpvbi5jb20g -SW5jLiBMPVNlYXR0bGUgU1Q9V2FzaGluZ3RvbiBDPVVTMB4XDTE2MDkxNzIxNDI1 -MVoXDTQ5MTIzMTIzNTk1OVowHjEcMBoGA1UEAwwTQVdTIElvVCBDZXJ0aWZpY2F0 -ZTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAIqu7JpefK48aLZbaswK -nNY/X20yzD5fa/duH8YZc6G0XskR3jKxtMQa/xz4qShFRyi/XEDWFeVBGe6zsOjo -7Qbf/Q6HzGOxfFbJNd+xo9t/6dxSX7nOFAa3o6EDH4ID6vQGg5S3kmDQWTNfq32Y -NuLfY4d9e8zHXVU/RCTUgbE+FPg7Fli5BoPWUufowsp5ZKPQNrIBwMrDuqY6PRFa -tOolybJi5MiQyaNWHAwoiozoZaJxuMTk+f/a658UYSJmkWLVhAAB1BwRqpJOErUx -dOylGCgcYAcgDJYL7CnJlWlsX/ydGHqu9qah192lpxeiv96oTnDiHbsRfsvjnIAI -XNcCAwEAAaNgMF4wHwYDVR0jBBgwFoAUQh4sEAJgaN2krLfjtGNaPSHeTcQwHQYD -VR0OBBYEFBUVF2WWXjLEdXCq9BSJghTbJzGpMAwGA1UdEwEB/wQCMAAwDgYDVR0P -AQH/BAQDAgeAMA0GCSqGSIb3DQEBCwUAA4IBAQB43Nf+StCdSbQMLFywxnyfcp3u -dZHCrtN45LAUmcJ64LAcr6vAWfdQzAInuv1eSpFl/IXW/k3gjijm6XEC2yCL3efB -6ckJ0vczkX6UrqSOtbltZM8fRjBM2rrRGuaL4M56J4XjEchVJdZ1UXKFtxkeWR2H -NTNQzqH4pEHCwraJA1odAFj1qmh7L2bAQFmC8rBi7y4bRaYPs6v3esxP9a93AZnn -g07camolyUlAMt9sFFqkWxGFST+MvZGN4LG8iytHhBwvp8CdwFXBIRPnJhlIz6uh -/a4p1NgUyFAvE07EhyESRQ9Uf+Og286sOfZPDhmw22VJzAkf3vz7Q8jY1EW0 ------END CERTIFICATE----- diff --git a/test-integration/Credentials/privateKey.pem.key b/test-integration/Credentials/privateKey.pem.key deleted file mode 100644 index 3bbb2c4..0000000 --- a/test-integration/Credentials/privateKey.pem.key +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCKruyaXnyuPGi2 -W2rMCpzWP19tMsw+X2v3bh/GGXOhtF7JEd4ysbTEGv8c+KkoRUcov1xA1hXlQRnu -s7Do6O0G3/0Oh8xjsXxWyTXfsaPbf+ncUl+5zhQGt6OhAx+CA+r0BoOUt5Jg0Fkz -X6t9mDbi32OHfXvMx11VP0Qk1IGxPhT4OxZYuQaD1lLn6MLKeWSj0DayAcDKw7qm -Oj0RWrTqJcmyYuTIkMmjVhwMKIqM6GWicbjE5Pn/2uufFGEiZpFi1YQAAdQcEaqS -ThK1MXTspRgoHGAHIAyWC+wpyZVpbF/8nRh6rvamodfdpacXor/eqE5w4h27EX7L -45yACFzXAgMBAAECggEAFWtS67ywMRDvc0rHQeBZvNVo3aq81N1UBZEioywHKfB7 -uz5hPR8pNiidQlI9fhTFGswDaZgnJZNad/2U00Q2Z5tssDPVI4ikFZtaEMSU9C/b -ZMXsNvxXJ3pxcUrUMPZ98VaYN/3qQ72qKT8bc3hw8bVi3+hHmKcqCSKOWcnghglA -OUyCFZiEWnxPRQ8VUkLWawmr1c/1fBCJT2HD9PnLiYv9FwqCU2gUuhR8yzkl0/xF -pxtgr6tUdflCD2MD4jjhICTFKuD75q1ZbkqHwCYSXw6NqayR9k2zID1h/E47FRA0 -6R50nt5c8jsXERo31KGCECEHXt3S9sIsU8HrruSO4QKBgQDMliNUIa/INc5ggo6H -9B15oxZB1pVJSw4ZREuznGJCy6XjjEtayiHhP+nqQQ2Ks5SxA1JXqPQOaoeKTV6i -fitkMMipt5hA3ZHdg6M1ElBveT2zba89TB5kxEJtBxkwK1Az1L7qtJCuw/gJqL27 -mLUyngAvKFRRslgz91GxCERSfwKBgQCtiPj4Ijs4dmEhdDcDt/RogIwOQ0aMIta6 -OIcclXNzTJe1G6GldmfSojdDQHWRGCX83qDtZkwrIg3t0nd+4DQCoSIDhEx4ZKhd -od1Yr+8R94DEk7l5dULfDroDNx56tSZKKx8IXBJZEY1/MPqCxIv1tyUTGHhNtsAx -2BrCumCZqQKBgQCxOY45H0VrJlE1AWP/GdU+vaxWNFD2QPJhqOv7F4l3W3rE949/ -goJ+4iL8LoZQlOhFvx7hmRZyNo5bnFJSaQGltSze+JAIAOiO/62uF8NeDaUJfgbE -DuB1Yh443GFRfPPpMm6AWxLKkjCYDXjuvYaZ5o06TLFeZCRMP/6gYQzueQKBgDY7 -RRs+j2VQ0YAD8qFt3qq96YcXjTeEN7jQq6DKL99Lg2ms7fJos3+HBGA3B8bFVjHV -XVNzkcA1D0dQa9mxtW9Q7fFNahyB0IAacrMhoEPsJkjIpcEIXVKaJpWUpyjP7dxl -53fUVkORkbe7Fb7CL78lcieqkQYwY5XwJETQvBmpAoGAMcC/2oWca/+An8CxKxpR -i5qyKsqner21+PbxISSTJPgfDdzkQ9CwdJnSSZW8bT9eeE/D2kqo43rRLpTo/DOm -18EEr6M3ED1sgzdOEYiCsKEVEr/MZNOv6155KkSN5mJknKWE+TOvu3Zr8k8Ejjim -Iouc5aXZwdb4ERaY7jNsTm8= ------END PRIVATE KEY----- diff --git a/test-integration/Credentials/rootCA.crt b/test-integration/Credentials/rootCA.crt deleted file mode 100644 index a6f3e92..0000000 --- a/test-integration/Credentials/rootCA.crt +++ /dev/null @@ -1,20 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDQTCCAimgAwIBAgITBmyfz5m/jAo54vB4ikPmljZbyjANBgkqhkiG9w0BAQsF -ADA5MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6 -b24gUm9vdCBDQSAxMB4XDTE1MDUyNjAwMDAwMFoXDTM4MDExNzAwMDAwMFowOTEL -MAkGA1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEZMBcGA1UEAxMQQW1hem9uIFJv -b3QgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALJ4gHHKeNXj -ca9HgFB0fW7Y14h29Jlo91ghYPl0hAEvrAIthtOgQ3pOsqTQNroBvo3bSMgHFzZM -9O6II8c+6zf1tRn4SWiw3te5djgdYZ6k/oI2peVKVuRF4fn9tBb6dNqcmzU5L/qw -IFAGbHrQgLKm+a/sRxmPUDgH3KKHOVj4utWp+UhnMJbulHheb4mjUcAwhmahRWa6 -VOujw5H5SNz/0egwLX0tdHA114gk957EWW67c4cX8jJGKLhD+rcdqsq08p8kDi1L -93FcXmn/6pUCyziKrlA4b9v7LWIbxcceVOF34GfID5yHI9Y/QCB/IIDEgEw+OyQm -jgSubJrIqg0CAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMC -AYYwHQYDVR0OBBYEFIQYzIU07LwMlJQuCFmcx7IQTgoIMA0GCSqGSIb3DQEBCwUA -A4IBAQCY8jdaQZChGsV2USggNiMOruYou6r4lK5IpDB/G/wkjUu0yKGX9rbxenDI -U5PMCCjjmCXPI6T53iHTfIUJrU6adTrCC2qJeHZERxhlbI1Bjjt/msv0tadQ1wUs -N+gDS63pYaACbvXy8MWy7Vu33PqUXHeeE6V/Uq2V8viTO96LXFvKWlJbYK8U90vv -o/ufQJVtMVT8QtPHRh8jrdkPSHCa2XV4cdFyQzR1bldZwgJcJmApzyMZFo6IQ6XU -5MsI+yMRQ+hDKXJioaldXgjUkK642M4UwtBV8ob2xJNDd2ZhwLnoQdeXeGADbkpy -rqXRfboQnoZsG4q5WTP468SQvvG5 ------END CERTIFICATE----- diff --git a/test-integration/run/run.sh b/test-integration/run/run.sh index bafdc8f..9af2bc0 100755 --- a/test-integration/run/run.sh +++ b/test-integration/run/run.sh @@ -32,96 +32,92 @@ # Scale number must also be specified (see usage) # Define const -USAGE="usage: run.sh " -OdinSetForMutualAuth_Desktop="com.amazon.aws.iot.sdk.desktop.certificate" -OdinSetForWebsocket_Desktop="com.amazon.aws.iot.sdk.desktop.websockets" -OdinSetForMutualAuth_TodWorker="com.amazonaws.iot.device.sdk.credentials.testing.certificates.us-east-1" +USAGE="usage: run.sh " + AWSMutualAuth_TodWorker_private_key="arn:aws:secretsmanager:us-east-1:123124136734:secret:V1IotSdkIntegrationTestPrivateKey-vNUQU8" AWSMutualAuth_TodWorker_certificate="arn:aws:secretsmanager:us-east-1:123124136734:secret:V1IotSdkIntegrationTestCertificate-vTRwjE" +AWSMutualAuth_Desktop_private_key="arn:aws:secretsmanager:us-east-1:123124136734:secret:V1IotSdkIntegrationTestDesktopPrivateKey-DdC7nv" +AWSMutualAuth_Desktop_certificate="arn:aws:secretsmanager:us-east-1:123124136734:secret:V1IotSdkIntegrationTestDesktopCertificate-IA4xbj" + +AWSGGDiscovery_TodWorker_private_key="arn:aws:secretsmanager:us-east-1:123124136734:secret:V1IotSdkIntegrationTestGGDiscoveryPrivateKey-YHQI1F" +AWSGGDiscovery_TodWorker_certificate="arn:aws:secretsmanager:us-east-1:123124136734:secret:V1IotSdkIntegrationTestGGDiscoveryCertificate-TwlAcS" + +AWSSecretForWebsocket_TodWorker_KeyId="arn:aws:secretsmanager:us-east-1:123124136734:secret:V1IotSdkIntegrationTestWebsocketAccessKeyId-1YdB9z" +AWSSecretForWebsocket_TodWorker_SecretKey="arn:aws:secretsmanager:us-east-1:123124136734:secret:V1IotSdkIntegrationTestWebsocketSecretAccessKey-MKTSaV" +AWSSecretForWebsocket_Desktop_KeyId="arn:aws:secretsmanager:us-east-1:123124136734:secret:V1IotSdkIntegrationTestWebsocketAccessKeyId-1YdB9z" +AWSSecretForWebsocket_Desktop_SecretKey="arn:aws:secretsmanager:us-east-1:123124136734:secret:V1IotSdkIntegrationTestWebsocketSecretAccessKey-MKTSaV" -OdinSetForGGDiscovery_TodWorker="com.amazonaws.iot.device.sdk.credentials.testing.gg.discovery.us-east-1" -OdinSetForWebsocket_TodWorker="com.amazonaws.iot.device.sdk.credentials.testing.websocket" -RetrieveOdinMaterial="./test-integration/Tools/retrieve-material.sh" RetrieveAWSKeys="./test-integration/Tools/retrieve-key.py" CREDENTIAL_DIR="./test-integration/Credentials/" TEST_DIR="./test-integration/IntegrationTests/" CA_CERT_URL="https://www.amazontrust.com/repository/AmazonRootCA1.pem" CA_CERT_PATH=${CREDENTIAL_DIR}rootCA.crt -ACCESS_KEY_ID_ARN="arn:aws:secretsmanager:us-east-1:123124136734:secret:V1IotSdkIntegrationTestWebsocketAccessKeyId-1YdB9z" -ACCESS_KEY_ARN="arn:aws:secretsmanager:us-east-1:123124136734:secret:V1IotSdkIntegrationTestWebsocketSecretAccessKey-MKTSaV" + # If input args not correct, echo usage -if [ $# -ne 5 ]; then +if [ $# -ne 4 ]; then echo ${USAGE} else # Description echo "[STEP] Start run.sh" echo "***************************************************" - echo "About to start integration tests for $1..." - echo "Test Mode: $2" + echo "About to start integration tests for IoTPySDK..." + echo "Test Mode: $1" # Determine the Python versions need to test for this SDK pythonExecutableArray=() - if [ "$1"x == "IoTYunSDK"x ]; then - pythonExecutableArray[0]="2" - elif [ "$1"x == "IoTPySDK"x ]; then - pythonExecutableArray[0]="3" - else - echo "Cannot determine Python executables for this unrecognized SDK!" - exit 1 - fi + pythonExecutableArray[0]="3" # Retrieve credentials as needed from Odin TestMode="" echo "[STEP] Retrieve credentials from Odin" echo "***************************************************" - if [ "$2"x == "MutualAuth"x -o "$2"x == "MutualAuthT"x ]; then - OdinSetName=${OdinSetForMutualAuth_TodWorker} + if [ "$1"x == "MutualAuth"x -o "$1"x == "MutualAuthT"x ]; then AWSSetName_privatekey=${AWSMutualAuth_TodWorker_private_key} AWSSetName_certificate=${AWSMutualAuth_TodWorker_certificate} - OdinSetDRSName=${OdinSetForGGDiscovery_TodWorker} + AWSDRSName_privatekey=${AWSGGDiscovery_TodWorker_private_key} + AWSDRSName_certificate=${AWSGGDiscovery_TodWorker_certificate} TestMode="MutualAuth" - if [ "$2"x == "MutualAuthT"x ]; then - OdinSetName=${OdinSetForMutualAuth_Desktop} + if [ "$1"x == "MutualAuthT"x ]; then + AWSSetName_privatekey=${AWSMutualAuth_Desktop_private_key} + AWSSetName_certificate=${AWSMutualAuth_Desktop_certificate} fi - OdinMaterialTypeCertificate="Certificate" - OdinMaterialTypeKey="PrivateKey" python ${RetrieveAWSKeys} ${AWSSetName_certificate} > ${CREDENTIAL_DIR}certificate.pem.crt python ${RetrieveAWSKeys} ${AWSSetName_privatekey} > ${CREDENTIAL_DIR}privateKey.pem.key curl -s "${CA_CERT_URL}" > ${CA_CERT_PATH} echo -e "URL retrieved certificate data:\n$(cat ${CA_CERT_PATH})\n" - #${RetrieveOdinMaterial} ${OdinSetDRSName} ${OdinMaterialTypeCertificate} > ${CREDENTIAL_DIR}certificate_drs.pem.crt - #${RetrieveOdinMaterial} ${OdinSetDRSName} ${OdinMaterialTypeKey} > ${CREDENTIAL_DIR}privateKey_drs.pem.key - elif [ "$2"x == "Websocket"x -o "$2"x == "WebsocketT"x ]; then - OdinSetName=${OdinSetForWebsocket_TodWorker} + python ${RetrieveAWSKeys} ${AWSDRSName_certificate} > ${CREDENTIAL_DIR}certificate_drs.pem.crt + python ${RetrieveAWSKeys} ${AWSDRSName_privatekey} > ${CREDENTIAL_DIR}privateKey_drs.pem.key + elif [ "$1"x == "Websocket"x -o "$1"x == "WebsocketT"x ]; then + ACCESS_KEY_ID_ARN=$(python ${RetrieveAWSKeys} ${AWSSecretForWebsocket_TodWorker_KeyId}) + ACCESS_SECRET_KEY_ARN=$(python ${RetrieveAWSKeys} ${AWSSecretForWebsocket_TodWorker_SecretKey}) TestMode="Websocket" - if [ "$2"x == "WebsocketT"x ]; then - OdinSetName=${OdinSetForWebsocket_Desktop} + if [ "$1"x == "WebsocketT"x ]; then + ACCESS_KEY_ID_ARN=$(python ${RetrieveAWSKeys} ${AWSSecretForWebsocket_Desktop_KeyId}) + ACCESS_SECRET_KEY_ARN=$(python ${RetrieveAWSKeys} ${AWSSecretForWebsocket_Desktop_SecretKey}) fi - OdinMaterialTypeID="Principal" - OdinMaterialTypeSecret="Credential" - python ${RetrieveAWSKeys} ${ACCESS_KEY_ID_ARN} > KEY_ID - python ${RetrieveAWSKeys} ${ACCESS_KEY_ARN} > SECRET_KEY - #SECRET_KEY=$(${RetrieveOdinMaterial} ${OdinSetName} ${OdinMaterialTypeSecret}) - export AWS_ACCESS_KEY_ID=${KEY_ID} - export AWS_SECRET_ACCESS_KEY=${SECRET_KEY} + echo ${ACCESS_KEY_ID_ARN} + echo ${ACCESS_SECRET_KEY_ARN} + export AWS_ACCESS_KEY_ID=${ACCESS_KEY_ID_ARN} + export AWS_SECRET_ACCESS_KEY=${ACCESS_SECRET_KEY_ARN} curl -s "${CA_CERT_URL}" > ${CA_CERT_PATH} echo -e "URL retrieved certificate data:\n$(cat ${CA_CERT_PATH})\n" - elif [ "$2"x == "ALPN"x -o "$2"x == "ALPNT"x ]; then - OdinSetName=${OdinSetForMutualAuth_TodWorker} - OdinSetDRSName=${OdinSetForGGDiscovery_TodWorker} + elif [ "$1"x == "ALPN"x -o "$1"x == "ALPNT"x ]; then + AWSSetName_privatekey=${AWSMutualAuth_TodWorker_private_key} + AWSSetName_certificate=${AWSMutualAuth_TodWorker_certificate} + AWSDRSName_privatekey=${AWSGGDiscovery_TodWorker_private_key} + AWSDRSName_certificate=${AWSGGDiscovery_TodWorker_certificate} TestMode="ALPN" - if [ "$2"x == "ALPNT"x ]; then - OdinSetName=${OdinSetForMutualAuth_Desktop} + if [ "$1"x == "ALPNT"x ]; then + AWSSetName_privatekey=${AWSMutualAuth_Desktop_private_key} + AWSSetName_certificate=${AWSMutualAuth_Desktop_certificate} fi - OdinMaterialTypeCertificate="Certificate" - OdinMaterialTypeKey="PrivateKey" - ${RetrieveOdinMaterial} ${OdinSetName} ${OdinMaterialTypeCertificate} > ${CREDENTIAL_DIR}certificate.pem.crt - ${RetrieveOdinMaterial} ${OdinSetName} ${OdinMaterialTypeKey} > ${CREDENTIAL_DIR}privateKey.pem.key + python ${RetrieveAWSKeys} ${AWSSetName_certificate} > ${CREDENTIAL_DIR}certificate.pem.crt + python ${RetrieveAWSKeys} ${AWSSetName_privatekey} > ${CREDENTIAL_DIR}privateKey.pem.key curl -s "${CA_CERT_URL}" > ${CA_CERT_PATH} echo -e "URL retrieved certificate data:\n$(cat ${CA_CERT_PATH})\n" - #${RetrieveOdinMaterial} ${OdinSetDRSName} ${OdinMaterialTypeCertificate} > ${CREDENTIAL_DIR}certificate_drs.pem.crt - #${RetrieveOdinMaterial} ${OdinSetDRSName} ${OdinMaterialTypeKey} > ${CREDENTIAL_DIR}privateKey_drs.pem.key + python ${RetrieveAWSKeys} ${AWSDRSName_certificate} > ${CREDENTIAL_DIR}certificate_drs.pem.crt + python ${RetrieveAWSKeys} ${AWSDRSName_privatekey} > ${CREDENTIAL_DIR}privateKey_drs.pem.key else echo "Mode not supported" exit 1 @@ -135,57 +131,48 @@ else echo "Cannot find SDK ZIP package" exit 2 fi - if [ "$1"x == "IoTYunSDK"x ]; then - unzip -q ${ZIPLocation} -d ./test-integration/IntegrationTests/TestToolLibrary/SDKPackage/ - elif [ "$1"x == "IoTPySDK"x ]; then - cp -R ${ZIPLocation} ./test-integration/IntegrationTests/TestToolLibrary/SDKPackage/ - else - echo "Error in getSDKZIP" - exit 2 - fi + cp -R ${ZIPLocation} ./test-integration/IntegrationTests/TestToolLibrary/SDKPackage/ # Obtain Python executable - for iii in "${pythonExecutableArray[@]}" + + echo "***************************************************" + for file in `ls ${TEST_DIR}` do - echo "***************************************************" - for file in `ls ${TEST_DIR}` - do - # if [ ${file}x == "IntegrationTestMQTTConnection.py"x ]; then - if [ ${file##*.}x == "py"x ]; then - echo "[SUB] Running test: ${file}..." + # if [ ${file}x == "IntegrationTestMQTTConnection.py"x ]; then + if [ ${file##*.}x == "py"x ]; then + echo "[SUB] Running test: ${file}..." - Scale=10 - case "$file" in - "IntegrationTestMQTTConnection.py") Scale=$3 - ;; - "IntegrationTestShadow.py") Scale=$4 - ;; - "IntegrationTestAutoReconnectResubscribe.py") Scale="" - ;; - "IntegrationTestProgressiveBackoff.py") Scale=$5 - ;; - "IntegrationTestConfigurablePublishMessageQueueing.py") Scale="" - ;; - "IntegrationTestDiscovery.py") Scale="" - ;; - "IntegrationTestAsyncAPIGeneralNotificationCallbacks.py") Scale="" - ;; - "IntegrationTestOfflineQueueingForSubscribeUnsubscribe.py") Scale="" - ;; - "IntegrationTestClientReusability.py") Scale="" - ;; - "IntegrationTestJobsClient.py") Scale="" - esac + Scale=10 + case "$file" in + "IntegrationTestMQTTConnection.py") Scale=$2 + ;; + "IntegrationTestShadow.py") Scale=$3 + ;; + "IntegrationTestAutoReconnectResubscribe.py") Scale="" + ;; + "IntegrationTestProgressiveBackoff.py") Scale=$4 + ;; + "IntegrationTestConfigurablePublishMessageQueueing.py") Scale="" + ;; + "IntegrationTestDiscovery.py") Scale="" + ;; + "IntegrationTestAsyncAPIGeneralNotificationCallbacks.py") Scale="" + ;; + "IntegrationTestOfflineQueueingForSubscribeUnsubscribe.py") Scale="" + ;; + "IntegrationTestClientReusability.py") Scale="" + ;; + "IntegrationTestJobsClient.py") Scale="" + esac - python3 ${TEST_DIR}${file} ${TestMode} ${Scale} - currentTestStatus=$? - echo "[SUB] Test: ${file} completed. Exiting with status: ${currentTestStatus}" - if [ ${currentTestStatus} -ne 0 ]; then - echo "!!!!!!!!!!!!!Test: ${file} in Python version ${iii}.x failed.!!!!!!!!!!!!!" - exit ${currentTestStatus} - fi - echo "" + python ${TEST_DIR}${file} ${TestMode} ${Scale} + currentTestStatus=$? + echo "[SUB] Test: ${file} completed. Exiting with status: ${currentTestStatus}" + if [ ${currentTestStatus} -ne 0 ]; then + echo "!!!!!!!!!!!!!Test: ${file} in Python version ${iii}.x failed.!!!!!!!!!!!!!" + exit ${currentTestStatus} fi - done - echo "All integration tests passed for Python version ${iii}.x." + echo "" + fi done + echo "All integration tests passed" fi From 8c44be6127a94df2520a40b37b4bb63998fe389b Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Tue, 5 Apr 2022 14:26:29 -0700 Subject: [PATCH 10/66] update integration test commands --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 064a5f7..d6704f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,4 +49,4 @@ jobs: pip install pytest pip install mock pip install boto3 - ./test-integration/run/run.sh IoTPySDK ${{ matrix.test-type }} 1000 100 7 \ No newline at end of file + ./test-integration/run/run.sh ${{ matrix.test-type }} 1000 100 7 \ No newline at end of file From d9affb83a5a1e81f0c727e7e2dc4d31a0c863c09 Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Tue, 5 Apr 2022 14:39:43 -0700 Subject: [PATCH 11/66] manully call command instead of use matrix --- .github/workflows/ci.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d6704f2..c95fb58 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,7 +37,7 @@ jobs: fail-fast: false matrix: test-type: [ MutualAuth, MutualAuthT , Websocket, ALPN, ALPNT] - python-version: [ '2.x', '3.x' ] + python-version: [ '3.x' ] #[MutualAuth, Websocket, ALPN] steps: - uses: actions/checkout@v2 @@ -49,4 +49,8 @@ jobs: pip install pytest pip install mock pip install boto3 - ./test-integration/run/run.sh ${{ matrix.test-type }} 1000 100 7 \ No newline at end of file + ./test-integration/run/run.sh MutualAuth 1000 100 7 + ./test-integration/run/run.sh MutualAuthT 1000 100 7 + ./test-integration/run/run.sh Websocket 1000 100 7 + ./test-integration/run/run.sh ALPN 1000 100 7 + ALPNT \ No newline at end of file From 37202c075aac0d4b114e2e3fdf545c32f33285bc Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Tue, 5 Apr 2022 14:40:08 -0700 Subject: [PATCH 12/66] manully call command instead of use matrix --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c95fb58..8e21c69 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,4 +53,4 @@ jobs: ./test-integration/run/run.sh MutualAuthT 1000 100 7 ./test-integration/run/run.sh Websocket 1000 100 7 ./test-integration/run/run.sh ALPN 1000 100 7 - ALPNT \ No newline at end of file + ./test-integration/run/run.sh ALPNT 1000 100 7 \ No newline at end of file From b35e010df70c0e4a9c11cd8293f850918b8f22c8 Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Tue, 5 Apr 2022 14:49:54 -0700 Subject: [PATCH 13/66] manully call command instead of use matrix --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8e21c69..b5cd4b2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,7 +36,7 @@ jobs: strategy: fail-fast: false matrix: - test-type: [ MutualAuth, MutualAuthT , Websocket, ALPN, ALPNT] + #test-type: [ MutualAuth, MutualAuthT , Websocket, ALPN, ALPNT] python-version: [ '3.x' ] #[MutualAuth, Websocket, ALPN] steps: From c3c45956bd2c6663c01d7c5dca7d99e446868f09 Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Tue, 5 Apr 2022 15:16:30 -0700 Subject: [PATCH 14/66] use concurrency to prevent github CI run simultaneously --- .github/workflows/ci.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b5cd4b2..3d7468c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,12 +32,14 @@ jobs: python3 -m pytest test integration-tests: + concurrency: + group: ${{ github.head_ref || github.run_id }} runs-on: ubuntu-latest strategy: fail-fast: false matrix: - #test-type: [ MutualAuth, MutualAuthT , Websocket, ALPN, ALPNT] - python-version: [ '3.x' ] + test-type: [ MutualAuth, MutualAuthT , Websocket, ALPN, ALPNT] + python-version: [ '2.x', '3.x' ] #[MutualAuth, Websocket, ALPN] steps: - uses: actions/checkout@v2 @@ -49,8 +51,4 @@ jobs: pip install pytest pip install mock pip install boto3 - ./test-integration/run/run.sh MutualAuth 1000 100 7 - ./test-integration/run/run.sh MutualAuthT 1000 100 7 - ./test-integration/run/run.sh Websocket 1000 100 7 - ./test-integration/run/run.sh ALPN 1000 100 7 - ./test-integration/run/run.sh ALPNT 1000 100 7 \ No newline at end of file + ./test-integration/run/run.sh ${{ matrix.test-type }} 1000 100 7 \ No newline at end of file From f1e7a705261ce2357e86f6a6ce152470b616b353 Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Tue, 5 Apr 2022 16:11:58 -0700 Subject: [PATCH 15/66] update concurrency group --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3d7468c..f72cfdc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,7 +33,7 @@ jobs: integration-tests: concurrency: - group: ${{ github.head_ref || github.run_id }} + group: integration-test runs-on: ubuntu-latest strategy: fail-fast: false From da268b7c300fc9f4f2d812739f84715921e9064c Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Tue, 5 Apr 2022 16:13:02 -0700 Subject: [PATCH 16/66] update cancle in process option --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f72cfdc..b689cea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,7 @@ jobs: integration-tests: concurrency: group: integration-test + cancel-in-progress: false runs-on: ubuntu-latest strategy: fail-fast: false From a1442d615b512ef7509913ce79397b978e0bd3e6 Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Tue, 5 Apr 2022 16:27:22 -0700 Subject: [PATCH 17/66] update .gitignore file --- test-integration/.gitignore | 0 test-integration/Credentials/.gitignore | 3 +++ test-integration/IntegrationTests/.gitignore | 1 + test-integration/Tools/.gitignore | 0 test-integration/run/.gitignore | 0 5 files changed, 4 insertions(+) delete mode 100644 test-integration/.gitignore delete mode 100644 test-integration/Tools/.gitignore delete mode 100644 test-integration/run/.gitignore diff --git a/test-integration/.gitignore b/test-integration/.gitignore deleted file mode 100644 index e69de29..0000000 diff --git a/test-integration/Credentials/.gitignore b/test-integration/Credentials/.gitignore index e69de29..e7a210e 100644 --- a/test-integration/Credentials/.gitignore +++ b/test-integration/Credentials/.gitignore @@ -0,0 +1,3 @@ +* +*/ +!.gitignore \ No newline at end of file diff --git a/test-integration/IntegrationTests/.gitignore b/test-integration/IntegrationTests/.gitignore index e69de29..a96cd33 100644 --- a/test-integration/IntegrationTests/.gitignore +++ b/test-integration/IntegrationTests/.gitignore @@ -0,0 +1 @@ +TestToolLibrary/SDKPackage \ No newline at end of file diff --git a/test-integration/Tools/.gitignore b/test-integration/Tools/.gitignore deleted file mode 100644 index e69de29..0000000 diff --git a/test-integration/run/.gitignore b/test-integration/run/.gitignore deleted file mode 100644 index e69de29..0000000 From 7409849907f1ccda6b53609e0de4838505d421e0 Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Tue, 5 Apr 2022 16:32:19 -0700 Subject: [PATCH 18/66] clean up code --- .github/workflows/ci.yml | 8 ++-- test-integration/Credentials/.gitignore | 2 +- test-integration/IntegrationTests/.gitignore | 2 +- test-integration/Tools/getSDKZIP.sh | 47 -------------------- test-integration/Tools/retrieve-key.py | 2 +- test-integration/run/run.sh | 22 ++++----- 6 files changed, 17 insertions(+), 66 deletions(-) delete mode 100644 test-integration/Tools/getSDKZIP.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b689cea..e75d9b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,12 +34,10 @@ jobs: integration-tests: concurrency: group: integration-test - cancel-in-progress: false runs-on: ubuntu-latest strategy: fail-fast: false matrix: - test-type: [ MutualAuth, MutualAuthT , Websocket, ALPN, ALPNT] python-version: [ '2.x', '3.x' ] #[MutualAuth, Websocket, ALPN] steps: @@ -52,4 +50,8 @@ jobs: pip install pytest pip install mock pip install boto3 - ./test-integration/run/run.sh ${{ matrix.test-type }} 1000 100 7 \ No newline at end of file + ./test-integration/run/run.sh MutualAuth 1000 100 7 + ./test-integration/run/run.sh MutualAuthT 1000 100 7 + ./test-integration/run/run.sh Websocket 1000 100 7 + ./test-integration/run/run.sh ALPN 1000 100 7 + ./test-integration/run/run.sh ALPNT 1000 100 7 \ No newline at end of file diff --git a/test-integration/Credentials/.gitignore b/test-integration/Credentials/.gitignore index e7a210e..94548af 100644 --- a/test-integration/Credentials/.gitignore +++ b/test-integration/Credentials/.gitignore @@ -1,3 +1,3 @@ * */ -!.gitignore \ No newline at end of file +!.gitignore diff --git a/test-integration/IntegrationTests/.gitignore b/test-integration/IntegrationTests/.gitignore index a96cd33..8147f60 100644 --- a/test-integration/IntegrationTests/.gitignore +++ b/test-integration/IntegrationTests/.gitignore @@ -1 +1 @@ -TestToolLibrary/SDKPackage \ No newline at end of file +TestToolLibrary/SDKPackage diff --git a/test-integration/Tools/getSDKZIP.sh b/test-integration/Tools/getSDKZIP.sh deleted file mode 100644 index 7cb939f..0000000 --- a/test-integration/Tools/getSDKZIP.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -# -# This script checks which environment this -# is, brazil or ToD Worker and trying to find -# ZIP package for the latest build of SDK in -# certain directory. -# Will print out the location of ZIP package -# and return code specifying environment type: -# 0 - Brazil -# 1 - ToD -# -1 - Not supported -# Otherwise print nothing. The environment is -# not supported. - -# Define usage -USAGE="usage: getSDKZIP.sh " -# if input args not correct, echo usage -if [ $# -ne 1 ]; then - echo ${USAGE} -else -# Find SDK ZIP package - ZIPLocation="" - if [ "$1"x == "IoTYunSDK"x ]; then - ZIPLocation=$(find ../IotSdkArduino/build/zipPackage -name AWS-IoT-Arduino-Yun-SDK-latest.zip) - if [ -n "$ZIPLocation" ]; then - echo ${ZIPLocation} - exit 0 - fi - ZIPLocation=$(find ./test-runtime/ -name AWS-IoT-Arduino-Yun-SDK-latest.zip) - if [ -n "$ZIPLocation" ]; then - echo ${ZIPLocation} - exit 1 - fi - elif [ "$1"x == "IoTPySDK"x ]; then - if [ -d "../../src/IotSdkPython/AWSIoTPythonSDK/" ]; then - echo "../../src/IotSdkPython/AWSIoTPythonSDK/" - exit 2 - fi - if [ -d "./test-runtime/aws-iot-device-sdk-python/AWSIoTPythonSDK" ]; then - echo "./test-runtime/aws-iot-device-sdk-python/AWSIoTPythonSDK" - exit 3 - fi - else - exit -1 - fi - exit -1 -fi diff --git a/test-integration/Tools/retrieve-key.py b/test-integration/Tools/retrieve-key.py index fd6336a..3884d7f 100644 --- a/test-integration/Tools/retrieve-key.py +++ b/test-integration/Tools/retrieve-key.py @@ -56,4 +56,4 @@ def main(): if __name__ == '__main__': - sys.exit(main()) # next section explains the use of sys.exit \ No newline at end of file + sys.exit(main()) # next section explains the use of sys.exit diff --git a/test-integration/run/run.sh b/test-integration/run/run.sh index 9af2bc0..755ae5d 100755 --- a/test-integration/run/run.sh +++ b/test-integration/run/run.sh @@ -11,18 +11,18 @@ # 4. Start the integration tests and check results # 5. Report any status returned. # To start the tests as TodWorker: -# > run.sh MutualAuth 1000 100 7 +# > run.sh MutualAuth 1000 100 7 # or -# > run.sh Websocket 1000 100 7 +# > run.sh Websocket 1000 100 7 # or -# > run.sh ALPN 1000 100 7 +# > run.sh ALPN 1000 100 7 # # To start the tests from desktop: -# > run.sh MutualAuthT 1000 100 7 +# > run.sh MutualAuthT 1000 100 7 # or -# > run.sh WebsocketT 1000 100 7 +# > run.sh WebsocketT 1000 100 7 # or -# > run.sh ALPNT 1000 100 7 +# > run.sh ALPNT 1000 100 7 # # 1000 MQTT messages, 100 bytes of random string # in length and 7 rounds of network failure for @@ -47,6 +47,7 @@ AWSSecretForWebsocket_TodWorker_SecretKey="arn:aws:secretsmanager:us-east-1:1231 AWSSecretForWebsocket_Desktop_KeyId="arn:aws:secretsmanager:us-east-1:123124136734:secret:V1IotSdkIntegrationTestWebsocketAccessKeyId-1YdB9z" AWSSecretForWebsocket_Desktop_SecretKey="arn:aws:secretsmanager:us-east-1:123124136734:secret:V1IotSdkIntegrationTestWebsocketSecretAccessKey-MKTSaV" +SDKLocation="./AWSIoTPythonSDK" RetrieveAWSKeys="./test-integration/Tools/retrieve-key.py" CREDENTIAL_DIR="./test-integration/Credentials/" TEST_DIR="./test-integration/IntegrationTests/" @@ -126,12 +127,7 @@ else echo ${TestMode} echo "[STEP] Obtain ZIP package" echo "***************************************************" - ZIPLocation="./AWSIoTPythonSDK" - if [ $? -eq "-1" ]; then - echo "Cannot find SDK ZIP package" - exit 2 - fi - cp -R ${ZIPLocation} ./test-integration/IntegrationTests/TestToolLibrary/SDKPackage/ + cp -R ${SDKLocation} ./test-integration/IntegrationTests/TestToolLibrary/SDKPackage/ # Obtain Python executable echo "***************************************************" @@ -168,7 +164,7 @@ else currentTestStatus=$? echo "[SUB] Test: ${file} completed. Exiting with status: ${currentTestStatus}" if [ ${currentTestStatus} -ne 0 ]; then - echo "!!!!!!!!!!!!!Test: ${file} in Python version ${iii}.x failed.!!!!!!!!!!!!!" + echo "!!!!!!!!!!!!!Test: ${file} failed.!!!!!!!!!!!!!" exit ${currentTestStatus} fi echo "" From 92501f6d1250992155b24d4b600df776de9d51e3 Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Wed, 6 Apr 2022 09:25:47 -0700 Subject: [PATCH 19/66] kick ci From 54dee767dc796ad4439a2a9e18ded11332e8f7cc Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Wed, 6 Apr 2022 10:45:48 -0700 Subject: [PATCH 20/66] update codebuild to run integration test --- .github/workflows/ci.yml | 27 +-------------------------- codebuild/linux-integration-tests.yml | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 26 deletions(-) create mode 100644 codebuild/linux-integration-tests.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e75d9b0..1ad9359 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,29 +29,4 @@ jobs: python3 setup.py install pip install pytest pip install mock - python3 -m pytest test - - integration-tests: - concurrency: - group: integration-test - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: [ '2.x', '3.x' ] - #[MutualAuth, Websocket, ALPN] - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Integration tests - run: | - pip install pytest - pip install mock - pip install boto3 - ./test-integration/run/run.sh MutualAuth 1000 100 7 - ./test-integration/run/run.sh MutualAuthT 1000 100 7 - ./test-integration/run/run.sh Websocket 1000 100 7 - ./test-integration/run/run.sh ALPN 1000 100 7 - ./test-integration/run/run.sh ALPNT 1000 100 7 \ No newline at end of file + python3 -m pytest test \ No newline at end of file diff --git a/codebuild/linux-integration-tests.yml b/codebuild/linux-integration-tests.yml new file mode 100644 index 0000000..f43f033 --- /dev/null +++ b/codebuild/linux-integration-tests.yml @@ -0,0 +1,23 @@ +version: 0.2 +#this build spec assumes the manylinux1 image for pypi +#additional packages we installed: cmake 3.5, libcrypto 1.1.0j, gcc 4.8.4 +phases: + install: + commands: + - apt-get install python3 -y + - pip install pytest + - pip install mock + - pip install boto3 + pre_build: + commands: + - echo Integration Test Starts... `date` + build: + commands: + - ./test-integration/run/run.sh MutualAuth 1000 100 7 + - ./test-integration/run/run.sh MutualAuthT 1000 100 7 + - ./test-integration/run/run.sh Websocket 1000 100 7 + - ./test-integration/run/run.sh ALPN 1000 100 7 + - ./test-integration/run/run.sh ALPNT 1000 100 7 + post_build: + commands: + - echo Integration Test completed on `date` \ No newline at end of file From 5b5a66d2a7418b1c1d439bf4b063e3d5d0c95d73 Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Wed, 6 Apr 2022 10:51:07 -0700 Subject: [PATCH 21/66] kick CI From 7113589843a3de502cabaef018820f2a83a66036 Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Wed, 6 Apr 2022 11:02:49 -0700 Subject: [PATCH 22/66] kick CI From ee1e3502eda96d085755e114b969ff8f29a97c2f Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Wed, 6 Apr 2022 11:28:20 -0700 Subject: [PATCH 23/66] update logic to use random topic to avoid test conflicts --- .github/workflows/ci.yml | 22 ++++++++++++++++++- ...tOfflineQueueingForSubscribeUnsubscribe.py | 11 +++++----- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1ad9359..d6704f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,4 +29,24 @@ jobs: python3 setup.py install pip install pytest pip install mock - python3 -m pytest test \ No newline at end of file + python3 -m pytest test + + integration-tests: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + test-type: [ MutualAuth, MutualAuthT , Websocket, ALPN, ALPNT] + python-version: [ '2.x', '3.x' ] + #[MutualAuth, Websocket, ALPN] + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Integration tests + run: | + pip install pytest + pip install mock + pip install boto3 + ./test-integration/run/run.sh ${{ matrix.test-type }} 1000 100 7 \ No newline at end of file diff --git a/test-integration/IntegrationTests/IntegrationTestOfflineQueueingForSubscribeUnsubscribe.py b/test-integration/IntegrationTests/IntegrationTestOfflineQueueingForSubscribeUnsubscribe.py index 67cdfb2..7e382b1 100644 --- a/test-integration/IntegrationTests/IntegrationTestOfflineQueueingForSubscribeUnsubscribe.py +++ b/test-integration/IntegrationTests/IntegrationTestOfflineQueueingForSubscribeUnsubscribe.py @@ -40,8 +40,11 @@ from TestToolLibrary.skip import Python3VersionLowerThan -TOPIC_A = "topic/test/offline_sub_unsub/a" -TOPIC_B = "topic/test/offline_sub_unsub/b" +def get_random_string(length): + return "".join(random.choice(string.ascii_lowercase) for i in range(length)) + +TOPIC_A = "topic/test/offline_sub_unsub/a" + get_random_string(4) +TOPIC_B = "topic/test/offline_sub_unsub/b" + get_random_string(4) MESSAGE_PREFIX = "MagicMessage-" NUMBER_OF_PUBLISHES = 3 HOST = "ajje7lpljulm4-ats.iot.us-east-1.amazonaws.com" @@ -54,10 +57,6 @@ EVENT_WAIT_TIME_OUT_SEC = 5 -def get_random_string(length): - return "".join(random.choice(string.ascii_lowercase) for i in range(length)) - - class DualClientRunner(object): def __init__(self, mode): From 96425fec6e0bea0931b2b9d2c1ca57cc6ed90799 Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Wed, 6 Apr 2022 11:48:57 -0700 Subject: [PATCH 24/66] add debug log for integration test --- .../IntegrationTests/IntegrationTestProgressiveBackoff.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test-integration/IntegrationTests/IntegrationTestProgressiveBackoff.py b/test-integration/IntegrationTests/IntegrationTestProgressiveBackoff.py index 7fb7139..9d57275 100644 --- a/test-integration/IntegrationTests/IntegrationTestProgressiveBackoff.py +++ b/test-integration/IntegrationTests/IntegrationTestProgressiveBackoff.py @@ -89,6 +89,7 @@ def _publish3Messages(self): while loopCount != 0: try: currentMessage = "Message" + str(self._pubCount) + print("Test publish to topic : " + self._targetedTopic) self._clientPub.publish(self._targetedTopic, currentMessage, 1, False) print("Thread A: Published new message: " + str(currentMessage)) self._publishMessagePool.add(currentMessage) @@ -106,6 +107,7 @@ def _publish3Messages(self): def threadBRuntime(self): # Subscribe to the topic try: + print("Test subscribe to topic : " + self._targetedTopic) self._clientSub.subscribe(self._targetedTopic, 1, self._messageCallback) except subscribeTimeoutException: print("Subscribe timeout!") From e4995d6550f15eac15c561f9398cef6d7d23fe1d Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Thu, 7 Apr 2022 10:00:11 -0700 Subject: [PATCH 25/66] remove empty files and improve comments --- codebuild/linux-integration-tests.yml | 23 ------------------- .../TestToolLibrary/SDKPackage/__init__.py | 1 - .../TestToolLibrary/__init__.py | 0 test-integration/run/run.sh | 11 ++++----- 4 files changed, 5 insertions(+), 30 deletions(-) delete mode 100644 codebuild/linux-integration-tests.yml delete mode 100644 test-integration/IntegrationTests/TestToolLibrary/SDKPackage/__init__.py delete mode 100644 test-integration/IntegrationTests/TestToolLibrary/__init__.py diff --git a/codebuild/linux-integration-tests.yml b/codebuild/linux-integration-tests.yml deleted file mode 100644 index f43f033..0000000 --- a/codebuild/linux-integration-tests.yml +++ /dev/null @@ -1,23 +0,0 @@ -version: 0.2 -#this build spec assumes the manylinux1 image for pypi -#additional packages we installed: cmake 3.5, libcrypto 1.1.0j, gcc 4.8.4 -phases: - install: - commands: - - apt-get install python3 -y - - pip install pytest - - pip install mock - - pip install boto3 - pre_build: - commands: - - echo Integration Test Starts... `date` - build: - commands: - - ./test-integration/run/run.sh MutualAuth 1000 100 7 - - ./test-integration/run/run.sh MutualAuthT 1000 100 7 - - ./test-integration/run/run.sh Websocket 1000 100 7 - - ./test-integration/run/run.sh ALPN 1000 100 7 - - ./test-integration/run/run.sh ALPNT 1000 100 7 - post_build: - commands: - - echo Integration Test completed on `date` \ No newline at end of file diff --git a/test-integration/IntegrationTests/TestToolLibrary/SDKPackage/__init__.py b/test-integration/IntegrationTests/TestToolLibrary/SDKPackage/__init__.py deleted file mode 100644 index 1ad354e..0000000 --- a/test-integration/IntegrationTests/TestToolLibrary/SDKPackage/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = "1.4.9" diff --git a/test-integration/IntegrationTests/TestToolLibrary/__init__.py b/test-integration/IntegrationTests/TestToolLibrary/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/test-integration/run/run.sh b/test-integration/run/run.sh index 755ae5d..f420f73 100755 --- a/test-integration/run/run.sh +++ b/test-integration/run/run.sh @@ -5,11 +5,10 @@ # SDK. The tests should be able to run both in # Brazil and ToD Worker environment. # The script will perform the following tasks: -# 1. Retrieve credentials as needed from Odin +# 1. Retrieve credentials as needed from AWS # 2. Obtain ZIP package and unzip it locally -# 3. Obtain Python executable -# 4. Start the integration tests and check results -# 5. Report any status returned. +# 3. Start the integration tests and check results +# 4. Report any status returned. # To start the tests as TodWorker: # > run.sh MutualAuth 1000 100 7 # or @@ -69,9 +68,9 @@ else # Determine the Python versions need to test for this SDK pythonExecutableArray=() pythonExecutableArray[0]="3" -# Retrieve credentials as needed from Odin +# Retrieve credentials as needed from AWS TestMode="" - echo "[STEP] Retrieve credentials from Odin" + echo "[STEP] Retrieve credentials from AWS" echo "***************************************************" if [ "$1"x == "MutualAuth"x -o "$1"x == "MutualAuthT"x ]; then AWSSetName_privatekey=${AWSMutualAuth_TodWorker_private_key} From 261d5639aaf90f721216cbf1f686e0fc8d0e0b9c Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Thu, 7 Apr 2022 10:06:30 -0700 Subject: [PATCH 26/66] update ignore file --- test-integration/IntegrationTests/.gitignore | 1 - .../IntegrationTests/TestToolLibrary/SDKPackage/.gitignore | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) delete mode 100644 test-integration/IntegrationTests/.gitignore create mode 100644 test-integration/IntegrationTests/TestToolLibrary/SDKPackage/.gitignore diff --git a/test-integration/IntegrationTests/.gitignore b/test-integration/IntegrationTests/.gitignore deleted file mode 100644 index 8147f60..0000000 --- a/test-integration/IntegrationTests/.gitignore +++ /dev/null @@ -1 +0,0 @@ -TestToolLibrary/SDKPackage diff --git a/test-integration/IntegrationTests/TestToolLibrary/SDKPackage/.gitignore b/test-integration/IntegrationTests/TestToolLibrary/SDKPackage/.gitignore new file mode 100644 index 0000000..ac23051 --- /dev/null +++ b/test-integration/IntegrationTests/TestToolLibrary/SDKPackage/.gitignore @@ -0,0 +1,2 @@ +*.* +!.gitignore \ No newline at end of file From 08f4b782d07269991fe9d45b0d73c2a3f64c6fea Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Thu, 7 Apr 2022 10:11:36 -0700 Subject: [PATCH 27/66] add back __init__.py for python2 --- .../IntegrationTests/TestToolLibrary/SDKPackage/.gitignore | 3 ++- .../IntegrationTests/TestToolLibrary/SDKPackage/__init__.py | 1 + test-integration/IntegrationTests/TestToolLibrary/__init__.py | 0 3 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 test-integration/IntegrationTests/TestToolLibrary/SDKPackage/__init__.py create mode 100644 test-integration/IntegrationTests/TestToolLibrary/__init__.py diff --git a/test-integration/IntegrationTests/TestToolLibrary/SDKPackage/.gitignore b/test-integration/IntegrationTests/TestToolLibrary/SDKPackage/.gitignore index ac23051..151aa74 100644 --- a/test-integration/IntegrationTests/TestToolLibrary/SDKPackage/.gitignore +++ b/test-integration/IntegrationTests/TestToolLibrary/SDKPackage/.gitignore @@ -1,2 +1,3 @@ *.* -!.gitignore \ No newline at end of file +!.gitignore +!__init__.py \ No newline at end of file diff --git a/test-integration/IntegrationTests/TestToolLibrary/SDKPackage/__init__.py b/test-integration/IntegrationTests/TestToolLibrary/SDKPackage/__init__.py new file mode 100644 index 0000000..1ad354e --- /dev/null +++ b/test-integration/IntegrationTests/TestToolLibrary/SDKPackage/__init__.py @@ -0,0 +1 @@ +__version__ = "1.4.9" diff --git a/test-integration/IntegrationTests/TestToolLibrary/__init__.py b/test-integration/IntegrationTests/TestToolLibrary/__init__.py new file mode 100644 index 0000000..e69de29 From 53efd56e004efd6ad05dc65d620fb38cbb398ab8 Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Thu, 7 Apr 2022 10:24:19 -0700 Subject: [PATCH 28/66] update subscribe topic --- .../IntegrationTestConfigurablePublishMessageQueueing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-integration/IntegrationTests/IntegrationTestConfigurablePublishMessageQueueing.py b/test-integration/IntegrationTests/IntegrationTestConfigurablePublishMessageQueueing.py index 32f4514..705df53 100644 --- a/test-integration/IntegrationTests/IntegrationTestConfigurablePublishMessageQueueing.py +++ b/test-integration/IntegrationTests/IntegrationTestConfigurablePublishMessageQueueing.py @@ -172,7 +172,7 @@ def _runtime(self): print("Type: " + str(type(e))) print("Message: " + str(e.message)) time.sleep(2.2) - print("Thread B: Subscribed to self._topic.") + print("Thread B: Subscribed to " + self._topic) print("Thread B: Now wait for Thread A.") # Scanning rate is 100 TPS while self._keepRunning: From 34734fb30658bdd2e4bead50bd3e49f52dc588f9 Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Thu, 7 Apr 2022 11:25:04 -0700 Subject: [PATCH 29/66] kick CI to verify the tests From b970c2d159ee2cf308be2d9d6a67f4bcb8b81c29 Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Mon, 11 Apr 2022 09:53:16 -0700 Subject: [PATCH 30/66] use a random id for client id to prevent conflicts between tests --- .../IntegrationTestAutoReconnectResubscribe.py | 6 ++++-- .../IntegrationTestConfigurablePublishMessageQueueing.py | 6 ++++-- .../IntegrationTests/IntegrationTestJobsClient.py | 3 ++- .../IntegrationTests/IntegrationTestMQTTConnection.py | 8 ++++++-- ...tegrationTestOfflineQueueingForSubscribeUnsubscribe.py | 4 ++-- .../IntegrationTests/IntegrationTestProgressiveBackoff.py | 6 ++++-- .../IntegrationTests/IntegrationTestShadow.py | 6 ++++-- 7 files changed, 26 insertions(+), 13 deletions(-) diff --git a/test-integration/IntegrationTests/IntegrationTestAutoReconnectResubscribe.py b/test-integration/IntegrationTests/IntegrationTestAutoReconnectResubscribe.py index 1ee251a..83b66c2 100644 --- a/test-integration/IntegrationTests/IntegrationTestAutoReconnectResubscribe.py +++ b/test-integration/IntegrationTests/IntegrationTestAutoReconnectResubscribe.py @@ -30,6 +30,8 @@ from TestToolLibrary.skip import Python2VersionLowerThan from TestToolLibrary.skip import Python3VersionLowerThan +CLIENT_ID_PUB = "integrationTestMQTT_ClientPub" + "".join(random.choice(string.ascii_lowercase) for i in range(4)) +CLIENT_ID_SUB = "integrationTestMQTT_ClientSub" + "".join(random.choice(string.ascii_lowercase) for i in range(4)) # Callback unit class callbackUnit: @@ -148,9 +150,9 @@ def threadBRuntime(self, pyCoreClient, callback): # Init Python core and connect myMQTTClientManager = MQTTClientManager.MQTTClientManager() -clientPub = myMQTTClientManager.create_connected_mqtt_core("integrationTestMQTT_ClientPub", host, rootCA, +clientPub = myMQTTClientManager.create_connected_mqtt_core(CLIENT_ID_PUB, host, rootCA, certificate, privateKey, mode=mode) -clientSub = myMQTTClientManager.create_connected_mqtt_core("integrationTestMQTT_ClientSub", host, rootCA, +clientSub = myMQTTClientManager.create_connected_mqtt_core(CLIENT_ID_SUB, host, rootCA, certificate, privateKey, mode=mode) if clientPub is None or clientSub is None: diff --git a/test-integration/IntegrationTests/IntegrationTestConfigurablePublishMessageQueueing.py b/test-integration/IntegrationTests/IntegrationTestConfigurablePublishMessageQueueing.py index 705df53..d6bfdc5 100644 --- a/test-integration/IntegrationTests/IntegrationTestConfigurablePublishMessageQueueing.py +++ b/test-integration/IntegrationTests/IntegrationTestConfigurablePublishMessageQueueing.py @@ -38,6 +38,8 @@ from TestToolLibrary.skip import Python2VersionLowerThan from TestToolLibrary.skip import Python3VersionLowerThan +CLIENT_ID_PUB = "integrationTestMQTT_ClientPub" + "".join(random.choice(string.ascii_lowercase) for i in range(4)) +CLIENT_ID_SUB = "integrationTestMQTT_ClientSub" + "".join(random.choice(string.ascii_lowercase) for i in range(4)) # Class that implements the publishing thread: Thread A, with network failure # This thread will publish 3 messages first, and then keep publishing @@ -287,9 +289,9 @@ def performConfigurableOfflinePublishQueueTest(clientPub, clientSub): # Init Python core and connect myMQTTClientManager = MQTTClientManager.MQTTClientManager() -clientPub = myMQTTClientManager.create_connected_mqtt_core("integrationTestMQTT_ClientPub", host, rootCA, +clientPub = myMQTTClientManager.create_connected_mqtt_core(CLIENT_ID_PUB, host, rootCA, certificate, privateKey, mode=mode) -clientSub = myMQTTClientManager.create_connected_mqtt_core("integrationTestMQTT_ClientSub", host, rootCA, +clientSub = myMQTTClientManager.create_connected_mqtt_core(CLIENT_ID_SUB, host, rootCA, certificate, privateKey, mode=mode) if clientPub is None or clientSub is None: diff --git a/test-integration/IntegrationTests/IntegrationTestJobsClient.py b/test-integration/IntegrationTests/IntegrationTestJobsClient.py index 9eb5690..18d8aa5 100644 --- a/test-integration/IntegrationTests/IntegrationTestJobsClient.py +++ b/test-integration/IntegrationTests/IntegrationTestJobsClient.py @@ -37,6 +37,7 @@ import json IOT_JOBS_MQTT_RESPONSE_WAIT_SECONDS = 5 +CLIENT_ID = "integrationTestMQTT_Client" + "".join(random.choice(string.ascii_lowercase) for i in range(4)) class JobsMessageProcessor(object): def __init__(self, awsIoTMQTTThingJobsClient, clientToken): @@ -168,7 +169,7 @@ def _test_send_response_confirm(self, sendResult): # Init Python core and connect myMQTTClientManager = MQTTClientManager.MQTTClientManager() -client = myMQTTClientManager.create_connected_mqtt_client(mode, "integrationTestJobs_Client", host, (rootCA, certificate, privateKey)) +client = myMQTTClientManager.create_connected_mqtt_client(mode, CLIENT_ID, host, (rootCA, certificate, privateKey)) clientId = 'AWSPythonkSDKTestThingClient' thingName = 'AWSPythonkSDKTestThing' diff --git a/test-integration/IntegrationTests/IntegrationTestMQTTConnection.py b/test-integration/IntegrationTests/IntegrationTestMQTTConnection.py index 9deb10e..252770f 100644 --- a/test-integration/IntegrationTests/IntegrationTestMQTTConnection.py +++ b/test-integration/IntegrationTests/IntegrationTestMQTTConnection.py @@ -30,6 +30,10 @@ API_TYPE_SYNC = "sync" API_TYPE_ASYNC = "async" +CLIENT_ID_PUB = "integrationTestMQTT_ClientPub" + "".join(random.choice(string.ascii_lowercase) for i in range(4)) +CLIENT_ID_SUB = "integrationTestMQTT_ClientSub" + "".join(random.choice(string.ascii_lowercase) for i in range(4)) + + # Callback unit for subscribe class callbackUnit: @@ -95,9 +99,9 @@ def _performPublish(self, pyCoreClient, topic, qos, payload): # Init Python core and connect myMQTTClientManager = MQTTClientManager.MQTTClientManager() -clientPub = myMQTTClientManager.create_connected_mqtt_core("integrationTestMQTT_ClientPub", host, rootCA, +clientPub = myMQTTClientManager.create_connected_mqtt_core(CLIENT_ID_PUB, host, rootCA, certificate, privateKey, mode=mode) -clientSub = myMQTTClientManager.create_connected_mqtt_core("integrationTestMQTT_ClientSub", host, rootCA, +clientSub = myMQTTClientManager.create_connected_mqtt_core(CLIENT_ID_SUB, host, rootCA, certificate, privateKey, mode=mode) if clientPub is None or clientSub is None: diff --git a/test-integration/IntegrationTests/IntegrationTestOfflineQueueingForSubscribeUnsubscribe.py b/test-integration/IntegrationTests/IntegrationTestOfflineQueueingForSubscribeUnsubscribe.py index 7e382b1..c06847d 100644 --- a/test-integration/IntegrationTests/IntegrationTestOfflineQueueingForSubscribeUnsubscribe.py +++ b/test-integration/IntegrationTests/IntegrationTestOfflineQueueingForSubscribeUnsubscribe.py @@ -51,8 +51,8 @@ def get_random_string(length): ROOT_CA = "./test-integration/Credentials/rootCA.crt" CERT = "./test-integration/Credentials/certificate.pem.crt" KEY = "./test-integration/Credentials/privateKey.pem.key" -CLIENT_PUB_ID = "PySdkIntegTest_OfflineSubUnsub_pub" -CLIENT_SUB_UNSUB_ID = "PySdkIntegTest_OfflineSubUnsub_subunsub" +CLIENT_PUB_ID = "PySdkIntegTest_OfflineSubUnsub_pub" + get_random_string(4) +CLIENT_SUB_UNSUB_ID = "PySdkIntegTest_OfflineSubUnsub_subunsub" + get_random_string(4) KEEP_ALIVE_SEC = 1 EVENT_WAIT_TIME_OUT_SEC = 5 diff --git a/test-integration/IntegrationTests/IntegrationTestProgressiveBackoff.py b/test-integration/IntegrationTests/IntegrationTestProgressiveBackoff.py index 9d57275..cd7b7ec 100644 --- a/test-integration/IntegrationTests/IntegrationTestProgressiveBackoff.py +++ b/test-integration/IntegrationTests/IntegrationTestProgressiveBackoff.py @@ -43,6 +43,8 @@ from TestToolLibrary.skip import Python2VersionLowerThan from TestToolLibrary.skip import Python3VersionLowerThan +CLIENT_ID_PUB = "integrationTestMQTT_ClientPub" + "".join(random.choice(string.ascii_lowercase) for i in range(4)) +CLIENT_ID_SUB = "integrationTestMQTT_ClientSub" + "".join(random.choice(string.ascii_lowercase) for i in range(4)) # Class that implements all the related threads in the test in a controllable manner class threadPool: @@ -234,9 +236,9 @@ def verifyBackoffTime(answerList, resultList): # Init Python core and connect myMQTTClientManager = MQTTClientManager.MQTTClientManager() -clientPub = myMQTTClientManager.create_connected_mqtt_core("integrationTestMQTT_ClientPub", host, rootCA, +clientPub = myMQTTClientManager.create_connected_mqtt_core(CLIENT_ID_PUB, host, rootCA, certificate, privateKey, mode=mode) -clientSub = myMQTTClientManager.create_connected_mqtt_core("integrationTestMQTT_ClientSub", host, rootCA, +clientSub = myMQTTClientManager.create_connected_mqtt_core(CLIENT_ID_SUB, host, rootCA, certificate, privateKey, mode=mode) if clientPub is None or clientSub is None: diff --git a/test-integration/IntegrationTests/IntegrationTestShadow.py b/test-integration/IntegrationTests/IntegrationTestShadow.py index 0b1885d..9e2c2a5 100644 --- a/test-integration/IntegrationTests/IntegrationTestShadow.py +++ b/test-integration/IntegrationTests/IntegrationTestShadow.py @@ -39,6 +39,8 @@ # Global configuration TPS = 1 # Update speed, Spectre does not tolerate high TPS shadow operations... +CLIENT_ID_PUB = "integrationTestMQTT_ClientPub" + "".join(random.choice(string.ascii_lowercase) for i in range(4)) +CLIENT_ID_SUB = "integrationTestMQTT_ClientSub" + "".join(random.choice(string.ascii_lowercase) for i in range(4)) # Class that manages the generation and chopping of the random string @@ -163,9 +165,9 @@ def randomString(lengthOfString): # Init Python core and connect myMQTTClientManager = MQTTClientManager.MQTTClientManager() -clientPub = myMQTTClientManager.create_connected_mqtt_core("integrationTestMQTT_ClientPub", host, rootCA, +clientPub = myMQTTClientManager.create_connected_mqtt_core(CLIENT_ID_PUB, host, rootCA, certificate, privateKey, mode=mode) -clientSub = myMQTTClientManager.create_connected_mqtt_core("integrationTestMQTT_ClientSub", host, rootCA, +clientSub = myMQTTClientManager.create_connected_mqtt_core(CLIENT_ID_SUB, host, rootCA, certificate, privateKey, mode=mode) if clientPub is None or clientSub is None: From 72e47e06cb9ea77498b7f42c1a561f07594b9bad Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Mon, 11 Apr 2022 09:56:22 -0700 Subject: [PATCH 31/66] add script to check version --- continuous-delivery/test_version_exists | 22 +++++++++++++++++++++ continuous-delivery/test_version_exists.yml | 21 ++++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 continuous-delivery/test_version_exists create mode 100644 continuous-delivery/test_version_exists.yml diff --git a/continuous-delivery/test_version_exists b/continuous-delivery/test_version_exists new file mode 100644 index 0000000..3579dbc --- /dev/null +++ b/continuous-delivery/test_version_exists @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -e +set -x +# force a failure if there's no tag +git describe --tags +# now get the tag +CURRENT_TAG=$(git describe --tags | cut -f2 -dv) +# convert v0.2.12-2-g50254a9 to 0.2.12 +CURRENT_TAG_VERSION=$(git describe --tags | cut -f1 -d'-' | cut -f2 -dv) +# if there's a hash on the tag, then this is not a release tagged commit +if [ "$CURRENT_TAG" != "$CURRENT_TAG_VERSION" ]; then + echo "Current tag version is not a release tag, cut a new release if you want to publish." + exit 1 +fi + +if python3 -m pip install --no-cache-dir -vvv AWSIoTPythonSDK==$CURRENT_TAG_VERSION; then + echo "$CURRENT_TAG_VERSION is already in pypi, cut a new tag if you want to upload another version." + exit 1 +fi + +echo "$CURRENT_TAG_VERSION currently does not exist in pypi, allowing pipeline to continue." +exit 0 diff --git a/continuous-delivery/test_version_exists.yml b/continuous-delivery/test_version_exists.yml new file mode 100644 index 0000000..70947c6 --- /dev/null +++ b/continuous-delivery/test_version_exists.yml @@ -0,0 +1,21 @@ +version: 0.2 +#this build spec assumes the ubuntu 14.04 trusty image +#this build run simply verifies we haven't published something at this tag yet. +#if we have we fail the build and stop the pipeline, if we haven't we allow the pipeline to run. +phases: + install: + commands: + - sudo apt-get update -y + - sudo apt-get install python3 python3-pip -y + - pip3 install --upgrade setuptools + pre_build: + commands: + build: + commands: + - echo Build started on `date` + - cd aws-iot-device-sdk-python + - bash ./continuous-delivery/test_version_exists + post_build: + commands: + - echo Build completed on `date` + From 99fe2e9f635a0126a0cdcc629aecda71b6d051ab Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Mon, 11 Apr 2022 09:58:09 -0700 Subject: [PATCH 32/66] Revert "add script to check version" This reverts commit 72e47e06cb9ea77498b7f42c1a561f07594b9bad. --- continuous-delivery/test_version_exists | 22 --------------------- continuous-delivery/test_version_exists.yml | 21 -------------------- 2 files changed, 43 deletions(-) delete mode 100644 continuous-delivery/test_version_exists delete mode 100644 continuous-delivery/test_version_exists.yml diff --git a/continuous-delivery/test_version_exists b/continuous-delivery/test_version_exists deleted file mode 100644 index 3579dbc..0000000 --- a/continuous-delivery/test_version_exists +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash -set -e -set -x -# force a failure if there's no tag -git describe --tags -# now get the tag -CURRENT_TAG=$(git describe --tags | cut -f2 -dv) -# convert v0.2.12-2-g50254a9 to 0.2.12 -CURRENT_TAG_VERSION=$(git describe --tags | cut -f1 -d'-' | cut -f2 -dv) -# if there's a hash on the tag, then this is not a release tagged commit -if [ "$CURRENT_TAG" != "$CURRENT_TAG_VERSION" ]; then - echo "Current tag version is not a release tag, cut a new release if you want to publish." - exit 1 -fi - -if python3 -m pip install --no-cache-dir -vvv AWSIoTPythonSDK==$CURRENT_TAG_VERSION; then - echo "$CURRENT_TAG_VERSION is already in pypi, cut a new tag if you want to upload another version." - exit 1 -fi - -echo "$CURRENT_TAG_VERSION currently does not exist in pypi, allowing pipeline to continue." -exit 0 diff --git a/continuous-delivery/test_version_exists.yml b/continuous-delivery/test_version_exists.yml deleted file mode 100644 index 70947c6..0000000 --- a/continuous-delivery/test_version_exists.yml +++ /dev/null @@ -1,21 +0,0 @@ -version: 0.2 -#this build spec assumes the ubuntu 14.04 trusty image -#this build run simply verifies we haven't published something at this tag yet. -#if we have we fail the build and stop the pipeline, if we haven't we allow the pipeline to run. -phases: - install: - commands: - - sudo apt-get update -y - - sudo apt-get install python3 python3-pip -y - - pip3 install --upgrade setuptools - pre_build: - commands: - build: - commands: - - echo Build started on `date` - - cd aws-iot-device-sdk-python - - bash ./continuous-delivery/test_version_exists - post_build: - commands: - - echo Build completed on `date` - From a09b436dc55ac61fb77b46a0ea5ccdc5065210b0 Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Mon, 11 Apr 2022 11:17:56 -0700 Subject: [PATCH 33/66] add continuous-delivery --- continuous-delivery/pip-install-with-retry.py | 39 +++++++++++++++++++ continuous-delivery/publish_to_prod_pypi.yml | 25 ++++++++++++ continuous-delivery/publish_to_test_pypi.yml | 25 ++++++++++++ continuous-delivery/test_prod_pypi.yml | 28 +++++++++++++ continuous-delivery/test_test_pypi.yml | 30 ++++++++++++++ continuous-delivery/test_version_exists | 22 +++++++++++ continuous-delivery/test_version_exists.yml | 21 ++++++++++ 7 files changed, 190 insertions(+) create mode 100644 continuous-delivery/pip-install-with-retry.py create mode 100644 continuous-delivery/publish_to_prod_pypi.yml create mode 100644 continuous-delivery/publish_to_test_pypi.yml create mode 100644 continuous-delivery/test_prod_pypi.yml create mode 100644 continuous-delivery/test_test_pypi.yml create mode 100644 continuous-delivery/test_version_exists create mode 100644 continuous-delivery/test_version_exists.yml diff --git a/continuous-delivery/pip-install-with-retry.py b/continuous-delivery/pip-install-with-retry.py new file mode 100644 index 0000000..347e0dc --- /dev/null +++ b/continuous-delivery/pip-install-with-retry.py @@ -0,0 +1,39 @@ +import time +import sys +import subprocess + +DOCS = """Given cmdline args, executes: python3 -m pip install [args...] +Keeps retrying until the new version becomes available in pypi (or we time out)""" +if len(sys.argv) < 2: + sys.exit(DOCS) + +RETRY_INTERVAL_SECS = 10 +GIVE_UP_AFTER_SECS = 60 * 15 + +pip_install_args = [sys.executable, '-m', 'pip', 'install'] + sys.argv[1:] + +start_time = time.time() +while True: + print(subprocess.list2cmdline(pip_install_args)) + result = subprocess.run(pip_install_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + + stdout = result.stdout.decode().strip() + if stdout: + print(stdout) + + if result.returncode == 0: + # success + sys.exit(0) + + if "could not find a version" in stdout.lower(): + elapsed_secs = time.time() - start_time + if elapsed_secs < GIVE_UP_AFTER_SECS: + # try again + print("Retrying in", RETRY_INTERVAL_SECS, "secs...") + time.sleep(RETRY_INTERVAL_SECS) + continue + else: + print("Giving up on retries after", int(elapsed_secs), "total secs.") + + # fail + sys.exit(result.returncode) diff --git a/continuous-delivery/publish_to_prod_pypi.yml b/continuous-delivery/publish_to_prod_pypi.yml new file mode 100644 index 0000000..c8d7885 --- /dev/null +++ b/continuous-delivery/publish_to_prod_pypi.yml @@ -0,0 +1,25 @@ +version: 0.2 +# this image assumes Ubuntu 14.04 base image +phases: + install: + commands: + - sudo apt-get update -y + - sudo apt-get install python3 python3-pip -y + - export PATH=$PATH:$HOME/.local/bin + - python3 -m pip install --user --upgrade pip + - python3 -m pip install --user --upgrade twine setuptools wheel awscli PyOpenSSL six + pre_build: + commands: + - cd aws-iot-device-sdk-python-v2 + - pypirc=$(aws secretsmanager get-secret-value --secret-id "prod/aws-crt-python/.pypirc" --query "SecretString" | cut -f2 -d\") && echo "$pypirc" > ~/.pypirc + - export PKG_VERSION=$(git describe --tags | cut -f2 -dv) + - echo "Updating package version to ${PKG_VERSION}" + - sed --in-place -E "s/__version__ = '.+'/__version__ = '${PKG_VERSION}'/" awsiot/__init__.py + build: + commands: + - echo Build started on `date` + - python3 setup.py sdist bdist_wheel + - python3 -m twine upload -r pypi dist/* + post_build: + commands: + - echo Build completed on `date` diff --git a/continuous-delivery/publish_to_test_pypi.yml b/continuous-delivery/publish_to_test_pypi.yml new file mode 100644 index 0000000..86cde41 --- /dev/null +++ b/continuous-delivery/publish_to_test_pypi.yml @@ -0,0 +1,25 @@ +version: 0.2 +# this image assumes Ubuntu 14.04 base image +phases: + install: + commands: + - sudo apt-get update -y + - sudo apt-get install python3 python3-pip -y + - export PATH=$PATH:$HOME/.local/bin + - python3 -m pip install --user --upgrade pip + - python3 -m pip install --user --upgrade twine setuptools wheel awscli PyOpenSSL six + pre_build: + commands: + - pypirc=$(aws secretsmanager get-secret-value --secret-id "alpha/aws-crt-python/.pypirc" --query "SecretString" | cut -f2 -d\") && echo "$pypirc" > ~/.pypirc + - cd aws-iot-device-sdk-python-v2 + - export PKG_VERSION=$(git describe --tags | cut -f2 -dv) + - echo "Updating package version to ${PKG_VERSION}" + - sed --in-place -E "s/__version__ = '.+'/__version__ = '${PKG_VERSION}'/" awsiot/__init__.py + build: + commands: + - echo Build started on `date` + - python3 setup.py sdist bdist_wheel + - python3 -m twine upload -r testpypi dist/* + post_build: + commands: + - echo Build completed on `date` diff --git a/continuous-delivery/test_prod_pypi.yml b/continuous-delivery/test_prod_pypi.yml new file mode 100644 index 0000000..181c074 --- /dev/null +++ b/continuous-delivery/test_prod_pypi.yml @@ -0,0 +1,28 @@ +version: 0.2 +# this image assumes Ubuntu 14.04 base image +phases: + install: + commands: + - sudo apt-get update -y + - sudo apt-get install python3 python3-pip -y + - python3 -m pip install --upgrade pip + - python3 -m pip install --upgrade setuptools + + pre_build: + commands: + - curl https://www.amazontrust.com/repository/AmazonRootCA1.pem --output /tmp/AmazonRootCA1.pem + - cert=$(aws secretsmanager get-secret-value --secret-id "unit-test/certificate" --query "SecretString" | cut -f2 -d":" | cut -f2 -d\") && echo "$cert" > /tmp/certificate.pem + - key=$(aws secretsmanager get-secret-value --secret-id "unit-test/privatekey" --query "SecretString" | cut -f2 -d":" | cut -f2 -d\") && echo "$key" > /tmp/privatekey.pem + - ENDPOINT=$(aws secretsmanager get-secret-value --secret-id "unit-test/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g') + build: + commands: + - echo Build started on `date` + - cd aws-iot-device-sdk-python-v2 + - CURRENT_TAG_VERSION=$(git describe --tags | cut -f2 -dv) + - python3 continuous-delivery/pip-install-with-retry.py --no-cache-dir --user awsiotsdk==$CURRENT_TAG_VERSION + - python3 samples/basic_discovery.py --region us-east-1 --cert /tmp/certificate.pem --key /tmp/privatekey.pem --root-ca /tmp/AmazonRootCA1.pem --thing-name aws-sdk-crt-unit-test --print-discover-resp-only -v Trace + + post_build: + commands: + - echo Build completed on `date` + diff --git a/continuous-delivery/test_test_pypi.yml b/continuous-delivery/test_test_pypi.yml new file mode 100644 index 0000000..1fd0e51 --- /dev/null +++ b/continuous-delivery/test_test_pypi.yml @@ -0,0 +1,30 @@ +version: 0.2 +# this image assumes Ubuntu 14.04 base image +phases: + install: + commands: + - sudo apt-get update -y + - sudo apt-get install python3 python3-pip -y + - python3 -m pip install --upgrade pip + - python3 -m pip install --upgrade setuptools + + pre_build: + commands: + - curl https://www.amazontrust.com/repository/AmazonRootCA1.pem --output /tmp/AmazonRootCA1.pem + - cert=$(aws secretsmanager get-secret-value --secret-id "unit-test/certificate" --query "SecretString" | cut -f2 -d":" | cut -f2 -d\") && echo "$cert" > /tmp/certificate.pem + - key=$(aws secretsmanager get-secret-value --secret-id "unit-test/privatekey" --query "SecretString" | cut -f2 -d":" | cut -f2 -d\") && echo "$key" > /tmp/privatekey.pem + - ENDPOINT=$(aws secretsmanager get-secret-value --secret-id "unit-test/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g') + build: + commands: + - echo Build started on `date` + - cd aws-iot-device-sdk-python-v2 + - CURRENT_TAG_VERSION=$(git describe --tags | cut -f2 -dv) + # this is here because typing isn't in testpypi, so pull it from prod instead + - python3 -m pip install typing + - python3 continuous-delivery/pip-install-with-retry.py -i https://testpypi.python.org/simple --user awsiotsdk==$CURRENT_TAG_VERSION + - python3 samples/basic_discovery.py --region us-east-1 --cert /tmp/certificate.pem --key /tmp/privatekey.pem --root-ca /tmp/AmazonRootCA1.pem --thing-name aws-sdk-crt-unit-test --print-discover-resp-only -v Trace + + post_build: + commands: + - echo Build completed on `date` + diff --git a/continuous-delivery/test_version_exists b/continuous-delivery/test_version_exists new file mode 100644 index 0000000..3579dbc --- /dev/null +++ b/continuous-delivery/test_version_exists @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -e +set -x +# force a failure if there's no tag +git describe --tags +# now get the tag +CURRENT_TAG=$(git describe --tags | cut -f2 -dv) +# convert v0.2.12-2-g50254a9 to 0.2.12 +CURRENT_TAG_VERSION=$(git describe --tags | cut -f1 -d'-' | cut -f2 -dv) +# if there's a hash on the tag, then this is not a release tagged commit +if [ "$CURRENT_TAG" != "$CURRENT_TAG_VERSION" ]; then + echo "Current tag version is not a release tag, cut a new release if you want to publish." + exit 1 +fi + +if python3 -m pip install --no-cache-dir -vvv AWSIoTPythonSDK==$CURRENT_TAG_VERSION; then + echo "$CURRENT_TAG_VERSION is already in pypi, cut a new tag if you want to upload another version." + exit 1 +fi + +echo "$CURRENT_TAG_VERSION currently does not exist in pypi, allowing pipeline to continue." +exit 0 diff --git a/continuous-delivery/test_version_exists.yml b/continuous-delivery/test_version_exists.yml new file mode 100644 index 0000000..ca67e64 --- /dev/null +++ b/continuous-delivery/test_version_exists.yml @@ -0,0 +1,21 @@ +version: 0.2 +#this build spec assumes the ubuntu 14.04 trusty image +#this build run simply verifies we haven't published something at this tag yet. +#if we have we fail the build and stop the pipeline, if we haven't we allow the pipeline to run. +phases: + install: + commands: + - sudo apt-get update -y + - sudo apt-get install python3 python3-pip -y + - pip3 install --upgrade setuptools + pre_build: + commands: + build: + commands: + - echo Build started on `date` + - cd aws-iot-device-sdk-python-v2 + - bash ./continuous-delivery/test_version_exists + post_build: + commands: + - echo Build completed on `date` + From bf9c0d1b2e0b154a226f9dc4bc932b41a83d4e5f Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Mon, 11 Apr 2022 13:50:02 -0700 Subject: [PATCH 34/66] update test_version_exists --- continuous-delivery/test_version_exists.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/continuous-delivery/test_version_exists.yml b/continuous-delivery/test_version_exists.yml index ca67e64..2704ba7 100644 --- a/continuous-delivery/test_version_exists.yml +++ b/continuous-delivery/test_version_exists.yml @@ -10,10 +10,10 @@ phases: - pip3 install --upgrade setuptools pre_build: commands: + - echo Build start on `date` build: commands: - - echo Build started on `date` - - cd aws-iot-device-sdk-python-v2 + - cd aws-iot-device-sdk-python - bash ./continuous-delivery/test_version_exists post_build: commands: From 7657abd2e0e563ea9964f30892543e39ea044f22 Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Mon, 11 Apr 2022 14:28:27 -0700 Subject: [PATCH 35/66] update deployment script --- continuous-delivery/publish_to_prod_pypi.yml | 2 +- continuous-delivery/test_prod_pypi.yml | 2 +- continuous-delivery/test_test_pypi.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/continuous-delivery/publish_to_prod_pypi.yml b/continuous-delivery/publish_to_prod_pypi.yml index c8d7885..a3d739f 100644 --- a/continuous-delivery/publish_to_prod_pypi.yml +++ b/continuous-delivery/publish_to_prod_pypi.yml @@ -14,7 +14,7 @@ phases: - pypirc=$(aws secretsmanager get-secret-value --secret-id "prod/aws-crt-python/.pypirc" --query "SecretString" | cut -f2 -d\") && echo "$pypirc" > ~/.pypirc - export PKG_VERSION=$(git describe --tags | cut -f2 -dv) - echo "Updating package version to ${PKG_VERSION}" - - sed --in-place -E "s/__version__ = '.+'/__version__ = '${PKG_VERSION}'/" awsiot/__init__.py + - sed --in-place -E "s/__version__ = '.+'/__version__ = '${PKG_VERSION}'/" AWSIoTPythonSDK/__init__.py build: commands: - echo Build started on `date` diff --git a/continuous-delivery/test_prod_pypi.yml b/continuous-delivery/test_prod_pypi.yml index 181c074..5897f38 100644 --- a/continuous-delivery/test_prod_pypi.yml +++ b/continuous-delivery/test_prod_pypi.yml @@ -17,7 +17,7 @@ phases: build: commands: - echo Build started on `date` - - cd aws-iot-device-sdk-python-v2 + - cd aws-iot-device-sdk-python - CURRENT_TAG_VERSION=$(git describe --tags | cut -f2 -dv) - python3 continuous-delivery/pip-install-with-retry.py --no-cache-dir --user awsiotsdk==$CURRENT_TAG_VERSION - python3 samples/basic_discovery.py --region us-east-1 --cert /tmp/certificate.pem --key /tmp/privatekey.pem --root-ca /tmp/AmazonRootCA1.pem --thing-name aws-sdk-crt-unit-test --print-discover-resp-only -v Trace diff --git a/continuous-delivery/test_test_pypi.yml b/continuous-delivery/test_test_pypi.yml index 1fd0e51..84830c1 100644 --- a/continuous-delivery/test_test_pypi.yml +++ b/continuous-delivery/test_test_pypi.yml @@ -17,7 +17,7 @@ phases: build: commands: - echo Build started on `date` - - cd aws-iot-device-sdk-python-v2 + - cd aws-iot-device-sdk-python - CURRENT_TAG_VERSION=$(git describe --tags | cut -f2 -dv) # this is here because typing isn't in testpypi, so pull it from prod instead - python3 -m pip install typing From d08006aa2f8c5c5356f9becfbac08c7e671a9ef2 Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Mon, 11 Apr 2022 15:12:27 -0700 Subject: [PATCH 36/66] update deployment test and scripts --- continuous-delivery/publish_to_prod_pypi.yml | 2 +- continuous-delivery/publish_to_test_pypi.yml | 4 ++-- continuous-delivery/test_prod_pypi.yml | 2 +- continuous-delivery/test_test_pypi.yml | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/continuous-delivery/publish_to_prod_pypi.yml b/continuous-delivery/publish_to_prod_pypi.yml index a3d739f..b6c63a4 100644 --- a/continuous-delivery/publish_to_prod_pypi.yml +++ b/continuous-delivery/publish_to_prod_pypi.yml @@ -10,7 +10,7 @@ phases: - python3 -m pip install --user --upgrade twine setuptools wheel awscli PyOpenSSL six pre_build: commands: - - cd aws-iot-device-sdk-python-v2 + - cd aws-iot-device-sdk-python - pypirc=$(aws secretsmanager get-secret-value --secret-id "prod/aws-crt-python/.pypirc" --query "SecretString" | cut -f2 -d\") && echo "$pypirc" > ~/.pypirc - export PKG_VERSION=$(git describe --tags | cut -f2 -dv) - echo "Updating package version to ${PKG_VERSION}" diff --git a/continuous-delivery/publish_to_test_pypi.yml b/continuous-delivery/publish_to_test_pypi.yml index 86cde41..ede62b8 100644 --- a/continuous-delivery/publish_to_test_pypi.yml +++ b/continuous-delivery/publish_to_test_pypi.yml @@ -11,10 +11,10 @@ phases: pre_build: commands: - pypirc=$(aws secretsmanager get-secret-value --secret-id "alpha/aws-crt-python/.pypirc" --query "SecretString" | cut -f2 -d\") && echo "$pypirc" > ~/.pypirc - - cd aws-iot-device-sdk-python-v2 + - cd aws-iot-device-sdk-python - export PKG_VERSION=$(git describe --tags | cut -f2 -dv) - echo "Updating package version to ${PKG_VERSION}" - - sed --in-place -E "s/__version__ = '.+'/__version__ = '${PKG_VERSION}'/" awsiot/__init__.py + - sed --in-place -E "s/__version__ = '.+'/__version__ = '${PKG_VERSION}'/" AWSIoTPythonSDK/__init__.py build: commands: - echo Build started on `date` diff --git a/continuous-delivery/test_prod_pypi.yml b/continuous-delivery/test_prod_pypi.yml index 5897f38..da416bd 100644 --- a/continuous-delivery/test_prod_pypi.yml +++ b/continuous-delivery/test_prod_pypi.yml @@ -20,7 +20,7 @@ phases: - cd aws-iot-device-sdk-python - CURRENT_TAG_VERSION=$(git describe --tags | cut -f2 -dv) - python3 continuous-delivery/pip-install-with-retry.py --no-cache-dir --user awsiotsdk==$CURRENT_TAG_VERSION - - python3 samples/basic_discovery.py --region us-east-1 --cert /tmp/certificate.pem --key /tmp/privatekey.pem --root-ca /tmp/AmazonRootCA1.pem --thing-name aws-sdk-crt-unit-test --print-discover-resp-only -v Trace + - python3 samples/greengrass/basicDiscovery.py -e ${ENDPOINT} -c /tmp/certificate.pem -k /tmp/privatekey.pem -r /tmp/AmazonRootCA1.pem -n aws-sdk-crt-unit-test post_build: commands: diff --git a/continuous-delivery/test_test_pypi.yml b/continuous-delivery/test_test_pypi.yml index 84830c1..5b8adc5 100644 --- a/continuous-delivery/test_test_pypi.yml +++ b/continuous-delivery/test_test_pypi.yml @@ -21,8 +21,8 @@ phases: - CURRENT_TAG_VERSION=$(git describe --tags | cut -f2 -dv) # this is here because typing isn't in testpypi, so pull it from prod instead - python3 -m pip install typing - - python3 continuous-delivery/pip-install-with-retry.py -i https://testpypi.python.org/simple --user awsiotsdk==$CURRENT_TAG_VERSION - - python3 samples/basic_discovery.py --region us-east-1 --cert /tmp/certificate.pem --key /tmp/privatekey.pem --root-ca /tmp/AmazonRootCA1.pem --thing-name aws-sdk-crt-unit-test --print-discover-resp-only -v Trace + - python3 continuous-delivery/pip-install-with-retry.py -i https://testpypi.python.org/simple --user AWSIoTPythonSDK==$CURRENT_TAG_VERSION + - python3 samples/greengrass/basicDiscovery.py -e ${ENDPOINT} -c /tmp/certificate.pem -k /tmp/privatekey.pem -r /tmp/AmazonRootCA1.pem -n aws-sdk-crt-unit-test post_build: commands: From bece8387e2d2f15f7e9702e8f9119ffd744b402d Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Mon, 11 Apr 2022 15:17:05 -0700 Subject: [PATCH 37/66] add --universal option for python2&3 --- continuous-delivery/publish_to_prod_pypi.yml | 2 +- continuous-delivery/publish_to_test_pypi.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/continuous-delivery/publish_to_prod_pypi.yml b/continuous-delivery/publish_to_prod_pypi.yml index b6c63a4..e2fd958 100644 --- a/continuous-delivery/publish_to_prod_pypi.yml +++ b/continuous-delivery/publish_to_prod_pypi.yml @@ -18,7 +18,7 @@ phases: build: commands: - echo Build started on `date` - - python3 setup.py sdist bdist_wheel + - python3 setup.py sdist bdist_wheel --universal - python3 -m twine upload -r pypi dist/* post_build: commands: diff --git a/continuous-delivery/publish_to_test_pypi.yml b/continuous-delivery/publish_to_test_pypi.yml index ede62b8..d8ab916 100644 --- a/continuous-delivery/publish_to_test_pypi.yml +++ b/continuous-delivery/publish_to_test_pypi.yml @@ -18,7 +18,7 @@ phases: build: commands: - echo Build started on `date` - - python3 setup.py sdist bdist_wheel + - python3 setup.py sdist bdist_wheel --universal - python3 -m twine upload -r testpypi dist/* post_build: commands: From a5e922a45ff788fb5871d5ecf8a3e1f835d9df75 Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Mon, 11 Apr 2022 15:19:04 -0700 Subject: [PATCH 38/66] remove version check for test --- continuous-delivery/test_version_exists | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/continuous-delivery/test_version_exists b/continuous-delivery/test_version_exists index 3579dbc..6d9bcb8 100644 --- a/continuous-delivery/test_version_exists +++ b/continuous-delivery/test_version_exists @@ -8,10 +8,10 @@ CURRENT_TAG=$(git describe --tags | cut -f2 -dv) # convert v0.2.12-2-g50254a9 to 0.2.12 CURRENT_TAG_VERSION=$(git describe --tags | cut -f1 -d'-' | cut -f2 -dv) # if there's a hash on the tag, then this is not a release tagged commit -if [ "$CURRENT_TAG" != "$CURRENT_TAG_VERSION" ]; then - echo "Current tag version is not a release tag, cut a new release if you want to publish." - exit 1 -fi +# if [ "$CURRENT_TAG" != "$CURRENT_TAG_VERSION" ]; then +# echo "Current tag version is not a release tag, cut a new release if you want to publish." +# exit 1 +# fi if python3 -m pip install --no-cache-dir -vvv AWSIoTPythonSDK==$CURRENT_TAG_VERSION; then echo "$CURRENT_TAG_VERSION is already in pypi, cut a new tag if you want to upload another version." From 533b588a0a5a806bb9c614305b4bf9fd65e7556f Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Tue, 12 Apr 2022 11:13:16 -0700 Subject: [PATCH 39/66] update package name --- continuous-delivery/test_prod_pypi.yml | 2 +- setup_test.py | 34 ++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 setup_test.py diff --git a/continuous-delivery/test_prod_pypi.yml b/continuous-delivery/test_prod_pypi.yml index da416bd..fa49265 100644 --- a/continuous-delivery/test_prod_pypi.yml +++ b/continuous-delivery/test_prod_pypi.yml @@ -19,7 +19,7 @@ phases: - echo Build started on `date` - cd aws-iot-device-sdk-python - CURRENT_TAG_VERSION=$(git describe --tags | cut -f2 -dv) - - python3 continuous-delivery/pip-install-with-retry.py --no-cache-dir --user awsiotsdk==$CURRENT_TAG_VERSION + - python3 continuous-delivery/pip-install-with-retry.py --no-cache-dir --user AWSIoTPythonSDK==$CURRENT_TAG_VERSION - python3 samples/greengrass/basicDiscovery.py -e ${ENDPOINT} -c /tmp/certificate.pem -k /tmp/privatekey.pem -r /tmp/AmazonRootCA1.pem -n aws-sdk-crt-unit-test post_build: diff --git a/setup_test.py b/setup_test.py new file mode 100644 index 0000000..f9d2a64 --- /dev/null +++ b/setup_test.py @@ -0,0 +1,34 @@ +import sys +sys.path.insert(0, 'AWSIoTPythonSDK') +import AWSIoTPythonSDK +currentVersion = AWSIoTPythonSDK.__version__ + +from distutils.core import setup +setup( + name = 'AWSIoTPythonSDK-V1', + packages=['AWSIoTPythonSDK', 'AWSIoTPythonSDK.core', + 'AWSIoTPythonSDK.core.util', 'AWSIoTPythonSDK.core.shadow', 'AWSIoTPythonSDK.core.protocol', + 'AWSIoTPythonSDK.core.jobs', + 'AWSIoTPythonSDK.core.protocol.paho', 'AWSIoTPythonSDK.core.protocol.internal', + 'AWSIoTPythonSDK.core.protocol.connection', 'AWSIoTPythonSDK.core.greengrass', + 'AWSIoTPythonSDK.core.greengrass.discovery', 'AWSIoTPythonSDK.exception'], + version = currentVersion, + description = 'SDK for connecting to AWS IoT using Python.', + author = 'Amazon Web Service', + author_email = '', + url = 'https://github.com/aws/aws-iot-device-sdk-python.git', + download_url = 'https://s3.amazonaws.com/aws-iot-device-sdk-python/aws-iot-device-sdk-python-latest.zip', + keywords = ['aws', 'iot', 'mqtt'], + classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Natural Language :: English", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5" + ] +) From cd8b757e4922d521c84799f4589dcacb8c278bfd Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Tue, 12 Apr 2022 11:14:08 -0700 Subject: [PATCH 40/66] update test setup --- continuous-delivery/publish_to_test_pypi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/continuous-delivery/publish_to_test_pypi.yml b/continuous-delivery/publish_to_test_pypi.yml index d8ab916..6a1db5c 100644 --- a/continuous-delivery/publish_to_test_pypi.yml +++ b/continuous-delivery/publish_to_test_pypi.yml @@ -18,7 +18,7 @@ phases: build: commands: - echo Build started on `date` - - python3 setup.py sdist bdist_wheel --universal + - python3 setup_test.py sdist bdist_wheel --universal - python3 -m twine upload -r testpypi dist/* post_build: commands: From 24e1a4d0048864d0e441b0962817e14e385142da Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Tue, 12 Apr 2022 12:01:51 -0700 Subject: [PATCH 41/66] add logs flag for test-pypi --- AWSIoTPythonSDK/__init__.py | 2 +- continuous-delivery/publish_to_test_pypi.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/AWSIoTPythonSDK/__init__.py b/AWSIoTPythonSDK/__init__.py index 1ad354e..5b60188 100755 --- a/AWSIoTPythonSDK/__init__.py +++ b/AWSIoTPythonSDK/__init__.py @@ -1 +1 @@ -__version__ = "1.4.9" +__version__ = "1.5.0" diff --git a/continuous-delivery/publish_to_test_pypi.yml b/continuous-delivery/publish_to_test_pypi.yml index 6a1db5c..c8f610b 100644 --- a/continuous-delivery/publish_to_test_pypi.yml +++ b/continuous-delivery/publish_to_test_pypi.yml @@ -19,7 +19,7 @@ phases: commands: - echo Build started on `date` - python3 setup_test.py sdist bdist_wheel --universal - - python3 -m twine upload -r testpypi dist/* + - python3 -m twine upload -r testpypi dist/* --verbose post_build: commands: - echo Build completed on `date` From 3d48971f26bcbaf482980128f0b0316dbe3b119e Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Tue, 12 Apr 2022 12:57:33 -0700 Subject: [PATCH 42/66] update test test-pypi for new package name --- continuous-delivery/test_test_pypi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/continuous-delivery/test_test_pypi.yml b/continuous-delivery/test_test_pypi.yml index 5b8adc5..7552e79 100644 --- a/continuous-delivery/test_test_pypi.yml +++ b/continuous-delivery/test_test_pypi.yml @@ -21,7 +21,7 @@ phases: - CURRENT_TAG_VERSION=$(git describe --tags | cut -f2 -dv) # this is here because typing isn't in testpypi, so pull it from prod instead - python3 -m pip install typing - - python3 continuous-delivery/pip-install-with-retry.py -i https://testpypi.python.org/simple --user AWSIoTPythonSDK==$CURRENT_TAG_VERSION + - python3 continuous-delivery/pip-install-with-retry.py -i https://testpypi.python.org/simple --user AWSIoTPythonSDK-V1==$CURRENT_TAG_VERSION - python3 samples/greengrass/basicDiscovery.py -e ${ENDPOINT} -c /tmp/certificate.pem -k /tmp/privatekey.pem -r /tmp/AmazonRootCA1.pem -n aws-sdk-crt-unit-test post_build: From 04ad731a947ea149a963366d08c77bc59ceb04ec Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Tue, 12 Apr 2022 13:13:49 -0700 Subject: [PATCH 43/66] update set command for version number update --- AWSIoTPythonSDK/__init__.py | 2 +- continuous-delivery/publish_to_prod_pypi.yml | 2 +- continuous-delivery/publish_to_test_pypi.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/AWSIoTPythonSDK/__init__.py b/AWSIoTPythonSDK/__init__.py index 5b60188..1ad354e 100755 --- a/AWSIoTPythonSDK/__init__.py +++ b/AWSIoTPythonSDK/__init__.py @@ -1 +1 @@ -__version__ = "1.5.0" +__version__ = "1.4.9" diff --git a/continuous-delivery/publish_to_prod_pypi.yml b/continuous-delivery/publish_to_prod_pypi.yml index e2fd958..d911029 100644 --- a/continuous-delivery/publish_to_prod_pypi.yml +++ b/continuous-delivery/publish_to_prod_pypi.yml @@ -14,7 +14,7 @@ phases: - pypirc=$(aws secretsmanager get-secret-value --secret-id "prod/aws-crt-python/.pypirc" --query "SecretString" | cut -f2 -d\") && echo "$pypirc" > ~/.pypirc - export PKG_VERSION=$(git describe --tags | cut -f2 -dv) - echo "Updating package version to ${PKG_VERSION}" - - sed --in-place -E "s/__version__ = '.+'/__version__ = '${PKG_VERSION}'/" AWSIoTPythonSDK/__init__.py + - sed --in-place -E "s/__version__ = \".+\"/__version__ = \"${PKG_VERSION}\"/" AWSIoTPythonSDK/__init__.py build: commands: - echo Build started on `date` diff --git a/continuous-delivery/publish_to_test_pypi.yml b/continuous-delivery/publish_to_test_pypi.yml index c8f610b..49a69c9 100644 --- a/continuous-delivery/publish_to_test_pypi.yml +++ b/continuous-delivery/publish_to_test_pypi.yml @@ -14,7 +14,7 @@ phases: - cd aws-iot-device-sdk-python - export PKG_VERSION=$(git describe --tags | cut -f2 -dv) - echo "Updating package version to ${PKG_VERSION}" - - sed --in-place -E "s/__version__ = '.+'/__version__ = '${PKG_VERSION}'/" AWSIoTPythonSDK/__init__.py + - sed --in-place -E "s/__version__ = \".+\"/__version__ = \"${PKG_VERSION}\"/" AWSIoTPythonSDK/__init__.py build: commands: - echo Build started on `date` From d908131009cf59ae259de4f945febbc5bb49f1a9 Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Tue, 12 Apr 2022 13:21:17 -0700 Subject: [PATCH 44/66] hard code the version number for test --- AWSIoTPythonSDK/__init__.py | 2 +- continuous-delivery/publish_to_test_pypi.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/AWSIoTPythonSDK/__init__.py b/AWSIoTPythonSDK/__init__.py index 1ad354e..0f228f2 100755 --- a/AWSIoTPythonSDK/__init__.py +++ b/AWSIoTPythonSDK/__init__.py @@ -1 +1 @@ -__version__ = "1.4.9" +__version__ = "1.5.1" diff --git a/continuous-delivery/publish_to_test_pypi.yml b/continuous-delivery/publish_to_test_pypi.yml index 49a69c9..32317d4 100644 --- a/continuous-delivery/publish_to_test_pypi.yml +++ b/continuous-delivery/publish_to_test_pypi.yml @@ -14,7 +14,7 @@ phases: - cd aws-iot-device-sdk-python - export PKG_VERSION=$(git describe --tags | cut -f2 -dv) - echo "Updating package version to ${PKG_VERSION}" - - sed --in-place -E "s/__version__ = \".+\"/__version__ = \"${PKG_VERSION}\"/" AWSIoTPythonSDK/__init__.py + #- sed --in-place -E "s/__version__ = \".+\"/__version__ = \"${PKG_VERSION}\"/" AWSIoTPythonSDK/__init__.py build: commands: - echo Build started on `date` From 0738f24b039ecf76a67170dad58c630b8f7b3a58 Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Tue, 12 Apr 2022 13:35:26 -0700 Subject: [PATCH 45/66] hard code the version number for test --- AWSIoTPythonSDK/__init__.py | 2 +- continuous-delivery/test_test_pypi.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/AWSIoTPythonSDK/__init__.py b/AWSIoTPythonSDK/__init__.py index 0f228f2..4963389 100755 --- a/AWSIoTPythonSDK/__init__.py +++ b/AWSIoTPythonSDK/__init__.py @@ -1 +1 @@ -__version__ = "1.5.1" +__version__ = "1.4.8" diff --git a/continuous-delivery/test_test_pypi.yml b/continuous-delivery/test_test_pypi.yml index 7552e79..295731a 100644 --- a/continuous-delivery/test_test_pypi.yml +++ b/continuous-delivery/test_test_pypi.yml @@ -21,7 +21,7 @@ phases: - CURRENT_TAG_VERSION=$(git describe --tags | cut -f2 -dv) # this is here because typing isn't in testpypi, so pull it from prod instead - python3 -m pip install typing - - python3 continuous-delivery/pip-install-with-retry.py -i https://testpypi.python.org/simple --user AWSIoTPythonSDK-V1==$CURRENT_TAG_VERSION + - python3 continuous-delivery/pip-install-with-retry.py -i https://testpypi.python.org/simple --user AWSIoTPythonSDK-V1==1.4.8 - python3 samples/greengrass/basicDiscovery.py -e ${ENDPOINT} -c /tmp/certificate.pem -k /tmp/privatekey.pem -r /tmp/AmazonRootCA1.pem -n aws-sdk-crt-unit-test post_build: From 5d989a16287eef624e56682b473d01294ea1bb4e Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Tue, 12 Apr 2022 15:04:52 -0700 Subject: [PATCH 46/66] fix test command --- AWSIoTPythonSDK/__init__.py | 2 +- continuous-delivery/test_test_pypi.yml | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/AWSIoTPythonSDK/__init__.py b/AWSIoTPythonSDK/__init__.py index 4963389..ac329c9 100755 --- a/AWSIoTPythonSDK/__init__.py +++ b/AWSIoTPythonSDK/__init__.py @@ -1 +1 @@ -__version__ = "1.4.8" +__version__ = "1.4.7" diff --git a/continuous-delivery/test_test_pypi.yml b/continuous-delivery/test_test_pypi.yml index 295731a..c358c75 100644 --- a/continuous-delivery/test_test_pypi.yml +++ b/continuous-delivery/test_test_pypi.yml @@ -12,7 +12,9 @@ phases: commands: - curl https://www.amazontrust.com/repository/AmazonRootCA1.pem --output /tmp/AmazonRootCA1.pem - cert=$(aws secretsmanager get-secret-value --secret-id "unit-test/certificate" --query "SecretString" | cut -f2 -d":" | cut -f2 -d\") && echo "$cert" > /tmp/certificate.pem + - echo /tmp/certificate.pem - key=$(aws secretsmanager get-secret-value --secret-id "unit-test/privatekey" --query "SecretString" | cut -f2 -d":" | cut -f2 -d\") && echo "$key" > /tmp/privatekey.pem + - echo /tmp/certificate.pem - ENDPOINT=$(aws secretsmanager get-secret-value --secret-id "unit-test/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g') build: commands: @@ -22,7 +24,7 @@ phases: # this is here because typing isn't in testpypi, so pull it from prod instead - python3 -m pip install typing - python3 continuous-delivery/pip-install-with-retry.py -i https://testpypi.python.org/simple --user AWSIoTPythonSDK-V1==1.4.8 - - python3 samples/greengrass/basicDiscovery.py -e ${ENDPOINT} -c /tmp/certificate.pem -k /tmp/privatekey.pem -r /tmp/AmazonRootCA1.pem -n aws-sdk-crt-unit-test + - python3 samples/greengrass/basicDiscovery.py -e ${ENDPOINT} -c /tmp/certificate.pem -k /tmp/privatekey.pem -r /tmp/AmazonRootCA1.pem post_build: commands: From 244e2df48ae572c7bfd687c5e91998e2b66aee32 Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Tue, 12 Apr 2022 16:07:29 -0700 Subject: [PATCH 47/66] clean up and revert test changes --- AWSIoTPythonSDK/__init__.py | 2 +- continuous-delivery/publish_to_test_pypi.yml | 2 +- continuous-delivery/test_prod_pypi.yml | 2 +- continuous-delivery/test_test_pypi.yml | 5 ++--- continuous-delivery/test_version_exists | 10 +++++----- 5 files changed, 10 insertions(+), 11 deletions(-) diff --git a/AWSIoTPythonSDK/__init__.py b/AWSIoTPythonSDK/__init__.py index ac329c9..1ad354e 100755 --- a/AWSIoTPythonSDK/__init__.py +++ b/AWSIoTPythonSDK/__init__.py @@ -1 +1 @@ -__version__ = "1.4.7" +__version__ = "1.4.9" diff --git a/continuous-delivery/publish_to_test_pypi.yml b/continuous-delivery/publish_to_test_pypi.yml index 32317d4..49a69c9 100644 --- a/continuous-delivery/publish_to_test_pypi.yml +++ b/continuous-delivery/publish_to_test_pypi.yml @@ -14,7 +14,7 @@ phases: - cd aws-iot-device-sdk-python - export PKG_VERSION=$(git describe --tags | cut -f2 -dv) - echo "Updating package version to ${PKG_VERSION}" - #- sed --in-place -E "s/__version__ = \".+\"/__version__ = \"${PKG_VERSION}\"/" AWSIoTPythonSDK/__init__.py + - sed --in-place -E "s/__version__ = \".+\"/__version__ = \"${PKG_VERSION}\"/" AWSIoTPythonSDK/__init__.py build: commands: - echo Build started on `date` diff --git a/continuous-delivery/test_prod_pypi.yml b/continuous-delivery/test_prod_pypi.yml index fa49265..6c1a817 100644 --- a/continuous-delivery/test_prod_pypi.yml +++ b/continuous-delivery/test_prod_pypi.yml @@ -20,7 +20,7 @@ phases: - cd aws-iot-device-sdk-python - CURRENT_TAG_VERSION=$(git describe --tags | cut -f2 -dv) - python3 continuous-delivery/pip-install-with-retry.py --no-cache-dir --user AWSIoTPythonSDK==$CURRENT_TAG_VERSION - - python3 samples/greengrass/basicDiscovery.py -e ${ENDPOINT} -c /tmp/certificate.pem -k /tmp/privatekey.pem -r /tmp/AmazonRootCA1.pem -n aws-sdk-crt-unit-test + - python3 samples/greengrass/basicDiscovery.py -e ${ENDPOINT} -c /tmp/certificate.pem -k /tmp/privatekey.pem -r /tmp/AmazonRootCA1.pem post_build: commands: diff --git a/continuous-delivery/test_test_pypi.yml b/continuous-delivery/test_test_pypi.yml index c358c75..deec2a4 100644 --- a/continuous-delivery/test_test_pypi.yml +++ b/continuous-delivery/test_test_pypi.yml @@ -12,9 +12,7 @@ phases: commands: - curl https://www.amazontrust.com/repository/AmazonRootCA1.pem --output /tmp/AmazonRootCA1.pem - cert=$(aws secretsmanager get-secret-value --secret-id "unit-test/certificate" --query "SecretString" | cut -f2 -d":" | cut -f2 -d\") && echo "$cert" > /tmp/certificate.pem - - echo /tmp/certificate.pem - key=$(aws secretsmanager get-secret-value --secret-id "unit-test/privatekey" --query "SecretString" | cut -f2 -d":" | cut -f2 -d\") && echo "$key" > /tmp/privatekey.pem - - echo /tmp/certificate.pem - ENDPOINT=$(aws secretsmanager get-secret-value --secret-id "unit-test/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g') build: commands: @@ -23,7 +21,8 @@ phases: - CURRENT_TAG_VERSION=$(git describe --tags | cut -f2 -dv) # this is here because typing isn't in testpypi, so pull it from prod instead - python3 -m pip install typing - - python3 continuous-delivery/pip-install-with-retry.py -i https://testpypi.python.org/simple --user AWSIoTPythonSDK-V1==1.4.8 + - python3 continuous-delivery/pip-install-with-retry.py -i https://testpypi.python.org/simple --user AWSIoTPythonSDK-V1==$CURRENT_TAG_VERSION + # The greengrass test failed for no discover the correct thing. However it is good enough to test if the AWSIoTPythonSDK installed successfully or not. - python3 samples/greengrass/basicDiscovery.py -e ${ENDPOINT} -c /tmp/certificate.pem -k /tmp/privatekey.pem -r /tmp/AmazonRootCA1.pem post_build: diff --git a/continuous-delivery/test_version_exists b/continuous-delivery/test_version_exists index 6d9bcb8..eaed9c7 100644 --- a/continuous-delivery/test_version_exists +++ b/continuous-delivery/test_version_exists @@ -7,11 +7,11 @@ git describe --tags CURRENT_TAG=$(git describe --tags | cut -f2 -dv) # convert v0.2.12-2-g50254a9 to 0.2.12 CURRENT_TAG_VERSION=$(git describe --tags | cut -f1 -d'-' | cut -f2 -dv) -# if there's a hash on the tag, then this is not a release tagged commit -# if [ "$CURRENT_TAG" != "$CURRENT_TAG_VERSION" ]; then -# echo "Current tag version is not a release tag, cut a new release if you want to publish." -# exit 1 -# fi +if there's a hash on the tag, then this is not a release tagged commit +if [ "$CURRENT_TAG" != "$CURRENT_TAG_VERSION" ]; then + echo "Current tag version is not a release tag, cut a new release if you want to publish." + exit 1 +fi if python3 -m pip install --no-cache-dir -vvv AWSIoTPythonSDK==$CURRENT_TAG_VERSION; then echo "$CURRENT_TAG_VERSION is already in pypi, cut a new tag if you want to upload another version." From e8fca898fc7dbeb5646ea6a75af51e661b0dda81 Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Tue, 12 Apr 2022 17:01:53 -0700 Subject: [PATCH 48/66] add print_only mode for deployment test --- continuous-delivery/test_prod_pypi.yml | 2 +- continuous-delivery/test_test_pypi.yml | 3 +-- samples/greengrass/basicDiscovery.py | 8 +++++++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/continuous-delivery/test_prod_pypi.yml b/continuous-delivery/test_prod_pypi.yml index 6c1a817..4575306 100644 --- a/continuous-delivery/test_prod_pypi.yml +++ b/continuous-delivery/test_prod_pypi.yml @@ -20,7 +20,7 @@ phases: - cd aws-iot-device-sdk-python - CURRENT_TAG_VERSION=$(git describe --tags | cut -f2 -dv) - python3 continuous-delivery/pip-install-with-retry.py --no-cache-dir --user AWSIoTPythonSDK==$CURRENT_TAG_VERSION - - python3 samples/greengrass/basicDiscovery.py -e ${ENDPOINT} -c /tmp/certificate.pem -k /tmp/privatekey.pem -r /tmp/AmazonRootCA1.pem + - python3 samples/greengrass/basicDiscovery.py -e ${ENDPOINT} -c /tmp/certificate.pem -k /tmp/privatekey.pem -r /tmp/AmazonRootCA1.pem --print_discover_resp_only post_build: commands: diff --git a/continuous-delivery/test_test_pypi.yml b/continuous-delivery/test_test_pypi.yml index deec2a4..c3aa47d 100644 --- a/continuous-delivery/test_test_pypi.yml +++ b/continuous-delivery/test_test_pypi.yml @@ -22,8 +22,7 @@ phases: # this is here because typing isn't in testpypi, so pull it from prod instead - python3 -m pip install typing - python3 continuous-delivery/pip-install-with-retry.py -i https://testpypi.python.org/simple --user AWSIoTPythonSDK-V1==$CURRENT_TAG_VERSION - # The greengrass test failed for no discover the correct thing. However it is good enough to test if the AWSIoTPythonSDK installed successfully or not. - - python3 samples/greengrass/basicDiscovery.py -e ${ENDPOINT} -c /tmp/certificate.pem -k /tmp/privatekey.pem -r /tmp/AmazonRootCA1.pem + - python3 samples/greengrass/basicDiscovery.py -e ${ENDPOINT} -c /tmp/certificate.pem -k /tmp/privatekey.pem -r /tmp/AmazonRootCA1.pem --print_discover_resp_only post_build: commands: diff --git a/samples/greengrass/basicDiscovery.py b/samples/greengrass/basicDiscovery.py index cd73e43..a6fcd61 100644 --- a/samples/greengrass/basicDiscovery.py +++ b/samples/greengrass/basicDiscovery.py @@ -47,6 +47,8 @@ def customOnMessage(message): help="Operation modes: %s"%str(AllowedActions)) parser.add_argument("-M", "--message", action="store", dest="message", default="Hello World!", help="Message to publish") +#--print_discover_resp_only used for delopyment testing. The test run will return 0 as long as the SDK installed correctly. +parser.add_argument("-p", "--print_discover_resp_only", action="store_true", dest="print_only", default=False) args = parser.parse_args() host = args.host @@ -56,6 +58,7 @@ def customOnMessage(message): clientId = args.thingName thingName = args.thingName topic = args.topic +print_only = args.print_only if args.mode not in AllowedActions: parser.error("Unknown --mode option %s. Must be one of %s" % (args.mode, str(AllowedActions))) @@ -94,7 +97,7 @@ def customOnMessage(message): discoveryInfoProvider.configureCredentials(rootCAPath, certificatePath, privateKeyPath) discoveryInfoProvider.configureTimeout(10) # 10 sec -retryCount = MAX_DISCOVERY_RETRIES +retryCount = MAX_DISCOVERY_RETRIES if not print_only else 1 discovered = False groupCA = None coreInfo = None @@ -136,6 +139,9 @@ def customOnMessage(message): backOffCore.backOff() if not discovered: + # With print_discover_resp_only flag, we only woud like to check if the API get called correctly. + if print_only: + sys.exit(0) print("Discovery failed after %d retries. Exiting...\n" % (MAX_DISCOVERY_RETRIES)) sys.exit(-1) From 55526970e6a97fea0c690e303f2c551b2b885d8c Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Wed, 13 Apr 2022 09:19:58 -0700 Subject: [PATCH 49/66] setup a fake version number for testing --- AWSIoTPythonSDK/__init__.py | 2 +- continuous-delivery/publish_to_test_pypi.yml | 2 +- continuous-delivery/test_version_exists | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/AWSIoTPythonSDK/__init__.py b/AWSIoTPythonSDK/__init__.py index 1ad354e..bde0031 100755 --- a/AWSIoTPythonSDK/__init__.py +++ b/AWSIoTPythonSDK/__init__.py @@ -1 +1 @@ -__version__ = "1.4.9" +__version__ = "1.4.6" diff --git a/continuous-delivery/publish_to_test_pypi.yml b/continuous-delivery/publish_to_test_pypi.yml index 49a69c9..32317d4 100644 --- a/continuous-delivery/publish_to_test_pypi.yml +++ b/continuous-delivery/publish_to_test_pypi.yml @@ -14,7 +14,7 @@ phases: - cd aws-iot-device-sdk-python - export PKG_VERSION=$(git describe --tags | cut -f2 -dv) - echo "Updating package version to ${PKG_VERSION}" - - sed --in-place -E "s/__version__ = \".+\"/__version__ = \"${PKG_VERSION}\"/" AWSIoTPythonSDK/__init__.py + #- sed --in-place -E "s/__version__ = \".+\"/__version__ = \"${PKG_VERSION}\"/" AWSIoTPythonSDK/__init__.py build: commands: - echo Build started on `date` diff --git a/continuous-delivery/test_version_exists b/continuous-delivery/test_version_exists index eaed9c7..6d9bcb8 100644 --- a/continuous-delivery/test_version_exists +++ b/continuous-delivery/test_version_exists @@ -7,11 +7,11 @@ git describe --tags CURRENT_TAG=$(git describe --tags | cut -f2 -dv) # convert v0.2.12-2-g50254a9 to 0.2.12 CURRENT_TAG_VERSION=$(git describe --tags | cut -f1 -d'-' | cut -f2 -dv) -if there's a hash on the tag, then this is not a release tagged commit -if [ "$CURRENT_TAG" != "$CURRENT_TAG_VERSION" ]; then - echo "Current tag version is not a release tag, cut a new release if you want to publish." - exit 1 -fi +# if there's a hash on the tag, then this is not a release tagged commit +# if [ "$CURRENT_TAG" != "$CURRENT_TAG_VERSION" ]; then +# echo "Current tag version is not a release tag, cut a new release if you want to publish." +# exit 1 +# fi if python3 -m pip install --no-cache-dir -vvv AWSIoTPythonSDK==$CURRENT_TAG_VERSION; then echo "$CURRENT_TAG_VERSION is already in pypi, cut a new tag if you want to upload another version." From 2f237bd555ee281730e875d448a711f9437d8e7a Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Wed, 13 Apr 2022 09:41:22 -0700 Subject: [PATCH 50/66] setup a fake version number for testing --- AWSIoTPythonSDK/__init__.py | 2 +- continuous-delivery/test_test_pypi.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/AWSIoTPythonSDK/__init__.py b/AWSIoTPythonSDK/__init__.py index bde0031..56dadec 100755 --- a/AWSIoTPythonSDK/__init__.py +++ b/AWSIoTPythonSDK/__init__.py @@ -1 +1 @@ -__version__ = "1.4.6" +__version__ = "1.4.5" diff --git a/continuous-delivery/test_test_pypi.yml b/continuous-delivery/test_test_pypi.yml index c3aa47d..42310e4 100644 --- a/continuous-delivery/test_test_pypi.yml +++ b/continuous-delivery/test_test_pypi.yml @@ -21,7 +21,7 @@ phases: - CURRENT_TAG_VERSION=$(git describe --tags | cut -f2 -dv) # this is here because typing isn't in testpypi, so pull it from prod instead - python3 -m pip install typing - - python3 continuous-delivery/pip-install-with-retry.py -i https://testpypi.python.org/simple --user AWSIoTPythonSDK-V1==$CURRENT_TAG_VERSION + - python3 continuous-delivery/pip-install-with-retry.py -i https://testpypi.python.org/simple --user AWSIoTPythonSDK-V1==1.4.5 - python3 samples/greengrass/basicDiscovery.py -e ${ENDPOINT} -c /tmp/certificate.pem -k /tmp/privatekey.pem -r /tmp/AmazonRootCA1.pem --print_discover_resp_only post_build: From 04432b1d20e43a9f4d0418290ca5e4dd8401c414 Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Wed, 13 Apr 2022 10:02:42 -0700 Subject: [PATCH 51/66] revert testing changes --- AWSIoTPythonSDK/__init__.py | 2 +- continuous-delivery/publish_to_test_pypi.yml | 2 +- continuous-delivery/test_test_pypi.yml | 2 +- continuous-delivery/test_version_exists | 10 +++++----- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/AWSIoTPythonSDK/__init__.py b/AWSIoTPythonSDK/__init__.py index 56dadec..1ad354e 100755 --- a/AWSIoTPythonSDK/__init__.py +++ b/AWSIoTPythonSDK/__init__.py @@ -1 +1 @@ -__version__ = "1.4.5" +__version__ = "1.4.9" diff --git a/continuous-delivery/publish_to_test_pypi.yml b/continuous-delivery/publish_to_test_pypi.yml index 32317d4..49a69c9 100644 --- a/continuous-delivery/publish_to_test_pypi.yml +++ b/continuous-delivery/publish_to_test_pypi.yml @@ -14,7 +14,7 @@ phases: - cd aws-iot-device-sdk-python - export PKG_VERSION=$(git describe --tags | cut -f2 -dv) - echo "Updating package version to ${PKG_VERSION}" - #- sed --in-place -E "s/__version__ = \".+\"/__version__ = \"${PKG_VERSION}\"/" AWSIoTPythonSDK/__init__.py + - sed --in-place -E "s/__version__ = \".+\"/__version__ = \"${PKG_VERSION}\"/" AWSIoTPythonSDK/__init__.py build: commands: - echo Build started on `date` diff --git a/continuous-delivery/test_test_pypi.yml b/continuous-delivery/test_test_pypi.yml index 42310e4..c3aa47d 100644 --- a/continuous-delivery/test_test_pypi.yml +++ b/continuous-delivery/test_test_pypi.yml @@ -21,7 +21,7 @@ phases: - CURRENT_TAG_VERSION=$(git describe --tags | cut -f2 -dv) # this is here because typing isn't in testpypi, so pull it from prod instead - python3 -m pip install typing - - python3 continuous-delivery/pip-install-with-retry.py -i https://testpypi.python.org/simple --user AWSIoTPythonSDK-V1==1.4.5 + - python3 continuous-delivery/pip-install-with-retry.py -i https://testpypi.python.org/simple --user AWSIoTPythonSDK-V1==$CURRENT_TAG_VERSION - python3 samples/greengrass/basicDiscovery.py -e ${ENDPOINT} -c /tmp/certificate.pem -k /tmp/privatekey.pem -r /tmp/AmazonRootCA1.pem --print_discover_resp_only post_build: diff --git a/continuous-delivery/test_version_exists b/continuous-delivery/test_version_exists index 6d9bcb8..eaed9c7 100644 --- a/continuous-delivery/test_version_exists +++ b/continuous-delivery/test_version_exists @@ -7,11 +7,11 @@ git describe --tags CURRENT_TAG=$(git describe --tags | cut -f2 -dv) # convert v0.2.12-2-g50254a9 to 0.2.12 CURRENT_TAG_VERSION=$(git describe --tags | cut -f1 -d'-' | cut -f2 -dv) -# if there's a hash on the tag, then this is not a release tagged commit -# if [ "$CURRENT_TAG" != "$CURRENT_TAG_VERSION" ]; then -# echo "Current tag version is not a release tag, cut a new release if you want to publish." -# exit 1 -# fi +if there's a hash on the tag, then this is not a release tagged commit +if [ "$CURRENT_TAG" != "$CURRENT_TAG_VERSION" ]; then + echo "Current tag version is not a release tag, cut a new release if you want to publish." + exit 1 +fi if python3 -m pip install --no-cache-dir -vvv AWSIoTPythonSDK==$CURRENT_TAG_VERSION; then echo "$CURRENT_TAG_VERSION is already in pypi, cut a new tag if you want to upload another version." From ce1b26adaa38cb1b0cd6f85dbb7638bacc942607 Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Wed, 13 Apr 2022 10:05:26 -0700 Subject: [PATCH 52/66] revert test changes --- continuous-delivery/test_version_exists | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/continuous-delivery/test_version_exists b/continuous-delivery/test_version_exists index eaed9c7..3579dbc 100644 --- a/continuous-delivery/test_version_exists +++ b/continuous-delivery/test_version_exists @@ -7,7 +7,7 @@ git describe --tags CURRENT_TAG=$(git describe --tags | cut -f2 -dv) # convert v0.2.12-2-g50254a9 to 0.2.12 CURRENT_TAG_VERSION=$(git describe --tags | cut -f1 -d'-' | cut -f2 -dv) -if there's a hash on the tag, then this is not a release tagged commit +# if there's a hash on the tag, then this is not a release tagged commit if [ "$CURRENT_TAG" != "$CURRENT_TAG_VERSION" ]; then echo "Current tag version is not a release tag, cut a new release if you want to publish." exit 1 From 6a7da4ea321a3b7007e34cf926cad8b0185d25c6 Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Wed, 13 Apr 2022 10:09:05 -0700 Subject: [PATCH 53/66] add comments to explain about test pypi package name --- setup_test.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/setup_test.py b/setup_test.py index f9d2a64..820d715 100644 --- a/setup_test.py +++ b/setup_test.py @@ -1,3 +1,7 @@ +# For test deployment with package AWSIoTPythonSDK. The package name has already taken. Therefore we used an +# alternative name for test pypi. +# prod_pypi : AWSIoTPythonSDK +# test_pypi : AWSIoTPythonSDK-V1 import sys sys.path.insert(0, 'AWSIoTPythonSDK') import AWSIoTPythonSDK From 8e964489ee21bf84dc3bee1a1435f77b5af526a0 Mon Sep 17 00:00:00 2001 From: Michael Graeb Date: Thu, 2 Jun 2022 10:11:28 -0700 Subject: [PATCH 54/66] Give explicit permissions to Github actions (#314) because actions in this repo no longer have write permission by default. --- .github/workflows/closed-issue-message.yml | 2 ++ .github/workflows/stale_issue.yml | 3 +++ 2 files changed, 5 insertions(+) diff --git a/.github/workflows/closed-issue-message.yml b/.github/workflows/closed-issue-message.yml index 3340afb..22bf2a7 100644 --- a/.github/workflows/closed-issue-message.yml +++ b/.github/workflows/closed-issue-message.yml @@ -5,6 +5,8 @@ on: jobs: auto_comment: runs-on: ubuntu-latest + permissions: + issues: write steps: - uses: aws-actions/closed-issue-message@v1 with: diff --git a/.github/workflows/stale_issue.yml b/.github/workflows/stale_issue.yml index 8c50bb7..804c1f0 100644 --- a/.github/workflows/stale_issue.yml +++ b/.github/workflows/stale_issue.yml @@ -9,6 +9,9 @@ jobs: cleanup: runs-on: ubuntu-latest name: Stale issue job + permissions: + issues: write + pull-requests: write steps: - uses: aws-actions/stale-issue-cleanup@v3 with: From f3521d104bd30e41ea88aef9a2a683146a22d044 Mon Sep 17 00:00:00 2001 From: Khem Raj Date: Tue, 14 Jun 2022 11:47:27 -0700 Subject: [PATCH 55/66] setup.py: Use setuptools instead of distutils (#305) distutils is deprecated and will be gone in 3.12+ Signed-off-by: Khem Raj --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3846bae..d0c2b8d 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ import AWSIoTPythonSDK currentVersion = AWSIoTPythonSDK.__version__ -from distutils.core import setup +from setuptools import setup setup( name = 'AWSIoTPythonSDK', packages=['AWSIoTPythonSDK', 'AWSIoTPythonSDK.core', From 39c35f61b30e089e413b833c8a44a40bfc3b2e95 Mon Sep 17 00:00:00 2001 From: TwistedTwigleg Date: Thu, 18 Aug 2022 08:48:28 -0400 Subject: [PATCH 56/66] Use PyPi token specific to the Python V1 SDK (#317) --- continuous-delivery/publish_to_prod_pypi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/continuous-delivery/publish_to_prod_pypi.yml b/continuous-delivery/publish_to_prod_pypi.yml index d911029..905d849 100644 --- a/continuous-delivery/publish_to_prod_pypi.yml +++ b/continuous-delivery/publish_to_prod_pypi.yml @@ -11,7 +11,7 @@ phases: pre_build: commands: - cd aws-iot-device-sdk-python - - pypirc=$(aws secretsmanager get-secret-value --secret-id "prod/aws-crt-python/.pypirc" --query "SecretString" | cut -f2 -d\") && echo "$pypirc" > ~/.pypirc + - pypirc=$(aws secretsmanager get-secret-value --secret-id "prod/aws-sdk-python-v1/.pypirc" --query "SecretString" | cut -f2 -d\") && echo "$pypirc" > ~/.pypirc - export PKG_VERSION=$(git describe --tags | cut -f2 -dv) - echo "Updating package version to ${PKG_VERSION}" - sed --in-place -E "s/__version__ = \".+\"/__version__ = \"${PKG_VERSION}\"/" AWSIoTPythonSDK/__init__.py From dcbd4e734864fbad10773d1d6ba53f88c5c8a4e0 Mon Sep 17 00:00:00 2001 From: TwistedTwigleg Date: Mon, 22 May 2023 18:42:34 -0400 Subject: [PATCH 57/66] Fix CI failing due to Python version (#325) --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d6704f2..8ff9adb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ env: jobs: unit-tests: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 strategy: fail-fast: false @@ -23,7 +23,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: '3.6' + python-version: '3.6.7' - name: Unit tests run: | python3 setup.py install @@ -49,4 +49,4 @@ jobs: pip install pytest pip install mock pip install boto3 - ./test-integration/run/run.sh ${{ matrix.test-type }} 1000 100 7 \ No newline at end of file + ./test-integration/run/run.sh ${{ matrix.test-type }} 1000 100 7 From fdda2ecc36c2ccf5be19281b2be49d353f4e93f4 Mon Sep 17 00:00:00 2001 From: Joseph Klix Date: Tue, 23 May 2023 07:20:53 -0700 Subject: [PATCH 58/66] update time to ancient (#324) Co-authored-by: Noah Beard --- .github/workflows/stale_issue.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale_issue.yml b/.github/workflows/stale_issue.yml index 804c1f0..cbcc8b4 100644 --- a/.github/workflows/stale_issue.yml +++ b/.github/workflows/stale_issue.yml @@ -35,7 +35,7 @@ jobs: # Issue timing days-before-stale: 7 days-before-close: 4 - days-before-ancient: 190 + days-before-ancient: 36500 # If you don't want to mark a issue as being ancient based on a # threshold of "upvotes", you can set this here. An "upvote" is From f78e330bfc4f007be5ecbd269b2429718e9b25e2 Mon Sep 17 00:00:00 2001 From: Shailja Khurana <117320115+khushail@users.noreply.github.com> Date: Thu, 20 Jul 2023 14:17:22 -0700 Subject: [PATCH 59/66] added workflow for handling answerable discussions (#328) --- .github/workflows/handle-stale-discussions.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .github/workflows/handle-stale-discussions.yml diff --git a/.github/workflows/handle-stale-discussions.yml b/.github/workflows/handle-stale-discussions.yml new file mode 100644 index 0000000..e92e660 --- /dev/null +++ b/.github/workflows/handle-stale-discussions.yml @@ -0,0 +1,18 @@ +name: HandleStaleDiscussions +on: +schedule: +- cron: '0 */4 * * *' +discussion_comment: +types: [created] + +jobs: +handle-stale-discussions: +name: Handle stale discussions +runs-on: ubuntu-latest +permissions: + discussions: write +steps: + - name: Stale discussions action + uses: aws-github-ops/handle-stale-discussions@v1 + env: + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} \ No newline at end of file From 800c16e7ef8b8c2ec84befdf2e9f76fa2f431fd7 Mon Sep 17 00:00:00 2001 From: Bret Ambrose Date: Wed, 14 Feb 2024 09:47:40 -0800 Subject: [PATCH 60/66] Use ssl context wrap socket in Python3.12+ (#338) * Use ssl context wrap socket in Python3.12+ * Remove redundant test modes --------- Co-authored-by: Bret Ambrose --- .github/workflows/ci.yml | 8 +-- .../core/greengrass/discovery/providers.py | 25 +++++--- AWSIoTPythonSDK/core/protocol/paho/client.py | 42 +++++++++---- test-integration/run/run.sh | 59 +++++++------------ 4 files changed, 75 insertions(+), 59 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8ff9adb..fd09c7e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,11 +36,10 @@ jobs: strategy: fail-fast: false matrix: - test-type: [ MutualAuth, MutualAuthT , Websocket, ALPN, ALPNT] - python-version: [ '2.x', '3.x' ] - #[MutualAuth, Websocket, ALPN] + test-type: [ MutualAuth, Websocket, ALPN ] + python-version: [ '3.7', '3.12' ] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} @@ -49,4 +48,5 @@ jobs: pip install pytest pip install mock pip install boto3 + python --version ./test-integration/run/run.sh ${{ matrix.test-type }} 1000 100 7 diff --git a/AWSIoTPythonSDK/core/greengrass/discovery/providers.py b/AWSIoTPythonSDK/core/greengrass/discovery/providers.py index b37b086..192f71a 100644 --- a/AWSIoTPythonSDK/core/greengrass/discovery/providers.py +++ b/AWSIoTPythonSDK/core/greengrass/discovery/providers.py @@ -261,17 +261,28 @@ def _create_ssl_connection(self, sock): ssl_sock = ssl_context.wrap_socket(sock, server_hostname=self._host, do_handshake_on_connect=False) ssl_sock.do_handshake() else: - ssl_sock = ssl.wrap_socket(sock, - certfile=self._cert_path, - keyfile=self._key_path, - ca_certs=self._ca_path, - cert_reqs=ssl.CERT_REQUIRED, - ssl_version=ssl_protocol_version) + # To keep the SSL Context update minimal, only apply forced ssl context to python3.12+ + force_ssl_context = sys.version_info[0] > 3 or (sys.version_info[0] == 3 and sys.version_info[1] >= 12) + if force_ssl_context: + ssl_context = ssl.SSLContext(ssl_protocol_version) + ssl_context.load_cert_chain(self._cert_path, self._key_path) + ssl_context.load_verify_locations(self._ca_path) + ssl_context.verify_mode = ssl.CERT_REQUIRED + + ssl_sock = ssl_context.wrap_socket(sock) + else: + ssl_sock = ssl.wrap_socket(sock, + certfile=self._cert_path, + keyfile=self._key_path, + ca_certs=self._ca_path, + cert_reqs=ssl.CERT_REQUIRED, + ssl_version=ssl_protocol_version) self._logger.debug("Matching host name...") if sys.version_info[0] < 3 or (sys.version_info[0] == 3 and sys.version_info[1] < 2): self._tls_match_hostname(ssl_sock) - else: + elif sys.version_info[0] == 3 and sys.version_info[1] < 7: + # host name verification is handled internally in Python3.7+ ssl.match_hostname(ssl_sock.getpeercert(), self._host) return ssl_sock diff --git a/AWSIoTPythonSDK/core/protocol/paho/client.py b/AWSIoTPythonSDK/core/protocol/paho/client.py index 4216829..0b637c5 100755 --- a/AWSIoTPythonSDK/core/protocol/paho/client.py +++ b/AWSIoTPythonSDK/core/protocol/paho/client.py @@ -793,11 +793,22 @@ def reconnect(self): verify_hostname = self._tls_insecure is False # Decide whether we need to verify hostname + # To keep the SSL Context update minimal, only apply forced ssl context to python3.12+ + force_ssl_context = sys.version_info[0] > 3 or (sys.version_info[0] == 3 and sys.version_info[1] >= 12) + if self._tls_ca_certs is not None: if self._useSecuredWebsocket: # Never assign to ._ssl before wss handshake is finished # Non-None value for ._ssl will allow ops before wss-MQTT connection is established - rawSSL = ssl.wrap_socket(sock, ca_certs=self._tls_ca_certs, cert_reqs=ssl.CERT_REQUIRED) # Add server certificate verification + if force_ssl_context: + ssl_context = ssl.SSLContext() + ssl_context.load_verify_locations(self._tls_ca_certs) + ssl_context.verify_mode = ssl.CERT_REQUIRED + + rawSSL = ssl_context.wrap_socket(sock) + else: + rawSSL = ssl.wrap_socket(sock, ca_certs=self._tls_ca_certs, cert_reqs=ssl.CERT_REQUIRED) # Add server certificate verification + rawSSL.setblocking(0) # Non-blocking socket self._ssl = SecuredWebSocketCore(rawSSL, self._host, self._port, self._AWSAccessKeyIDCustomConfig, self._AWSSecretAccessKeyCustomConfig, self._AWSSessionTokenCustomConfig) # Override the _ssl socket # self._ssl.enableDebug() @@ -816,19 +827,30 @@ def reconnect(self): verify_hostname = False # Since check_hostname in SSLContext is already set to True, no need to verify it again self._ssl.do_handshake() else: - self._ssl = ssl.wrap_socket( - sock, - certfile=self._tls_certfile, - keyfile=self._tls_keyfile, - ca_certs=self._tls_ca_certs, - cert_reqs=self._tls_cert_reqs, - ssl_version=self._tls_version, - ciphers=self._tls_ciphers) + if force_ssl_context: + ssl_context = ssl.SSLContext(self._tls_version) + ssl_context.load_cert_chain(self._tls_certfile, self._tls_keyfile) + ssl_context.load_verify_locations(self._tls_ca_certs) + ssl_context.verify_mode = self._tls_cert_reqs + if self._tls_ciphers is not None: + ssl_context.set_ciphers(self._tls_ciphers) + + self._ssl = ssl_context.wrap_socket(sock) + else: + self._ssl = ssl.wrap_socket( + sock, + certfile=self._tls_certfile, + keyfile=self._tls_keyfile, + ca_certs=self._tls_ca_certs, + cert_reqs=self._tls_cert_reqs, + ssl_version=self._tls_version, + ciphers=self._tls_ciphers) if verify_hostname: if sys.version_info[0] < 3 or (sys.version_info[0] == 3 and sys.version_info[1] < 5): # No IP host match before 3.5.x self._tls_match_hostname() - else: + elif sys.version_info[0] == 3 and sys.version_info[1] < 7: + # host name verification is handled internally in Python3.7+ ssl.match_hostname(self._ssl.getpeercert(), self._host) self._sock = sock diff --git a/test-integration/run/run.sh b/test-integration/run/run.sh index f420f73..0eb933b 100755 --- a/test-integration/run/run.sh +++ b/test-integration/run/run.sh @@ -35,16 +35,13 @@ USAGE="usage: run.sh ${CREDENTIAL_DIR}certificate.pem.crt - python ${RetrieveAWSKeys} ${AWSSetName_privatekey} > ${CREDENTIAL_DIR}privateKey.pem.key + python ${RetrieveAWSKeys} ${AWSSetName_certificate} > ${CREDENTIAL_DIR}certificate.pem.crt + python ${RetrieveAWSKeys} ${AWSSetName_privatekey} > ${CREDENTIAL_DIR}privateKey.pem.key curl -s "${CA_CERT_URL}" > ${CA_CERT_PATH} - echo -e "URL retrieved certificate data:\n$(cat ${CA_CERT_PATH})\n" - python ${RetrieveAWSKeys} ${AWSDRSName_certificate} > ${CREDENTIAL_DIR}certificate_drs.pem.crt - python ${RetrieveAWSKeys} ${AWSDRSName_privatekey} > ${CREDENTIAL_DIR}privateKey_drs.pem.key - elif [ "$1"x == "Websocket"x -o "$1"x == "WebsocketT"x ]; then - ACCESS_KEY_ID_ARN=$(python ${RetrieveAWSKeys} ${AWSSecretForWebsocket_TodWorker_KeyId}) + echo -e "URL retrieved certificate data\n" + python ${RetrieveAWSKeys} ${AWSDRSName_certificate} > ${CREDENTIAL_DIR}certificate_drs.pem.crt + python ${RetrieveAWSKeys} ${AWSDRSName_privatekey} > ${CREDENTIAL_DIR}privateKey_drs.pem.key + elif [ "$1"x == "Websocket"x ]; then + ACCESS_KEY_ID_ARN=$(python ${RetrieveAWSKeys} ${AWSSecretForWebsocket_TodWorker_KeyId}) ACCESS_SECRET_KEY_ARN=$(python ${RetrieveAWSKeys} ${AWSSecretForWebsocket_TodWorker_SecretKey}) TestMode="Websocket" - if [ "$1"x == "WebsocketT"x ]; then - ACCESS_KEY_ID_ARN=$(python ${RetrieveAWSKeys} ${AWSSecretForWebsocket_Desktop_KeyId}) - ACCESS_SECRET_KEY_ARN=$(python ${RetrieveAWSKeys} ${AWSSecretForWebsocket_Desktop_SecretKey}) - fi - echo ${ACCESS_KEY_ID_ARN} - echo ${ACCESS_SECRET_KEY_ARN} export AWS_ACCESS_KEY_ID=${ACCESS_KEY_ID_ARN} export AWS_SECRET_ACCESS_KEY=${ACCESS_SECRET_KEY_ARN} curl -s "${CA_CERT_URL}" > ${CA_CERT_PATH} - echo -e "URL retrieved certificate data:\n$(cat ${CA_CERT_PATH})\n" - elif [ "$1"x == "ALPN"x -o "$1"x == "ALPNT"x ]; then + echo -e "URL retrieved certificate data\n" + elif [ "$1"x == "ALPN"x ]; then AWSSetName_privatekey=${AWSMutualAuth_TodWorker_private_key} - AWSSetName_certificate=${AWSMutualAuth_TodWorker_certificate} - AWSDRSName_privatekey=${AWSGGDiscovery_TodWorker_private_key} + AWSSetName_certificate=${AWSMutualAuth_TodWorker_certificate} + AWSDRSName_privatekey=${AWSGGDiscovery_TodWorker_private_key} AWSDRSName_certificate=${AWSGGDiscovery_TodWorker_certificate} TestMode="ALPN" - if [ "$1"x == "ALPNT"x ]; then - AWSSetName_privatekey=${AWSMutualAuth_Desktop_private_key} - AWSSetName_certificate=${AWSMutualAuth_Desktop_certificate} - fi python ${RetrieveAWSKeys} ${AWSSetName_certificate} > ${CREDENTIAL_DIR}certificate.pem.crt - python ${RetrieveAWSKeys} ${AWSSetName_privatekey} > ${CREDENTIAL_DIR}privateKey.pem.key + python ${RetrieveAWSKeys} ${AWSSetName_privatekey} > ${CREDENTIAL_DIR}privateKey.pem.key curl -s "${CA_CERT_URL}" > ${CA_CERT_PATH} - echo -e "URL retrieved certificate data:\n$(cat ${CA_CERT_PATH})\n" - python ${RetrieveAWSKeys} ${AWSDRSName_certificate} > ${CREDENTIAL_DIR}certificate_drs.pem.crt - python ${RetrieveAWSKeys} ${AWSDRSName_privatekey} > ${CREDENTIAL_DIR}privateKey_drs.pem.key + echo -e "URL retrieved certificate data\n" + python ${RetrieveAWSKeys} ${AWSDRSName_certificate} > ${CREDENTIAL_DIR}certificate_drs.pem.crt + python ${RetrieveAWSKeys} ${AWSDRSName_privatekey} > ${CREDENTIAL_DIR}privateKey_drs.pem.key else - echo "Mode not supported" - exit 1 + echo "Mode not supported" + exit 1 fi # Obtain ZIP package and unzip it locally echo ${TestMode} From 3865ac9eec6274dac41f37cabd86e896e0e3bdbc Mon Sep 17 00:00:00 2001 From: Bret Ambrose Date: Wed, 14 Feb 2024 11:24:23 -0800 Subject: [PATCH 61/66] Manual version update for pending release (#339) Co-authored-by: Bret Ambrose --- AWSIoTPythonSDK/__init__.py | 2 +- CHANGELOG.rst | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/AWSIoTPythonSDK/__init__.py b/AWSIoTPythonSDK/__init__.py index 1ad354e..a06ff4e 100755 --- a/AWSIoTPythonSDK/__init__.py +++ b/AWSIoTPythonSDK/__init__.py @@ -1 +1 @@ -__version__ = "1.4.9" +__version__ = "1.5.3" diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e63030c..5ebb619 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,9 @@ CHANGELOG ========= +1.5.3 +* improvement: Support Python3.12+ by conditionally removing deprecated API usage + 1.4.9 ===== * bugfix: Fixing possible race condition with timer in deviceShadow. From 567e3dbd0e027e990e4ecfc423b98629e13d37f3 Mon Sep 17 00:00:00 2001 From: Bret Ambrose Date: Wed, 14 Feb 2024 13:24:09 -0800 Subject: [PATCH 62/66] Iterate flushing CD pipeline (#340) Co-authored-by: Bret Ambrose --- AWSIoTPythonSDK/__init__.py | 2 +- CHANGELOG.rst | 3 +++ continuous-delivery/publish_to_test_pypi.yml | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/AWSIoTPythonSDK/__init__.py b/AWSIoTPythonSDK/__init__.py index a06ff4e..3a384fb 100755 --- a/AWSIoTPythonSDK/__init__.py +++ b/AWSIoTPythonSDK/__init__.py @@ -1 +1 @@ -__version__ = "1.5.3" +__version__ = "1.5.4" diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 5ebb619..15ad4b5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,9 @@ CHANGELOG ========= +1.5.4 +* chore: CD pipeline flushing try #1 + 1.5.3 * improvement: Support Python3.12+ by conditionally removing deprecated API usage diff --git a/continuous-delivery/publish_to_test_pypi.yml b/continuous-delivery/publish_to_test_pypi.yml index 49a69c9..c435e5e 100644 --- a/continuous-delivery/publish_to_test_pypi.yml +++ b/continuous-delivery/publish_to_test_pypi.yml @@ -10,7 +10,7 @@ phases: - python3 -m pip install --user --upgrade twine setuptools wheel awscli PyOpenSSL six pre_build: commands: - - pypirc=$(aws secretsmanager get-secret-value --secret-id "alpha/aws-crt-python/.pypirc" --query "SecretString" | cut -f2 -d\") && echo "$pypirc" > ~/.pypirc + - pypirc=$(aws secretsmanager get-secret-value --secret-id "alpha/aws-sdk-python-v1/.pypirc" --query "SecretString" | cut -f2 -d\") && echo "$pypirc" > ~/.pypirc - cd aws-iot-device-sdk-python - export PKG_VERSION=$(git describe --tags | cut -f2 -dv) - echo "Updating package version to ${PKG_VERSION}" From 37de5162375eafde639742b113ebb2925170b51e Mon Sep 17 00:00:00 2001 From: Ashish Dhingra <67916761+ashishdhingra@users.noreply.github.com> Date: Fri, 25 Oct 2024 15:40:13 -0700 Subject: [PATCH 63/66] chore: Modified bug issue template to add checkbox to report potential regression. (#344) --- .github/ISSUE_TEMPLATE/bug-report.yml | 8 +++++ .../workflows/issue-regression-labeler.yml | 32 +++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 .github/workflows/issue-regression-labeler.yml diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml index 94d19fb..946fd2f 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.yml +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -12,6 +12,14 @@ body: description: What is the problem? A clear and concise description of the bug. validations: required: true + - type: checkboxes + id: regression + attributes: + label: Regression Issue + description: What is a regression? If it worked in a previous version but doesn't in the latest version, it's considered a regression. In this case, please provide specific version number in the report. + options: + - label: Select this option if this issue appears to be a regression. + required: false - type: textarea id: expected attributes: diff --git a/.github/workflows/issue-regression-labeler.yml b/.github/workflows/issue-regression-labeler.yml new file mode 100644 index 0000000..bd00071 --- /dev/null +++ b/.github/workflows/issue-regression-labeler.yml @@ -0,0 +1,32 @@ +# Apply potential regression label on issues +name: issue-regression-label +on: + issues: + types: [opened, edited] +jobs: + add-regression-label: + runs-on: ubuntu-latest + permissions: + issues: write + steps: + - name: Fetch template body + id: check_regression + uses: actions/github-script@v7 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TEMPLATE_BODY: ${{ github.event.issue.body }} + with: + script: | + const regressionPattern = /\[x\] Select this option if this issue appears to be a regression\./i; + const template = `${process.env.TEMPLATE_BODY}` + const match = regressionPattern.test(template); + core.setOutput('is_regression', match); + - name: Manage regression label + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + if [ "${{ steps.check_regression.outputs.is_regression }}" == "true" ]; then + gh issue edit ${{ github.event.issue.number }} --add-label "potential-regression" -R ${{ github.repository }} + else + gh issue edit ${{ github.event.issue.number }} --remove-label "potential-regression" -R ${{ github.repository }} + fi From b1f495dd556e670af147009d964ad707e98a458a Mon Sep 17 00:00:00 2001 From: Bret Ambrose Date: Mon, 28 Oct 2024 09:50:14 -0700 Subject: [PATCH 64/66] Update and clarify Python minimum version (#346) Co-authored-by: Bret Ambrose --- .github/workflows/ci.yml | 10 +++++----- CHANGELOG.rst | 6 ++++++ README.rst | 16 ++-------------- setup.py | 8 ++------ setup_test.py | 6 +----- 5 files changed, 16 insertions(+), 30 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fd09c7e..f1d8dfa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,10 +20,10 @@ jobs: fail-fast: false steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: - python-version: '3.6.7' + python-version: '3.8' - name: Unit tests run: | python3 setup.py install @@ -37,10 +37,10 @@ jobs: fail-fast: false matrix: test-type: [ MutualAuth, Websocket, ALPN ] - python-version: [ '3.7', '3.12' ] + python-version: [ '3.8', '3.13' ] steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Integration tests diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 15ad4b5..765c557 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,10 +2,16 @@ CHANGELOG ========= +1.5.5 +===== +* chore: Update minimum Python version based on current supportable levels + 1.5.4 +===== * chore: CD pipeline flushing try #1 1.5.3 +===== * improvement: Support Python3.12+ by conditionally removing deprecated API usage 1.4.9 diff --git a/README.rst b/README.rst index 991007f..ba88218 100755 --- a/README.rst +++ b/README.rst @@ -70,22 +70,10 @@ also allows the use of the same connection for shadow operations and non-shadow, Installation ~~~~~~~~~~~~ -Minimum Requirements +Requirements ____________________ -- Python 2.7+ or Python 3.3+ for X.509 certificate-based mutual authentication via port 8883 - and MQTT over WebSocket protocol with AWS Signature Version 4 authentication -- Python 2.7.10+ or Python 3.5+ for X.509 certificate-based mutual authentication via port 443 -- OpenSSL version 1.0.1+ (TLS version 1.2) compiled with the Python executable for - X.509 certificate-based mutual authentication - - To check your version of OpenSSL, use the following command in a Python interpreter: - - .. code-block:: python - - >>> import ssl - >>> ssl.OPENSSL_VERSION - +- Python3.8+. The SDK has worked for older Python versions in the past, but they are no longer formally supported. Over time, expect the minimum Python version to loosely track the minimum non-end-of-life version. Install from pip ________________ diff --git a/setup.py b/setup.py index d0c2b8d..0ca4cfa 100644 --- a/setup.py +++ b/setup.py @@ -20,15 +20,11 @@ download_url = 'https://s3.amazonaws.com/aws-iot-device-sdk-python/aws-iot-device-sdk-python-latest.zip', keywords = ['aws', 'iot', 'mqtt'], classifiers = [ - "Development Status :: 5 - Production/Stable", + "Development Status :: 6 - Mature", "Intended Audience :: Developers", "Natural Language :: English", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python", - "Programming Language :: Python :: 2.7", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.3", - "Programming Language :: Python :: 3.4", - "Programming Language :: Python :: 3.5" + "Programming Language :: Python :: 3" ] ) diff --git a/setup_test.py b/setup_test.py index 820d715..2a4e78e 100644 --- a/setup_test.py +++ b/setup_test.py @@ -24,15 +24,11 @@ download_url = 'https://s3.amazonaws.com/aws-iot-device-sdk-python/aws-iot-device-sdk-python-latest.zip', keywords = ['aws', 'iot', 'mqtt'], classifiers = [ - "Development Status :: 5 - Production/Stable", + "Development Status :: 6 - Mature", "Intended Audience :: Developers", "Natural Language :: English", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python", - "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.3", - "Programming Language :: Python :: 3.4", - "Programming Language :: Python :: 3.5" ] ) From 33016f1ee7f76423805a63a68385b94e8a7015f8 Mon Sep 17 00:00:00 2001 From: Steve Kim <86316075+sbSteveK@users.noreply.github.com> Date: Tue, 26 Nov 2024 12:02:46 -0800 Subject: [PATCH 65/66] Switch to using a role for CI (#348) * switch to using a role for integrated tests * separate python * setup permision&test against CI_PubSub_Role * test against iot account * update host * test host * use a ci job role for test * update role permission * update tests to use host argument * quick run discovery test * quick test for gg discovery * migrate gg discovery test * more comments --------- Co-authored-by: Vera Xia --- .github/workflows/ci.yml | 11 +++++-- .../workflows/handle-stale-discussions.yml | 29 ++++++++++--------- ...estAsyncAPIGeneralNotificationCallbacks.py | 6 ++-- ...IntegrationTestAutoReconnectResubscribe.py | 4 +-- .../IntegrationTestClientReusability.py | 8 ++--- ...nTestConfigurablePublishMessageQueueing.py | 4 +-- .../IntegrationTestDiscovery.py | 11 +++---- .../IntegrationTestJobsClient.py | 4 +-- .../IntegrationTestMQTTConnection.py | 4 +-- ...tOfflineQueueingForSubscribeUnsubscribe.py | 6 ++-- .../IntegrationTestProgressiveBackoff.py | 4 +-- .../IntegrationTests/IntegrationTestShadow.py | 4 +-- .../TestToolLibrary/checkInManager.py | 6 ++-- test-integration/run/run.sh | 27 +++++++++-------- 14 files changed, 69 insertions(+), 59 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f1d8dfa..195bf2d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,8 +8,8 @@ on: env: RUN: ${{ github.run_id }}-${{ github.run_number }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: us-east-1 + CI_SDK_V1_ROLE: arn:aws:iam::180635532705:role/CI_SDK_V1_ROLE PACKAGE_NAME: aws-iot-device-sdk-python AWS_EC2_METADATA_DISABLED: true @@ -33,6 +33,9 @@ jobs: integration-tests: runs-on: ubuntu-latest + permissions: + id-token: write # This is required for requesting the JWT + contents: read # This is required for actions/checkout strategy: fail-fast: false matrix: @@ -43,6 +46,10 @@ jobs: - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + - uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ env.CI_SDK_V1_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: Integration tests run: | pip install pytest diff --git a/.github/workflows/handle-stale-discussions.yml b/.github/workflows/handle-stale-discussions.yml index e92e660..4fbcd70 100644 --- a/.github/workflows/handle-stale-discussions.yml +++ b/.github/workflows/handle-stale-discussions.yml @@ -1,18 +1,19 @@ name: HandleStaleDiscussions on: -schedule: -- cron: '0 */4 * * *' -discussion_comment: -types: [created] + schedule: + - cron: '0 */4 * * *' + discussion_comment: + types: [created] jobs: -handle-stale-discussions: -name: Handle stale discussions -runs-on: ubuntu-latest -permissions: - discussions: write -steps: - - name: Stale discussions action - uses: aws-github-ops/handle-stale-discussions@v1 - env: - GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} \ No newline at end of file + handle-stale-discussions: + name: Handle stale discussions + runs-on: ubuntu-latest + permissions: + discussions: write + steps: + - name: Stale discussions action + uses: aws-github-ops/handle-stale-discussions@v1 + env: + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} + \ No newline at end of file diff --git a/test-integration/IntegrationTests/IntegrationTestAsyncAPIGeneralNotificationCallbacks.py b/test-integration/IntegrationTests/IntegrationTestAsyncAPIGeneralNotificationCallbacks.py index b69fdcd..577c5fa 100644 --- a/test-integration/IntegrationTests/IntegrationTestAsyncAPIGeneralNotificationCallbacks.py +++ b/test-integration/IntegrationTests/IntegrationTestAsyncAPIGeneralNotificationCallbacks.py @@ -32,7 +32,6 @@ TOPIC = "topic/test/async_cb/" MESSAGE_PREFIX = "MagicMessage-" NUMBER_OF_PUBLISHES = 3 -HOST = "ajje7lpljulm4-ats.iot.us-east-1.amazonaws.com" ROOT_CA = "./test-integration/Credentials/rootCA.crt" CERT = "./test-integration/Credentials/certificate.pem.crt" KEY = "./test-integration/Credentials/privateKey.pem.key" @@ -102,9 +101,10 @@ def get_random_string(length): ############################################################################ # Main # # Check inputs -my_check_in_manager = checkInManager(1) +my_check_in_manager = checkInManager(2) my_check_in_manager.verify(sys.argv) mode = my_check_in_manager.mode +host = my_check_in_manager.host skip_when_match(ModeIsALPN(mode).And( Python2VersionLowerThan((2, 7, 10)).Or(Python3VersionLowerThan((3, 5, 0))) @@ -115,7 +115,7 @@ def get_random_string(length): print("Connecting...") callback_manager = CallbackManager() sdk_mqtt_client = MQTTClientManager()\ - .create_nonconnected_mqtt_client(mode, CLIENT_ID, HOST, (ROOT_CA, CERT, KEY), callback_manager) + .create_nonconnected_mqtt_client(mode, CLIENT_ID, host, (ROOT_CA, CERT, KEY), callback_manager) sdk_mqtt_client.connectAsync(keepAliveIntervalSecond=1, ackCallback=callback_manager.connack) # Add callback print("Wait some time to make sure we are connected...") time.sleep(10) # 10 sec diff --git a/test-integration/IntegrationTests/IntegrationTestAutoReconnectResubscribe.py b/test-integration/IntegrationTests/IntegrationTestAutoReconnectResubscribe.py index 83b66c2..e6c1bee 100644 --- a/test-integration/IntegrationTests/IntegrationTestAutoReconnectResubscribe.py +++ b/test-integration/IntegrationTests/IntegrationTestAutoReconnectResubscribe.py @@ -135,14 +135,14 @@ def threadBRuntime(self, pyCoreClient, callback): ############################################################################ # Main # # Check inputs -myCheckInManager = checkInManager.checkInManager(1) +myCheckInManager = checkInManager.checkInManager(2) myCheckInManager.verify(sys.argv) -host = "ajje7lpljulm4-ats.iot.us-east-1.amazonaws.com" rootCA = "./test-integration/Credentials/rootCA.crt" certificate = "./test-integration/Credentials/certificate.pem.crt" privateKey = "./test-integration/Credentials/privateKey.pem.key" mode = myCheckInManager.mode +host = myCheckInManager.host skip_when_match(ModeIsALPN(mode).And( Python2VersionLowerThan((2, 7, 10)).Or(Python3VersionLowerThan((3, 5, 0))) diff --git a/test-integration/IntegrationTests/IntegrationTestClientReusability.py b/test-integration/IntegrationTests/IntegrationTestClientReusability.py index f747e19..56e77b8 100644 --- a/test-integration/IntegrationTests/IntegrationTestClientReusability.py +++ b/test-integration/IntegrationTests/IntegrationTestClientReusability.py @@ -40,7 +40,6 @@ NUMBER_OF_MESSAGES_PER_LOOP = 3 NUMBER_OF_LOOPS = 3 SUB_WAIT_TIME_OUT_SEC = 20 -HOST = "ajje7lpljulm4-ats.iot.us-east-1.amazonaws.com" ROOT_CA = "./test-integration/Credentials/rootCA.crt" CERT = "./test-integration/Credentials/certificate.pem.crt" KEY = "./test-integration/Credentials/privateKey.pem.key" @@ -94,9 +93,10 @@ def verify(self): ############################################################################ # Main # -my_check_in_manager = checkInManager(1) +my_check_in_manager = checkInManager(2) my_check_in_manager.verify(sys.argv) mode = my_check_in_manager.mode +host = my_check_in_manager.host skip_when_match(ModeIsALPN(mode).And( Python2VersionLowerThan((2, 7, 10)).Or(Python3VersionLowerThan((3, 5, 0))) @@ -104,9 +104,9 @@ def verify(self): simple_thread_manager = simpleThreadManager() -client_pub = MQTTClientManager().create_nonconnected_mqtt_client(mode, CLIENT_ID_PUB, HOST, (ROOT_CA, CERT, KEY)) +client_pub = MQTTClientManager().create_nonconnected_mqtt_client(mode, CLIENT_ID_PUB, host, (ROOT_CA, CERT, KEY)) print("Client publisher initialized.") -client_sub = MQTTClientManager().create_nonconnected_mqtt_client(mode, CLIENT_ID_SUB, HOST, (ROOT_CA, CERT, KEY)) +client_sub = MQTTClientManager().create_nonconnected_mqtt_client(mode, CLIENT_ID_SUB, host, (ROOT_CA, CERT, KEY)) print("Client subscriber initialized.") client_twins = ClientTwins(client_pub, client_sub) print("Client twins initialized.") diff --git a/test-integration/IntegrationTests/IntegrationTestConfigurablePublishMessageQueueing.py b/test-integration/IntegrationTests/IntegrationTestConfigurablePublishMessageQueueing.py index d6bfdc5..0d78f4f 100644 --- a/test-integration/IntegrationTests/IntegrationTestConfigurablePublishMessageQueueing.py +++ b/test-integration/IntegrationTests/IntegrationTestConfigurablePublishMessageQueueing.py @@ -274,10 +274,10 @@ def performConfigurableOfflinePublishQueueTest(clientPub, clientSub): # Check inputs -myCheckInManager = checkInManager.checkInManager(1) +myCheckInManager = checkInManager.checkInManager(2) myCheckInManager.verify(sys.argv) -host = "ajje7lpljulm4-ats.iot.us-east-1.amazonaws.com" +host = myCheckInManager.host rootCA = "./test-integration/Credentials/rootCA.crt" certificate = "./test-integration/Credentials/certificate.pem.crt" privateKey = "./test-integration/Credentials/privateKey.pem.key" diff --git a/test-integration/IntegrationTests/IntegrationTestDiscovery.py b/test-integration/IntegrationTests/IntegrationTestDiscovery.py index 8f23aa9..2fac25b 100644 --- a/test-integration/IntegrationTests/IntegrationTestDiscovery.py +++ b/test-integration/IntegrationTests/IntegrationTestDiscovery.py @@ -8,13 +8,13 @@ from TestToolLibrary.skip import ModeIsWebSocket -HOST = "arc9d2oott9lj-ats.iot.us-east-1.amazonaws.com" # 003261610643 PORT = 8443 CA = "./test-integration/Credentials/rootCA.crt" CERT = "./test-integration/Credentials/certificate_drs.pem.crt" KEY = "./test-integration/Credentials/privateKey_drs.pem.key" TIME_OUT_SEC = 30 # This is a pre-generated test data from DRS integration tests +# The test resources point to account # 003261610643 ID_PREFIX = "Id-" GGC_ARN = "arn:aws:iot:us-east-1:003261610643:thing/DRS_GGC_0kegiNGA_0" GGC_PORT_NUMBER_BASE = 8080 @@ -108,10 +108,14 @@ } ''' +my_check_in_manager = checkInManager(2) +my_check_in_manager.verify(sys.argv) +mode = my_check_in_manager.mode +host = my_check_in_manager.host def create_discovery_info_provider(): discovery_info_provider = DiscoveryInfoProvider() - discovery_info_provider.configureEndpoint(HOST, PORT) + discovery_info_provider.configureEndpoint(host, PORT) discovery_info_provider.configureCredentials(CA, CERT, KEY) discovery_info_provider.configureTimeout(TIME_OUT_SEC) return discovery_info_provider @@ -196,9 +200,6 @@ def verify_group_object(discovery_info): ############################################################################ # Main # -my_check_in_manager = checkInManager(1) -my_check_in_manager.verify(sys.argv) -mode = my_check_in_manager.mode skip_when_match(ModeIsWebSocket(mode), "This test is not applicable for mode: %s. Skipping..." % mode) diff --git a/test-integration/IntegrationTests/IntegrationTestJobsClient.py b/test-integration/IntegrationTests/IntegrationTestJobsClient.py index 18d8aa5..3653725 100644 --- a/test-integration/IntegrationTests/IntegrationTestJobsClient.py +++ b/test-integration/IntegrationTests/IntegrationTestJobsClient.py @@ -154,10 +154,10 @@ def _test_send_response_confirm(self, sendResult): ############################################################################ # Main # # Check inputs -myCheckInManager = checkInManager.checkInManager(1) +myCheckInManager = checkInManager.checkInManager(2) myCheckInManager.verify(sys.argv) -host = "ajje7lpljulm4-ats.iot.us-east-1.amazonaws.com" +host = myCheckInManager.host rootCA = "./test-integration/Credentials/rootCA.crt" certificate = "./test-integration/Credentials/certificate.pem.crt" privateKey = "./test-integration/Credentials/privateKey.pem.key" diff --git a/test-integration/IntegrationTests/IntegrationTestMQTTConnection.py b/test-integration/IntegrationTests/IntegrationTestMQTTConnection.py index 252770f..9adc38c 100644 --- a/test-integration/IntegrationTests/IntegrationTestMQTTConnection.py +++ b/test-integration/IntegrationTests/IntegrationTestMQTTConnection.py @@ -84,10 +84,10 @@ def _performPublish(self, pyCoreClient, topic, qos, payload): ############################################################################ # Main # # Check inputs -myCheckInManager = checkInManager.checkInManager(2) +myCheckInManager = checkInManager.checkInManager(3) myCheckInManager.verify(sys.argv) -host = "ajje7lpljulm4-ats.iot.us-east-1.amazonaws.com" +host = myCheckInManager.host rootCA = "./test-integration/Credentials/rootCA.crt" certificate = "./test-integration/Credentials/certificate.pem.crt" privateKey = "./test-integration/Credentials/privateKey.pem.key" diff --git a/test-integration/IntegrationTests/IntegrationTestOfflineQueueingForSubscribeUnsubscribe.py b/test-integration/IntegrationTests/IntegrationTestOfflineQueueingForSubscribeUnsubscribe.py index c06847d..37c1862 100644 --- a/test-integration/IntegrationTests/IntegrationTestOfflineQueueingForSubscribeUnsubscribe.py +++ b/test-integration/IntegrationTests/IntegrationTestOfflineQueueingForSubscribeUnsubscribe.py @@ -47,7 +47,6 @@ def get_random_string(length): TOPIC_B = "topic/test/offline_sub_unsub/b" + get_random_string(4) MESSAGE_PREFIX = "MagicMessage-" NUMBER_OF_PUBLISHES = 3 -HOST = "ajje7lpljulm4-ats.iot.us-east-1.amazonaws.com" ROOT_CA = "./test-integration/Credentials/rootCA.crt" CERT = "./test-integration/Credentials/certificate.pem.crt" KEY = "./test-integration/Credentials/privateKey.pem.key" @@ -74,7 +73,7 @@ def __init__(self, mode): time.sleep(2) # Make sure the subscription is valid def _create_connected_client(self, id_prefix): - return MQTTClientManager().create_connected_mqtt_client(self.__mode, id_prefix, HOST, (ROOT_CA, CERT, KEY)) + return MQTTClientManager().create_connected_mqtt_client(self.__mode, id_prefix, host, (ROOT_CA, CERT, KEY)) def start(self): thread_client_sub_unsub = Thread(target=self._thread_client_sub_unsub_runtime) @@ -192,9 +191,10 @@ def verify(self): ############################################################################ # Main # # Check inputs -my_check_in_manager = checkInManager(1) +my_check_in_manager = checkInManager(2) my_check_in_manager.verify(sys.argv) mode = my_check_in_manager.mode +host = my_check_in_manager.host skip_when_match(ModeIsALPN(mode).And( Python2VersionLowerThan((2, 7, 10)).Or(Python3VersionLowerThan((3, 5, 0))) diff --git a/test-integration/IntegrationTests/IntegrationTestProgressiveBackoff.py b/test-integration/IntegrationTests/IntegrationTestProgressiveBackoff.py index cd7b7ec..fc937ef 100644 --- a/test-integration/IntegrationTests/IntegrationTestProgressiveBackoff.py +++ b/test-integration/IntegrationTests/IntegrationTestProgressiveBackoff.py @@ -220,11 +220,11 @@ def verifyBackoffTime(answerList, resultList): ############################################################################ # Main # # Check inputs -myCheckInManager = checkInManager.checkInManager(2) +myCheckInManager = checkInManager.checkInManager(3) myCheckInManager.verify(sys.argv) #host via describe-endpoint on this OdinMS: com.amazonaws.iot.device.sdk.credentials.testing.websocket -host = "ajje7lpljulm4-ats.iot.us-east-1.amazonaws.com" +host = myCheckInManager.host rootCA = "./test-integration/Credentials/rootCA.crt" certificate = "./test-integration/Credentials/certificate.pem.crt" privateKey = "./test-integration/Credentials/privateKey.pem.key" diff --git a/test-integration/IntegrationTests/IntegrationTestShadow.py b/test-integration/IntegrationTests/IntegrationTestShadow.py index 9e2c2a5..9b2d85a 100644 --- a/test-integration/IntegrationTests/IntegrationTestShadow.py +++ b/test-integration/IntegrationTests/IntegrationTestShadow.py @@ -150,10 +150,10 @@ def randomString(lengthOfString): ############################################################################ # Main # # Check inputs -myCheckInManager = checkInManager.checkInManager(2) +myCheckInManager = checkInManager.checkInManager(3) myCheckInManager.verify(sys.argv) -host = "ajje7lpljulm4-ats.iot.us-east-1.amazonaws.com" +host = myCheckInManager.host rootCA = "./test-integration/Credentials/rootCA.crt" certificate = "./test-integration/Credentials/certificate.pem.crt" privateKey = "./test-integration/Credentials/privateKey.pem.key" diff --git a/test-integration/IntegrationTests/TestToolLibrary/checkInManager.py b/test-integration/IntegrationTests/TestToolLibrary/checkInManager.py index 2faaa02..aeeedd9 100644 --- a/test-integration/IntegrationTests/TestToolLibrary/checkInManager.py +++ b/test-integration/IntegrationTests/TestToolLibrary/checkInManager.py @@ -7,6 +7,7 @@ class checkInManager: def __init__(self, numberOfInputParameters): self._numberOfInputParameters = numberOfInputParameters self.mode = None + self.host = None self.customParameter = None def verify(self, args): @@ -14,5 +15,6 @@ def verify(self, args): if len(args) != self._numberOfInputParameters + 1: exit(4) self.mode = str(args[1]) - if self._numberOfInputParameters + 1 > 2: - self.customParameter = int(args[2]) + self.host = str(args[2]) + if self._numberOfInputParameters + 1 > 3: + self.customParameter = int(args[3]) diff --git a/test-integration/run/run.sh b/test-integration/run/run.sh index 0eb933b..8e23c91 100755 --- a/test-integration/run/run.sh +++ b/test-integration/run/run.sh @@ -33,14 +33,14 @@ # Define const USAGE="usage: run.sh " -AWSMutualAuth_TodWorker_private_key="arn:aws:secretsmanager:us-east-1:123124136734:secret:V1IotSdkIntegrationTestPrivateKey-vNUQU8" -AWSMutualAuth_TodWorker_certificate="arn:aws:secretsmanager:us-east-1:123124136734:secret:V1IotSdkIntegrationTestCertificate-vTRwjE" +UnitTestHostArn="arn:aws:secretsmanager:us-east-1:180635532705:secret:unit-test/endpoint-HSpeEu" +GreenGrassHostArn="arn:aws:secretsmanager:us-east-1:180635532705:secret:ci/greengrassv1/endpoint-DgM00X" -AWSGGDiscovery_TodWorker_private_key="arn:aws:secretsmanager:us-east-1:123124136734:secret:V1IotSdkIntegrationTestGGDiscoveryPrivateKey-YHQI1F" -AWSGGDiscovery_TodWorker_certificate="arn:aws:secretsmanager:us-east-1:123124136734:secret:V1IotSdkIntegrationTestGGDiscoveryCertificate-TwlAcS" +AWSMutualAuth_TodWorker_private_key="arn:aws:secretsmanager:us-east-1:180635532705:secret:ci/mqtt5/us/Mqtt5Prod/key-kqgyvf" +AWSMutualAuth_TodWorker_certificate="arn:aws:secretsmanager:us-east-1:180635532705:secret:ci/mqtt5/us/Mqtt5Prod/cert-VDI1Gd" -AWSSecretForWebsocket_TodWorker_KeyId="arn:aws:secretsmanager:us-east-1:123124136734:secret:V1IotSdkIntegrationTestWebsocketAccessKeyId-1YdB9z" -AWSSecretForWebsocket_TodWorker_SecretKey="arn:aws:secretsmanager:us-east-1:123124136734:secret:V1IotSdkIntegrationTestWebsocketSecretAccessKey-MKTSaV" +AWSGGDiscovery_TodWorker_private_key="arn:aws:secretsmanager:us-east-1:180635532705:secret:V1IotSdkIntegrationTestGGDiscoveryPrivateKey-BsLvNP" +AWSGGDiscovery_TodWorker_certificate="arn:aws:secretsmanager:us-east-1:180635532705:secret:V1IotSdkIntegrationTestGGDiscoveryCertificate-DSwdhA" SDKLocation="./AWSIoTPythonSDK" @@ -49,6 +49,8 @@ CREDENTIAL_DIR="./test-integration/Credentials/" TEST_DIR="./test-integration/IntegrationTests/" CA_CERT_URL="https://www.amazontrust.com/repository/AmazonRootCA1.pem" CA_CERT_PATH=${CREDENTIAL_DIR}rootCA.crt +TestHost=$(python ${RetrieveAWSKeys} ${UnitTestHostArn}) +GreengrassHost=$(python ${RetrieveAWSKeys} ${GreenGrassHostArn}) @@ -82,11 +84,7 @@ else python ${RetrieveAWSKeys} ${AWSDRSName_certificate} > ${CREDENTIAL_DIR}certificate_drs.pem.crt python ${RetrieveAWSKeys} ${AWSDRSName_privatekey} > ${CREDENTIAL_DIR}privateKey_drs.pem.key elif [ "$1"x == "Websocket"x ]; then - ACCESS_KEY_ID_ARN=$(python ${RetrieveAWSKeys} ${AWSSecretForWebsocket_TodWorker_KeyId}) - ACCESS_SECRET_KEY_ARN=$(python ${RetrieveAWSKeys} ${AWSSecretForWebsocket_TodWorker_SecretKey}) TestMode="Websocket" - export AWS_ACCESS_KEY_ID=${ACCESS_KEY_ID_ARN} - export AWS_SECRET_ACCESS_KEY=${ACCESS_SECRET_KEY_ARN} curl -s "${CA_CERT_URL}" > ${CA_CERT_PATH} echo -e "URL retrieved certificate data\n" elif [ "$1"x == "ALPN"x ]; then @@ -115,11 +113,11 @@ else echo "***************************************************" for file in `ls ${TEST_DIR}` do - # if [ ${file}x == "IntegrationTestMQTTConnection.py"x ]; then if [ ${file##*.}x == "py"x ]; then echo "[SUB] Running test: ${file}..." - + Scale=10 + Host=TestHost case "$file" in "IntegrationTestMQTTConnection.py") Scale=$2 ;; @@ -131,7 +129,8 @@ else ;; "IntegrationTestConfigurablePublishMessageQueueing.py") Scale="" ;; - "IntegrationTestDiscovery.py") Scale="" + "IntegrationTestDiscovery.py") Scale="" + Host=${GreengrassHost} ;; "IntegrationTestAsyncAPIGeneralNotificationCallbacks.py") Scale="" ;; @@ -142,7 +141,7 @@ else "IntegrationTestJobsClient.py") Scale="" esac - python ${TEST_DIR}${file} ${TestMode} ${Scale} + python ${TEST_DIR}${file} ${TestMode} ${TestHost} ${Scale} currentTestStatus=$? echo "[SUB] Test: ${file} completed. Exiting with status: ${currentTestStatus}" if [ ${currentTestStatus} -ne 0 ]; then From 057fda5720281407d6d054932a3f6b03c5b495fb Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Mon, 4 Aug 2025 09:10:47 -0700 Subject: [PATCH 66/66] Handle Suback Packet (#349) * handle ack packet * update suback packet * update runner image * improve error structure * add suback test --- .github/workflows/ci.yml | 2 +- AWSIoTPythonSDK/core/protocol/mqtt_core.py | 25 +++++++++++--- AWSIoTPythonSDK/core/protocol/paho/client.py | 4 +++ AWSIoTPythonSDK/exception/AWSIoTExceptions.py | 5 +++ test/core/protocol/test_mqtt_core.py | 33 ++++++++++++++++++- 5 files changed, 62 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 195bf2d..8776272 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ env: jobs: unit-tests: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest strategy: fail-fast: false diff --git a/AWSIoTPythonSDK/core/protocol/mqtt_core.py b/AWSIoTPythonSDK/core/protocol/mqtt_core.py index fbdd6bf..bff6456 100644 --- a/AWSIoTPythonSDK/core/protocol/mqtt_core.py +++ b/AWSIoTPythonSDK/core/protocol/mqtt_core.py @@ -28,7 +28,7 @@ from AWSIoTPythonSDK.core.protocol.internal.defaults import METRICS_PREFIX from AWSIoTPythonSDK.core.protocol.internal.defaults import ALPN_PROTCOLS from AWSIoTPythonSDK.core.protocol.internal.events import FixedEventMids -from AWSIoTPythonSDK.core.protocol.paho.client import MQTT_ERR_SUCCESS +from AWSIoTPythonSDK.core.protocol.paho.client import MQTT_ERR_SUCCESS, SUBACK_ERROR from AWSIoTPythonSDK.exception.AWSIoTExceptions import connectError from AWSIoTPythonSDK.exception.AWSIoTExceptions import connectTimeoutException from AWSIoTPythonSDK.exception.AWSIoTExceptions import disconnectError @@ -41,7 +41,7 @@ from AWSIoTPythonSDK.exception.AWSIoTExceptions import subscribeQueueDisabledException from AWSIoTPythonSDK.exception.AWSIoTExceptions import unsubscribeQueueFullException from AWSIoTPythonSDK.exception.AWSIoTExceptions import unsubscribeQueueDisabledException -from AWSIoTPythonSDK.exception.AWSIoTExceptions import subscribeError +from AWSIoTPythonSDK.exception.AWSIoTExceptions import subscribeError, subackError from AWSIoTPythonSDK.exception.AWSIoTExceptions import subscribeTimeoutException from AWSIoTPythonSDK.exception.AWSIoTExceptions import unsubscribeError from AWSIoTPythonSDK.exception.AWSIoTExceptions import unsubscribeTimeoutException @@ -58,6 +58,12 @@ from queue import Queue +class SubackPacket(object): + def __init__(self): + self.event = Event() + self.data = None + + class MqttCore(object): _logger = logging.getLogger(__name__) @@ -298,12 +304,15 @@ def subscribe(self, topic, qos, message_callback=None): if ClientStatus.STABLE != self._client_status.get_status(): self._handle_offline_request(RequestTypes.SUBSCRIBE, (topic, qos, message_callback, None)) else: - event = Event() - rc, mid = self._subscribe_async(topic, qos, self._create_blocking_ack_callback(event), message_callback) - if not event.wait(self._operation_timeout_sec): + suback = SubackPacket() + rc, mid = self._subscribe_async(topic, qos, self._create_blocking_suback_callback(suback), message_callback) + if not suback.event.wait(self._operation_timeout_sec): self._internal_async_client.remove_event_callback(mid) self._logger.error("Subscribe timed out") raise subscribeTimeoutException() + if suback.data and suback.data[0] == SUBACK_ERROR: + self._logger.error(f"Suback error return code: {suback.data[0]}") + raise subackError(suback=suback.data) ret = True return ret @@ -361,6 +370,12 @@ def ack_callback(mid, data=None): event.set() return ack_callback + def _create_blocking_suback_callback(self, ack: SubackPacket): + def ack_callback(mid, data=None): + ack.data = data + ack.event.set() + return ack_callback + def _handle_offline_request(self, type, data): self._logger.info("Offline request detected!") offline_request = QueueableRequest(type, data) diff --git a/AWSIoTPythonSDK/core/protocol/paho/client.py b/AWSIoTPythonSDK/core/protocol/paho/client.py index 0b637c5..1c60c81 100755 --- a/AWSIoTPythonSDK/core/protocol/paho/client.py +++ b/AWSIoTPythonSDK/core/protocol/paho/client.py @@ -97,6 +97,9 @@ CONNACK_REFUSED_BAD_USERNAME_PASSWORD = 4 CONNACK_REFUSED_NOT_AUTHORIZED = 5 +# SUBACK codes +SUBACK_ERROR = 0x80 + # Connection state mqtt_cs_new = 0 mqtt_cs_connected = 1 @@ -137,6 +140,7 @@ MSG_QUEUEING_DROP_OLDEST = 0 MSG_QUEUEING_DROP_NEWEST = 1 + if sys.version_info[0] < 3: sockpair_data = "0" else: diff --git a/AWSIoTPythonSDK/exception/AWSIoTExceptions.py b/AWSIoTPythonSDK/exception/AWSIoTExceptions.py index 0de5401..d5b6d63 100755 --- a/AWSIoTPythonSDK/exception/AWSIoTExceptions.py +++ b/AWSIoTPythonSDK/exception/AWSIoTExceptions.py @@ -79,6 +79,11 @@ class subscribeError(operationError.operationError): def __init__(self, errorCode): self.message = "Subscribe Error: " + str(errorCode) +class subackError(operationError.operationError): + def __init__(self, suback=None): + self.message = "Received Error suback. Subscription failed." + self.suback = suback + class subscribeQueueFullException(operationError.operationError): def __init__(self): diff --git a/test/core/protocol/test_mqtt_core.py b/test/core/protocol/test_mqtt_core.py index 8469ea6..3cb0cb0 100644 --- a/test/core/protocol/test_mqtt_core.py +++ b/test/core/protocol/test_mqtt_core.py @@ -1,5 +1,6 @@ import AWSIoTPythonSDK from AWSIoTPythonSDK.core.protocol.mqtt_core import MqttCore +from AWSIoTPythonSDK.core.protocol.mqtt_core import SubackPacket from AWSIoTPythonSDK.core.protocol.internal.clients import InternalAsyncMqttClient from AWSIoTPythonSDK.core.protocol.internal.clients import ClientStatusContainer from AWSIoTPythonSDK.core.protocol.internal.clients import ClientStatus @@ -20,6 +21,7 @@ from AWSIoTPythonSDK.exception.AWSIoTExceptions import publishQueueFullException from AWSIoTPythonSDK.exception.AWSIoTExceptions import publishQueueDisabledException from AWSIoTPythonSDK.exception.AWSIoTExceptions import subscribeError +from AWSIoTPythonSDK.exception.AWSIoTExceptions import subackError from AWSIoTPythonSDK.exception.AWSIoTExceptions import subscribeTimeoutException from AWSIoTPythonSDK.exception.AWSIoTExceptions import subscribeQueueFullException from AWSIoTPythonSDK.exception.AWSIoTExceptions import subscribeQueueDisabledException @@ -29,6 +31,7 @@ from AWSIoTPythonSDK.exception.AWSIoTExceptions import unsubscribeQueueDisabledException from AWSIoTPythonSDK.core.protocol.paho.client import MQTT_ERR_SUCCESS from AWSIoTPythonSDK.core.protocol.paho.client import MQTT_ERR_ERRNO +from AWSIoTPythonSDK.core.protocol.paho.client import SUBACK_ERROR from AWSIoTPythonSDK.core.protocol.paho.client import MQTTv311 from AWSIoTPythonSDK.core.protocol.internal.defaults import ALPN_PROTCOLS try: @@ -61,6 +64,7 @@ KEY_EXPECTED_QUEUE_APPEND_RESULT = "ExpectedQueueAppendResult" KEY_EXPECTED_REQUEST_MID_OVERRIDE = "ExpectedRequestMidOverride" KEY_EXPECTED_REQUEST_TIMEOUT = "ExpectedRequestTimeout" +KEY_EXPECTED_ACK_RESULT = "ExpectedAckPacketResult" SUCCESS_RC_EXPECTED_VALUES = { KEY_EXPECTED_REQUEST_RC : DUMMY_SUCCESS_RC } @@ -73,6 +77,10 @@ NO_TIMEOUT_EXPECTED_VALUES = { KEY_EXPECTED_REQUEST_TIMEOUT : False } +ERROR_SUBACK_EXPECTED_VALUES = { + KEY_EXPECTED_ACK_RESULT : (SUBACK_ERROR, None) +} + QUEUED_EXPECTED_VALUES = { KEY_EXPECTED_QUEUE_APPEND_RESULT : AppendResults.APPEND_SUCCESS } @@ -121,6 +129,9 @@ def setup_class(cls): RequestTypes.SUBSCRIBE: subscribeError, RequestTypes.UNSUBSCRIBE: unsubscribeError } + cls.ack_error = { + RequestTypes.SUBSCRIBE : subackError, + } cls.request_queue_full = { RequestTypes.PUBLISH : publishQueueFullException, RequestTypes.SUBSCRIBE: subscribeQueueFullException, @@ -518,6 +529,9 @@ def test_subscribe_success(self): def test_subscribe_timeout(self): self._internal_test_sync_api_with(RequestTypes.SUBSCRIBE, TIMEOUT_EXPECTED_VALUES) + + def test_subscribe_error_suback(self): + self._internal_test_sync_api_with(RequestTypes.SUBSCRIBE, ERROR_SUBACK_EXPECTED_VALUES) def test_subscribe_queued(self): self._internal_test_sync_api_with(RequestTypes.SUBSCRIBE, QUEUED_EXPECTED_VALUES) @@ -547,6 +561,7 @@ def _internal_test_sync_api_with(self, request_type, expected_values): expected_request_mid = expected_values.get(KEY_EXPECTED_REQUEST_MID_OVERRIDE) expected_timeout = expected_values.get(KEY_EXPECTED_REQUEST_TIMEOUT) expected_append_result = expected_values.get(KEY_EXPECTED_QUEUE_APPEND_RESULT) + expected_suback_result = expected_values.get(KEY_EXPECTED_ACK_RESULT) if expected_request_mid is None: expected_request_mid = DUMMY_REQUEST_MID @@ -562,7 +577,16 @@ def _internal_test_sync_api_with(self, request_type, expected_values): self.invoke_mqtt_core_sync_api[request_type](self, message_callback) else: self.python_event_mock.wait.return_value = True - assert self.invoke_mqtt_core_sync_api[request_type](self, message_callback) is True + if expected_suback_result is not None: + self._use_mock_python_suback() + # mock the suback with expected suback result + self.python_suback_mock.data = expected_suback_result + if expected_suback_result[0] == SUBACK_ERROR: + with pytest.raises(self.ack_error[request_type]): + self.invoke_mqtt_core_sync_api[request_type](self, message_callback) + self.python_suback_patcher.stop() + else: + assert self.invoke_mqtt_core_sync_api[request_type](self, message_callback) is True if expected_append_result is not None: self.client_status_mock.get_status.return_value = ClientStatus.ABNORMAL_DISCONNECT @@ -583,3 +607,10 @@ def _use_mock_python_event(self): self.python_event_constructor = self.python_event_patcher.start() self.python_event_mock = MagicMock() self.python_event_constructor.return_value = self.python_event_mock + + # Create a SubackPacket mock, which would mock the data in SubackPacket + def _use_mock_python_suback(self): + self.python_suback_patcher = patch(PATCH_MODULE_LOCATION + "SubackPacket", spec=SubackPacket) + self.python_suback_constructor = self.python_suback_patcher.start() + self.python_suback_mock = MagicMock() + self.python_suback_constructor.return_value = self.python_suback_mock