diff --git a/README.md b/README.md index ca1a0f9..248fde4 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ You can include this package in your preferred base image to make that base imag ## Requirements The Python Runtime Interface Client package currently supports Python versions: - - 3.7.x up to and including 3.11.x + - 3.7.x up to and including 3.12.x ## Usage diff --git a/RELEASE.CHANGELOG.md b/RELEASE.CHANGELOG.md index 8ae5c33..e14c364 100644 --- a/RELEASE.CHANGELOG.md +++ b/RELEASE.CHANGELOG.md @@ -1,3 +1,12 @@ +### October 30, 2023 + +`2.0.8`: + +- Onboarded Python3.12 ([#118](https://github.com/aws/aws-lambda-python-runtime-interface-client/pull/118)) +- Fix runtime_client blocking main thread from SIGTERM being handled. Enabled by default only for Python3.12 ([#115](https://github.com/aws/aws-lambda-python-runtime-interface-client/pull/115)) ([#124](https://github.com/aws/aws-lambda-python-runtime-interface-client/pull/124)) ([#125](https://github.com/aws/aws-lambda-python-runtime-interface-client/pull/125)) +- Use unicode chars instead of escape sequences in json encoder output. Enabled by default only for Python3.12 ([#88](https://github.com/aws/aws-lambda-python-runtime-interface-client/pull/88)) ([#122](https://github.com/aws/aws-lambda-python-runtime-interface-client/pull/122)) +- Cold start improvements ([#121](https://github.com/aws/aws-lambda-python-runtime-interface-client/pull/121)) + ### August 29, 2023 `2.0.7`: diff --git a/awslambdaric/__init__.py b/awslambdaric/__init__.py index b0184d1..f9e4637 100644 --- a/awslambdaric/__init__.py +++ b/awslambdaric/__init__.py @@ -2,4 +2,4 @@ Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. """ -__version__ = "2.0.7" +__version__ = "2.0.8" diff --git a/awslambdaric/bootstrap.py b/awslambdaric/bootstrap.py index a3da58c..f87ee1b 100644 --- a/awslambdaric/bootstrap.py +++ b/awslambdaric/bootstrap.py @@ -462,8 +462,14 @@ def run(app_root, handler, lambda_runtime_api_addr): sys.stdout = Unbuffered(sys.stdout) sys.stderr = Unbuffered(sys.stderr) + use_thread_for_polling_next = ( + os.environ.get("AWS_EXECUTION_ENV") == "AWS_Lambda_python3.12" + ) + with create_log_sink() as log_sink: - lambda_runtime_client = LambdaRuntimeClient(lambda_runtime_api_addr) + lambda_runtime_client = LambdaRuntimeClient( + lambda_runtime_api_addr, use_thread_for_polling_next + ) try: _setup_logging(_AWS_LAMBDA_LOG_FORMAT, _AWS_LAMBDA_LOG_LEVEL, log_sink) diff --git a/awslambdaric/lambda_runtime_client.py b/awslambdaric/lambda_runtime_client.py index 2066f6c..ba85902 100644 --- a/awslambdaric/lambda_runtime_client.py +++ b/awslambdaric/lambda_runtime_client.py @@ -2,10 +2,9 @@ Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. """ -import http -import http.client import sys from awslambdaric import __version__ +from .lambda_runtime_exception import FaultException def _user_agent(): @@ -50,10 +49,22 @@ class LambdaRuntimeClient(object): and response. It allows for function authors to override the the default implementation, LambdaMarshaller which unmarshals and marshals JSON, to an instance of a class that implements the same interface.""" - def __init__(self, lambda_runtime_address): + def __init__(self, lambda_runtime_address, use_thread_for_polling_next=False): self.lambda_runtime_address = lambda_runtime_address + self.use_thread_for_polling_next = use_thread_for_polling_next + if self.use_thread_for_polling_next: + # Conditionally import only for the case when TPE is used in this class. + from concurrent.futures import ThreadPoolExecutor + + # Not defining symbol as global to avoid relying on TPE being imported unconditionally. + self.ThreadPoolExecutor = ThreadPoolExecutor def post_init_error(self, error_response_data): + # These imports are heavy-weight. They implicitly trigger `import ssl, hashlib`. + # Importing them lazily to speed up critical path of a common case. + import http + import http.client + runtime_connection = http.client.HTTPConnection(self.lambda_runtime_address) runtime_connection.connect() endpoint = "/2018-06-01/runtime/init/error" @@ -65,7 +76,22 @@ def post_init_error(self, error_response_data): raise LambdaRuntimeClientError(endpoint, response.code, response_body) def wait_next_invocation(self): - response_body, headers = runtime_client.next() + # Calling runtime_client.next() from a separate thread unblocks the main thread, + # which can then process signals. + if self.use_thread_for_polling_next: + try: + # TPE class is supposed to be registered at construction time and be ready to use. + with self.ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit(runtime_client.next) + response_body, headers = future.result() + except Exception as e: + raise FaultException( + FaultException.LAMBDA_RUNTIME_CLIENT_ERROR, + "LAMBDA_RUNTIME Failed to get next invocation: {}".format(str(e)), + None, + ) + else: + response_body, headers = runtime_client.next() return InvocationRequest( invoke_id=headers.get("Lambda-Runtime-Aws-Request-Id"), x_amzn_trace_id=headers.get("Lambda-Runtime-Trace-Id"), diff --git a/awslambdaric/lambda_runtime_exception.py b/awslambdaric/lambda_runtime_exception.py index 416327e..e09af70 100644 --- a/awslambdaric/lambda_runtime_exception.py +++ b/awslambdaric/lambda_runtime_exception.py @@ -12,6 +12,7 @@ class FaultException(Exception): BUILT_IN_MODULE_CONFLICT = "Runtime.BuiltInModuleConflict" MALFORMED_HANDLER_NAME = "Runtime.MalformedHandlerName" LAMBDA_CONTEXT_UNMARSHAL_ERROR = "Runtime.LambdaContextUnmarshalError" + LAMBDA_RUNTIME_CLIENT_ERROR = "Runtime.LambdaRuntimeClientError" def __init__(self, exception_type, msg, trace=None): self.msg = msg diff --git a/awslambdaric/lambda_runtime_marshaller.py b/awslambdaric/lambda_runtime_marshaller.py index 7eee25d..42ee127 100644 --- a/awslambdaric/lambda_runtime_marshaller.py +++ b/awslambdaric/lambda_runtime_marshaller.py @@ -4,7 +4,7 @@ import decimal import math - +import os import simplejson as json from .lambda_runtime_exception import FaultException @@ -12,9 +12,13 @@ # simplejson's Decimal encoding allows '-NaN' as an output, which is a parse error for json.loads # to get the good parts of Decimal support, we'll special-case NaN decimals and otherwise duplicate the encoding for decimals the same way simplejson does +# We also set 'ensure_ascii=False' so that the encoded json contains unicode characters instead of unicode escape sequences class Encoder(json.JSONEncoder): def __init__(self): - super().__init__(use_decimal=False) + if os.environ.get("AWS_EXECUTION_ENV") == "AWS_Lambda_python3.12": + super().__init__(use_decimal=False, ensure_ascii=False) + else: + super().__init__(use_decimal=False) def default(self, obj): if isinstance(obj, decimal.Decimal): diff --git a/requirements/dev.txt b/requirements/dev.txt index c432413..68377ce 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -9,3 +9,4 @@ bandit>=1.6.2 # Test requirements pytest>=3.0.7 mock>=2.0.0 +parameterized>=0.9.0 \ No newline at end of file diff --git a/setup.py b/setup.py index a69c646..2544b21 100644 --- a/setup.py +++ b/setup.py @@ -90,6 +90,7 @@ def read_requirements(req="base.txt"): "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", ], diff --git a/tests/integration/codebuild/buildspec.os.alpine.yml b/tests/integration/codebuild/buildspec.os.alpine.yml index eba1d14..da09a26 100644 --- a/tests/integration/codebuild/buildspec.os.alpine.yml +++ b/tests/integration/codebuild/buildspec.os.alpine.yml @@ -24,6 +24,7 @@ batch: - "3.9" - "3.10" - "3.11" + - "3.12" phases: pre_build: commands: diff --git a/tests/integration/codebuild/buildspec.os.amazonlinux.1.yml b/tests/integration/codebuild/buildspec.os.amazonlinux.1.yml index 5ec01d2..91bb021 100644 --- a/tests/integration/codebuild/buildspec.os.amazonlinux.1.yml +++ b/tests/integration/codebuild/buildspec.os.amazonlinux.1.yml @@ -22,6 +22,7 @@ batch: - "3.9" - "3.10" - "3.11" + - "3.12" phases: pre_build: commands: diff --git a/tests/integration/codebuild/buildspec.os.amazonlinux.2.yml b/tests/integration/codebuild/buildspec.os.amazonlinux.2.yml index 18cabc9..38f2509 100644 --- a/tests/integration/codebuild/buildspec.os.amazonlinux.2.yml +++ b/tests/integration/codebuild/buildspec.os.amazonlinux.2.yml @@ -22,6 +22,7 @@ batch: - "3.9" - "3.10" - "3.11" + - "3.12" phases: pre_build: commands: diff --git a/tests/integration/codebuild/buildspec.os.centos.yml b/tests/integration/codebuild/buildspec.os.centos.yml index f993c7d..4058a1e 100644 --- a/tests/integration/codebuild/buildspec.os.centos.yml +++ b/tests/integration/codebuild/buildspec.os.centos.yml @@ -22,6 +22,7 @@ batch: - "3.9" - "3.10" - "3.11" + - "3.12" phases: pre_build: commands: diff --git a/tests/integration/codebuild/buildspec.os.debian.yml b/tests/integration/codebuild/buildspec.os.debian.yml index 48305bb..628fd95 100644 --- a/tests/integration/codebuild/buildspec.os.debian.yml +++ b/tests/integration/codebuild/buildspec.os.debian.yml @@ -23,6 +23,7 @@ batch: - "3.9" - "3.10" - "3.11" + - "3.12" phases: pre_build: commands: diff --git a/tests/integration/codebuild/buildspec.os.ubuntu.yml b/tests/integration/codebuild/buildspec.os.ubuntu.yml index 7c66865..b876817 100644 --- a/tests/integration/codebuild/buildspec.os.ubuntu.yml +++ b/tests/integration/codebuild/buildspec.os.ubuntu.yml @@ -23,6 +23,7 @@ batch: - "3.9" - "3.10" - "3.11" + - "3.12" phases: pre_build: commands: diff --git a/tests/test_bootstrap.py b/tests/test_bootstrap.py index ca367fd..83d31ee 100644 --- a/tests/test_bootstrap.py +++ b/tests/test_bootstrap.py @@ -589,10 +589,11 @@ def raise_exception_handler(json_input, lambda_context): self.assertEqual(mock_stdout.getvalue(), error_logs) - @patch("sys.stdout", new_callable=StringIO) + # The order of patches matter. Using MagicMock resets sys.stdout to the default. @patch("importlib.import_module") + @patch("sys.stdout", new_callable=StringIO) def test_handle_event_request_fault_exception_logging_syntax_error( - self, mock_import_module, mock_stdout + self, mock_stdout, mock_import_module ): try: eval("-") diff --git a/tests/test_lambda_runtime_client.py b/tests/test_lambda_runtime_client.py index 47d95cf..b0eae4a 100644 --- a/tests/test_lambda_runtime_client.py +++ b/tests/test_lambda_runtime_client.py @@ -84,6 +84,21 @@ def test_wait_next_invocation(self, mock_runtime_client): self.assertEqual(event_request.content_type, "application/json") self.assertEqual(event_request.event_body, response_body) + # Using ThreadPoolExecutor to polling next() + runtime_client = LambdaRuntimeClient("localhost:1234", True) + + event_request = runtime_client.wait_next_invocation() + + self.assertIsNotNone(event_request) + self.assertEqual(event_request.invoke_id, "RID1234") + self.assertEqual(event_request.x_amzn_trace_id, "TID1234") + self.assertEqual(event_request.invoked_function_arn, "FARN1234") + self.assertEqual(event_request.deadline_time_in_ms, 12) + self.assertEqual(event_request.client_context, "client_context") + self.assertEqual(event_request.cognito_identity, "cognito_identity") + self.assertEqual(event_request.content_type, "application/json") + self.assertEqual(event_request.event_body, response_body) + @patch("http.client.HTTPConnection", autospec=http.client.HTTPConnection) def test_post_init_error(self, MockHTTPConnection): mock_conn = MockHTTPConnection.return_value diff --git a/tests/test_lambda_runtime_marshaller.py b/tests/test_lambda_runtime_marshaller.py index 8268de1..7cd73b4 100644 --- a/tests/test_lambda_runtime_marshaller.py +++ b/tests/test_lambda_runtime_marshaller.py @@ -3,12 +3,35 @@ """ import decimal +import os import unittest - +from parameterized import parameterized from awslambdaric.lambda_runtime_marshaller import to_json class TestLambdaRuntimeMarshaller(unittest.TestCase): + execution_envs = ( + "AWS_Lambda_python3.12", + "AWS_Lambda_python3.11", + "AWS_Lambda_python3.10", + "AWS_Lambda_python3.9", + ) + + envs_lambda_marshaller_ensure_ascii_false = {"AWS_Lambda_python3.12"} + + execution_envs_lambda_marshaller_ensure_ascii_true = tuple( + set(execution_envs).difference(envs_lambda_marshaller_ensure_ascii_false) + ) + execution_envs_lambda_marshaller_ensure_ascii_false = tuple( + envs_lambda_marshaller_ensure_ascii_false + ) + + def setUp(self): + self.org_os_environ = os.environ + + def tearDown(self): + os.environ = self.org_os_environ + def test_to_json_decimal_encoding(self): response = to_json({"pi": decimal.Decimal("3.14159")}) self.assertEqual('{"pi": 3.14159}', response) @@ -37,3 +60,23 @@ def test_json_serializer_is_not_default_json(self): self.assertTrue(hasattr(internal_json, "YOLO")) self.assertFalse(hasattr(stock_json, "YOLO")) self.assertTrue(hasattr(simplejson, "YOLO")) + + @parameterized.expand(execution_envs_lambda_marshaller_ensure_ascii_false) + def test_to_json_unicode_not_escaped_encoding(self, execution_env): + os.environ = {"AWS_EXECUTION_ENV": execution_env} + response = to_json({"price": "£1.00"}) + self.assertEqual('{"price": "£1.00"}', response) + self.assertNotEqual('{"price": "\\u00a31.00"}', response) + self.assertEqual( + 19, len(response.encode("utf-8")) + ) # would be 23 bytes if a unicode escape was returned + + @parameterized.expand(execution_envs_lambda_marshaller_ensure_ascii_true) + def test_to_json_unicode_is_escaped_encoding(self, execution_env): + os.environ = {"AWS_EXECUTION_ENV": execution_env} + response = to_json({"price": "£1.00"}) + self.assertEqual('{"price": "\\u00a31.00"}', response) + self.assertNotEqual('{"price": "£1.00"}', response) + self.assertEqual( + 23, len(response.encode("utf-8")) + ) # would be 19 bytes if a escaped was returned