diff --git a/.github/workflows/test-on-push-and-pr.yml b/.github/workflows/test-on-push-and-pr.yml index fe17cda..5b80d23 100644 --- a/.github/workflows/test-on-push-and-pr.yml +++ b/.github/workflows/test-on-push-and-pr.yml @@ -11,12 +11,38 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - env: - GITHUB_WORKSPACE: / - - name: Set up python - uses: actions/setup-python@v2 - with: - python-version: '3.8' + - uses: actions/checkout@v4 - name: Run 'pr' target run: make pr + + alpine: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Run alpine integration tests + run: DISTRO=alpine make test-integ + + amazonlinux: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Run amazonlinux integration tests + run: DISTRO=amazonlinux make test-integ + + debian: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Run debian integration tests + run: DISTRO=debian make test-integ + + ubuntu: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Run ubuntu integration tests + run: DISTRO=ubuntu make test-integ \ No newline at end of file diff --git a/.gitignore b/.gitignore index bd5c9b7..9d46e4c 100644 --- a/.gitignore +++ b/.gitignore @@ -148,3 +148,8 @@ cython_debug/ # Test files generated tmp*.py + +# dependencies +deps/artifacts/ +deps/aws-lambda-cpp-*/ +deps/curl-*/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dcabbf4..bb66b03 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,5 +3,5 @@ repos: rev: 19.3b0 hooks: - id: black - language_version: python3.6 + language_version: python3.9 exclude_types: ['markdown', 'ini', 'toml', 'rst'] diff --git a/Makefile b/Makefile index ab2ba46..521b61c 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ test-smoke: setup-codebuild-agent .PHONY: test-integ test-integ: setup-codebuild-agent - CODEBUILD_IMAGE_TAG=codebuild-agent tests/integration/codebuild-local/test_all.sh tests/integration/codebuild/. + CODEBUILD_IMAGE_TAG=codebuild-agent DISTRO="$(DISTRO)" tests/integration/codebuild-local/test_all.sh tests/integration/codebuild/. .PHONY: check-security check-security: @@ -41,7 +41,10 @@ dev: init test # Verifications to run before sending a pull request .PHONY: pr -pr: init check-format check-security dev test-smoke +pr: init check-format check-security dev + +codebuild: setup-codebuild-agent + CODEBUILD_IMAGE_TAG=codebuild-agent DISTRO="$(DISTRO)" tests/integration/codebuild-local/test_all.sh tests/integration/codebuild .PHONY: clean clean: diff --git a/README.md b/README.md index ca1a0f9..4a96a3f 100644 --- a/README.md +++ b/README.md @@ -5,12 +5,12 @@ We have open-sourced a set of software packages, Runtime Interface Clients (RIC) base images to be Lambda compatible. The Lambda Runtime Interface Client is a lightweight interface that allows your runtime to receive requests from and send requests to the Lambda service. -The Lambda Python Runtime Interface Client is vended through [pip](https://pypi.org/project/awslambdaric). +The Lambda Python Runtime Interface Client is vended through [pip](https://pypi.org/project/awslambdaric). You can include this package in your preferred base image to make that base image Lambda compatible. ## Requirements The Python Runtime Interface Client package currently supports Python versions: - - 3.7.x up to and including 3.11.x + - 3.9.x up to and including 3.13.x ## Usage @@ -19,7 +19,6 @@ First step is to choose the base image to be used. The supported Linux OS distri - Amazon Linux 2 - Alpine - - CentOS - Debian - Ubuntu @@ -104,18 +103,18 @@ def handler(event, context): ### Local Testing -To make it easy to locally test Lambda functions packaged as container images we open-sourced a lightweight web-server, Lambda Runtime Interface Emulator (RIE), which allows your function packaged as a container image to accept HTTP requests. You can install the [AWS Lambda Runtime Interface Emulator](https://github.com/aws/aws-lambda-runtime-interface-emulator) on your local machine to test your function. Then when you run the image function, you set the entrypoint to be the emulator. +To make it easy to locally test Lambda functions packaged as container images we open-sourced a lightweight web-server, Lambda Runtime Interface Emulator (RIE), which allows your function packaged as a container image to accept HTTP requests. You can install the [AWS Lambda Runtime Interface Emulator](https://github.com/aws/aws-lambda-runtime-interface-emulator) on your local machine to test your function. Then when you run the image function, you set the entrypoint to be the emulator. *To install the emulator and test your Lambda function* -1) From your project directory, run the following command to download the RIE from GitHub and install it on your local machine. +1) From your project directory, run the following command to download the RIE from GitHub and install it on your local machine. ```shell script mkdir -p ~/.aws-lambda-rie && \ curl -Lo ~/.aws-lambda-rie/aws-lambda-rie https://github.com/aws/aws-lambda-runtime-interface-emulator/releases/latest/download/aws-lambda-rie && \ chmod +x ~/.aws-lambda-rie/aws-lambda-rie ``` -2) Run your Lambda image function using the docker run command. +2) Run your Lambda image function using the docker run command. ```shell script docker run -d -v ~/.aws-lambda-rie:/aws-lambda -p 9000:8080 \ @@ -124,9 +123,9 @@ docker run -d -v ~/.aws-lambda-rie:/aws-lambda -p 9000:8080 \ /usr/local/bin/python -m awslambdaric app.handler ``` -This runs the image as a container and starts up an endpoint locally at `http://localhost:9000/2015-03-31/functions/function/invocations`. +This runs the image as a container and starts up an endpoint locally at `http://localhost:9000/2015-03-31/functions/function/invocations`. -3) Post an event to the following endpoint using a curl command: +3) Post an event to the following endpoint using a curl command: ```shell script curl -XPOST "http://localhost:9000/2015-03-31/functions/function/invocations" -d '{}' @@ -175,4 +174,4 @@ If you discover a potential security issue in this project we ask that you notif ## License -This project is licensed under the Apache-2.0 License. \ No newline at end of file +This project is licensed under the Apache-2.0 License. diff --git a/RELEASE.CHANGELOG.md b/RELEASE.CHANGELOG.md index a6bdfc8..fc45791 100644 --- a/RELEASE.CHANGELOG.md +++ b/RELEASE.CHANGELOG.md @@ -1,3 +1,79 @@ +### May 26, 2025 +`3.1.1` +- Move unhandled exception warning message to init errors. ([#189](https://github.com/aws/aws-lambda-python-runtime-interface-client/pull/189)) + +### May 21, 2025 +`3.1.0` +- Add support for multi tenancy ([#187](https://github.com/aws/aws-lambda-python-runtime-interface-client/pull/187)) + +### February 27, 2024 +`3.0.2` +- Update `simplejson` to `3.20.1`([#184](https://github.com/aws/aws-lambda-python-runtime-interface-client/pull/184)) + +### January 27, 2024 +`3.0.1` +- Don't enforce text format on uncaught exception warning message ([#182](https://github.com/aws/aws-lambda-python-runtime-interface-client/pull/182)) + +### November 19, 2024 +`3.0.0` +- Drop support for deprecated python versions ([#179](https://github.com/aws/aws-lambda-python-runtime-interface-client/pull/179)) +- Add support for snapstart runtime hooks ([#176](https://github.com/aws/aws-lambda-python-runtime-interface-client/pull/176)) + +### August 23, 2024 +`2.2.1`: +- Patch libcurl configure.ac to work with later versions of autoconf ([#166](https://github.com/aws/aws-lambda-python-runtime-interface-client/pull/168)) + +### August 8, 2024 + +`2.2.0`: + +- Propogate error type in header when reporting init error to RAPID ([#166](https://github.com/aws/aws-lambda-python-runtime-interface-client/pull/166)) + +### July 31, 2024 + +`2.1.0`: + +- Raise all init errors in init instead of suppressing them until the first invoke ([#163](https://github.com/aws/aws-lambda-python-runtime-interface-client/pull/163)) + +### June 19, 2024 + +`2.0.12`: + +- Relax simplejson dependency and keep it backwards compatible ([#153](https://github.com/aws/aws-lambda-python-runtime-interface-client/pull/152)) + +### March 27, 2024 + +`2.0.11`: + +- Upgrade simplejson to 3.18.4 ([#136](https://github.com/aws/aws-lambda-python-runtime-interface-client/pull/136)) + +### February 13, 2024 + +`2.0.10`: + +- Update format of unhandled exception warning message. ([#132](https://github.com/aws/aws-lambda-python-runtime-interface-client/pull/132)) + +### February 01, 2024 + +`2.0.9`: + +- Log warning on unhandled exceptions. ([#120](https://github.com/aws/aws-lambda-python-runtime-interface-client/pull/120)) + +### October 30, 2023 + +`2.0.8`: + +- Onboarded Python3.12 ([#118](https://github.com/aws/aws-lambda-python-runtime-interface-client/pull/118)) +- Fix runtime_client blocking main thread from SIGTERM being handled. Enabled by default only for Python3.12 ([#115](https://github.com/aws/aws-lambda-python-runtime-interface-client/pull/115)) ([#124](https://github.com/aws/aws-lambda-python-runtime-interface-client/pull/124)) ([#125](https://github.com/aws/aws-lambda-python-runtime-interface-client/pull/125)) +- Use unicode chars instead of escape sequences in json encoder output. Enabled by default only for Python3.12 ([#88](https://github.com/aws/aws-lambda-python-runtime-interface-client/pull/88)) ([#122](https://github.com/aws/aws-lambda-python-runtime-interface-client/pull/122)) +- Cold start improvements ([#121](https://github.com/aws/aws-lambda-python-runtime-interface-client/pull/121)) + +### August 29, 2023 + +`2.0.7`: + +- Allow already structured logs in text format to use level-specific headers for logging protocol ([#111](https://github.com/aws/aws-lambda-python-runtime-interface-client/pull/111)) + ### August 22, 2023 `2.0.6`: diff --git a/awslambdaric/__init__.py b/awslambdaric/__init__.py index b3f0ada..5605903 100644 --- a/awslambdaric/__init__.py +++ b/awslambdaric/__init__.py @@ -2,4 +2,4 @@ Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. """ -__version__ = "2.0.6" +__version__ = "3.1.1" diff --git a/awslambdaric/bootstrap.py b/awslambdaric/bootstrap.py index 5ad7bb5..cb8d5c3 100644 --- a/awslambdaric/bootstrap.py +++ b/awslambdaric/bootstrap.py @@ -17,51 +17,52 @@ _DATETIME_FORMAT, _DEFAULT_FRAME_TYPE, _JSON_FRAME_TYPES, + _TEXT_FRAME_TYPES, JsonFormatter, LogFormat, + _format_log_level, + _get_log_level_from_env_var, ) from .lambda_runtime_marshaller import to_json ERROR_LOG_LINE_TERMINATE = "\r" ERROR_LOG_IDENT = "\u00a0" # NO-BREAK SPACE U+00A0 _AWS_LAMBDA_LOG_FORMAT = LogFormat.from_str(os.environ.get("AWS_LAMBDA_LOG_FORMAT")) -_AWS_LAMBDA_LOG_LEVEL = os.environ.get("AWS_LAMBDA_LOG_LEVEL", "").upper() +_AWS_LAMBDA_LOG_LEVEL = _get_log_level_from_env_var( + os.environ.get("AWS_LAMBDA_LOG_LEVEL") +) +AWS_LAMBDA_INITIALIZATION_TYPE = "AWS_LAMBDA_INITIALIZATION_TYPE" +INIT_TYPE_SNAP_START = "snap-start" def _get_handler(handler): try: (modname, fname) = handler.rsplit(".", 1) except ValueError as e: - fault = FaultException( + raise FaultException( FaultException.MALFORMED_HANDLER_NAME, "Bad handler '{}': {}".format(handler, str(e)), ) - return make_fault_handler(fault) try: if modname.split(".")[0] in sys.builtin_module_names: - fault = FaultException( + raise FaultException( FaultException.BUILT_IN_MODULE_CONFLICT, "Cannot use built-in module {} as a handler module".format(modname), ) - return make_fault_handler(fault) m = importlib.import_module(modname.replace("/", ".")) except ImportError as e: - fault = FaultException( + raise FaultException( FaultException.IMPORT_MODULE_ERROR, "Unable to import module '{}': {}".format(modname, str(e)), ) - request_handler = make_fault_handler(fault) - return request_handler except SyntaxError as e: trace = [' File "%s" Line %s\n %s' % (e.filename, e.lineno, e.text)] - fault = FaultException( + raise FaultException( FaultException.USER_CODE_SYNTAX_ERROR, "Syntax error in module '{}': {}".format(modname, str(e)), trace, ) - request_handler = make_fault_handler(fault) - return request_handler try: request_handler = getattr(m, fname) @@ -71,15 +72,8 @@ def _get_handler(handler): "Handler '{}' missing on module '{}'".format(fname, modname), None, ) - request_handler = make_fault_handler(fault) - return request_handler - - -def make_fault_handler(fault): - def result(*args): raise fault - - return result + return request_handler def make_error( @@ -122,7 +116,7 @@ def log_error(error_result, log_sink): ) else: - _ERROR_FRAME_TYPE = _DEFAULT_FRAME_TYPE + _ERROR_FRAME_TYPE = _TEXT_FRAME_TYPES[logging.ERROR] def log_error(error_result, log_sink): error_description = "[ERROR]" @@ -164,6 +158,7 @@ def handle_event_request( cognito_identity_json, invoked_function_arn, epoch_deadline_time_in_ms, + tenant_id, log_sink, ): error_result = None @@ -174,6 +169,7 @@ def handle_event_request( epoch_deadline_time_in_ms, invoke_id, invoked_function_arn, + tenant_id, ) event = lambda_runtime_client.marshaller.unmarshal_request( event_body, content_type @@ -205,6 +201,7 @@ def handle_event_request( ) if error_result is not None: + log_error(error_result, log_sink) lambda_runtime_client.post_invocation_error( invoke_id, to_json(error_result), to_json(xray_fault) @@ -232,6 +229,7 @@ def create_lambda_context( epoch_deadline_time_in_ms, invoke_id, invoked_function_arn, + tenant_id, ): client_context = None if client_context_json: @@ -246,6 +244,7 @@ def create_lambda_context( cognito_identity, epoch_deadline_time_in_ms, invoked_function_arn, + tenant_id, ) @@ -289,6 +288,29 @@ def extract_traceback(tb): ] +def on_init_complete(lambda_runtime_client, log_sink): + from . import lambda_runtime_hooks_runner + + try: + lambda_runtime_hooks_runner.run_before_snapshot() + lambda_runtime_client.restore_next() + except: + error_result = build_fault_result(sys.exc_info(), None) + log_error(error_result, log_sink) + lambda_runtime_client.post_init_error( + error_result, FaultException.BEFORE_SNAPSHOT_ERROR + ) + sys.exit(64) + + try: + lambda_runtime_hooks_runner.run_after_restore() + except: + error_result = build_fault_result(sys.exc_info(), None) + log_error(error_result, log_sink) + lambda_runtime_client.report_restore_error(error_result) + sys.exit(65) + + class LambdaLoggerHandler(logging.Handler): def __init__(self, log_sink): logging.Handler.__init__(self) @@ -296,13 +318,28 @@ def __init__(self, log_sink): def emit(self, record): msg = self.format(record) + self.log_sink.log(msg) + + +class LambdaLoggerHandlerWithFrameType(logging.Handler): + def __init__(self, log_sink): + super().__init__() + self.log_sink = log_sink - self.log_sink.log(msg, frame_type=getattr(record, "_frame_type", None)) + def emit(self, record): + self.log_sink.log( + self.format(record), + frame_type=( + getattr(record, "_frame_type", None) + or _TEXT_FRAME_TYPES.get(_format_log_level(record)) + ), + ) class LambdaLoggerFilter(logging.Filter): def filter(self, record): record.aws_request_id = _GLOBAL_AWS_REQUEST_ID or "" + record.tenant_id = _GLOBAL_TENANT_ID return True @@ -411,18 +448,20 @@ def create_log_sink(): _GLOBAL_AWS_REQUEST_ID = None +_GLOBAL_TENANT_ID = None def _setup_logging(log_format, log_level, log_sink): logging.Formatter.converter = time.gmtime logger = logging.getLogger() - logger_handler = LambdaLoggerHandler(log_sink) + + if log_format == LogFormat.JSON or log_level: + logger_handler = LambdaLoggerHandlerWithFrameType(log_sink) + else: + logger_handler = LambdaLoggerHandler(log_sink) + if log_format == LogFormat.JSON: logger_handler.setFormatter(JsonFormatter()) - - logging.addLevelName(logging.DEBUG, "TRACE") - if log_level in logging._nameToLevel: - logger.setLevel(log_level) else: logger_handler.setFormatter( logging.Formatter( @@ -431,6 +470,9 @@ def _setup_logging(log_format, log_level, log_sink): ) ) + if log_level in logging._nameToLevel: + logger.setLevel(log_level) + logger_handler.addFilter(LambdaLoggerFilter()) logger.addHandler(logger_handler) @@ -439,26 +481,48 @@ def run(app_root, handler, lambda_runtime_api_addr): sys.stdout = Unbuffered(sys.stdout) sys.stderr = Unbuffered(sys.stderr) + use_thread_for_polling_next = os.environ.get("AWS_EXECUTION_ENV") in { + "AWS_Lambda_python3.12", + "AWS_Lambda_python3.13", + } + with create_log_sink() as log_sink: - lambda_runtime_client = LambdaRuntimeClient(lambda_runtime_api_addr) + lambda_runtime_client = LambdaRuntimeClient( + lambda_runtime_api_addr, use_thread_for_polling_next + ) + error_result = None try: _setup_logging(_AWS_LAMBDA_LOG_FORMAT, _AWS_LAMBDA_LOG_LEVEL, log_sink) - global _GLOBAL_AWS_REQUEST_ID + global _GLOBAL_AWS_REQUEST_ID, _GLOBAL_TENANT_ID request_handler = _get_handler(handler) + except FaultException as e: + error_result = make_error( + e.msg, + e.exception_type, + e.trace, + ) except Exception: error_result = build_fault_result(sys.exc_info(), None) + if error_result is not None: + from .lambda_literals import lambda_unhandled_exception_warning_message + + logging.warning(lambda_unhandled_exception_warning_message) log_error(error_result, log_sink) - lambda_runtime_client.post_init_error(to_json(error_result)) + lambda_runtime_client.post_init_error(error_result) sys.exit(1) + if os.environ.get(AWS_LAMBDA_INITIALIZATION_TYPE) == INIT_TYPE_SNAP_START: + on_init_complete(lambda_runtime_client, log_sink) + while True: event_request = lambda_runtime_client.wait_next_invocation() _GLOBAL_AWS_REQUEST_ID = event_request.invoke_id + _GLOBAL_TENANT_ID = event_request.tenant_id update_xray_env_variable(event_request.x_amzn_trace_id) @@ -472,5 +536,6 @@ def run(app_root, handler, lambda_runtime_api_addr): event_request.cognito_identity, event_request.invoked_function_arn, event_request.deadline_time_in_ms, + event_request.tenant_id, log_sink, ) diff --git a/awslambdaric/lambda_context.py b/awslambdaric/lambda_context.py index 1465827..e0a3363 100644 --- a/awslambdaric/lambda_context.py +++ b/awslambdaric/lambda_context.py @@ -16,6 +16,7 @@ def __init__( cognito_identity, epoch_deadline_time_in_ms, invoked_function_arn=None, + tenant_id=None, ): self.aws_request_id = invoke_id self.log_group_name = os.environ.get("AWS_LAMBDA_LOG_GROUP_NAME") @@ -24,6 +25,7 @@ def __init__( self.memory_limit_in_mb = os.environ.get("AWS_LAMBDA_FUNCTION_MEMORY_SIZE") self.function_version = os.environ.get("AWS_LAMBDA_FUNCTION_VERSION") self.invoked_function_arn = invoked_function_arn + self.tenant_id = tenant_id self.client_context = make_obj_from_dict(ClientContext, client_context) if self.client_context is not None: @@ -65,7 +67,8 @@ def __repr__(self): f"function_version={self.function_version}," f"invoked_function_arn={self.invoked_function_arn}," f"client_context={self.client_context}," - f"identity={self.identity}" + f"identity={self.identity}," + f"tenant_id={self.tenant_id}" "])" ) diff --git a/awslambdaric/lambda_literals.py b/awslambdaric/lambda_literals.py new file mode 100644 index 0000000..2585b89 --- /dev/null +++ b/awslambdaric/lambda_literals.py @@ -0,0 +1,17 @@ +""" +Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved. +""" + +lambda_warning = "LAMBDA_WARNING" + +# Holds warning message that is emitted when an unhandled exception is raised during function invocation. +lambda_unhandled_exception_warning_message = str( + f"{lambda_warning}: " + "Unhandled exception. " + "The most likely cause is an issue in the function code. " + "However, in rare cases, a Lambda runtime update can cause unexpected function behavior. " + "For functions using managed runtimes, runtime updates can be triggered by a function change, or can be applied automatically. " + "To determine if the runtime has been updated, check the runtime version in the INIT_START log entry. " + "If this error correlates with a change in the runtime version, you may be able to mitigate this error by temporarily rolling back to the previous runtime version. " + "For more information, see https://docs.aws.amazon.com/lambda/latest/dg/runtimes-update.html\r" +) diff --git a/awslambdaric/lambda_runtime_client.py b/awslambdaric/lambda_runtime_client.py index 2066f6c..ba4ad92 100644 --- a/awslambdaric/lambda_runtime_client.py +++ b/awslambdaric/lambda_runtime_client.py @@ -2,10 +2,12 @@ Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. """ -import http -import http.client import sys from awslambdaric import __version__ +from .lambda_runtime_exception import FaultException +from .lambda_runtime_marshaller import to_json + +ERROR_TYPE_HEADER = "Lambda-Runtime-Function-Error-Type" def _user_agent(): @@ -50,22 +52,84 @@ class LambdaRuntimeClient(object): and response. It allows for function authors to override the the default implementation, LambdaMarshaller which unmarshals and marshals JSON, to an instance of a class that implements the same interface.""" - def __init__(self, lambda_runtime_address): + def __init__(self, lambda_runtime_address, use_thread_for_polling_next=False): self.lambda_runtime_address = lambda_runtime_address + self.use_thread_for_polling_next = use_thread_for_polling_next + if self.use_thread_for_polling_next: + # Conditionally import only for the case when TPE is used in this class. + from concurrent.futures import ThreadPoolExecutor + + # Not defining symbol as global to avoid relying on TPE being imported unconditionally. + self.ThreadPoolExecutor = ThreadPoolExecutor + + def call_rapid( + self, http_method, endpoint, expected_http_code, payload=None, headers=None + ): + # These imports are heavy-weight. They implicitly trigger `import ssl, hashlib`. + # Importing them lazily to speed up critical path of a common case. + import http.client - def post_init_error(self, error_response_data): runtime_connection = http.client.HTTPConnection(self.lambda_runtime_address) runtime_connection.connect() - endpoint = "/2018-06-01/runtime/init/error" - runtime_connection.request("POST", endpoint, error_response_data) + if http_method == "GET": + runtime_connection.request(http_method, endpoint) + else: + runtime_connection.request( + http_method, endpoint, to_json(payload), headers=headers + ) + response = runtime_connection.getresponse() response_body = response.read() - - if response.code != http.HTTPStatus.ACCEPTED: + if response.code != expected_http_code: raise LambdaRuntimeClientError(endpoint, response.code, response_body) + def post_init_error(self, error_response_data, error_type_override=None): + import http + + endpoint = "/2018-06-01/runtime/init/error" + headers = { + ERROR_TYPE_HEADER: ( + error_type_override + if error_type_override + else error_response_data["errorType"] + ) + } + self.call_rapid( + "POST", endpoint, http.HTTPStatus.ACCEPTED, error_response_data, headers + ) + + def restore_next(self): + import http + + endpoint = "/2018-06-01/runtime/restore/next" + self.call_rapid("GET", endpoint, http.HTTPStatus.OK) + + def report_restore_error(self, restore_error_data): + import http + + endpoint = "/2018-06-01/runtime/restore/error" + headers = {ERROR_TYPE_HEADER: FaultException.AFTER_RESTORE_ERROR} + self.call_rapid( + "POST", endpoint, http.HTTPStatus.ACCEPTED, restore_error_data, headers + ) + def wait_next_invocation(self): - response_body, headers = runtime_client.next() + # Calling runtime_client.next() from a separate thread unblocks the main thread, + # which can then process signals. + if self.use_thread_for_polling_next: + try: + # TPE class is supposed to be registered at construction time and be ready to use. + with self.ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit(runtime_client.next) + response_body, headers = future.result() + except Exception as e: + raise FaultException( + FaultException.LAMBDA_RUNTIME_CLIENT_ERROR, + "LAMBDA_RUNTIME Failed to get next invocation: {}".format(str(e)), + None, + ) + else: + response_body, headers = runtime_client.next() return InvocationRequest( invoke_id=headers.get("Lambda-Runtime-Aws-Request-Id"), x_amzn_trace_id=headers.get("Lambda-Runtime-Trace-Id"), @@ -73,6 +137,7 @@ def wait_next_invocation(self): deadline_time_in_ms=headers.get("Lambda-Runtime-Deadline-Ms"), client_context=headers.get("Lambda-Runtime-Client-Context"), cognito_identity=headers.get("Lambda-Runtime-Cognito-Identity"), + tenant_id=headers.get("Lambda-Runtime-Aws-Tenant-Id"), content_type=headers.get("Content-Type"), event_body=response_body, ) @@ -82,9 +147,11 @@ def post_invocation_result( ): runtime_client.post_invocation_result( invoke_id, - result_data - if isinstance(result_data, bytes) - else result_data.encode("utf-8"), + ( + result_data + if isinstance(result_data, bytes) + else result_data.encode("utf-8") + ), content_type, ) diff --git a/awslambdaric/lambda_runtime_exception.py b/awslambdaric/lambda_runtime_exception.py index 416327e..3ea5b29 100644 --- a/awslambdaric/lambda_runtime_exception.py +++ b/awslambdaric/lambda_runtime_exception.py @@ -11,7 +11,10 @@ class FaultException(Exception): IMPORT_MODULE_ERROR = "Runtime.ImportModuleError" BUILT_IN_MODULE_CONFLICT = "Runtime.BuiltInModuleConflict" MALFORMED_HANDLER_NAME = "Runtime.MalformedHandlerName" + BEFORE_SNAPSHOT_ERROR = "Runtime.BeforeSnapshotError" + AFTER_RESTORE_ERROR = "Runtime.AfterRestoreError" LAMBDA_CONTEXT_UNMARSHAL_ERROR = "Runtime.LambdaContextUnmarshalError" + LAMBDA_RUNTIME_CLIENT_ERROR = "Runtime.LambdaRuntimeClientError" def __init__(self, exception_type, msg, trace=None): self.msg = msg diff --git a/awslambdaric/lambda_runtime_hooks_runner.py b/awslambdaric/lambda_runtime_hooks_runner.py new file mode 100644 index 0000000..8aee181 --- /dev/null +++ b/awslambdaric/lambda_runtime_hooks_runner.py @@ -0,0 +1,18 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +from snapshot_restore_py import get_before_snapshot, get_after_restore + + +def run_before_snapshot(): + before_snapshot_callables = get_before_snapshot() + while before_snapshot_callables: + # Using pop as before checkpoint callables are executed in the reverse order of their registration + func, args, kwargs = before_snapshot_callables.pop() + func(*args, **kwargs) + + +def run_after_restore(): + after_restore_callables = get_after_restore() + for func, args, kwargs in after_restore_callables: + func(*args, **kwargs) diff --git a/awslambdaric/lambda_runtime_log_utils.py b/awslambdaric/lambda_runtime_log_utils.py index f140253..9ddbcfb 100644 --- a/awslambdaric/lambda_runtime_log_utils.py +++ b/awslambdaric/lambda_runtime_log_utils.py @@ -30,6 +30,7 @@ "processName", "process", "aws_request_id", + "tenant_id", "_frame_type", } @@ -45,6 +46,10 @@ def from_str(cls, value: str): return cls.TEXT.value +def _get_log_level_from_env_var(log_level): + return {None: "", "TRACE": "DEBUG"}.get(log_level, log_level).upper() + + _JSON_FRAME_TYPES = { logging.NOTSET: 0xA55A0002.to_bytes(4, "big"), logging.DEBUG: 0xA55A000A.to_bytes(4, "big"), @@ -53,12 +58,24 @@ def from_str(cls, value: str): logging.ERROR: 0xA55A0016.to_bytes(4, "big"), logging.CRITICAL: 0xA55A001A.to_bytes(4, "big"), } -_DEFAULT_FRAME_TYPE = 0xA55A0003.to_bytes(4, "big") +_TEXT_FRAME_TYPES = { + logging.NOTSET: 0xA55A0003.to_bytes(4, "big"), + logging.DEBUG: 0xA55A000B.to_bytes(4, "big"), + logging.INFO: 0xA55A000F.to_bytes(4, "big"), + logging.WARNING: 0xA55A0013.to_bytes(4, "big"), + logging.ERROR: 0xA55A0017.to_bytes(4, "big"), + logging.CRITICAL: 0xA55A001B.to_bytes(4, "big"), +} +_DEFAULT_FRAME_TYPE = _TEXT_FRAME_TYPES[logging.NOTSET] _json_encoder = json.JSONEncoder(ensure_ascii=False) _encode_json = _json_encoder.encode +def _format_log_level(record: logging.LogRecord) -> int: + return min(50, max(0, record.levelno)) // 10 * 10 + + class JsonFormatter(logging.Formatter): def __init__(self): super().__init__(datefmt=_DATETIME_FORMAT) @@ -90,13 +107,9 @@ def __format_location(record: logging.LogRecord): return f"{record.pathname}:{record.funcName}:{record.lineno}" - @staticmethod - def __format_log_level(record: logging.LogRecord): - record.levelno = min(50, max(0, record.levelno)) // 10 * 10 - record.levelname = logging.getLevelName(record.levelno) - def format(self, record: logging.LogRecord) -> str: - self.__format_log_level(record) + record.levelno = _format_log_level(record) + record.levelname = logging.getLevelName(record.levelno) record._frame_type = _JSON_FRAME_TYPES.get( record.levelno, _JSON_FRAME_TYPES[logging.NOTSET] ) @@ -112,6 +125,9 @@ def format(self, record: logging.LogRecord) -> str: "requestId": getattr(record, "aws_request_id", None), "location": self.__format_location(record), } + if hasattr(record, "tenant_id") and record.tenant_id is not None: + result["tenantId"] = record.tenant_id + result.update( (key, value) for key, value in record.__dict__.items() diff --git a/awslambdaric/lambda_runtime_marshaller.py b/awslambdaric/lambda_runtime_marshaller.py index 7eee25d..4256066 100644 --- a/awslambdaric/lambda_runtime_marshaller.py +++ b/awslambdaric/lambda_runtime_marshaller.py @@ -4,7 +4,7 @@ import decimal import math - +import os import simplejson as json from .lambda_runtime_exception import FaultException @@ -12,9 +12,16 @@ # simplejson's Decimal encoding allows '-NaN' as an output, which is a parse error for json.loads # to get the good parts of Decimal support, we'll special-case NaN decimals and otherwise duplicate the encoding for decimals the same way simplejson does +# We also set 'ensure_ascii=False' so that the encoded json contains unicode characters instead of unicode escape sequences class Encoder(json.JSONEncoder): def __init__(self): - super().__init__(use_decimal=False) + if os.environ.get("AWS_EXECUTION_ENV") in { + "AWS_Lambda_python3.12", + "AWS_Lambda_python3.13", + }: + super().__init__(use_decimal=False, ensure_ascii=False, allow_nan=True) + else: + super().__init__(use_decimal=False, allow_nan=True) def default(self, obj): if isinstance(obj, decimal.Decimal): diff --git a/awslambdaric/runtime_client.cpp b/awslambdaric/runtime_client.cpp index 66252bf..7fb2e95 100644 --- a/awslambdaric/runtime_client.cpp +++ b/awslambdaric/runtime_client.cpp @@ -52,9 +52,10 @@ static PyObject *method_next(PyObject *self) { auto client_context = response.client_context.c_str(); auto content_type = response.content_type.c_str(); auto cognito_id = response.cognito_identity.c_str(); + auto tenant_id = response.tenant_id.c_str(); PyObject *payload_bytes = PyBytes_FromStringAndSize(payload.c_str(), payload.length()); - PyObject *result = Py_BuildValue("(O,{s:s,s:s,s:s,s:l,s:s,s:s,s:s})", + PyObject *result = Py_BuildValue("(O,{s:s,s:s,s:s,s:l,s:s,s:s,s:s,s:s})", payload_bytes, //Py_BuildValue() increments reference counter "Lambda-Runtime-Aws-Request-Id", request_id, "Lambda-Runtime-Trace-Id", NULL_IF_EMPTY(trace_id), @@ -62,7 +63,8 @@ static PyObject *method_next(PyObject *self) { "Lambda-Runtime-Deadline-Ms", deadline, "Lambda-Runtime-Client-Context", NULL_IF_EMPTY(client_context), "Content-Type", NULL_IF_EMPTY(content_type), - "Lambda-Runtime-Cognito-Identity", NULL_IF_EMPTY(cognito_id) + "Lambda-Runtime-Cognito-Identity", NULL_IF_EMPTY(cognito_id), + "Lambda-Runtime-Aws-Tenant-Id", NULL_IF_EMPTY(tenant_id) ); Py_XDECREF(payload_bytes); diff --git a/deps/aws-lambda-cpp-0.2.6.tar.gz b/deps/aws-lambda-cpp-0.2.6.tar.gz index 26fa498..51d7f51 100644 Binary files a/deps/aws-lambda-cpp-0.2.6.tar.gz and b/deps/aws-lambda-cpp-0.2.6.tar.gz differ diff --git a/deps/curl-7.83.1.tar.gz b/deps/curl-7.83.1.tar.gz index b71926a..305bb4f 100644 Binary files a/deps/curl-7.83.1.tar.gz and b/deps/curl-7.83.1.tar.gz differ diff --git a/deps/patches/aws-lambda-cpp-add-tenant-id.patch b/deps/patches/aws-lambda-cpp-add-tenant-id.patch new file mode 100644 index 0000000..a7b7172 --- /dev/null +++ b/deps/patches/aws-lambda-cpp-add-tenant-id.patch @@ -0,0 +1,39 @@ +diff --git a/include/aws/lambda-runtime/runtime.h b/include/aws/lambda-runtime/runtime.h +index 7812ff6..96be869 100644 +--- a/include/aws/lambda-runtime/runtime.h ++++ b/include/aws/lambda-runtime/runtime.h +@@ -61,6 +61,11 @@ struct invocation_request { + */ + std::string content_type; + ++ /** ++ * The Tenant ID of the current invocation. ++ */ ++ std::string tenant_id; ++ + /** + * Function execution deadline counted in milliseconds since the Unix epoch. + */ +diff --git a/src/runtime.cpp b/src/runtime.cpp +index e53b2b8..9763282 100644 +--- a/src/runtime.cpp ++++ b/src/runtime.cpp +@@ -40,6 +40,7 @@ static constexpr auto CLIENT_CONTEXT_HEADER = "lambda-runtime-client-context"; + static constexpr auto COGNITO_IDENTITY_HEADER = "lambda-runtime-cognito-identity"; + static constexpr auto DEADLINE_MS_HEADER = "lambda-runtime-deadline-ms"; + static constexpr auto FUNCTION_ARN_HEADER = "lambda-runtime-invoked-function-arn"; ++static constexpr auto TENANT_ID_HEADER = "lambda-runtime-aws-tenant-id"; + + enum Endpoints { + INIT, +@@ -289,6 +290,10 @@ runtime::next_outcome runtime::get_next() + req.function_arn = resp.get_header(FUNCTION_ARN_HEADER); + } + ++ if (resp.has_header(TENANT_ID_HEADER)) { ++ req.tenant_id = resp.get_header(TENANT_ID_HEADER); ++ } ++ + if (resp.has_header(DEADLINE_MS_HEADER)) { + auto const& deadline_string = resp.get_header(DEADLINE_MS_HEADER); + constexpr int base = 10; diff --git a/deps/patches/libcurl-configure-template.patch b/deps/patches/libcurl-configure-template.patch new file mode 100644 index 0000000..e26be47 --- /dev/null +++ b/deps/patches/libcurl-configure-template.patch @@ -0,0 +1,131 @@ +diff --git a/configure.ac b/configure.ac +index d24daea..64aca7f 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -193,87 +193,96 @@ AS_HELP_STRING([--with-schannel],[enable Windows native SSL/TLS]), + + OPT_SECURETRANSPORT=no + AC_ARG_WITH(secure-transport,dnl +-AS_HELP_STRING([--with-secure-transport],[enable Apple OS native SSL/TLS]), ++AS_HELP_STRING([--with-secure-transport],[enable Apple OS native SSL/TLS]),[ + OPT_SECURETRANSPORT=$withval + test -z "TLSCHOICE" || TLSCHOICE="${TLSCHOICE:+$TLSCHOICE, }Secure-Transport" +-) ++]) + + OPT_AMISSL=no + AC_ARG_WITH(amissl,dnl +-AS_HELP_STRING([--with-amissl],[enable Amiga native SSL/TLS (AmiSSL)]), ++AS_HELP_STRING([--with-amissl],[enable Amiga native SSL/TLS (AmiSSL)]),[ + OPT_AMISSL=$withval +- test -z "TLSCHOICE" || TLSCHOICE="${TLSCHOICE:+$TLSCHOICE, }AmiSSL") ++ test -z "TLSCHOICE" || TLSCHOICE="${TLSCHOICE:+$TLSCHOICE, }AmiSSL" ++]) ++ + + OPT_OPENSSL=no + dnl Default to no CA bundle + ca="no" + AC_ARG_WITH(ssl,dnl + AS_HELP_STRING([--with-ssl=PATH],[old version of --with-openssl]) +-AS_HELP_STRING([--without-ssl], [build without any TLS library]), ++AS_HELP_STRING([--without-ssl], [build without any TLS library]),[ + OPT_SSL=$withval + OPT_OPENSSL=$withval + if test X"$withval" != Xno; then +- test -z "TLSCHOICE" || TLSCHOICE="${TLSCHOICE:+$TLSCHOICE, }OpenSSL") ++ test -z "TLSCHOICE" || TLSCHOICE="${TLSCHOICE:+$TLSCHOICE, }OpenSSL" + fi ++]) + + AC_ARG_WITH(openssl,dnl +-AS_HELP_STRING([--with-openssl=PATH],[Where to look for OpenSSL, PATH points to the SSL installation (default: /usr/local/ssl); when possible, set the PKG_CONFIG_PATH environment variable instead of using this option]), ++AS_HELP_STRING([--with-openssl=PATH],[Where to look for OpenSSL, PATH points to the SSL installation (default: /usr/local/ssl); when possible, set the PKG_CONFIG_PATH environment variable instead of using this option]),[ + OPT_OPENSSL=$withval + if test X"$withval" != Xno; then +- test -z "TLSCHOICE" || TLSCHOICE="${TLSCHOICE:+$TLSCHOICE, }OpenSSL") ++ test -z "TLSCHOICE" || TLSCHOICE="${TLSCHOICE:+$TLSCHOICE, }OpenSSL" + fi ++]) + + OPT_GNUTLS=no + AC_ARG_WITH(gnutls,dnl +-AS_HELP_STRING([--with-gnutls=PATH],[where to look for GnuTLS, PATH points to the installation root]), ++AS_HELP_STRING([--with-gnutls=PATH],[where to look for GnuTLS, PATH points to the installation root]),[ + OPT_GNUTLS=$withval + if test X"$withval" != Xno; then +- test -z "TLSCHOICE" || TLSCHOICE="${TLSCHOICE:+$TLSCHOICE, }GnuTLS") ++ test -z "TLSCHOICE" || TLSCHOICE="${TLSCHOICE:+$TLSCHOICE, }GnuTLS" + fi ++]) + + OPT_MBEDTLS=no + AC_ARG_WITH(mbedtls,dnl +-AS_HELP_STRING([--with-mbedtls=PATH],[where to look for mbedTLS, PATH points to the installation root]), ++AS_HELP_STRING([--with-mbedtls=PATH],[where to look for mbedTLS, PATH points to the installation root]),[ + OPT_MBEDTLS=$withval + if test X"$withval" != Xno; then +- test -z "TLSCHOICE" || TLSCHOICE="${TLSCHOICE:+$TLSCHOICE, }mbedTLS") ++ test -z "TLSCHOICE" || TLSCHOICE="${TLSCHOICE:+$TLSCHOICE, }mbedTLS" + fi ++]) + + OPT_WOLFSSL=no + AC_ARG_WITH(wolfssl,dnl +-AS_HELP_STRING([--with-wolfssl=PATH],[where to look for WolfSSL, PATH points to the installation root (default: system lib default)]), ++AS_HELP_STRING([--with-wolfssl=PATH],[where to look for WolfSSL, PATH points to the installation root (default: system lib default)]),[ + OPT_WOLFSSL=$withval + if test X"$withval" != Xno; then +- test -z "TLSCHOICE" || TLSCHOICE="${TLSCHOICE:+$TLSCHOICE, }wolfSSL") ++ test -z "TLSCHOICE" || TLSCHOICE="${TLSCHOICE:+$TLSCHOICE, }wolfSSL" + fi ++]) + + OPT_BEARSSL=no + AC_ARG_WITH(bearssl,dnl +-AS_HELP_STRING([--with-bearssl=PATH],[where to look for BearSSL, PATH points to the installation root]), ++AS_HELP_STRING([--with-bearssl=PATH],[where to look for BearSSL, PATH points to the installation root]),[ + OPT_BEARSSL=$withval + if test X"$withval" != Xno; then +- test -z "TLSCHOICE" || TLSCHOICE="${TLSCHOICE:+$TLSCHOICE, }BearSSL") ++ test -z "TLSCHOICE" || TLSCHOICE="${TLSCHOICE:+$TLSCHOICE, }BearSSL" + fi ++]) + + OPT_RUSTLS=no + AC_ARG_WITH(rustls,dnl +-AS_HELP_STRING([--with-rustls=PATH],[where to look for rustls, PATH points to the installation root]), ++AS_HELP_STRING([--with-rustls=PATH],[where to look for rustls, PATH points to the installation root]),[ + OPT_RUSTLS=$withval + if test X"$withval" != Xno; then +- test -z "TLSCHOICE" || TLSCHOICE="${TLSCHOICE:+$TLSCHOICE, }rustls") ++ test -z "TLSCHOICE" || TLSCHOICE="${TLSCHOICE:+$TLSCHOICE, }rustls" + fi ++]) + + OPT_NSS_AWARE=no + AC_ARG_WITH(nss-deprecated,dnl +-AS_HELP_STRING([--with-nss-deprecated],[confirm you realize NSS is going away]), ++AS_HELP_STRING([--with-nss-deprecated],[confirm you realize NSS is going away]),[ + if test X"$withval" != Xno; then + OPT_NSS_AWARE=$withval + fi +-) ++]) + + OPT_NSS=no + AC_ARG_WITH(nss,dnl +-AS_HELP_STRING([--with-nss=PATH],[where to look for NSS, PATH points to the installation root]), ++AS_HELP_STRING([--with-nss=PATH],[where to look for NSS, PATH points to the installation root]),[ + OPT_NSS=$withval + if test X"$withval" != Xno; then + +@@ -283,7 +292,7 @@ AS_HELP_STRING([--with-nss=PATH],[where to look for NSS, PATH points to the inst + + test -z "TLSCHOICE" || TLSCHOICE="${TLSCHOICE:+$TLSCHOICE, }NSS" + fi +-) ++]) + + dnl If no TLS choice has been made, check if it was explicitly disabled or + dnl error out to force the user to decide. diff --git a/requirements/base.txt b/requirements/base.txt index 614d435..4bb251e 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1 +1,2 @@ -simplejson==3.17.2 +simplejson>=3.20.1 +snapshot-restore-py>=1.0.0 diff --git a/requirements/dev.txt b/requirements/dev.txt index c432413..68377ce 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -9,3 +9,4 @@ bandit>=1.6.2 # Test requirements pytest>=3.0.7 mock>=2.0.0 +parameterized>=0.9.0 \ No newline at end of file diff --git a/scripts/update_deps.sh b/scripts/update_deps.sh index 4ec4ec1..4799a6f 100755 --- a/scripts/update_deps.sh +++ b/scripts/update_deps.sh @@ -1,6 +1,6 @@ #!/bin/bash # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. -set -e +set -x cd deps source versions @@ -8,8 +8,17 @@ source versions # Clean up old files rm -f aws-lambda-cpp-*.tar.gz && rm -f curl-*.tar.gz + +LIBCURL="curl-${CURL_MAJOR_VERSION}.${CURL_MINOR_VERSION}.${CURL_PATCH_VERSION}" + # Grab Curl -wget -c "https://github.com/curl/curl/releases/download/curl-${CURL_MAJOR_VERSION}_${CURL_MINOR_VERSION}_${CURL_PATCH_VERSION}/curl-${CURL_MAJOR_VERSION}.${CURL_MINOR_VERSION}.${CURL_PATCH_VERSION}.tar.gz" +wget -c "https://github.com/curl/curl/releases/download/curl-${CURL_MAJOR_VERSION}_${CURL_MINOR_VERSION}_${CURL_PATCH_VERSION}/$LIBCURL.tar.gz" -O - | tar -xz +( + cd $LIBCURL && \ + patch -p1 < ../patches/libcurl-configure-template.patch +) + +tar -czf $LIBCURL.tar.gz $LIBCURL --no-same-owner && rm -rf $LIBCURL # Grab aws-lambda-cpp wget -c https://github.com/awslabs/aws-lambda-cpp/archive/v$AWS_LAMBDA_CPP_RELEASE.tar.gz -O - | tar -xz @@ -21,9 +30,10 @@ wget -c https://github.com/awslabs/aws-lambda-cpp/archive/v$AWS_LAMBDA_CPP_RELEA patch -p1 < ../patches/aws-lambda-cpp-posting-init-errors.patch && \ patch -p1 < ../patches/aws-lambda-cpp-make-the-runtime-client-user-agent-overrideable.patch && \ patch -p1 < ../patches/aws-lambda-cpp-make-lto-optional.patch && \ - patch -p1 < ../patches/aws-lambda-cpp-add-content-type.patch + patch -p1 < ../patches/aws-lambda-cpp-add-content-type.patch && \ + patch -p1 < ../patches/aws-lambda-cpp-add-tenant-id.patch ) ## Pack again and remove the folder -tar -czvf aws-lambda-cpp-$AWS_LAMBDA_CPP_RELEASE.tar.gz aws-lambda-cpp-$AWS_LAMBDA_CPP_RELEASE --no-same-owner && \ +tar -czf aws-lambda-cpp-$AWS_LAMBDA_CPP_RELEASE.tar.gz aws-lambda-cpp-$AWS_LAMBDA_CPP_RELEASE --no-same-owner && \ rm -rf aws-lambda-cpp-$AWS_LAMBDA_CPP_RELEASE diff --git a/setup.py b/setup.py index a69c646..2bf28ef 100644 --- a/setup.py +++ b/setup.py @@ -84,16 +84,15 @@ def read_requirements(req="base.txt"): "Intended Audience :: Developers", "Natural Language :: English", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", ], - python_requires=">=3.6", + python_requires=">=3.9", ext_modules=get_runtime_client_extension(), test_suite="tests", ) diff --git a/tests/integration/codebuild-local/codebuild_build.sh b/tests/integration/codebuild-local/codebuild_build.sh index ffadfa3..45329d2 100755 --- a/tests/integration/codebuild-local/codebuild_build.sh +++ b/tests/integration/codebuild-local/codebuild_build.sh @@ -36,6 +36,7 @@ function usage { echo " -a Used to specify an artifact output directory." echo "Options:" echo " -l IMAGE Used to override the default local agent image." + echo " -r Used to specify a report output directory." echo " -s Used to specify source information. Defaults to the current working directory for primary source." echo " * First (-s) is for primary source" echo " * Use additional (-s) in : format for secondary source" @@ -61,10 +62,11 @@ awsconfig_flag=false mount_src_dir_flag=false docker_privileged_mode_flag=false -while getopts "cmdi:a:s:b:e:l:p:h" opt; do +while getopts "cmdi:a:r:s:b:e:l:p:h" opt; do case $opt in i ) image_flag=true; image_name=$OPTARG;; a ) artifact_flag=true; artifact_dir=$OPTARG;; + r ) report_dir=$OPTARG;; b ) buildspec=$OPTARG;; c ) awsconfig_flag=true;; m ) mount_src_dir_flag=true;; @@ -106,6 +108,11 @@ fi docker_command+="\"IMAGE_NAME=$image_name\" -e \ \"ARTIFACTS=$(allOSRealPath "$artifact_dir")\"" +if [ -n "$report_dir" ] +then + docker_command+=" -e \"REPORTS=$(allOSRealPath "$report_dir")\"" +fi + if [ -z "$source_dirs" ] then docker_command+=" -e \"SOURCE=$(allOSRealPath "$PWD")\"" @@ -136,11 +143,6 @@ then docker_command+=" -v \"$environment_variable_file_dir:/LocalBuild/envFile/\" -e \"ENV_VAR_FILE=$environment_variable_file_basename\"" fi -if [ -n "$local_agent_image" ] -then - docker_command+=" -e \"LOCAL_AGENT_IMAGE_NAME=$local_agent_image\"" -fi - if $awsconfig_flag then if [ -d "$HOME/.aws" ] @@ -176,7 +178,12 @@ else docker_command+=" -e \"INITIATOR=$USER\"" fi -docker_command+=" amazon/aws-codebuild-local:latest" +if [ -n "$local_agent_image" ] +then + docker_command+=" $local_agent_image" +else + docker_command+=" public.ecr.aws/codebuild/local-builds:latest" +fi # Note we do not expose the AWS_SECRET_ACCESS_KEY or the AWS_SESSION_TOKEN exposed_command=$docker_command @@ -191,4 +198,4 @@ echo "" echo $exposed_command echo "" -eval $docker_command +eval $docker_command \ No newline at end of file diff --git a/tests/integration/codebuild-local/test_all.sh b/tests/integration/codebuild-local/test_all.sh index 0c5168c..1a09241 100755 --- a/tests/integration/codebuild-local/test_all.sh +++ b/tests/integration/codebuild-local/test_all.sh @@ -5,6 +5,7 @@ set -euo pipefail CODEBUILD_IMAGE_TAG="${CODEBUILD_IMAGE_TAG:-al2/x86_64/standard/3.0}" DRYRUN="${DRYRUN-0}" +DISTRO="${DISTRO:=""}" function usage { echo "usage: test_all.sh buildspec_yml_dir" @@ -51,10 +52,12 @@ main() { usage exit 1 fi - + BUILDSPEC_YML_DIR="$1" + echo $DISTRO $BUILDSPEC_YML_DIR + ls $BUILDSPEC_YML_DIR HAS_YML=0 - for f in "$BUILDSPEC_YML_DIR"/*.yml ; do + for f in "$BUILDSPEC_YML_DIR"/*"$DISTRO"*.yml ; do [ -f "$f" ] || continue; do_one_yaml "$f" HAS_YML=1 diff --git a/tests/integration/codebuild/buildspec.os.alpine.yml b/tests/integration/codebuild/buildspec.os.alpine.yml index eba1d14..8b290f5 100644 --- a/tests/integration/codebuild/buildspec.os.alpine.yml +++ b/tests/integration/codebuild/buildspec.os.alpine.yml @@ -15,15 +15,14 @@ batch: env: variables: DISTRO_VERSION: - - "3.13" - - "3.14" - - "3.15" + - "3.19" + - "3.20" RUNTIME_VERSION: - - "3.7" - - "3.8" - "3.9" - "3.10" - "3.11" + - "3.12" + - "3.13" phases: pre_build: commands: @@ -52,20 +51,16 @@ phases: echo "COPY ${SCRATCH_DIR}/${RIE} /usr/bin/${RIE}" >> \ "${SCRATCH_DIR}/Dockerfile.echo.${OS_DISTRIBUTION}.tmp" - > - if [[ -z "${DOCKERHUB_USERNAME}" && -z "${DOCKERHUB_PASSWORD}" ]]; - then - echo "DockerHub credentials not set as CodeBuild environment variables. Continuing without docker login." - else - echo "Performing DockerHub login . . ." - docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD - fi + echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' > /etc/docker/daemon.json + service docker restart - echo "Building image ${IMAGE_TAG}" - > docker build . \ -f "${SCRATCH_DIR}/Dockerfile.echo.${OS_DISTRIBUTION}.tmp" \ -t "${IMAGE_TAG}" \ --build-arg RUNTIME_VERSION="${RUNTIME_VERSION}" \ - --build-arg DISTRO_VERSION="${DISTRO_VERSION}" + --build-arg DISTRO_VERSION="${DISTRO_VERSION}" \ + --load build: commands: - set -x diff --git a/tests/integration/codebuild/buildspec.os.amazonlinux.1.yml b/tests/integration/codebuild/buildspec.os.amazonlinux.1.yml deleted file mode 100644 index 5ec01d2..0000000 --- a/tests/integration/codebuild/buildspec.os.amazonlinux.1.yml +++ /dev/null @@ -1,103 +0,0 @@ -version: 0.2 - -env: - variables: - OS_DISTRIBUTION: amazonlinux - PYTHON_LOCATION: "/usr/local/bin/python3" - TEST_NAME: "aws-lambda-python-rtc-amazonlinux-test" -batch: - build-matrix: - static: - ignore-failure: false - env: - privileged-mode: true - dynamic: - env: - variables: - DISTRO_VERSION: - - "1" - RUNTIME_VERSION: - - "3.7" - - "3.8" - - "3.9" - - "3.10" - - "3.11" -phases: - pre_build: - commands: - - export IMAGE_TAG="python-${OS_DISTRIBUTION}-${DISTRO_VERSION}:${RUNTIME_VERSION}" - - echo "Extracting and including the Runtime Interface Emulator" - - SCRATCH_DIR=".scratch" - - mkdir "${SCRATCH_DIR}" - - ARCHITECTURE=$(arch) - - tar -xvf tests/integration/resources/aws-lambda-rie.tar.gz --directory "${SCRATCH_DIR}" - - > - cp "tests/integration/docker/Dockerfile.echo.${OS_DISTRIBUTION}" \ - "${SCRATCH_DIR}/Dockerfile.echo.${OS_DISTRIBUTION}.tmp" - - > - echo "COPY ${SCRATCH_DIR}/aws-lambda-rie /usr/bin/aws-lambda-rie" >> \ - "${SCRATCH_DIR}/Dockerfile.echo.${OS_DISTRIBUTION}.tmp" - - > - if [[ -z "${DOCKERHUB_USERNAME}" && -z "${DOCKERHUB_PASSWORD}" ]]; - then - echo "DockerHub credentials not set as CodeBuild environment variables. Continuing without docker login." - else - echo "Performing DockerHub login . . ." - docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD - fi - - echo "Building image ${IMAGE_TAG}" - - > - docker build . \ - -f "${SCRATCH_DIR}/Dockerfile.echo.${OS_DISTRIBUTION}.tmp" \ - -t "${IMAGE_TAG}" \ - --build-arg RUNTIME_VERSION="${RUNTIME_VERSION}" \ - --build-arg DISTRO_VERSION="${DISTRO_VERSION}" \ - --build-arg ARCHITECTURE="${ARCHITECTURE}" - build: - commands: - - set -x - - echo "Running Image ${IMAGE_TAG}" - - docker network create "${TEST_NAME}-network" - - > - docker run \ - --detach \ - -e "PYTHON_LOCATION=${PYTHON_LOCATION}" \ - --name "${TEST_NAME}-app" \ - --network "${TEST_NAME}-network" \ - --entrypoint="" \ - "${IMAGE_TAG}" \ - sh -c '/usr/bin/aws-lambda-rie ${PYTHON_LOCATION} -m awslambdaric app.handler' - - sleep 2 - - > - docker run \ - --name "${TEST_NAME}-tester" \ - --env "TARGET=${TEST_NAME}-app" \ - --network "${TEST_NAME}-network" \ - --entrypoint="" \ - "${IMAGE_TAG}" \ - sh -c 'curl -X POST "http://${TARGET}:8080/2015-03-31/functions/function/invocations" -d "{}" --max-time 10' - - actual="$(docker logs --tail 1 "${TEST_NAME}-tester" | xargs)" - - expected='success' - - | - echo "Response: ${actual}" - if [[ "$actual" != "$expected" ]]; then - echo "fail! runtime: $RUNTIME - expected output $expected - got $actual" - echo "---------Container Logs: ${TEST_NAME}-app----------" - echo - docker logs "${TEST_NAME}-app" || true - echo - echo "---------------------------------------------------" - echo "--------Container Logs: ${TEST_NAME}-tester--------" - echo - docker logs "${TEST_NAME}-tester" || true - echo - echo "---------------------------------------------------" - exit -1 - fi - finally: - - echo "Cleaning up..." - - docker stop "${TEST_NAME}-app" || true - - docker rm --force "${TEST_NAME}-app" || true - - docker stop "${TEST_NAME}-tester" || true - - docker rm --force "${TEST_NAME}-tester" || true - - docker network rm "${TEST_NAME}-network" || true diff --git a/tests/integration/codebuild/buildspec.os.amazonlinux.2.yml b/tests/integration/codebuild/buildspec.os.amazonlinux.2.yml index 18cabc9..05722bb 100644 --- a/tests/integration/codebuild/buildspec.os.amazonlinux.2.yml +++ b/tests/integration/codebuild/buildspec.os.amazonlinux.2.yml @@ -2,7 +2,7 @@ version: 0.2 env: variables: - OS_DISTRIBUTION: amazonlinux + OS_DISTRIBUTION: amazonlinux2 PYTHON_LOCATION: "/usr/local/bin/python3" TEST_NAME: "aws-lambda-python-rtc-amazonlinux-test" batch: @@ -17,8 +17,6 @@ batch: DISTRO_VERSION: - "2" RUNTIME_VERSION: - - "3.7" - - "3.8" - "3.9" - "3.10" - "3.11" @@ -47,13 +45,8 @@ phases: echo "COPY ${SCRATCH_DIR}/${RIE} /usr/bin/${RIE}" >> \ "${SCRATCH_DIR}/Dockerfile.echo.${OS_DISTRIBUTION}.tmp" - > - if [[ -z "${DOCKERHUB_USERNAME}" && -z "${DOCKERHUB_PASSWORD}" ]]; - then - echo "DockerHub credentials not set as CodeBuild environment variables. Continuing without docker login." - else - echo "Performing DockerHub login . . ." - docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD - fi + echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' > /etc/docker/daemon.json + service docker restart - echo "Building image ${IMAGE_TAG}" - > docker build . \ @@ -61,7 +54,8 @@ phases: -t "${IMAGE_TAG}" \ --build-arg RUNTIME_VERSION="${RUNTIME_VERSION}" \ --build-arg DISTRO_VERSION="${DISTRO_VERSION}" \ - --build-arg ARCHITECTURE="${ARCHITECTURE}" + --build-arg ARCHITECTURE="${ARCHITECTURE}" \ + --load build: commands: - set -x diff --git a/tests/integration/codebuild/buildspec.os.centos.yml b/tests/integration/codebuild/buildspec.os.amazonlinux.2023.yml similarity index 84% rename from tests/integration/codebuild/buildspec.os.centos.yml rename to tests/integration/codebuild/buildspec.os.amazonlinux.2023.yml index f993c7d..9d6d20f 100644 --- a/tests/integration/codebuild/buildspec.os.centos.yml +++ b/tests/integration/codebuild/buildspec.os.amazonlinux.2023.yml @@ -2,9 +2,9 @@ version: 0.2 env: variables: - OS_DISTRIBUTION: centos + OS_DISTRIBUTION: amazonlinux2023 PYTHON_LOCATION: "/usr/local/bin/python3" - TEST_NAME: "aws-lambda-python-rtc-centos-test" + TEST_NAME: "aws-lambda-python-rtc-amazonlinux-test" batch: build-matrix: static: @@ -15,13 +15,10 @@ batch: env: variables: DISTRO_VERSION: - - "7" + - "2023" RUNTIME_VERSION: - - "3.7" - - "3.8" - - "3.9" - - "3.10" - - "3.11" + - "3.12" + - "3.13" phases: pre_build: commands: @@ -47,13 +44,8 @@ phases: echo "COPY ${SCRATCH_DIR}/${RIE} /usr/bin/${RIE}" >> \ "${SCRATCH_DIR}/Dockerfile.echo.${OS_DISTRIBUTION}.tmp" - > - if [[ -z "${DOCKERHUB_USERNAME}" && -z "${DOCKERHUB_PASSWORD}" ]]; - then - echo "DockerHub credentials not set as CodeBuild environment variables. Continuing without docker login." - else - echo "Performing DockerHub login . . ." - docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD - fi + echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' > /etc/docker/daemon.json + service docker restart - echo "Building image ${IMAGE_TAG}" - > docker build . \ @@ -61,7 +53,8 @@ phases: -t "${IMAGE_TAG}" \ --build-arg RUNTIME_VERSION="${RUNTIME_VERSION}" \ --build-arg DISTRO_VERSION="${DISTRO_VERSION}" \ - --build-arg ARCHITECTURE="${ARCHITECTURE}" + --build-arg ARCHITECTURE="${ARCHITECTURE}" \ + --load build: commands: - set -x diff --git a/tests/integration/codebuild/buildspec.os.debian.yml b/tests/integration/codebuild/buildspec.os.debian.yml index 48305bb..44c061f 100644 --- a/tests/integration/codebuild/buildspec.os.debian.yml +++ b/tests/integration/codebuild/buildspec.os.debian.yml @@ -15,14 +15,14 @@ batch: env: variables: DISTRO_VERSION: - - "buster" + - "bookworm" - "bullseye" RUNTIME_VERSION: - - "3.7" - - "3.8" - "3.9" - "3.10" - "3.11" + - "3.12" + - "3.13" phases: pre_build: commands: @@ -51,20 +51,16 @@ phases: echo "RUN apt-get update && apt-get install -y curl" >> \ "${SCRATCH_DIR}/Dockerfile.echo.${OS_DISTRIBUTION}.tmp" - > - if [[ -z "${DOCKERHUB_USERNAME}" && -z "${DOCKERHUB_PASSWORD}" ]]; - then - echo "DockerHub credentials not set as CodeBuild environment variables. Continuing without docker login." - else - echo "Performing DockerHub login . . ." - docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD - fi + echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' > /etc/docker/daemon.json + service docker restart - echo "Building image ${IMAGE_TAG}" - > docker build . \ -f "${SCRATCH_DIR}/Dockerfile.echo.${OS_DISTRIBUTION}.tmp" \ -t "${IMAGE_TAG}" \ --build-arg RUNTIME_VERSION="${RUNTIME_VERSION}" \ - --build-arg DISTRO_VERSION="${DISTRO_VERSION}" + --build-arg DISTRO_VERSION="${DISTRO_VERSION}" \ + --load build: commands: - set -x diff --git a/tests/integration/codebuild/buildspec.os.ubuntu.yml b/tests/integration/codebuild/buildspec.os.ubuntu.yml index 7c66865..a6e556d 100644 --- a/tests/integration/codebuild/buildspec.os.ubuntu.yml +++ b/tests/integration/codebuild/buildspec.os.ubuntu.yml @@ -15,14 +15,14 @@ batch: env: variables: DISTRO_VERSION: - - "20.04" - "22.04" + - "24.04" RUNTIME_VERSION: - - "3.7" - - "3.8" - "3.9" - "3.10" - "3.11" + - "3.12" + - "3.13" phases: pre_build: commands: @@ -48,20 +48,16 @@ phases: echo "COPY ${SCRATCH_DIR}/${RIE} /usr/bin/${RIE}" >> \ "${SCRATCH_DIR}/Dockerfile.echo.${OS_DISTRIBUTION}.tmp" - > - if [[ -z "${DOCKERHUB_USERNAME}" && -z "${DOCKERHUB_PASSWORD}" ]]; - then - echo "DockerHub credentials not set as CodeBuild environment variables. Continuing without docker login." - else - echo "Performing DockerHub login . . ." - docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD - fi + echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' > /etc/docker/daemon.json + service docker restart - echo "Building image ${IMAGE_TAG}" - > docker build . \ -f "${SCRATCH_DIR}/Dockerfile.echo.${OS_DISTRIBUTION}.tmp" \ -t "${IMAGE_TAG}" \ --build-arg RUNTIME_VERSION="${RUNTIME_VERSION}" \ - --build-arg DISTRO_VERSION="${DISTRO_VERSION}" + --build-arg DISTRO_VERSION="${DISTRO_VERSION}" \ + --load build: commands: - set -x diff --git a/tests/integration/docker/Dockerfile.echo.alpine b/tests/integration/docker/Dockerfile.echo.alpine index 9b239e4..f6790fa 100644 --- a/tests/integration/docker/Dockerfile.echo.alpine +++ b/tests/integration/docker/Dockerfile.echo.alpine @@ -7,21 +7,22 @@ ARG DISTRO_VERSION FROM public.ecr.aws/docker/library/python:${RUNTIME_VERSION}-alpine${DISTRO_VERSION} AS python-alpine # Install libstdc++ RUN apk add --no-cache \ - libstdc++ + libstdc++ \ + binutils # Stage 2 - build function and dependencies FROM python-alpine AS build-image # Install aws-lambda-cpp build dependencies RUN apk add --no-cache \ - build-base \ - libtool \ - autoconf \ - automake \ - libexecinfo-dev \ - make \ - cmake \ - libcurl + build-base \ + libtool \ + autoconf \ + automake \ + elfutils-dev \ + make \ + cmake \ + libcurl # Include global args in this stage of the build ARG RIC_BUILD_DIR="/home/build/" @@ -30,6 +31,7 @@ RUN mkdir -p ${RIC_BUILD_DIR} # Copy function code and Runtime Interface Client .tgz WORKDIR ${RIC_BUILD_DIR} COPY . . +RUN pip3 install setuptools RUN make init build test && \ mv ./dist/awslambdaric-*.tar.gz ./dist/awslambdaric-test.tar.gz @@ -39,15 +41,12 @@ ARG FUNCTION_DIR="/home/app/" RUN mkdir -p ${FUNCTION_DIR} # Copy function code COPY tests/integration/test-handlers/echo/* ${FUNCTION_DIR} -# Copy Runtime Interface Client .tgz -RUN cp ./dist/awslambdaric-test.tar.gz ${FUNCTION_DIR}/awslambdaric-test.tar.gz # Install the function's dependencies WORKDIR ${FUNCTION_DIR} RUN python${RUNTIME_VERSION} -m pip install \ - awslambdaric-test.tar.gz \ - --target ${FUNCTION_DIR} && \ - rm awslambdaric-test.tar.gz + ${RIC_BUILD_DIR}/dist/awslambdaric-test.tar.gz \ + --target ${FUNCTION_DIR} # Stage 3 - final runtime interface client image diff --git a/tests/integration/docker/Dockerfile.echo.amazonlinux b/tests/integration/docker/Dockerfile.echo.amazonlinux2 similarity index 98% rename from tests/integration/docker/Dockerfile.echo.amazonlinux rename to tests/integration/docker/Dockerfile.echo.amazonlinux2 index 188de01..be05aa1 100644 --- a/tests/integration/docker/Dockerfile.echo.amazonlinux +++ b/tests/integration/docker/Dockerfile.echo.amazonlinux2 @@ -17,8 +17,10 @@ RUN yum install -y \ freetype-devel \ yum-utils \ findutils \ - openssl-devel \ wget \ + openssl11 \ + openssl11-devel \ + bzip2-devel \ libffi-devel \ sqlite-devel @@ -78,6 +80,7 @@ RUN mkdir -p ${RIC_BUILD_DIR} # Copy function code and Runtime Interface Client .tgz WORKDIR ${RIC_BUILD_DIR} COPY . . + RUN make init build test && \ mv ./dist/awslambdaric-*.tar.gz ./dist/awslambdaric-test.tar.gz diff --git a/tests/integration/docker/Dockerfile.echo.centos b/tests/integration/docker/Dockerfile.echo.amazonlinux2023 similarity index 77% rename from tests/integration/docker/Dockerfile.echo.centos rename to tests/integration/docker/Dockerfile.echo.amazonlinux2023 index e2dd3d0..16bbc79 100644 --- a/tests/integration/docker/Dockerfile.echo.centos +++ b/tests/integration/docker/Dockerfile.echo.amazonlinux2023 @@ -1,13 +1,12 @@ ARG DISTRO_VERSION - # Stage 1 - bundle base image + runtime interface client # Grab a fresh copy of the image and install Python -FROM public.ecr.aws/docker/library/centos:${DISTRO_VERSION} AS python-centos-builder +FROM public.ecr.aws/amazonlinux/amazonlinux:${DISTRO_VERSION} AS python-amazonlinux-builder ARG RUNTIME_VERSION # Install apt dependencies -RUN yum install -y \ +RUN dnf install -y \ gcc \ gcc-c++ \ tar \ @@ -18,8 +17,10 @@ RUN yum install -y \ freetype-devel \ yum-utils \ findutils \ - openssl-devel \ wget \ + openssl \ + openssl-devel \ + bzip2-devel \ libffi-devel \ sqlite-devel @@ -39,21 +40,21 @@ RUN RUNTIME_LATEST_VERSION=${RUNTIME_VERSION}.$(curl -s https://www.python.org/f && ln -s /usr/local/bin/python${RUNTIME_VERSION} /usr/local/bin/python${RUNTIME_LATEST_VERSION} # Stage 2 - clean python build dependencies -FROM public.ecr.aws/docker/library/centos:${DISTRO_VERSION} AS python-centos -RUN yum install -y \ +FROM public.ecr.aws/amazonlinux/amazonlinux:${DISTRO_VERSION} AS python-amazonlinux +RUN dnf install -y \ libffi-devel # Copy the compiled python to /usr/local -COPY --from=python-centos-builder /usr/local /usr/local +COPY --from=python-amazonlinux-builder /usr/local /usr/local ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH # Stage 3 - build function and dependencies -FROM python-centos-builder AS build-image +FROM python-amazonlinux-builder AS build-image ARG RUNTIME_VERSION ARG ARCHITECTURE # Install aws-lambda-cpp build dependencies -RUN yum install -y \ +RUN dnf install -y \ tar \ gzip \ make \ @@ -62,7 +63,8 @@ RUN yum install -y \ libtool \ libcurl-devel \ gcc-c++ \ - wget + wget \ + sqlite-devel # Install a modern CMake RUN wget --quiet -O cmake-install https://github.com/Kitware/CMake/releases/download/v3.20.0/cmake-3.20.0-linux-${ARCHITECTURE}.sh && \ @@ -79,8 +81,19 @@ RUN mkdir -p ${RIC_BUILD_DIR} # Copy function code and Runtime Interface Client .tgz WORKDIR ${RIC_BUILD_DIR} COPY . . -RUN make init build && \ - mv ./dist/awslambdaric-*.tar.gz ./dist/awslambdaric-test.tar.gz + +# distutils no longer available in python3.12 and later +# https://docs.python.org/3/whatsnew/3.12.html +# https://peps.python.org/pep-0632/ +RUN pip3 install setuptools +RUN make init build + +RUN mv ./dist/awslambdaric-*.tar.gz ./dist/awslambdaric-test.tar.gz +RUN python${RUNTIME_VERSION} -m pip install \ + ./dist/awslambdaric-test.tar.gz \ + --target ${RIC_BUILD_DIR} + +RUN make test # Include global args in this stage of the build ARG FUNCTION_DIR="/home/app/" @@ -93,19 +106,16 @@ RUN cp ./dist/awslambdaric-test.tar.gz ${FUNCTION_DIR}/awslambdaric-test.tar.gz # Install the function's dependencies WORKDIR ${FUNCTION_DIR} -ARG ENABLE_LTO=OFF -ENV ENABLE_LTO ${ENABLE_LTO} RUN python${RUNTIME_VERSION} -m pip install \ - awslambdaric-test.tar.gz \ - --verbose \ - --target ${FUNCTION_DIR} && \ + awslambdaric-test.tar.gz \ + --target ${FUNCTION_DIR} && \ rm awslambdaric-test.tar.gz # Stage 4 - final runtime interface client image # Grab a fresh copy of the Python image -FROM python-centos - +FROM python-amazonlinux +RUN dnf install -y brotli # Include global arg in this stage of the build ARG FUNCTION_DIR="/home/app/" # Set working directory to function root directory diff --git a/tests/integration/docker/Dockerfile.echo.debian b/tests/integration/docker/Dockerfile.echo.debian index 8ac660b..bf0f4fa 100644 --- a/tests/integration/docker/Dockerfile.echo.debian +++ b/tests/integration/docker/Dockerfile.echo.debian @@ -19,6 +19,7 @@ RUN mkdir -p ${RIC_BUILD_DIR} # Copy function code and Runtime Interface Client .tgz WORKDIR ${RIC_BUILD_DIR} COPY . . +RUN pip3 install setuptools RUN make init build test && \ mv ./dist/awslambdaric-*.tar.gz ./dist/awslambdaric-test.tar.gz diff --git a/tests/integration/docker/Dockerfile.echo.ubuntu b/tests/integration/docker/Dockerfile.echo.ubuntu index 692b3f2..0ce3000 100644 --- a/tests/integration/docker/Dockerfile.echo.ubuntu +++ b/tests/integration/docker/Dockerfile.echo.ubuntu @@ -9,40 +9,41 @@ ENV DEBIAN_FRONTEND=noninteractive ARG RUNTIME_VERSION # Install python and pip -RUN apt-get update && \ - apt-get install -y \ - software-properties-common +RUN apt-get update && apt-get install -y software-properties-common RUN add-apt-repository ppa:deadsnakes/ppa RUN apt-get update && \ - apt-get install -y \ - curl \ - python${RUNTIME_VERSION} \ - python${RUNTIME_VERSION}-distutils + apt-get install -y \ + curl \ + python${RUNTIME_VERSION} \ + python3-pip \ + python3-virtualenv + +# python3xx-distutils is needed for python < 3.12 +RUN if [ $(echo ${RUNTIME_VERSION} | cut -d '.' -f 2) -lt 12 ]; then \ + apt-get install -y python${RUNTIME_VERSION}-distutils; \ + fi +RUN virtualenv --python /usr/bin/python${RUNTIME_VERSION} --no-setuptools /home/venv + -RUN ln -s /usr/bin/python${RUNTIME_VERSION} /usr/local/bin/python3 # Stage 2 - build function and dependencies FROM python-image AS python-ubuntu-builder ARG RUNTIME_VERSION -RUN curl "https://bootstrap.pypa.io/get-pip.py" -o "get-pip.py" -RUN python${RUNTIME_VERSION} get-pip.py - # Install aws-lambda-cpp build dependencies -RUN apt-get update && \ - apt-get install -y \ - g++ \ - gcc \ - tar \ - gzip \ - make \ - cmake \ - autoconf \ - automake \ - libtool \ - libcurl4-openssl-dev \ - python${RUNTIME_VERSION}-dev +RUN apt-get install -y \ + g++ \ + gcc \ + tar \ + gzip \ + make \ + cmake \ + autoconf \ + automake \ + libtool \ + libcurl4-openssl-dev \ + python${RUNTIME_VERSION}-dev # Include global args in this stage of the build ARG RIC_BUILD_DIR="/home/build/" @@ -51,27 +52,28 @@ RUN mkdir -p ${RIC_BUILD_DIR} # Copy function code and Runtime Interface Client .tgz WORKDIR ${RIC_BUILD_DIR} COPY . . -RUN make init build test && \ +RUN . /home/venv/bin/activate && \ + pip install setuptools && \ + make init build test && \ mv ./dist/awslambdaric-*.tar.gz ./dist/awslambdaric-test.tar.gz + + # Include global args in this stage of the build ARG FUNCTION_DIR="/home/app/" # Create function directory RUN mkdir -p ${FUNCTION_DIR} # Copy function code COPY tests/integration/test-handlers/echo/* ${FUNCTION_DIR} -# Copy Runtime Interface Client .tgz -RUN cp ./dist/awslambdaric-test.tar.gz ${FUNCTION_DIR}/awslambdaric-test.tar.gz - # Install the function's dependencies WORKDIR ${FUNCTION_DIR} -RUN python${RUNTIME_VERSION} -m pip install \ - awslambdaric-test.tar.gz \ - --target ${FUNCTION_DIR} && \ - rm awslambdaric-test.tar.gz +RUN . /home/venv/bin/activate && \ + pip install ${RIC_BUILD_DIR}/dist/awslambdaric-test.tar.gz --target ${FUNCTION_DIR} + + -# Stage 4 - final runtime interface client image +# Stage 3 - final runtime interface client image # Grab a fresh copy of the Python image FROM python-image diff --git a/tests/test_bootstrap.py b/tests/test_bootstrap.py index 5614a2e..33afb1c 100644 --- a/tests/test_bootstrap.py +++ b/tests/test_bootstrap.py @@ -14,12 +14,20 @@ import unittest from io import StringIO from tempfile import NamedTemporaryFile -from unittest.mock import MagicMock, Mock, patch +from unittest.mock import MagicMock, Mock, patch, ANY import awslambdaric.bootstrap as bootstrap from awslambdaric.lambda_runtime_exception import FaultException -from awslambdaric.lambda_runtime_log_utils import LogFormat +from awslambdaric.lambda_runtime_log_utils import ( + LogFormat, + _get_log_level_from_env_var, + JsonFormatter, +) from awslambdaric.lambda_runtime_marshaller import LambdaMarshaller +from awslambdaric.lambda_literals import ( + lambda_unhandled_exception_warning_message, +) +import snapshot_restore_py class TestUpdateXrayEnv(unittest.TestCase): @@ -57,6 +65,14 @@ def setUp(self): self.event_body = '"event_body"' self.working_directory = os.getcwd() + logging.getLogger().handlers.clear() + + def tearDown(self) -> None: + logging.getLogger().handlers.clear() + logging.getLogger().level = logging.NOTSET + + return super().tearDown() + @staticmethod def dummy_handler(json_input, lambda_context): return {"input": json_input, "aws_request_id": lambda_context.aws_request_id} @@ -72,6 +88,7 @@ def test_handle_event_request_happy_case(self): {}, "invoked_function_arn", 0, + "tenant_id", bootstrap.StandardLogSink(), ) self.lambda_runtime.post_invocation_result.assert_called_once_with( @@ -95,6 +112,7 @@ def test_handle_event_request_invalid_client_context(self): {}, "invoked_function_arn", 0, + "tenant_id", bootstrap.StandardLogSink(), ) args, _ = self.lambda_runtime.post_invocation_error.call_args @@ -136,6 +154,7 @@ def test_handle_event_request_invalid_cognito_idenity(self): "invalid_cognito_identity", "invoked_function_arn", 0, + "tenant_id", bootstrap.StandardLogSink(), ) args, _ = self.lambda_runtime.post_invocation_error.call_args @@ -178,6 +197,7 @@ def test_handle_event_request_invalid_event_body(self): {}, "invoked_function_arn", 0, + "tenant_id", bootstrap.StandardLogSink(), ) args, _ = self.lambda_runtime.post_invocation_error.call_args @@ -222,6 +242,7 @@ def invalid_json_response(json_input, lambda_context): {}, "invoked_function_arn", 0, + "tenant_id", bootstrap.StandardLogSink(), ) args, _ = self.lambda_runtime.post_invocation_error.call_args @@ -267,6 +288,7 @@ def __init__(self, message): {}, "invoked_function_arn", 0, + "tenant_id", bootstrap.StandardLogSink(), ) args, _ = self.lambda_runtime.post_invocation_error.call_args @@ -319,6 +341,7 @@ def __init__(self, message): {}, "invoked_function_arn", 0, + "tenant_id", bootstrap.StandardLogSink(), ) args, _ = self.lambda_runtime.post_invocation_error.call_args @@ -370,6 +393,7 @@ def unable_to_import_module(json_input, lambda_context): {}, "invoked_function_arn", 0, + "tenant_id", bootstrap.StandardLogSink(), ) args, _ = self.lambda_runtime.post_invocation_error.call_args @@ -409,6 +433,7 @@ def raise_exception_handler(json_input, lambda_context): {}, "invoked_function_arn", 0, + "tenant_id", bootstrap.StandardLogSink(), ) args, _ = self.lambda_runtime.post_invocation_error.call_args @@ -447,6 +472,8 @@ def raise_exception_handler(json_input, lambda_context): ), ) + logging.getLogger().addHandler(logging.StreamHandler(mock_stdout)) + bootstrap.handle_event_request( self.lambda_runtime, raise_exception_handler, @@ -457,6 +484,7 @@ def raise_exception_handler(json_input, lambda_context): {}, "invoked_function_arn", 0, + "tenant_id", bootstrap.StandardLogSink(), ) @@ -480,6 +508,8 @@ def raise_exception_handler(json_input, lambda_context): "FaultExceptionType", "Fault exception msg", None ) + logging.getLogger().addHandler(logging.StreamHandler(mock_stdout)) + bootstrap.handle_event_request( self.lambda_runtime, raise_exception_handler, @@ -490,6 +520,7 @@ def raise_exception_handler(json_input, lambda_context): {}, "invoked_function_arn", 0, + "tenant_id", bootstrap.StandardLogSink(), ) error_logs = "[ERROR] FaultExceptionType: Fault exception msg\rTraceback (most recent call last):\n" @@ -506,6 +537,8 @@ def raise_exception_handler(json_input, lambda_context): except ImportError: raise bootstrap.FaultException("FaultExceptionType", None, None) + logging.getLogger().addHandler(logging.StreamHandler(mock_stdout)) + bootstrap.handle_event_request( self.lambda_runtime, raise_exception_handler, @@ -516,6 +549,7 @@ def raise_exception_handler(json_input, lambda_context): {}, "invoked_function_arn", 0, + "tenant_id", bootstrap.StandardLogSink(), ) error_logs = "[ERROR] FaultExceptionType\rTraceback (most recent call last):\n" @@ -532,6 +566,8 @@ def raise_exception_handler(json_input, lambda_context): except ImportError: raise bootstrap.FaultException(None, "Fault exception msg", None) + logging.getLogger().addHandler(logging.StreamHandler(mock_stdout)) + bootstrap.handle_event_request( self.lambda_runtime, raise_exception_handler, @@ -542,6 +578,7 @@ def raise_exception_handler(json_input, lambda_context): {}, "invoked_function_arn", 0, + "tenant_id", bootstrap.StandardLogSink(), ) error_logs = "[ERROR] Fault exception msg\rTraceback (most recent call last):\n" @@ -567,6 +604,8 @@ def raise_exception_handler(json_input, lambda_context): ), ) + logging.getLogger().addHandler(logging.StreamHandler(mock_stdout)) + bootstrap.handle_event_request( self.lambda_runtime, raise_exception_handler, @@ -577,9 +616,9 @@ def raise_exception_handler(json_input, lambda_context): {}, "invoked_function_arn", 0, + "tenant_id", bootstrap.StandardLogSink(), ) - error_logs = "[ERROR]\r" error_logs += "Traceback (most recent call last):\r" error_logs += '  File "spam.py", line 3, in \r' @@ -590,22 +629,20 @@ def raise_exception_handler(json_input, lambda_context): self.assertEqual(mock_stdout.getvalue(), error_logs) @patch("sys.stdout", new_callable=StringIO) - @patch("importlib.import_module") - def test_handle_event_request_fault_exception_logging_syntax_error( - self, mock_import_module, mock_stdout - ): - try: - eval("-") - except SyntaxError as e: - syntax_error = e - - mock_import_module.side_effect = syntax_error + def test_handle_event_request_fault_exception_logging_in_json(self, mock_stdout): + def raise_exception_handler(json_input, lambda_context): + try: + import invalid_module # noqa: F401 + except ImportError: + raise bootstrap.FaultException("FaultExceptionType", None, None) - response_handler = bootstrap._get_handler("a.b") + logging_handler = logging.StreamHandler(mock_stdout) + logging_handler.setFormatter(JsonFormatter()) + logging.getLogger().addHandler(logging_handler) bootstrap.handle_event_request( self.lambda_runtime, - response_handler, + raise_exception_handler, "invoke_id", self.event_body, "application/json", @@ -613,15 +650,16 @@ def test_handle_event_request_fault_exception_logging_syntax_error( {}, "invoked_function_arn", 0, + "tenant_id", bootstrap.StandardLogSink(), ) - error_logs = f"[ERROR] Runtime.UserCodeSyntaxError: Syntax error in module 'a': {syntax_error}\r" - error_logs += "Traceback (most recent call last):\r" - error_logs += '  File "" Line 1\r' - error_logs += "    -\n" + stdout_value = mock_stdout.getvalue() - self.assertEqual(mock_stdout.getvalue(), error_logs) + # this line is not in json because of the way the test runtime is bootstrapped + error_logs = "[ERROR] FaultExceptionType\rTraceback (most recent call last):\n" + + self.assertEqual(stdout_value, error_logs) class TestXrayFault(unittest.TestCase): @@ -700,10 +738,8 @@ def __eq__(self, other): def test_get_event_handler_bad_handler(self): handler_name = "bad_handler" - response_handler = bootstrap._get_handler(handler_name) with self.assertRaises(FaultException) as cm: - response_handler() - + response_handler = bootstrap._get_handler(handler_name) returned_exception = cm.exception self.assertEqual( self.FaultExceptionMatcher( @@ -715,9 +751,8 @@ def test_get_event_handler_bad_handler(self): def test_get_event_handler_import_error(self): handler_name = "no_module.handler" - response_handler = bootstrap._get_handler(handler_name) with self.assertRaises(FaultException) as cm: - response_handler() + response_handler = bootstrap._get_handler(handler_name) returned_exception = cm.exception self.assertEqual( self.FaultExceptionMatcher( @@ -740,10 +775,9 @@ def test_get_event_handler_syntax_error(self): filename_w_ext = os.path.basename(tmp_file.name) filename, _ = os.path.splitext(filename_w_ext) handler_name = "{}.syntax_error".format(filename) - response_handler = bootstrap._get_handler(handler_name) with self.assertRaises(FaultException) as cm: - response_handler() + response_handler = bootstrap._get_handler(handler_name) returned_exception = cm.exception self.assertEqual( self.FaultExceptionMatcher( @@ -765,9 +799,8 @@ def test_get_event_handler_missing_error(self): filename_w_ext = os.path.basename(tmp_file.name) filename, _ = os.path.splitext(filename_w_ext) handler_name = "{}.my_handler".format(filename) - response_handler = bootstrap._get_handler(handler_name) with self.assertRaises(FaultException) as cm: - response_handler() + response_handler = bootstrap._get_handler(handler_name) returned_exception = cm.exception self.assertEqual( self.FaultExceptionMatcher( @@ -784,9 +817,8 @@ def test_get_event_handler_slash(self): response_handler() def test_get_event_handler_build_in_conflict(self): - response_handler = bootstrap._get_handler("sys.hello") with self.assertRaises(FaultException) as cm: - response_handler() + response_handler = bootstrap._get_handler("sys.hello") returned_exception = cm.exception self.assertEqual( self.FaultExceptionMatcher( @@ -825,6 +857,7 @@ def test_application_json(self): cognito_identity_json=None, invoked_function_arn="invocation-arn", epoch_deadline_time_in_ms=1415836801003, + tenant_id=None, log_sink=bootstrap.StandardLogSink(), ) @@ -844,6 +877,7 @@ def test_binary_request_binary_response(self): cognito_identity_json=None, invoked_function_arn="invocation-arn", epoch_deadline_time_in_ms=1415836801003, + tenant_id=None, log_sink=bootstrap.StandardLogSink(), ) @@ -863,6 +897,7 @@ def test_json_request_binary_response(self): cognito_identity_json=None, invoked_function_arn="invocation-arn", epoch_deadline_time_in_ms=1415836801003, + tenant_id=None, log_sink=bootstrap.StandardLogSink(), ) @@ -881,6 +916,7 @@ def test_binary_with_application_json(self): cognito_identity_json=None, invoked_function_arn="invocation-arn", epoch_deadline_time_in_ms=1415836801003, + tenant_id=None, log_sink=bootstrap.StandardLogSink(), ) @@ -927,7 +963,7 @@ def test_log_error_framed_log_sink(self): content = f.read() frame_type = int.from_bytes(content[:4], "big") - self.assertEqual(frame_type, 0xA55A0003) + self.assertEqual(frame_type, 0xA55A0017) length = int.from_bytes(content[4:8], "big") self.assertEqual(length, len(expected_logged_error.encode("utf8"))) @@ -973,7 +1009,7 @@ def test_log_error_indentation_framed_log_sink(self): content = f.read() frame_type = int.from_bytes(content[:4], "big") - self.assertEqual(frame_type, 0xA55A0003) + self.assertEqual(frame_type, 0xA55A0017) length = int.from_bytes(content[4:8], "big") self.assertEqual(length, len(expected_logged_error.encode("utf8"))) @@ -1016,7 +1052,7 @@ def test_log_error_empty_stacktrace_line_framed_log_sink(self): content = f.read() frame_type = int.from_bytes(content[:4], "big") - self.assertEqual(frame_type, 0xA55A0003) + self.assertEqual(frame_type, 0xA55A0017) length = int.from_bytes(content[4:8], "big") self.assertEqual(length, len(expected_logged_error)) @@ -1053,7 +1089,7 @@ def test_log_error_invokeId_line_framed_log_sink(self): content = f.read() frame_type = int.from_bytes(content[:4], "big") - self.assertEqual(frame_type, 0xA55A0003) + self.assertEqual(frame_type, 0xA55A0017) length = int.from_bytes(content[4:8], "big") self.assertEqual(length, len(expected_logged_error)) @@ -1179,14 +1215,13 @@ def test_log_level(self) -> None: (LogFormat.JSON, "WARN", logging.WARNING), (LogFormat.JSON, "ERROR", logging.ERROR), (LogFormat.JSON, "FATAL", logging.CRITICAL), - # Log level is set only for Json format - (LogFormat.TEXT, "TRACE", logging.NOTSET), - (LogFormat.TEXT, "DEBUG", logging.NOTSET), - (LogFormat.TEXT, "INFO", logging.NOTSET), - (LogFormat.TEXT, "WARN", logging.NOTSET), - (LogFormat.TEXT, "ERROR", logging.NOTSET), - (LogFormat.TEXT, "FATAL", logging.NOTSET), - ("Unknown format", "INFO", logging.NOTSET), + (LogFormat.TEXT, "TRACE", logging.DEBUG), + (LogFormat.TEXT, "DEBUG", logging.DEBUG), + (LogFormat.TEXT, "INFO", logging.INFO), + (LogFormat.TEXT, "WARN", logging.WARN), + (LogFormat.TEXT, "ERROR", logging.ERROR), + (LogFormat.TEXT, "FATAL", logging.CRITICAL), + ("Unknown format", "INFO", logging.INFO), # if level is unknown fall back to default (LogFormat.JSON, "Unknown level", logging.NOTSET), ] @@ -1196,11 +1231,67 @@ def test_log_level(self) -> None: logging.getLogger().handlers.clear() logging.getLogger().level = logging.NOTSET - bootstrap._setup_logging(fmt, log_level, bootstrap.StandardLogSink()) + bootstrap._setup_logging( + fmt, + _get_log_level_from_env_var(log_level), + bootstrap.StandardLogSink(), + ) self.assertEqual(expected_level, logging.getLogger().level) +class TestLambdaLoggerHandlerSetup(unittest.TestCase): + @classmethod + def tearDownClass(cls): + importlib.reload(bootstrap) + logging.getLogger().handlers.clear() + logging.getLogger().level = logging.NOTSET + + def test_handler_setup(self, *_): + test_cases = [ + (62, 0xA55A0003, 46, {}), + (133, 0xA55A001A, 117, {"AWS_LAMBDA_LOG_FORMAT": "JSON"}), + (62, 0xA55A001B, 46, {"AWS_LAMBDA_LOG_LEVEL": "INFO"}), + ] + + for total_length, header, message_length, env_vars in test_cases: + with patch.dict( + os.environ, env_vars, clear=True + ), NamedTemporaryFile() as temp_file: + importlib.reload(bootstrap) + logging.getLogger().handlers.clear() + logging.getLogger().level = logging.NOTSET + + before = int(time.time_ns() / 1000) + with bootstrap.FramedTelemetryLogSink( + os.open(temp_file.name, os.O_CREAT | os.O_RDWR) + ) as ls: + bootstrap._setup_logging( + bootstrap._AWS_LAMBDA_LOG_FORMAT, + bootstrap._AWS_LAMBDA_LOG_LEVEL, + ls, + ) + logger = logging.getLogger() + logger.critical("critical") + after = int(time.time_ns() / 1000) + + content = open(temp_file.name, "rb").read() + self.assertEqual(len(content), total_length) + + pos = 0 + frame_type = int.from_bytes(content[pos : pos + 4], "big") + self.assertEqual(frame_type, header) + pos += 4 + + length = int.from_bytes(content[pos : pos + 4], "big") + self.assertEqual(length, message_length) + pos += 4 + + timestamp = int.from_bytes(content[pos : pos + 8], "big") + self.assertTrue(before <= timestamp) + self.assertTrue(timestamp <= after) + + class TestLogging(unittest.TestCase): @classmethod def setUpClass(cls) -> None: @@ -1259,6 +1350,31 @@ def test_json_formatter(self, mock_stderr): ) self.assertEqual(mock_stderr.getvalue(), "") + @patch("awslambdaric.bootstrap._GLOBAL_TENANT_ID", "test-tenant-id") + @patch("sys.stderr", new_callable=StringIO) + def test_json_formatter_with_tenant_id(self, mock_stderr): + logger = logging.getLogger("a.b") + level = logging.INFO + message = "Test json formatting with tenant id" + expected = { + "level": "INFO", + "logger": "a.b", + "message": message, + "requestId": "", + "tenantId": "test-tenant-id", + } + + with patch("sys.stdout", new_callable=StringIO) as mock_stdout: + logger.log(level, message) + + data = json.loads(mock_stdout.getvalue()) + data.pop("timestamp") + self.assertEqual( + data, + expected, + ) + self.assertEqual(mock_stderr.getvalue(), "") + @patch("sys.stdout", new_callable=StringIO) @patch("sys.stderr", new_callable=StringIO) def test_exception(self, mock_stderr, mock_stdout): @@ -1380,9 +1496,8 @@ def test_set_log_level_with_dictConfig(self, mock_stderr, mock_stdout): class TestBootstrapModule(unittest.TestCase): - @patch("awslambdaric.bootstrap.handle_event_request") @patch("awslambdaric.bootstrap.LambdaRuntimeClient") - def test_run(self, mock_runtime_client, mock_handle_event_request): + def test_run(self, mock_runtime_client): expected_app_root = "/tmp/test/app_root" expected_handler = "app.my_test_handler" expected_lambda_runtime_api_addr = "test_addr" @@ -1395,22 +1510,22 @@ def test_run(self, mock_runtime_client, mock_handle_event_request): MagicMock(), ] - with self.assertRaises(TypeError): + with self.assertRaises(SystemExit) as cm: bootstrap.run( expected_app_root, expected_handler, expected_lambda_runtime_api_addr ) - mock_handle_event_request.assert_called_once() + self.assertEqual(cm.exception.code, 1) @patch( "awslambdaric.bootstrap.LambdaLoggerHandler", Mock(side_effect=Exception("Boom!")), ) - @patch("awslambdaric.bootstrap.build_fault_result", MagicMock()) + @patch("awslambdaric.bootstrap.build_fault_result") @patch("awslambdaric.bootstrap.log_error", MagicMock()) @patch("awslambdaric.bootstrap.LambdaRuntimeClient", MagicMock()) @patch("awslambdaric.bootstrap.sys") - def test_run_exception(self, mock_sys): + def test_run_exception(self, mock_sys, mock_build_fault_result): class TestException(Exception): pass @@ -1418,6 +1533,7 @@ class TestException(Exception): expected_handler = "app.my_test_handler" expected_lambda_runtime_api_addr = "test_addr" + mock_build_fault_result.return_value = {} mock_sys.exit.side_effect = TestException("Boom!") with self.assertRaises(TestException): @@ -1428,5 +1544,53 @@ class TestException(Exception): mock_sys.exit.assert_called_once_with(1) +class TestOnInitComplete(unittest.TestCase): + def tearDown(self): + # We are accessing private filed for cleaning up + snapshot_restore_py._before_snapshot_registry = [] + snapshot_restore_py._after_restore_registry = [] + + # We are using ANY over here as the main thing we want to test is teh errorType propogation and stack trace generation + error_result = { + "errorMessage": "This is a Dummy type error", + "errorType": "TypeError", + "requestId": "", + "stackTrace": ANY, + } + + def raise_type_error(self): + raise TypeError("This is a Dummy type error") + + @patch("awslambdaric.bootstrap.LambdaRuntimeClient") + def test_before_snapshot_exception(self, mock_runtime_client): + snapshot_restore_py.register_before_snapshot(self.raise_type_error) + + with self.assertRaises(SystemExit) as cm: + bootstrap.on_init_complete( + mock_runtime_client, log_sink=bootstrap.StandardLogSink() + ) + + self.assertEqual(cm.exception.code, 64) + mock_runtime_client.post_init_error.assert_called_once_with( + self.error_result, + FaultException.BEFORE_SNAPSHOT_ERROR, + ) + + @patch("awslambdaric.bootstrap.LambdaRuntimeClient") + def test_after_restore_exception(self, mock_runtime_client): + snapshot_restore_py.register_after_restore(self.raise_type_error) + + with self.assertRaises(SystemExit) as cm: + bootstrap.on_init_complete( + mock_runtime_client, log_sink=bootstrap.StandardLogSink() + ) + + self.assertEqual(cm.exception.code, 65) + mock_runtime_client.restore_next.assert_called_once() + mock_runtime_client.report_restore_error.assert_called_once_with( + self.error_result + ) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_lambda_context.py b/tests/test_lambda_context.py index 34d59da..f7959ab 100644 --- a/tests/test_lambda_context.py +++ b/tests/test_lambda_context.py @@ -37,6 +37,7 @@ def test_init(self): self.assertEqual(context.memory_limit_in_mb, "1234") self.assertEqual(context.function_version, "version1") self.assertEqual(context.invoked_function_arn, "arn:test1") + self.assertEqual(context.tenant_id, None) self.assertEqual(context.identity.cognito_identity_id, None) self.assertEqual(context.identity.cognito_identity_pool_id, None) self.assertEqual(context.client_context.client.installation_id, None) @@ -74,6 +75,21 @@ def test_init_cognito(self): self.assertEqual(context.identity.cognito_identity_id, "id1") self.assertEqual(context.identity.cognito_identity_pool_id, "poolid1") + def test_init_tenant_id(self): + client_context = {} + cognito_identity = {} + tenant_id = "blue" + + context = LambdaContext( + "invoke-id1", + client_context, + cognito_identity, + 1415836801000, + "arn:test", + tenant_id, + ) + self.assertEqual(context.tenant_id, "blue") + def test_init_client_context(self): client_context = { "client": { diff --git a/tests/test_lambda_runtime_client.py b/tests/test_lambda_runtime_client.py index 47d95cf..fc4af65 100644 --- a/tests/test_lambda_runtime_client.py +++ b/tests/test_lambda_runtime_client.py @@ -14,6 +14,7 @@ LambdaRuntimeClientError, _user_agent, ) +from awslambdaric.lambda_runtime_marshaller import to_json class TestInvocationRequest(unittest.TestCase): @@ -25,6 +26,7 @@ def test_constructor(self): deadline_time_in_ms="Lambda-Runtime-Deadline-Ms", client_context="Lambda-Runtime-Client-Context", cognito_identity="Lambda-Runtime-Cognito-Identity", + tenant_id="Lambda-Runtime-Aws-Tenant-Id", content_type="Content-Type", event_body="response_body", ) @@ -36,6 +38,7 @@ def test_constructor(self): deadline_time_in_ms="Lambda-Runtime-Deadline-Ms", client_context="Lambda-Runtime-Client-Context", cognito_identity="Lambda-Runtime-Cognito-Identity", + tenant_id="Lambda-Runtime-Aws-Tenant-Id", content_type="Content-Type", event_body="response_body", ) @@ -47,6 +50,7 @@ def test_constructor(self): deadline_time_in_ms="Lambda-Runtime-Deadline-Ms", client_context="Lambda-Runtime-Client-Context", cognito_identity="Lambda-Runtime-Cognito-Identity", + tenant_id="Lambda-Runtime-Aws-Tenant-Id", content_type="Content-Type", event_body="another_response_body", ) @@ -67,6 +71,7 @@ def test_wait_next_invocation(self, mock_runtime_client): "Lambda-Runtime-Deadline-Ms": 12, "Lambda-Runtime-Client-Context": "client_context", "Lambda-Runtime-Cognito-Identity": "cognito_identity", + "Lambda-Runtime-Aws-Tenant-Id": "tenant_id", "Content-Type": "application/json", } mock_runtime_client.next.return_value = response_body, headears @@ -81,9 +86,117 @@ def test_wait_next_invocation(self, mock_runtime_client): self.assertEqual(event_request.deadline_time_in_ms, 12) self.assertEqual(event_request.client_context, "client_context") self.assertEqual(event_request.cognito_identity, "cognito_identity") + self.assertEqual(event_request.tenant_id, "tenant_id") self.assertEqual(event_request.content_type, "application/json") self.assertEqual(event_request.event_body, response_body) + # Using ThreadPoolExecutor to polling next() + runtime_client = LambdaRuntimeClient("localhost:1234", True) + + event_request = runtime_client.wait_next_invocation() + + self.assertIsNotNone(event_request) + self.assertEqual(event_request.invoke_id, "RID1234") + self.assertEqual(event_request.x_amzn_trace_id, "TID1234") + self.assertEqual(event_request.invoked_function_arn, "FARN1234") + self.assertEqual(event_request.deadline_time_in_ms, 12) + self.assertEqual(event_request.client_context, "client_context") + self.assertEqual(event_request.cognito_identity, "cognito_identity") + self.assertEqual(event_request.tenant_id, "tenant_id") + self.assertEqual(event_request.content_type, "application/json") + self.assertEqual(event_request.event_body, response_body) + + @patch("awslambdaric.lambda_runtime_client.runtime_client") + def test_wait_next_invocation_without_tenant_id_header(self, mock_runtime_client): + response_body = b"{}" + headers = { + "Lambda-Runtime-Aws-Request-Id": "RID1234", + "Lambda-Runtime-Trace-Id": "TID1234", + "Lambda-Runtime-Invoked-Function-Arn": "FARN1234", + "Lambda-Runtime-Deadline-Ms": 12, + "Lambda-Runtime-Client-Context": "client_context", + "Lambda-Runtime-Cognito-Identity": "cognito_identity", + "Content-Type": "application/json", + } + mock_runtime_client.next.return_value = response_body, headers + runtime_client = LambdaRuntimeClient("localhost:1234") + + event_request = runtime_client.wait_next_invocation() + + self.assertIsNotNone(event_request) + self.assertIsNone(event_request.tenant_id) + self.assertEqual(event_request.event_body, response_body) + + @patch("awslambdaric.lambda_runtime_client.runtime_client") + def test_wait_next_invocation_with_null_tenant_id_header(self, mock_runtime_client): + response_body = b"{}" + headers = { + "Lambda-Runtime-Aws-Request-Id": "RID1234", + "Lambda-Runtime-Trace-Id": "TID1234", + "Lambda-Runtime-Invoked-Function-Arn": "FARN1234", + "Lambda-Runtime-Deadline-Ms": 12, + "Lambda-Runtime-Client-Context": "client_context", + "Lambda-Runtime-Cognito-Identity": "cognito_identity", + "Lambda-Runtime-Aws-Tenant-Id": None, + "Content-Type": "application/json", + } + mock_runtime_client.next.return_value = response_body, headers + runtime_client = LambdaRuntimeClient("localhost:1234") + + event_request = runtime_client.wait_next_invocation() + + self.assertIsNotNone(event_request) + self.assertIsNone(event_request.tenant_id) + self.assertEqual(event_request.event_body, response_body) + + @patch("awslambdaric.lambda_runtime_client.runtime_client") + def test_wait_next_invocation_with_empty_tenant_id_header( + self, mock_runtime_client + ): + response_body = b"{}" + headers = { + "Lambda-Runtime-Aws-Request-Id": "RID1234", + "Lambda-Runtime-Trace-Id": "TID1234", + "Lambda-Runtime-Invoked-Function-Arn": "FARN1234", + "Lambda-Runtime-Deadline-Ms": 12, + "Lambda-Runtime-Client-Context": "client_context", + "Lambda-Runtime-Cognito-Identity": "cognito_identity", + "Lambda-Runtime-Aws-Tenant-Id": "", + "Content-Type": "application/json", + } + mock_runtime_client.next.return_value = response_body, headers + runtime_client = LambdaRuntimeClient("localhost:1234") + + event_request = runtime_client.wait_next_invocation() + + self.assertIsNotNone(event_request) + self.assertEqual(event_request.tenant_id, "") + self.assertEqual(event_request.event_body, response_body) + + error_result = { + "errorMessage": "Dummy message", + "errorType": "Runtime.DummyError", + "requestId": "", + "stackTrace": [], + } + + headers = {"Lambda-Runtime-Function-Error-Type": error_result["errorType"]} + + restore_error_result = { + "errorMessage": "Dummy Restore error", + "errorType": "Runtime.DummyRestoreError", + "requestId": "", + "stackTrace": [], + } + + restore_error_header = { + "Lambda-Runtime-Function-Error-Type": "Runtime.AfterRestoreError" + } + + before_snapshot_error_header = { + "Lambda-Runtime-Function-Error-Type": "Runtime.BeforeSnapshotError" + } + @patch("http.client.HTTPConnection", autospec=http.client.HTTPConnection) def test_post_init_error(self, MockHTTPConnection): mock_conn = MockHTTPConnection.return_value @@ -93,11 +206,14 @@ def test_post_init_error(self, MockHTTPConnection): mock_response.code = http.HTTPStatus.ACCEPTED runtime_client = LambdaRuntimeClient("localhost:1234") - runtime_client.post_init_error("error_data") + runtime_client.post_init_error(self.error_result) MockHTTPConnection.assert_called_with("localhost:1234") mock_conn.request.assert_called_once_with( - "POST", "/2018-06-01/runtime/init/error", "error_data" + "POST", + "/2018-06-01/runtime/init/error", + to_json(self.error_result), + headers=self.headers, ) mock_response.read.assert_called_once() @@ -112,7 +228,7 @@ def test_post_init_error_non_accepted_status_code(self, MockHTTPConnection): runtime_client = LambdaRuntimeClient("localhost:1234") with self.assertRaises(LambdaRuntimeClientError) as cm: - runtime_client.post_init_error("error_data") + runtime_client.post_init_error(self.error_result) returned_exception = cm.exception self.assertEqual(returned_exception.endpoint, "/2018-06-01/runtime/init/error") @@ -197,15 +313,73 @@ def test_post_invocation_error_with_too_large_xray_cause(self, mock_runtime_clie invoke_id, error_data, "" ) + @patch("http.client.HTTPConnection", autospec=http.client.HTTPConnection) + def test_restore_next(self, MockHTTPConnection): + mock_conn = MockHTTPConnection.return_value + mock_response = MagicMock(autospec=http.client.HTTPResponse) + mock_conn.getresponse.return_value = mock_response + mock_response.read.return_value = b"" + mock_response.code = http.HTTPStatus.OK + + runtime_client = LambdaRuntimeClient("localhost:1234") + runtime_client.restore_next() + + MockHTTPConnection.assert_called_with("localhost:1234") + mock_conn.request.assert_called_once_with( + "GET", + "/2018-06-01/runtime/restore/next", + ) + mock_response.read.assert_called_once() + + @patch("http.client.HTTPConnection", autospec=http.client.HTTPConnection) + def test_restore_error(self, MockHTTPConnection): + mock_conn = MockHTTPConnection.return_value + mock_response = MagicMock(autospec=http.client.HTTPResponse) + mock_conn.getresponse.return_value = mock_response + mock_response.read.return_value = b"" + mock_response.code = http.HTTPStatus.ACCEPTED + + runtime_client = LambdaRuntimeClient("localhost:1234") + runtime_client.report_restore_error(self.restore_error_result) + + MockHTTPConnection.assert_called_with("localhost:1234") + mock_conn.request.assert_called_once_with( + "POST", + "/2018-06-01/runtime/restore/error", + to_json(self.restore_error_result), + headers=self.restore_error_header, + ) + mock_response.read.assert_called_once() + + @patch("http.client.HTTPConnection", autospec=http.client.HTTPConnection) + def test_init_before_snapshot_error(self, MockHTTPConnection): + mock_conn = MockHTTPConnection.return_value + mock_response = MagicMock(autospec=http.client.HTTPResponse) + mock_conn.getresponse.return_value = mock_response + mock_response.read.return_value = b"" + mock_response.code = http.HTTPStatus.ACCEPTED + + runtime_client = LambdaRuntimeClient("localhost:1234") + runtime_client.post_init_error(self.error_result, "Runtime.BeforeSnapshotError") + + MockHTTPConnection.assert_called_with("localhost:1234") + mock_conn.request.assert_called_once_with( + "POST", + "/2018-06-01/runtime/init/error", + to_json(self.error_result), + headers=self.before_snapshot_error_header, + ) + mock_response.read.assert_called_once() + def test_connection_refused(self): with self.assertRaises(ConnectionRefusedError): runtime_client = LambdaRuntimeClient("127.0.0.1:1") - runtime_client.post_init_error("error") + runtime_client.post_init_error(self.error_result) def test_invalid_addr(self): with self.assertRaises(OSError): runtime_client = LambdaRuntimeClient("::::") - runtime_client.post_init_error("error") + runtime_client.post_init_error(self.error_result) def test_lambdaric_version(self): self.assertTrue(_user_agent().endswith(__version__)) diff --git a/tests/test_lambda_runtime_marshaller.py b/tests/test_lambda_runtime_marshaller.py index 8268de1..843bcee 100644 --- a/tests/test_lambda_runtime_marshaller.py +++ b/tests/test_lambda_runtime_marshaller.py @@ -3,12 +3,39 @@ """ import decimal +import os import unittest - +from parameterized import parameterized from awslambdaric.lambda_runtime_marshaller import to_json class TestLambdaRuntimeMarshaller(unittest.TestCase): + execution_envs = ( + "AWS_Lambda_python3.13", + "AWS_Lambda_python3.12", + "AWS_Lambda_python3.11", + "AWS_Lambda_python3.10", + "AWS_Lambda_python3.9", + ) + + envs_lambda_marshaller_ensure_ascii_false = { + "AWS_Lambda_python3.12", + "AWS_Lambda_python3.13", + } + + execution_envs_lambda_marshaller_ensure_ascii_true = tuple( + set(execution_envs).difference(envs_lambda_marshaller_ensure_ascii_false) + ) + execution_envs_lambda_marshaller_ensure_ascii_false = tuple( + envs_lambda_marshaller_ensure_ascii_false + ) + + def setUp(self): + self.org_os_environ = os.environ + + def tearDown(self): + os.environ = self.org_os_environ + def test_to_json_decimal_encoding(self): response = to_json({"pi": decimal.Decimal("3.14159")}) self.assertEqual('{"pi": 3.14159}', response) @@ -37,3 +64,23 @@ def test_json_serializer_is_not_default_json(self): self.assertTrue(hasattr(internal_json, "YOLO")) self.assertFalse(hasattr(stock_json, "YOLO")) self.assertTrue(hasattr(simplejson, "YOLO")) + + @parameterized.expand(execution_envs_lambda_marshaller_ensure_ascii_false) + def test_to_json_unicode_not_escaped_encoding(self, execution_env): + os.environ = {"AWS_EXECUTION_ENV": execution_env} + response = to_json({"price": "£1.00"}) + self.assertEqual('{"price": "£1.00"}', response) + self.assertNotEqual('{"price": "\\u00a31.00"}', response) + self.assertEqual( + 19, len(response.encode("utf-8")) + ) # would be 23 bytes if a unicode escape was returned + + @parameterized.expand(execution_envs_lambda_marshaller_ensure_ascii_true) + def test_to_json_unicode_is_escaped_encoding(self, execution_env): + os.environ = {"AWS_EXECUTION_ENV": execution_env} + response = to_json({"price": "£1.00"}) + self.assertEqual('{"price": "\\u00a31.00"}', response) + self.assertNotEqual('{"price": "£1.00"}', response) + self.assertEqual( + 23, len(response.encode("utf-8")) + ) # would be 19 bytes if a escaped was returned diff --git a/tests/test_runtime_hooks.py b/tests/test_runtime_hooks.py new file mode 100644 index 0000000..e73204f --- /dev/null +++ b/tests/test_runtime_hooks.py @@ -0,0 +1,65 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +import unittest +from unittest.mock import patch, call +from awslambdaric import lambda_runtime_hooks_runner +import snapshot_restore_py + + +def fun_test1(): + print("In function ONE") + + +def fun_test2(): + print("In function TWO") + + +def fun_with_args_kwargs(x, y, **kwargs): + print("Here are the args:", x, y) + print("Here are the keyword args:", kwargs) + + +class TestRuntimeHooks(unittest.TestCase): + def tearDown(self): + # We are accessing private filed for cleaning up + snapshot_restore_py._before_snapshot_registry = [] + snapshot_restore_py._after_restore_registry = [] + + @patch("builtins.print") + def test_before_snapshot_execution_order(self, mock_print): + snapshot_restore_py.register_before_snapshot( + fun_with_args_kwargs, 5, 7, arg1="Lambda", arg2="SnapStart" + ) + snapshot_restore_py.register_before_snapshot(fun_test2) + snapshot_restore_py.register_before_snapshot(fun_test1) + + lambda_runtime_hooks_runner.run_before_snapshot() + + calls = [] + calls.append(call("In function ONE")) + calls.append(call("In function TWO")) + calls.append(call("Here are the args:", 5, 7)) + calls.append( + call("Here are the keyword args:", {"arg1": "Lambda", "arg2": "SnapStart"}) + ) + self.assertEqual(calls, mock_print.mock_calls) + + @patch("builtins.print") + def test_after_restore_execution_order(self, mock_print): + snapshot_restore_py.register_after_restore( + fun_with_args_kwargs, 11, 13, arg1="Lambda", arg2="SnapStart" + ) + snapshot_restore_py.register_after_restore(fun_test2) + snapshot_restore_py.register_after_restore(fun_test1) + + lambda_runtime_hooks_runner.run_after_restore() + + calls = [] + calls.append(call("Here are the args:", 11, 13)) + calls.append( + call("Here are the keyword args:", {"arg1": "Lambda", "arg2": "SnapStart"}) + ) + calls.append(call("In function TWO")) + calls.append(call("In function ONE")) + self.assertEqual(calls, mock_print.mock_calls)