From 503014c16ef4dc0170afdbd1800ad33be65417f4 Mon Sep 17 00:00:00 2001 From: Sean O Brien Date: Thu, 10 Oct 2024 16:37:39 +0000 Subject: [PATCH 01/11] Add python3.13. --- README.md | 2 +- awslambdaric/bootstrap.py | 7 ++++--- awslambdaric/lambda_runtime_marshaller.py | 5 ++++- setup.py | 1 + tests/integration/codebuild/buildspec.os.alpine.yml | 1 + tests/integration/codebuild/buildspec.os.amazonlinux.2.yml | 1 + tests/integration/codebuild/buildspec.os.debian.yml | 1 + tests/integration/codebuild/buildspec.os.ubuntu.yml | 1 + tests/test_lambda_runtime_marshaller.py | 6 +++++- 9 files changed, 19 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 8a448bd..882bf4d 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ You can include this package in your preferred base image to make that base imag ## Requirements The Python Runtime Interface Client package currently supports Python versions: - - 3.7.x up to and including 3.12.x + - 3.7.x up to and including 3.13.x ## Usage diff --git a/awslambdaric/bootstrap.py b/awslambdaric/bootstrap.py index 0f19f56..30fce8b 100644 --- a/awslambdaric/bootstrap.py +++ b/awslambdaric/bootstrap.py @@ -454,9 +454,10 @@ def run(app_root, handler, lambda_runtime_api_addr): sys.stdout = Unbuffered(sys.stdout) sys.stderr = Unbuffered(sys.stderr) - use_thread_for_polling_next = ( - os.environ.get("AWS_EXECUTION_ENV") == "AWS_Lambda_python3.12" - ) + use_thread_for_polling_next = os.environ.get("AWS_EXECUTION_ENV") in [ + "AWS_Lambda_python3.12", + "AWS_Lambda_python3.13", + ] with create_log_sink() as log_sink: lambda_runtime_client = LambdaRuntimeClient( diff --git a/awslambdaric/lambda_runtime_marshaller.py b/awslambdaric/lambda_runtime_marshaller.py index 3b28313..4256066 100644 --- a/awslambdaric/lambda_runtime_marshaller.py +++ b/awslambdaric/lambda_runtime_marshaller.py @@ -15,7 +15,10 @@ # We also set 'ensure_ascii=False' so that the encoded json contains unicode characters instead of unicode escape sequences class Encoder(json.JSONEncoder): def __init__(self): - if os.environ.get("AWS_EXECUTION_ENV") == "AWS_Lambda_python3.12": + if os.environ.get("AWS_EXECUTION_ENV") in { + "AWS_Lambda_python3.12", + "AWS_Lambda_python3.13", + }: super().__init__(use_decimal=False, ensure_ascii=False, allow_nan=True) else: super().__init__(use_decimal=False, allow_nan=True) diff --git a/setup.py b/setup.py index 2544b21..dda16dc 100644 --- a/setup.py +++ b/setup.py @@ -91,6 +91,7 @@ def read_requirements(req="base.txt"): "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", ], diff --git a/tests/integration/codebuild/buildspec.os.alpine.yml b/tests/integration/codebuild/buildspec.os.alpine.yml index f3c53e1..8b290f5 100644 --- a/tests/integration/codebuild/buildspec.os.alpine.yml +++ b/tests/integration/codebuild/buildspec.os.alpine.yml @@ -22,6 +22,7 @@ batch: - "3.10" - "3.11" - "3.12" + - "3.13" phases: pre_build: commands: diff --git a/tests/integration/codebuild/buildspec.os.amazonlinux.2.yml b/tests/integration/codebuild/buildspec.os.amazonlinux.2.yml index f8b7126..b7577c4 100644 --- a/tests/integration/codebuild/buildspec.os.amazonlinux.2.yml +++ b/tests/integration/codebuild/buildspec.os.amazonlinux.2.yml @@ -21,6 +21,7 @@ batch: - "3.10" - "3.11" - "3.12" + - "3.13" phases: pre_build: commands: diff --git a/tests/integration/codebuild/buildspec.os.debian.yml b/tests/integration/codebuild/buildspec.os.debian.yml index 008e6e8..44c061f 100644 --- a/tests/integration/codebuild/buildspec.os.debian.yml +++ b/tests/integration/codebuild/buildspec.os.debian.yml @@ -22,6 +22,7 @@ batch: - "3.10" - "3.11" - "3.12" + - "3.13" phases: pre_build: commands: diff --git a/tests/integration/codebuild/buildspec.os.ubuntu.yml b/tests/integration/codebuild/buildspec.os.ubuntu.yml index ac7c6db..a6e556d 100644 --- a/tests/integration/codebuild/buildspec.os.ubuntu.yml +++ b/tests/integration/codebuild/buildspec.os.ubuntu.yml @@ -22,6 +22,7 @@ batch: - "3.10" - "3.11" - "3.12" + - "3.13" phases: pre_build: commands: diff --git a/tests/test_lambda_runtime_marshaller.py b/tests/test_lambda_runtime_marshaller.py index 7cd73b4..843bcee 100644 --- a/tests/test_lambda_runtime_marshaller.py +++ b/tests/test_lambda_runtime_marshaller.py @@ -11,13 +11,17 @@ class TestLambdaRuntimeMarshaller(unittest.TestCase): execution_envs = ( + "AWS_Lambda_python3.13", "AWS_Lambda_python3.12", "AWS_Lambda_python3.11", "AWS_Lambda_python3.10", "AWS_Lambda_python3.9", ) - envs_lambda_marshaller_ensure_ascii_false = {"AWS_Lambda_python3.12"} + envs_lambda_marshaller_ensure_ascii_false = { + "AWS_Lambda_python3.12", + "AWS_Lambda_python3.13", + } execution_envs_lambda_marshaller_ensure_ascii_true = tuple( set(execution_envs).difference(envs_lambda_marshaller_ensure_ascii_false) From 4f33ba6a0cf10eb34a7b0d66c5bb7f4602898b0b Mon Sep 17 00:00:00 2001 From: Maxime David Date: Fri, 1 Nov 2024 15:51:06 +0000 Subject: [PATCH 02/11] test --- .../codebuild/buildspec.os.amazonlinux.2.yml | 10 +--- .../docker/Dockerfile.echo.amazonlinux | 49 ++++++++----------- 2 files changed, 23 insertions(+), 36 deletions(-) diff --git a/tests/integration/codebuild/buildspec.os.amazonlinux.2.yml b/tests/integration/codebuild/buildspec.os.amazonlinux.2.yml index b7577c4..49ddca1 100644 --- a/tests/integration/codebuild/buildspec.os.amazonlinux.2.yml +++ b/tests/integration/codebuild/buildspec.os.amazonlinux.2.yml @@ -15,12 +15,8 @@ batch: env: variables: DISTRO_VERSION: - - "2" + - "2023" RUNTIME_VERSION: - - "3.9" - - "3.10" - - "3.11" - - "3.12" - "3.13" phases: pre_build: @@ -43,9 +39,6 @@ phases: - > cp "tests/integration/docker/Dockerfile.echo.${OS_DISTRIBUTION}" \ "${SCRATCH_DIR}/Dockerfile.echo.${OS_DISTRIBUTION}.tmp" - - > - echo "COPY ${SCRATCH_DIR}/${RIE} /usr/bin/${RIE}" >> \ - "${SCRATCH_DIR}/Dockerfile.echo.${OS_DISTRIBUTION}.tmp" - > echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' > /etc/docker/daemon.json service docker restart @@ -57,6 +50,7 @@ phases: --build-arg RUNTIME_VERSION="${RUNTIME_VERSION}" \ --build-arg DISTRO_VERSION="${DISTRO_VERSION}" \ --build-arg ARCHITECTURE="${ARCHITECTURE}" \ + --build-arg RIE="${RIE}" \ --load build: commands: diff --git a/tests/integration/docker/Dockerfile.echo.amazonlinux b/tests/integration/docker/Dockerfile.echo.amazonlinux index 168c6a2..4c3ac68 100644 --- a/tests/integration/docker/Dockerfile.echo.amazonlinux +++ b/tests/integration/docker/Dockerfile.echo.amazonlinux @@ -2,11 +2,10 @@ ARG DISTRO_VERSION # Stage 1 - bundle base image + runtime interface client # Grab a fresh copy of the image and install Python FROM public.ecr.aws/amazonlinux/amazonlinux:${DISTRO_VERSION} AS python-amazonlinux-builder - ARG RUNTIME_VERSION # Install apt dependencies -RUN yum install -y \ +RUN dnf install -y \ gcc \ gcc-c++ \ tar \ @@ -18,12 +17,11 @@ RUN yum install -y \ yum-utils \ findutils \ wget \ - openssl11 \ - openssl11-devel \ + openssl \ + openssl-devel \ bzip2-devel \ libffi-devel \ sqlite-devel - RUN RUNTIME_LATEST_VERSION=${RUNTIME_VERSION}.$(curl -s https://www.python.org/ftp/python/ | \ grep -oE "href=\"$(echo ${RUNTIME_VERSION} | sed "s/\\./\\\./g")\.[0-9]+" | \ cut -d. -f3 | \ @@ -38,23 +36,20 @@ RUN RUNTIME_LATEST_VERSION=${RUNTIME_VERSION}.$(curl -s https://www.python.org/f && make \ && make install \ && ln -s /usr/local/bin/python${RUNTIME_VERSION} /usr/local/bin/python${RUNTIME_LATEST_VERSION} - # Stage 2 - clean python build dependencies FROM public.ecr.aws/amazonlinux/amazonlinux:${DISTRO_VERSION} AS python-amazonlinux -RUN yum install -y \ +RUN dnf install -y \ libffi-devel - # Copy the compiled python to /usr/local COPY --from=python-amazonlinux-builder /usr/local /usr/local ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH - # Stage 3 - build function and dependencies FROM python-amazonlinux-builder AS build-image ARG RUNTIME_VERSION ARG ARCHITECTURE - +ARG RIE # Install aws-lambda-cpp build dependencies -RUN yum install -y \ +RUN dnf install -y \ tar \ gzip \ make \ @@ -63,16 +58,13 @@ RUN yum install -y \ libtool \ libcurl-devel \ gcc-c++ \ - wget - + wget \ + sqlite-devel # Install a modern CMake RUN wget --quiet -O cmake-install https://github.com/Kitware/CMake/releases/download/v3.20.0/cmake-3.20.0-linux-${ARCHITECTURE}.sh && \ sh cmake-install --skip-license --prefix=/usr --exclude-subdirectory; - ENV PATH=/usr/local/bin:$PATH ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH - - # Include global args in this stage of the build ARG RIC_BUILD_DIR="/home/build/" # Create function directory @@ -81,12 +73,16 @@ RUN mkdir -p ${RIC_BUILD_DIR} WORKDIR ${RIC_BUILD_DIR} COPY . . -# distutils no longer available in python3.12 and later -# https://docs.python.org/3/whatsnew/3.12.html -# https://peps.python.org/pep-0632/ -RUN if [ $(cut -d '.' -f 2 <<< ${RUNTIME_VERSION}) -ge 12 ]; then pip3 install setuptools; fi -RUN make init build test && \ - mv ./dist/awslambdaric-*.tar.gz ./dist/awslambdaric-test.tar.gz +RUN pip3 install setuptools +RUN make init +RUN make build + +RUN mv ./dist/awslambdaric-*.tar.gz ./dist/awslambdaric-test.tar.gz +RUN python${RUNTIME_VERSION} -m pip install \ + ./dist/awslambdaric-test.tar.gz \ + --target ${RIC_BUILD_DIR} + +RUN make test # Include global args in this stage of the build ARG FUNCTION_DIR="/home/app/" @@ -96,25 +92,22 @@ RUN mkdir -p ${FUNCTION_DIR} COPY tests/integration/test-handlers/echo/* ${FUNCTION_DIR} # Copy Runtime Interface Client .tgz RUN cp ./dist/awslambdaric-test.tar.gz ${FUNCTION_DIR}/awslambdaric-test.tar.gz - # Install the function's dependencies WORKDIR ${FUNCTION_DIR} RUN python${RUNTIME_VERSION} -m pip install \ awslambdaric-test.tar.gz \ --target ${FUNCTION_DIR} && \ rm awslambdaric-test.tar.gz - - # Stage 4 - final runtime interface client image # Grab a fresh copy of the Python image FROM python-amazonlinux - +RUN dnf install -y brotli # Include global arg in this stage of the build ARG FUNCTION_DIR="/home/app/" # Set working directory to function root directory WORKDIR ${FUNCTION_DIR} # Copy in the built dependencies COPY --from=build-image ${FUNCTION_DIR} ${FUNCTION_DIR} - +COPY .scratch/${RIE} /usr/bin/${RIE} ENTRYPOINT [ "/usr/local/bin/python3", "-m", "awslambdaric" ] -CMD [ "app.handler" ] +CMD [ "app.handler" ] \ No newline at end of file From 2ae43e8234553a771f2abea15d89065aeb2c11b5 Mon Sep 17 00:00:00 2001 From: Maxime David Date: Fri, 1 Nov 2024 15:53:32 +0000 Subject: [PATCH 03/11] test --- .github/workflows/test-on-push-and-pr.yml | 26 +---------------------- 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/.github/workflows/test-on-push-and-pr.yml b/.github/workflows/test-on-push-and-pr.yml index 5b80d23..a776128 100644 --- a/.github/workflows/test-on-push-and-pr.yml +++ b/.github/workflows/test-on-push-and-pr.yml @@ -15,34 +15,10 @@ jobs: - name: Run 'pr' target run: make pr - alpine: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - name: Run alpine integration tests - run: DISTRO=alpine make test-integ - amazonlinux: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Run amazonlinux integration tests - run: DISTRO=amazonlinux make test-integ - - debian: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - name: Run debian integration tests - run: DISTRO=debian make test-integ - - ubuntu: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - name: Run ubuntu integration tests - run: DISTRO=ubuntu make test-integ \ No newline at end of file + run: DISTRO=amazonlinux make test-integ \ No newline at end of file From 088a43210933fd9de6c946db3c38031a5cf8f0af Mon Sep 17 00:00:00 2001 From: Maxime David Date: Fri, 1 Nov 2024 15:54:18 +0000 Subject: [PATCH 04/11] trigger build --- .github/workflows/test-on-push-and-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-on-push-and-pr.yml b/.github/workflows/test-on-push-and-pr.yml index a776128..0cb8681 100644 --- a/.github/workflows/test-on-push-and-pr.yml +++ b/.github/workflows/test-on-push-and-pr.yml @@ -2,7 +2,7 @@ name: test-on-push-and-pr on: push: - branches: [ main ] + branches: [ '*' ] pull_request: branches: [ '*' ] From 2137898dc4273231fdf839a34eb995f59755bc7e Mon Sep 17 00:00:00 2001 From: Maxime David Date: Fri, 1 Nov 2024 16:43:39 +0000 Subject: [PATCH 05/11] cleanup --- .github/workflows/test-on-push-and-pr.yml | 28 ++++- .../codebuild/buildspec.os.amazonlinux.2.yml | 11 +- .../buildspec.os.amazonlinux.2023.yml | 102 ++++++++++++++++ .../docker/Dockerfile.echo.amazonlinux2023 | 113 ++++++++++++++++++ 4 files changed, 249 insertions(+), 5 deletions(-) create mode 100644 tests/integration/codebuild/buildspec.os.amazonlinux.2023.yml create mode 100644 tests/integration/docker/Dockerfile.echo.amazonlinux2023 diff --git a/.github/workflows/test-on-push-and-pr.yml b/.github/workflows/test-on-push-and-pr.yml index 0cb8681..5b80d23 100644 --- a/.github/workflows/test-on-push-and-pr.yml +++ b/.github/workflows/test-on-push-and-pr.yml @@ -2,7 +2,7 @@ name: test-on-push-and-pr on: push: - branches: [ '*' ] + branches: [ main ] pull_request: branches: [ '*' ] @@ -15,10 +15,34 @@ jobs: - name: Run 'pr' target run: make pr + alpine: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Run alpine integration tests + run: DISTRO=alpine make test-integ + amazonlinux: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Run amazonlinux integration tests - run: DISTRO=amazonlinux make test-integ \ No newline at end of file + run: DISTRO=amazonlinux make test-integ + + debian: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Run debian integration tests + run: DISTRO=debian make test-integ + + ubuntu: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Run ubuntu integration tests + run: DISTRO=ubuntu make test-integ \ No newline at end of file diff --git a/tests/integration/codebuild/buildspec.os.amazonlinux.2.yml b/tests/integration/codebuild/buildspec.os.amazonlinux.2.yml index 49ddca1..f8b7126 100644 --- a/tests/integration/codebuild/buildspec.os.amazonlinux.2.yml +++ b/tests/integration/codebuild/buildspec.os.amazonlinux.2.yml @@ -15,9 +15,12 @@ batch: env: variables: DISTRO_VERSION: - - "2023" + - "2" RUNTIME_VERSION: - - "3.13" + - "3.9" + - "3.10" + - "3.11" + - "3.12" phases: pre_build: commands: @@ -39,6 +42,9 @@ phases: - > cp "tests/integration/docker/Dockerfile.echo.${OS_DISTRIBUTION}" \ "${SCRATCH_DIR}/Dockerfile.echo.${OS_DISTRIBUTION}.tmp" + - > + echo "COPY ${SCRATCH_DIR}/${RIE} /usr/bin/${RIE}" >> \ + "${SCRATCH_DIR}/Dockerfile.echo.${OS_DISTRIBUTION}.tmp" - > echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' > /etc/docker/daemon.json service docker restart @@ -50,7 +56,6 @@ phases: --build-arg RUNTIME_VERSION="${RUNTIME_VERSION}" \ --build-arg DISTRO_VERSION="${DISTRO_VERSION}" \ --build-arg ARCHITECTURE="${ARCHITECTURE}" \ - --build-arg RIE="${RIE}" \ --load build: commands: diff --git a/tests/integration/codebuild/buildspec.os.amazonlinux.2023.yml b/tests/integration/codebuild/buildspec.os.amazonlinux.2023.yml new file mode 100644 index 0000000..3e7ae9c --- /dev/null +++ b/tests/integration/codebuild/buildspec.os.amazonlinux.2023.yml @@ -0,0 +1,102 @@ +version: 0.2 + +env: + variables: + OS_DISTRIBUTION: amazonlinux2023 + PYTHON_LOCATION: "/usr/local/bin/python3" + TEST_NAME: "aws-lambda-python-rtc-amazonlinux-test" +batch: + build-matrix: + static: + ignore-failure: false + env: + privileged-mode: true + dynamic: + env: + variables: + DISTRO_VERSION: + - "2023" + RUNTIME_VERSION: + - "3.13" +phases: + pre_build: + commands: + - export IMAGE_TAG="python-${OS_DISTRIBUTION}-${DISTRO_VERSION}:${RUNTIME_VERSION}" + - echo "Extracting and including the Runtime Interface Emulator" + - SCRATCH_DIR=".scratch" + - mkdir "${SCRATCH_DIR}" + - ARCHITECTURE=$(arch) + - > + if [[ "$ARCHITECTURE" == "x86_64" ]]; then + RIE="aws-lambda-rie" + elif [[ "$ARCHITECTURE" == "aarch64" ]]; then + RIE="aws-lambda-rie-arm64" + else + echo "Architecture $ARCHITECTURE is not currently supported." + exit 1 + fi + - tar -xvf tests/integration/resources/${RIE}.tar.gz --directory "${SCRATCH_DIR}" + - > + cp "tests/integration/docker/Dockerfile.echo.${OS_DISTRIBUTION}" \ + "${SCRATCH_DIR}/Dockerfile.echo.${OS_DISTRIBUTION}.tmp" + - > + echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' > /etc/docker/daemon.json + service docker restart + - echo "Building image ${IMAGE_TAG}" + - > + docker build . \ + -f "${SCRATCH_DIR}/Dockerfile.echo.${OS_DISTRIBUTION}.tmp" \ + -t "${IMAGE_TAG}" \ + --build-arg RUNTIME_VERSION="${RUNTIME_VERSION}" \ + --build-arg DISTRO_VERSION="${DISTRO_VERSION}" \ + --build-arg ARCHITECTURE="${ARCHITECTURE}" \ + --build-arg RIE="${RIE}" \ + --load + build: + commands: + - set -x + - echo "Running Image ${IMAGE_TAG}" + - docker network create "${TEST_NAME}-network" + - > + docker run \ + --detach \ + --name "${TEST_NAME}-app" \ + --network "${TEST_NAME}-network" \ + --entrypoint="" \ + "${IMAGE_TAG}" \ + sh -c "/usr/bin/${RIE} ${PYTHON_LOCATION} -m awslambdaric app.handler" + - sleep 2 + - > + docker run \ + --name "${TEST_NAME}-tester" \ + --env "TARGET=${TEST_NAME}-app" \ + --network "${TEST_NAME}-network" \ + --entrypoint="" \ + "${IMAGE_TAG}" \ + sh -c 'curl -X POST "http://${TARGET}:8080/2015-03-31/functions/function/invocations" -d "{}" --max-time 10' + - actual="$(docker logs --tail 1 "${TEST_NAME}-tester" | xargs)" + - expected='success' + - | + echo "Response: ${actual}" + if [[ "$actual" != "$expected" ]]; then + echo "fail! runtime: $RUNTIME - expected output $expected - got $actual" + exit -1 + fi + finally: + - | + echo "---------Container Logs: ${TEST_NAME}-app----------" + echo + docker logs "${TEST_NAME}-app" || true + echo + echo "---------------------------------------------------" + echo "--------Container Logs: ${TEST_NAME}-tester--------" + echo + docker logs "${TEST_NAME}-tester" || true + echo + echo "---------------------------------------------------" + - echo "Cleaning up..." + - docker stop "${TEST_NAME}-app" || true + - docker rm --force "${TEST_NAME}-app" || true + - docker stop "${TEST_NAME}-tester" || true + - docker rm --force "${TEST_NAME}-tester" || true + - docker network rm "${TEST_NAME}-network" || true diff --git a/tests/integration/docker/Dockerfile.echo.amazonlinux2023 b/tests/integration/docker/Dockerfile.echo.amazonlinux2023 new file mode 100644 index 0000000..4c3ac68 --- /dev/null +++ b/tests/integration/docker/Dockerfile.echo.amazonlinux2023 @@ -0,0 +1,113 @@ +ARG DISTRO_VERSION +# Stage 1 - bundle base image + runtime interface client +# Grab a fresh copy of the image and install Python +FROM public.ecr.aws/amazonlinux/amazonlinux:${DISTRO_VERSION} AS python-amazonlinux-builder +ARG RUNTIME_VERSION + +# Install apt dependencies +RUN dnf install -y \ + gcc \ + gcc-c++ \ + tar \ + gzip \ + make \ + autoconf \ + automake \ + freetype-devel \ + yum-utils \ + findutils \ + wget \ + openssl \ + openssl-devel \ + bzip2-devel \ + libffi-devel \ + sqlite-devel +RUN RUNTIME_LATEST_VERSION=${RUNTIME_VERSION}.$(curl -s https://www.python.org/ftp/python/ | \ + grep -oE "href=\"$(echo ${RUNTIME_VERSION} | sed "s/\\./\\\./g")\.[0-9]+" | \ + cut -d. -f3 | \ + sort -rn | \ + while read -r patch; do \ + $(wget -c https://www.python.org/ftp/python/${RUNTIME_VERSION}.$patch/Python-${RUNTIME_VERSION}.$patch.tgz -O Python-${RUNTIME_VERSION}.$patch.tgz); \ + [ $? -eq 0 ] && echo $patch && break; \ + done) \ + && tar -xzf Python-${RUNTIME_LATEST_VERSION}.tgz \ + && cd Python-${RUNTIME_LATEST_VERSION} \ + && ./configure --prefix=/usr/local --enable-shared \ + && make \ + && make install \ + && ln -s /usr/local/bin/python${RUNTIME_VERSION} /usr/local/bin/python${RUNTIME_LATEST_VERSION} +# Stage 2 - clean python build dependencies +FROM public.ecr.aws/amazonlinux/amazonlinux:${DISTRO_VERSION} AS python-amazonlinux +RUN dnf install -y \ + libffi-devel +# Copy the compiled python to /usr/local +COPY --from=python-amazonlinux-builder /usr/local /usr/local +ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH +# Stage 3 - build function and dependencies +FROM python-amazonlinux-builder AS build-image +ARG RUNTIME_VERSION +ARG ARCHITECTURE +ARG RIE +# Install aws-lambda-cpp build dependencies +RUN dnf install -y \ + tar \ + gzip \ + make \ + autoconf \ + automake \ + libtool \ + libcurl-devel \ + gcc-c++ \ + wget \ + sqlite-devel +# Install a modern CMake +RUN wget --quiet -O cmake-install https://github.com/Kitware/CMake/releases/download/v3.20.0/cmake-3.20.0-linux-${ARCHITECTURE}.sh && \ + sh cmake-install --skip-license --prefix=/usr --exclude-subdirectory; +ENV PATH=/usr/local/bin:$PATH +ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH +# Include global args in this stage of the build +ARG RIC_BUILD_DIR="/home/build/" +# Create function directory +RUN mkdir -p ${RIC_BUILD_DIR} +# Copy function code and Runtime Interface Client .tgz +WORKDIR ${RIC_BUILD_DIR} +COPY . . + +RUN pip3 install setuptools +RUN make init +RUN make build + +RUN mv ./dist/awslambdaric-*.tar.gz ./dist/awslambdaric-test.tar.gz +RUN python${RUNTIME_VERSION} -m pip install \ + ./dist/awslambdaric-test.tar.gz \ + --target ${RIC_BUILD_DIR} + +RUN make test + +# Include global args in this stage of the build +ARG FUNCTION_DIR="/home/app/" +# Create function directory +RUN mkdir -p ${FUNCTION_DIR} +# Copy function code +COPY tests/integration/test-handlers/echo/* ${FUNCTION_DIR} +# Copy Runtime Interface Client .tgz +RUN cp ./dist/awslambdaric-test.tar.gz ${FUNCTION_DIR}/awslambdaric-test.tar.gz +# Install the function's dependencies +WORKDIR ${FUNCTION_DIR} +RUN python${RUNTIME_VERSION} -m pip install \ + awslambdaric-test.tar.gz \ + --target ${FUNCTION_DIR} && \ + rm awslambdaric-test.tar.gz +# Stage 4 - final runtime interface client image +# Grab a fresh copy of the Python image +FROM python-amazonlinux +RUN dnf install -y brotli +# Include global arg in this stage of the build +ARG FUNCTION_DIR="/home/app/" +# Set working directory to function root directory +WORKDIR ${FUNCTION_DIR} +# Copy in the built dependencies +COPY --from=build-image ${FUNCTION_DIR} ${FUNCTION_DIR} +COPY .scratch/${RIE} /usr/bin/${RIE} +ENTRYPOINT [ "/usr/local/bin/python3", "-m", "awslambdaric" ] +CMD [ "app.handler" ] \ No newline at end of file From 371b3c8a961dab6e2a588955aa160dc48fea67c2 Mon Sep 17 00:00:00 2001 From: Maxime David Date: Fri, 1 Nov 2024 16:44:23 +0000 Subject: [PATCH 06/11] cleanup --- .github/workflows/test-on-push-and-pr.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/test-on-push-and-pr.yml b/.github/workflows/test-on-push-and-pr.yml index 5b80d23..08ef44f 100644 --- a/.github/workflows/test-on-push-and-pr.yml +++ b/.github/workflows/test-on-push-and-pr.yml @@ -31,6 +31,14 @@ jobs: - name: Run amazonlinux integration tests run: DISTRO=amazonlinux make test-integ + amazonlinux2023: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Run amazonlinux integration tests + run: DISTRO=amazonlinux2023 make test-integ + debian: runs-on: ubuntu-latest From 4649818d341de715f0dbc3a5ca9c0c7f5cdc3d20 Mon Sep 17 00:00:00 2001 From: Maxime David Date: Fri, 1 Nov 2024 16:45:39 +0000 Subject: [PATCH 07/11] cleaup --- .../docker/Dockerfile.echo.amazonlinux | 49 +++++++++++-------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/tests/integration/docker/Dockerfile.echo.amazonlinux b/tests/integration/docker/Dockerfile.echo.amazonlinux index 4c3ac68..168c6a2 100644 --- a/tests/integration/docker/Dockerfile.echo.amazonlinux +++ b/tests/integration/docker/Dockerfile.echo.amazonlinux @@ -2,10 +2,11 @@ ARG DISTRO_VERSION # Stage 1 - bundle base image + runtime interface client # Grab a fresh copy of the image and install Python FROM public.ecr.aws/amazonlinux/amazonlinux:${DISTRO_VERSION} AS python-amazonlinux-builder + ARG RUNTIME_VERSION # Install apt dependencies -RUN dnf install -y \ +RUN yum install -y \ gcc \ gcc-c++ \ tar \ @@ -17,11 +18,12 @@ RUN dnf install -y \ yum-utils \ findutils \ wget \ - openssl \ - openssl-devel \ + openssl11 \ + openssl11-devel \ bzip2-devel \ libffi-devel \ sqlite-devel + RUN RUNTIME_LATEST_VERSION=${RUNTIME_VERSION}.$(curl -s https://www.python.org/ftp/python/ | \ grep -oE "href=\"$(echo ${RUNTIME_VERSION} | sed "s/\\./\\\./g")\.[0-9]+" | \ cut -d. -f3 | \ @@ -36,20 +38,23 @@ RUN RUNTIME_LATEST_VERSION=${RUNTIME_VERSION}.$(curl -s https://www.python.org/f && make \ && make install \ && ln -s /usr/local/bin/python${RUNTIME_VERSION} /usr/local/bin/python${RUNTIME_LATEST_VERSION} + # Stage 2 - clean python build dependencies FROM public.ecr.aws/amazonlinux/amazonlinux:${DISTRO_VERSION} AS python-amazonlinux -RUN dnf install -y \ +RUN yum install -y \ libffi-devel + # Copy the compiled python to /usr/local COPY --from=python-amazonlinux-builder /usr/local /usr/local ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH + # Stage 3 - build function and dependencies FROM python-amazonlinux-builder AS build-image ARG RUNTIME_VERSION ARG ARCHITECTURE -ARG RIE + # Install aws-lambda-cpp build dependencies -RUN dnf install -y \ +RUN yum install -y \ tar \ gzip \ make \ @@ -58,13 +63,16 @@ RUN dnf install -y \ libtool \ libcurl-devel \ gcc-c++ \ - wget \ - sqlite-devel + wget + # Install a modern CMake RUN wget --quiet -O cmake-install https://github.com/Kitware/CMake/releases/download/v3.20.0/cmake-3.20.0-linux-${ARCHITECTURE}.sh && \ sh cmake-install --skip-license --prefix=/usr --exclude-subdirectory; + ENV PATH=/usr/local/bin:$PATH ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH + + # Include global args in this stage of the build ARG RIC_BUILD_DIR="/home/build/" # Create function directory @@ -73,16 +81,12 @@ RUN mkdir -p ${RIC_BUILD_DIR} WORKDIR ${RIC_BUILD_DIR} COPY . . -RUN pip3 install setuptools -RUN make init -RUN make build - -RUN mv ./dist/awslambdaric-*.tar.gz ./dist/awslambdaric-test.tar.gz -RUN python${RUNTIME_VERSION} -m pip install \ - ./dist/awslambdaric-test.tar.gz \ - --target ${RIC_BUILD_DIR} - -RUN make test +# distutils no longer available in python3.12 and later +# https://docs.python.org/3/whatsnew/3.12.html +# https://peps.python.org/pep-0632/ +RUN if [ $(cut -d '.' -f 2 <<< ${RUNTIME_VERSION}) -ge 12 ]; then pip3 install setuptools; fi +RUN make init build test && \ + mv ./dist/awslambdaric-*.tar.gz ./dist/awslambdaric-test.tar.gz # Include global args in this stage of the build ARG FUNCTION_DIR="/home/app/" @@ -92,22 +96,25 @@ RUN mkdir -p ${FUNCTION_DIR} COPY tests/integration/test-handlers/echo/* ${FUNCTION_DIR} # Copy Runtime Interface Client .tgz RUN cp ./dist/awslambdaric-test.tar.gz ${FUNCTION_DIR}/awslambdaric-test.tar.gz + # Install the function's dependencies WORKDIR ${FUNCTION_DIR} RUN python${RUNTIME_VERSION} -m pip install \ awslambdaric-test.tar.gz \ --target ${FUNCTION_DIR} && \ rm awslambdaric-test.tar.gz + + # Stage 4 - final runtime interface client image # Grab a fresh copy of the Python image FROM python-amazonlinux -RUN dnf install -y brotli + # Include global arg in this stage of the build ARG FUNCTION_DIR="/home/app/" # Set working directory to function root directory WORKDIR ${FUNCTION_DIR} # Copy in the built dependencies COPY --from=build-image ${FUNCTION_DIR} ${FUNCTION_DIR} -COPY .scratch/${RIE} /usr/bin/${RIE} + ENTRYPOINT [ "/usr/local/bin/python3", "-m", "awslambdaric" ] -CMD [ "app.handler" ] \ No newline at end of file +CMD [ "app.handler" ] From bd107ffe03822b5fdbf0d10f4f35fff1101d949c Mon Sep 17 00:00:00 2001 From: Maxime David Date: Fri, 1 Nov 2024 16:49:52 +0000 Subject: [PATCH 08/11] test --- ...ildspec.os.amazonlinux.2.yml => buildspec.os.amazonlinux2.yml} | 0 ...c.os.amazonlinux.2023.yml => buildspec.os.amazonlinux2023.yml} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/integration/codebuild/{buildspec.os.amazonlinux.2.yml => buildspec.os.amazonlinux2.yml} (100%) rename tests/integration/codebuild/{buildspec.os.amazonlinux.2023.yml => buildspec.os.amazonlinux2023.yml} (100%) diff --git a/tests/integration/codebuild/buildspec.os.amazonlinux.2.yml b/tests/integration/codebuild/buildspec.os.amazonlinux2.yml similarity index 100% rename from tests/integration/codebuild/buildspec.os.amazonlinux.2.yml rename to tests/integration/codebuild/buildspec.os.amazonlinux2.yml diff --git a/tests/integration/codebuild/buildspec.os.amazonlinux.2023.yml b/tests/integration/codebuild/buildspec.os.amazonlinux2023.yml similarity index 100% rename from tests/integration/codebuild/buildspec.os.amazonlinux.2023.yml rename to tests/integration/codebuild/buildspec.os.amazonlinux2023.yml From 36b4796a3ba3c21f579b58099f1c5c5f554d3402 Mon Sep 17 00:00:00 2001 From: Maxime David Date: Fri, 1 Nov 2024 16:52:20 +0000 Subject: [PATCH 09/11] test --- ...dspec.os.amazonlinux2.yml => buildspec.os.amazonlinux.2.yml} | 0 ...os.amazonlinux2023.yml => buildspec.os.amazonlinux.2023.yml} | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename tests/integration/codebuild/{buildspec.os.amazonlinux2.yml => buildspec.os.amazonlinux.2.yml} (100%) rename tests/integration/codebuild/{buildspec.os.amazonlinux2023.yml => buildspec.os.amazonlinux.2023.yml} (98%) diff --git a/tests/integration/codebuild/buildspec.os.amazonlinux2.yml b/tests/integration/codebuild/buildspec.os.amazonlinux.2.yml similarity index 100% rename from tests/integration/codebuild/buildspec.os.amazonlinux2.yml rename to tests/integration/codebuild/buildspec.os.amazonlinux.2.yml diff --git a/tests/integration/codebuild/buildspec.os.amazonlinux2023.yml b/tests/integration/codebuild/buildspec.os.amazonlinux.2023.yml similarity index 98% rename from tests/integration/codebuild/buildspec.os.amazonlinux2023.yml rename to tests/integration/codebuild/buildspec.os.amazonlinux.2023.yml index 3e7ae9c..49ddca1 100644 --- a/tests/integration/codebuild/buildspec.os.amazonlinux2023.yml +++ b/tests/integration/codebuild/buildspec.os.amazonlinux.2023.yml @@ -2,7 +2,7 @@ version: 0.2 env: variables: - OS_DISTRIBUTION: amazonlinux2023 + OS_DISTRIBUTION: amazonlinux PYTHON_LOCATION: "/usr/local/bin/python3" TEST_NAME: "aws-lambda-python-rtc-amazonlinux-test" batch: From 2c77fa69941ddd8e185bb5ee19dab7b7e9bddacc Mon Sep 17 00:00:00 2001 From: Maxime David Date: Fri, 1 Nov 2024 16:56:47 +0000 Subject: [PATCH 10/11] test --- .github/workflows/test-on-push-and-pr.yml | 2 +- ...dspec.os.amazonlinux.2.yml => buildspec.os.amazonlinux2.yml} | 0 ...os.amazonlinux.2023.yml => buildspec.os.amazonlinux2023.yml} | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename tests/integration/codebuild/{buildspec.os.amazonlinux.2.yml => buildspec.os.amazonlinux2.yml} (100%) rename tests/integration/codebuild/{buildspec.os.amazonlinux.2023.yml => buildspec.os.amazonlinux2023.yml} (100%) diff --git a/.github/workflows/test-on-push-and-pr.yml b/.github/workflows/test-on-push-and-pr.yml index 08ef44f..28b7a8e 100644 --- a/.github/workflows/test-on-push-and-pr.yml +++ b/.github/workflows/test-on-push-and-pr.yml @@ -29,7 +29,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Run amazonlinux integration tests - run: DISTRO=amazonlinux make test-integ + run: DISTRO=amazonlinux2 make test-integ amazonlinux2023: runs-on: ubuntu-latest diff --git a/tests/integration/codebuild/buildspec.os.amazonlinux.2.yml b/tests/integration/codebuild/buildspec.os.amazonlinux2.yml similarity index 100% rename from tests/integration/codebuild/buildspec.os.amazonlinux.2.yml rename to tests/integration/codebuild/buildspec.os.amazonlinux2.yml diff --git a/tests/integration/codebuild/buildspec.os.amazonlinux.2023.yml b/tests/integration/codebuild/buildspec.os.amazonlinux2023.yml similarity index 100% rename from tests/integration/codebuild/buildspec.os.amazonlinux.2023.yml rename to tests/integration/codebuild/buildspec.os.amazonlinux2023.yml From 2ed9ea55690b5ceb2186abf2276729adf558bc97 Mon Sep 17 00:00:00 2001 From: Maxime David Date: Fri, 1 Nov 2024 17:02:43 +0000 Subject: [PATCH 11/11] test --- tests/integration/codebuild/buildspec.os.amazonlinux2023.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/codebuild/buildspec.os.amazonlinux2023.yml b/tests/integration/codebuild/buildspec.os.amazonlinux2023.yml index 49ddca1..404fd28 100644 --- a/tests/integration/codebuild/buildspec.os.amazonlinux2023.yml +++ b/tests/integration/codebuild/buildspec.os.amazonlinux2023.yml @@ -37,7 +37,7 @@ phases: fi - tar -xvf tests/integration/resources/${RIE}.tar.gz --directory "${SCRATCH_DIR}" - > - cp "tests/integration/docker/Dockerfile.echo.${OS_DISTRIBUTION}" \ + cp "tests/integration/docker/Dockerfile.echo.${OS_DISTRIBUTION}${DISTRO_VERSION}" \ "${SCRATCH_DIR}/Dockerfile.echo.${OS_DISTRIBUTION}.tmp" - > echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' > /etc/docker/daemon.json