From 4bee949f894ffd1130062c138ff693ae5655b8bb Mon Sep 17 00:00:00 2001 From: Aryahi Date: Sun, 10 Aug 2025 10:56:55 +0530 Subject: [PATCH 1/3] #7123 Tech debt: Improve documentation of Event model fields in DynamoDB parser models --- .../utilities/parser/models/dynamodb.py | 144 ++++++++++++++---- 1 file changed, 112 insertions(+), 32 deletions(-) diff --git a/aws_lambda_powertools/utilities/parser/models/dynamodb.py b/aws_lambda_powertools/utilities/parser/models/dynamodb.py index 99d82c7853d..d46d9b4e7ed 100644 --- a/aws_lambda_powertools/utilities/parser/models/dynamodb.py +++ b/aws_lambda_powertools/utilities/parser/models/dynamodb.py @@ -2,53 +2,133 @@ from datetime import datetime from typing import Any, Dict, List, Literal, Optional, Type, Union -from pydantic import BaseModel, field_validator - +from pydantic import BaseModel, Field, field_validator from aws_lambda_powertools.shared.dynamodb_deserializer import TypeDeserializer _DESERIALIZER = TypeDeserializer() class DynamoDBStreamChangedRecordModel(BaseModel): - ApproximateCreationDateTime: Optional[datetime] = None - Keys: Dict[str, Any] - NewImage: Optional[Union[Dict[str, Any], Type[BaseModel], BaseModel]] = None - OldImage: Optional[Union[Dict[str, Any], Type[BaseModel], BaseModel]] = None - SequenceNumber: str - SizeBytes: int - StreamViewType: Literal["NEW_AND_OLD_IMAGES", "KEYS_ONLY", "NEW_IMAGE", "OLD_IMAGE"] - - # context on why it's commented: https://github.com/aws-powertools/powertools-lambda-python/pull/118 - # since both images are optional, they can both be None. However, at least one must - # exist in a legal model of NEW_AND_OLD_IMAGES type - # @root_validator - # def check_one_image_exists(cls, values): # noqa: ERA001 - # new_img, old_img = values.get("NewImage"), values.get("OldImage") # noqa: ERA001 - # stream_type = values.get("StreamViewType") # noqa: ERA001 - # if stream_type == "NEW_AND_OLD_IMAGES" and not new_img and not old_img: # noqa: ERA001 - # raise TypeError("DynamoDB streams model failed validation, missing both new & old stream images") # noqa: ERA001,E501 - # return values # noqa: ERA001 + ApproximateCreationDateTime: Optional[float] = Field( # AWS sends this as Unix epoch float + default=None, + description="The approximate date and time when the stream record was created (Unix epoch time).", + examples=[1693997155.0] + ) + Keys: Dict[str, Any] = Field( + description="Primary key attributes for the item.", + examples=[{"Id": {"N": "101"}}] + ) + NewImage: Optional[Union[Dict[str, Any], Type[BaseModel], BaseModel]] = Field( + default=None, + description="The item after modifications, in DynamoDB attribute-value format.", + examples=[{ + "Message": {"S": "New item!"}, + "Id": {"N": "101"} + }] + ) + OldImage: Optional[Union[Dict[str, Any], Type[BaseModel], BaseModel]] = Field( + default=None, + description="The item before modifications, in DynamoDB attribute-value format.", + examples=[{ + "Message": {"S": "Old item!"}, + "Id": {"N": "100"} + }] + ) + SequenceNumber: str = Field( + description="A unique identifier for the stream record.", + examples=["222"] + ) + SizeBytes: int = Field( + description="The size of the stream record, in bytes.", + examples=[26] + ) + StreamViewType: Literal["NEW_AND_OLD_IMAGES", "KEYS_ONLY", "NEW_IMAGE", "OLD_IMAGE"] = Field( + description="The type of data included in the stream record.", + examples=["NEW_AND_OLD_IMAGES"] + ) @field_validator("Keys", "NewImage", "OldImage", mode="before") def deserialize_field(cls, value): - return {k: _DESERIALIZER.deserialize(v) for k, v in value.items()} + return {k: _DESERIALIZER.deserialize(v) for k, v in value.items()} if value else value class UserIdentity(BaseModel): - type: Literal["Service"] # noqa: VNE003, A003 - principalId: Literal["dynamodb.amazonaws.com"] + type: Literal["Service"] = Field( + default="Service", + description="The type of identity that made the request, which is always 'Service' for DynamoDB streams.", + examples=["Service"] + ) + principalId: str = Field( + description="The unique identifier for the principal that made the request.", + examples=["dynamodb.amazonaws.com"] + ) class DynamoDBStreamRecordModel(BaseModel): - eventID: str - eventName: Literal["INSERT", "MODIFY", "REMOVE"] - eventVersion: float - eventSource: Literal["aws:dynamodb"] - awsRegion: str - eventSourceARN: str - dynamodb: DynamoDBStreamChangedRecordModel - userIdentity: Optional[UserIdentity] = None + eventID: str = Field( + description="A unique identifier for the event.", + examples=["1"] + ) + eventName: Literal["INSERT", "MODIFY", "REMOVE"] = Field( + description="The type of operation that was performed on the item.", + examples=["INSERT"] + ) + eventVersion: str = Field( + default="1.0", + description="The version of the stream record format.", + examples=["1.0"] + ) + eventSource: str = Field( + default="aws:dynamodb", + description="The source of the event, which is always 'aws:dynamodb' for DynamoDB streams.", + examples=["aws:dynamodb"] + ) + awsRegion: str = Field( + description="The AWS region where the stream record was generated.", + examples=["us-west-2"] + ) + eventSourceARN: str = Field( + description="The Amazon Resource Name (ARN) of the DynamoDB stream.", + examples=["arn:aws:dynamodb:us-west-2:123456789012:table/ExampleTable/stream/2021-01-01T00:00:00.000"] + ) + dynamodb: DynamoDBStreamChangedRecordModel = Field( + description="Contains the details of the DynamoDB stream record.", + examples=[{ + "ApproximateCreationDateTime": 1693997155.0, + "Keys": {"Id": {"N": "101"}}, + "NewImage": {"Message": {"S": "New item!"}, "Id": {"N": "101"}}, + "OldImage": {"Message": {"S": "Old item!"}, "Id": {"N": "100"}}, + "SequenceNumber": "222", + "SizeBytes": 26, + "StreamViewType": "NEW_AND_OLD_IMAGES" + }] + ) + userIdentity: Optional[UserIdentity] = Field( + default=None, + description="Information about the identity that made the request.", + examples=[{"type": "Service", "principalId": "dynamodb.amazonaws.com"}] + ) class DynamoDBStreamModel(BaseModel): - Records: List[DynamoDBStreamRecordModel] + Records: List[DynamoDBStreamRecordModel] = Field( + description="A list of records that contain the details of the DynamoDB stream events.", + examples=[{ + "eventID": "1", + "eventName": "INSERT", + "eventVersion": "1.0", + "eventSource": "aws:dynamodb", + "awsRegion": "us-west-2", + "eventSourceARN": "arn:aws:dynamodb:us-west-2:123456789012:table/ExampleTable/stream/2021-01-01T00:00:00.000", + "dynamodb": { + "ApproximateCreationDateTime": 1693997155.0, + "Keys": {"Id": {"N": "101"}}, + "NewImage": {"Message": {"S": "New item!"}, "Id": {"N": "101"}}, + "OldImage": {"Message": {"S": "Old item!"}, "Id": {"N": "100"}}, + "SequenceNumber": "222", + "SizeBytes": 26, + "StreamViewType": "NEW_AND_OLD_IMAGES" + }, + "userIdentity": {"type": "Service", "principalId": "dynamodb.amazonaws.com"} + }] + ) \ No newline at end of file From 4e0e0975ba10ef98f8f1ff5d394b0cfb8e10b4d4 Mon Sep 17 00:00:00 2001 From: Aryahi Date: Fri, 15 Aug 2025 15:55:09 +0530 Subject: [PATCH 2/3] refactor(parser): address PR review feedback for DynamoDB models --- .../utilities/parser/models/dynamodb.py | 128 ++++++++---------- 1 file changed, 53 insertions(+), 75 deletions(-) diff --git a/aws_lambda_powertools/utilities/parser/models/dynamodb.py b/aws_lambda_powertools/utilities/parser/models/dynamodb.py index d46d9b4e7ed..9675d3f4d45 100644 --- a/aws_lambda_powertools/utilities/parser/models/dynamodb.py +++ b/aws_lambda_powertools/utilities/parser/models/dynamodb.py @@ -3,132 +3,110 @@ from typing import Any, Dict, List, Literal, Optional, Type, Union from pydantic import BaseModel, Field, field_validator + from aws_lambda_powertools.shared.dynamodb_deserializer import TypeDeserializer _DESERIALIZER = TypeDeserializer() class DynamoDBStreamChangedRecordModel(BaseModel): - ApproximateCreationDateTime: Optional[float] = Field( # AWS sends this as Unix epoch float + ApproximateCreationDateTime: Optional[datetime] = Field( # AWS sends this as Unix epoch float default=None, description="The approximate date and time when the stream record was created (Unix epoch time).", - examples=[1693997155.0] - ) - Keys: Dict[str, Any] = Field( - description="Primary key attributes for the item.", - examples=[{"Id": {"N": "101"}}] + examples=[1693997155.0], ) + Keys: Dict[str, Any] = Field(description="Primary key attributes for the item.", examples=[{"Id": {"N": "101"}}]) NewImage: Optional[Union[Dict[str, Any], Type[BaseModel], BaseModel]] = Field( default=None, description="The item after modifications, in DynamoDB attribute-value format.", - examples=[{ - "Message": {"S": "New item!"}, - "Id": {"N": "101"} - }] + examples=[{"Message": {"S": "New item!"}, "Id": {"N": "101"}}], ) OldImage: Optional[Union[Dict[str, Any], Type[BaseModel], BaseModel]] = Field( default=None, description="The item before modifications, in DynamoDB attribute-value format.", - examples=[{ - "Message": {"S": "Old item!"}, - "Id": {"N": "100"} - }] - ) - SequenceNumber: str = Field( - description="A unique identifier for the stream record.", - examples=["222"] - ) - SizeBytes: int = Field( - description="The size of the stream record, in bytes.", - examples=[26] + examples=[{"Message": {"S": "Old item!"}, "Id": {"N": "100"}}], ) + SequenceNumber: str = Field(description="A unique identifier for the stream record.", examples=["222"]) + SizeBytes: int = Field(description="The size of the stream record, in bytes.", examples=[26]) StreamViewType: Literal["NEW_AND_OLD_IMAGES", "KEYS_ONLY", "NEW_IMAGE", "OLD_IMAGE"] = Field( - description="The type of data included in the stream record.", - examples=["NEW_AND_OLD_IMAGES"] + description="The type of data included in the stream record.", examples=["NEW_AND_OLD_IMAGES"] ) @field_validator("Keys", "NewImage", "OldImage", mode="before") def deserialize_field(cls, value): - return {k: _DESERIALIZER.deserialize(v) for k, v in value.items()} if value else value + return {k: _DESERIALIZER.deserialize(v) for k, v in value.items()} class UserIdentity(BaseModel): type: Literal["Service"] = Field( default="Service", description="The type of identity that made the request, which is always 'Service' for DynamoDB streams.", - examples=["Service"] + examples=["Service"], ) - principalId: str = Field( + principalId: Literal["dynamodb.amazonaws.com"] = Field( description="The unique identifier for the principal that made the request.", - examples=["dynamodb.amazonaws.com"] + examples=["dynamodb.amazonaws.com"], ) class DynamoDBStreamRecordModel(BaseModel): - eventID: str = Field( - description="A unique identifier for the event.", - examples=["1"] - ) + eventID: str = Field(description="A unique identifier for the event.", examples=["1"]) eventName: Literal["INSERT", "MODIFY", "REMOVE"] = Field( - description="The type of operation that was performed on the item.", - examples=["INSERT"] + description="The type of operation that was performed on the item.", examples=["INSERT"] ) - eventVersion: str = Field( - default="1.0", - description="The version of the stream record format.", - examples=["1.0"] - ) - eventSource: str = Field( + eventVersion: float = Field(default="1.0", description="The version of the stream record format.", examples=["1.0"]) + eventSource: Literal["aws:dynamodb"] = Field( default="aws:dynamodb", description="The source of the event, which is always 'aws:dynamodb' for DynamoDB streams.", - examples=["aws:dynamodb"] - ) - awsRegion: str = Field( - description="The AWS region where the stream record was generated.", - examples=["us-west-2"] + examples=["aws:dynamodb"], ) + awsRegion: str = Field(description="The AWS region where the stream record was generated.", examples=["us-west-2"]) eventSourceARN: str = Field( description="The Amazon Resource Name (ARN) of the DynamoDB stream.", - examples=["arn:aws:dynamodb:us-west-2:123456789012:table/ExampleTable/stream/2021-01-01T00:00:00.000"] + examples=["arn:aws:dynamodb:us-west-2:123456789012:table/ExampleTable/stream/2021-01-01T00:00:00.000"], ) dynamodb: DynamoDBStreamChangedRecordModel = Field( description="Contains the details of the DynamoDB stream record.", - examples=[{ - "ApproximateCreationDateTime": 1693997155.0, - "Keys": {"Id": {"N": "101"}}, - "NewImage": {"Message": {"S": "New item!"}, "Id": {"N": "101"}}, - "OldImage": {"Message": {"S": "Old item!"}, "Id": {"N": "100"}}, - "SequenceNumber": "222", - "SizeBytes": 26, - "StreamViewType": "NEW_AND_OLD_IMAGES" - }] + examples=[ + { + "ApproximateCreationDateTime": 1693997155.0, + "Keys": {"Id": {"N": "101"}}, + "NewImage": {"Message": {"S": "New item!"}, "Id": {"N": "101"}}, + "OldImage": {"Message": {"S": "Old item!"}, "Id": {"N": "100"}}, + "SequenceNumber": "222", + "SizeBytes": 26, + "StreamViewType": "NEW_AND_OLD_IMAGES", + } + ], ) userIdentity: Optional[UserIdentity] = Field( default=None, description="Information about the identity that made the request.", - examples=[{"type": "Service", "principalId": "dynamodb.amazonaws.com"}] + examples=[{"type": "Service", "principalId": "dynamodb.amazonaws.com"}], ) class DynamoDBStreamModel(BaseModel): Records: List[DynamoDBStreamRecordModel] = Field( description="A list of records that contain the details of the DynamoDB stream events.", - examples=[{ - "eventID": "1", - "eventName": "INSERT", - "eventVersion": "1.0", - "eventSource": "aws:dynamodb", - "awsRegion": "us-west-2", - "eventSourceARN": "arn:aws:dynamodb:us-west-2:123456789012:table/ExampleTable/stream/2021-01-01T00:00:00.000", - "dynamodb": { - "ApproximateCreationDateTime": 1693997155.0, - "Keys": {"Id": {"N": "101"}}, - "NewImage": {"Message": {"S": "New item!"}, "Id": {"N": "101"}}, - "OldImage": {"Message": {"S": "Old item!"}, "Id": {"N": "100"}}, - "SequenceNumber": "222", - "SizeBytes": 26, - "StreamViewType": "NEW_AND_OLD_IMAGES" - }, - "userIdentity": {"type": "Service", "principalId": "dynamodb.amazonaws.com"} - }] - ) \ No newline at end of file + examples=[ + { + "eventID": "1", + "eventName": "INSERT", + "eventVersion": "1.0", + "eventSource": "aws:dynamodb", + "awsRegion": "us-west-2", + "eventSourceARN": "arn:aws:dynamodb:us-west-2:123456789012:table/ExampleTable/stream/2021-01-01T00:00:00.000", + "dynamodb": { + "ApproximateCreationDateTime": 1693997155.0, + "Keys": {"Id": {"N": "101"}}, + "NewImage": {"Message": {"S": "New item!"}, "Id": {"N": "101"}}, + "OldImage": {"Message": {"S": "Old item!"}, "Id": {"N": "100"}}, + "SequenceNumber": "222", + "SizeBytes": 26, + "StreamViewType": "NEW_AND_OLD_IMAGES", + }, + "userIdentity": {"type": "Service", "principalId": "dynamodb.amazonaws.com"}, + } + ], + ) From 436761d5ea0cd16c49231cb6189f55db4fe0945e Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Mon, 18 Aug 2025 09:26:51 +0100 Subject: [PATCH 3/3] Addressing changes --- .../utilities/parser/models/dynamodb.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/aws_lambda_powertools/utilities/parser/models/dynamodb.py b/aws_lambda_powertools/utilities/parser/models/dynamodb.py index 9675d3f4d45..e3c3dd4544f 100644 --- a/aws_lambda_powertools/utilities/parser/models/dynamodb.py +++ b/aws_lambda_powertools/utilities/parser/models/dynamodb.py @@ -29,7 +29,8 @@ class DynamoDBStreamChangedRecordModel(BaseModel): SequenceNumber: str = Field(description="A unique identifier for the stream record.", examples=["222"]) SizeBytes: int = Field(description="The size of the stream record, in bytes.", examples=[26]) StreamViewType: Literal["NEW_AND_OLD_IMAGES", "KEYS_ONLY", "NEW_IMAGE", "OLD_IMAGE"] = Field( - description="The type of data included in the stream record.", examples=["NEW_AND_OLD_IMAGES"] + description="The type of data included in the stream record.", + examples=["NEW_AND_OLD_IMAGES"], ) @field_validator("Keys", "NewImage", "OldImage", mode="before") @@ -39,7 +40,6 @@ def deserialize_field(cls, value): class UserIdentity(BaseModel): type: Literal["Service"] = Field( - default="Service", description="The type of identity that made the request, which is always 'Service' for DynamoDB streams.", examples=["Service"], ) @@ -52,11 +52,11 @@ class UserIdentity(BaseModel): class DynamoDBStreamRecordModel(BaseModel): eventID: str = Field(description="A unique identifier for the event.", examples=["1"]) eventName: Literal["INSERT", "MODIFY", "REMOVE"] = Field( - description="The type of operation that was performed on the item.", examples=["INSERT"] + description="The type of operation that was performed on the item.", + examples=["INSERT"], ) - eventVersion: float = Field(default="1.0", description="The version of the stream record format.", examples=["1.0"]) + eventVersion: float = Field(description="The version of the stream record format.", examples=["1.0"]) eventSource: Literal["aws:dynamodb"] = Field( - default="aws:dynamodb", description="The source of the event, which is always 'aws:dynamodb' for DynamoDB streams.", examples=["aws:dynamodb"], ) @@ -76,7 +76,7 @@ class DynamoDBStreamRecordModel(BaseModel): "SequenceNumber": "222", "SizeBytes": 26, "StreamViewType": "NEW_AND_OLD_IMAGES", - } + }, ], ) userIdentity: Optional[UserIdentity] = Field( @@ -96,7 +96,7 @@ class DynamoDBStreamModel(BaseModel): "eventVersion": "1.0", "eventSource": "aws:dynamodb", "awsRegion": "us-west-2", - "eventSourceARN": "arn:aws:dynamodb:us-west-2:123456789012:table/ExampleTable/stream/2021-01-01T00:00:00.000", + "eventSourceARN": "arn:aws:dynamodb:us-west-2:123456789012:table/ExampleTable/stream/2021-01-01T00:00:00.000", # noqa E501 "dynamodb": { "ApproximateCreationDateTime": 1693997155.0, "Keys": {"Id": {"N": "101"}}, @@ -107,6 +107,6 @@ class DynamoDBStreamModel(BaseModel): "StreamViewType": "NEW_AND_OLD_IMAGES", }, "userIdentity": {"type": "Service", "principalId": "dynamodb.amazonaws.com"}, - } + }, ], )