diff --git a/.cfnlintrc.yaml b/.cfnlintrc.yaml new file mode 100644 index 00000000000..3909b9bb437 --- /dev/null +++ b/.cfnlintrc.yaml @@ -0,0 +1,2 @@ +ignore_templates: + - examples/event_handler_appsync_events/sam/getting_started_with_appsync_events.yaml diff --git a/.chglog/CHANGELOG.tpl.md b/.chglog/CHANGELOG.tpl.md new file mode 100755 index 00000000000..beb340ad645 --- /dev/null +++ b/.chglog/CHANGELOG.tpl.md @@ -0,0 +1,67 @@ + + + +{{ if .Versions -}} + +# Unreleased + +{{ if .Unreleased.CommitGroups -}} +{{ range .Unreleased.CommitGroups -}} +## {{ .Title }} + +{{ range .Commits -}} +{{ if and (not (hasPrefix .Subject "changelog rebuild")) (not (hasPrefix .Subject "layer docs update")) (not (hasPrefix .Subject "bump version to")) -}} +* {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }} +{{ end -}} +{{ end }} +{{ end -}} +{{ end -}} +{{ end -}} + +{{ range .Versions }} + +## {{ if .Tag.Previous }}[{{ .Tag.Name }}]{{ else }}{{ .Tag.Name }}{{ end }} - {{ datetime "2006-01-02" .Tag.Date }} +{{ range .CommitGroups -}} + +## {{ .Title }} + +{{ range .Commits -}} +{{ if and (not (hasPrefix .Subject "changelog rebuild")) (not (hasPrefix .Subject "layer docs update")) (not (hasPrefix .Subject "bump version to")) -}} +* {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }} +{{ end -}} +{{ end }} +{{ end -}} + +{{- if .RevertCommits -}} +## Reverts +{{ range .RevertCommits -}} +* {{ .Revert.Header }} +{{ end }} +{{ end -}} + +{{- if .MergeCommits -}} +## Pull Requests + +{{ range .MergeCommits -}} +* {{ .Header }} +{{ end }} +{{ end -}} + +{{- if .NoteGroups -}} +{{ range .NoteGroups -}} +## {{ .Title }} +{{ range .Notes }} +{{ .Body }} +{{ end }} +{{ end -}} +{{ end -}} +{{ end -}} + +{{- if .Versions }} +[Unreleased]: {{ .Info.RepositoryURL }}/compare/{{ $latest := index .Versions 0 }}{{ $latest.Tag.Name }}...HEAD +{{ range .Versions -}} +{{ if .Tag.Previous -}} +[{{ .Tag.Name }}]: {{ $.Info.RepositoryURL }}/compare/{{ .Tag.Previous.Name }}...{{ .Tag.Name }} +{{ end -}} +{{ end -}} +{{ end -}} diff --git a/.chglog/config.yml b/.chglog/config.yml new file mode 100755 index 00000000000..9ec1c8ef31e --- /dev/null +++ b/.chglog/config.yml @@ -0,0 +1,37 @@ +style: github +template: CHANGELOG.tpl.md +info: + title: CHANGELOG + repository_url: https://github.com/aws-powertools/powertools-lambda-python +options: + commits: + filters: + Type: + - feat + - fix + - perf + - refactor + - docs + - chore + - revert + commit_groups: + title_maps: + feat: Features + fix: Bug Fixes + perf: Performance Improvements + refactor: Code Refactoring + docs: Documentation + chore: Maintenance + revert: Regression + header: + pattern: "^(\\w*)(?:\\(([\\w\\$\\.\\-\\*\\s]*)\\))?\\:\\s(.*)$" + pattern_maps: + - Type + - Scope + - Subject + notes: + keywords: + - BREAKING CHANGE + # issues: + # prefix: + # - # diff --git a/.clusterfuzzlite/Dockerfile b/.clusterfuzzlite/Dockerfile new file mode 100644 index 00000000000..19d3018f3ac --- /dev/null +++ b/.clusterfuzzlite/Dockerfile @@ -0,0 +1,12 @@ +FROM gcr.io/oss-fuzz-base/base-builder-python + +# Copy project source +COPY . $SRC/powertools + +WORKDIR $SRC/powertools + +# Install project dependencies +RUN pip3 install -e ".[all]" + +# Copy build script +COPY .clusterfuzzlite/build.sh $SRC/ diff --git a/.clusterfuzzlite/build.sh b/.clusterfuzzlite/build.sh new file mode 100644 index 00000000000..e5aadd80335 --- /dev/null +++ b/.clusterfuzzlite/build.sh @@ -0,0 +1,6 @@ +#!/bin/bash -eu + +# Build fuzz targets from tests/fuzz/ +for fuzzer in $(find $SRC/powertools/tests/fuzz -name 'fuzz_*.py'); do + compile_python_fuzzer "$fuzzer" +done diff --git a/.clusterfuzzlite/project.yaml b/.clusterfuzzlite/project.yaml new file mode 100644 index 00000000000..de5f07bb82a --- /dev/null +++ b/.clusterfuzzlite/project.yaml @@ -0,0 +1,4 @@ +language: python +main_repo: https://github.com/aws-powertools/powertools-lambda-python +sanitizers: + - address diff --git a/.flake8 b/.flake8 index 6c0c78fa967..1db8406d9e4 100644 --- a/.flake8 +++ b/.flake8 @@ -3,6 +3,11 @@ exclude = docs, .eggs, setup.py, example, .aws-sam, .git, dist, *.md, *.yaml, ex ignore = E203, E266, W503, BLK100, W291, I004 max-line-length = 120 max-complexity = 15 +; flake8-builtins isn't honouring inline ignore (A003) +per-file-ignores = + tests/e2e/utils/data_builder/__init__.py:F401 + tests/e2e/utils/data_fetcher/__init__.py:F401 + aws_lambda_powertools/utilities/data_classes/s3_event.py:A003 [isort] multi_line_output = 3 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000000..1062320f5b1 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,3 @@ +# https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners + +* @aws-powertools/lambda-python-core diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index 43ec3443e8b..00000000000 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,46 +0,0 @@ ---- -name: Bug report -about: Create a report to help us improve -title: '' -labels: bug, triage -assignees: '' - ---- - - - - -**What were you trying to accomplish?** - -## Expected Behavior - - - -## Current Behavior - - - -## Possible Solution - - - -## Steps to Reproduce (for bugs) - - -1. -2. -3. -4. - -## Environment - -* **Powertools version used**: -* **Packaging format (Layers, PyPi)**: -* **AWS Lambda function runtime:** -* **Debugging logs** - -> [How to enable debug mode](https://awslabs.github.io/aws-lambda-powertools-python/#debug-mode)** - -```python -# paste logs here -``` diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 00000000000..21a8f3b035c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,92 @@ +name: Bug report +description: Report a reproducible bug to help us improve +title: "Bug: TITLE" +labels: ["bug", "triage"] +body: + - type: markdown + attributes: + value: | + Thank you for submitting a bug report. Please add as much information as possible to help us reproduce, and remove any potential sensitive data. + + Please become familiar with [our definition of bug](https://github.com/aws-powertools/powertools-lambda-python/blob/develop/MAINTAINERS.md#is-that-a-bug). + - type: textarea + id: expected_behaviour + attributes: + label: Expected Behaviour + description: Please share details on the behaviour you expected + validations: + required: true + - type: textarea + id: current_behaviour + attributes: + label: Current Behaviour + description: Please share details on the current issue + validations: + required: true + - type: textarea + id: code_snippet + attributes: + label: Code snippet + description: Please share a code snippet to help us reproduce the issue + render: python + validations: + required: true + - type: textarea + id: solution + attributes: + label: Possible Solution + description: If known, please suggest a potential resolution + validations: + required: false + - type: textarea + id: steps + attributes: + label: Steps to Reproduce + description: Please share how we might be able to reproduce this issue + validations: + required: true + - type: input + id: version + attributes: + label: Powertools for AWS Lambda (Python) version + placeholder: "latest, 1.25.6" + value: latest + validations: + required: true + - type: dropdown + id: runtime + attributes: + label: AWS Lambda function runtime + options: + - "3.10" + - "3.11" + - "3.12" + - "3.13" + - "3.14" + validations: + required: true + - type: dropdown + id: packaging + attributes: + label: Packaging format used + options: + - Lambda Layers + - Serverless Application Repository (SAR) App + - PyPi + multiple: true + validations: + required: true + - type: textarea + id: logs + attributes: + label: Debugging logs + description: If available, please share [debugging logs](https://docs.powertools.aws.dev/lambda/python/#debug-mode) + render: python + validations: + required: false + - type: markdown + attributes: + value: | + --- + + **Disclaimer**: We value your time and bandwidth. As such, any pull requests created on non-triaged issues might not be successful. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000000..8acf5081f3b --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: false +contact_links: + - name: Ask a question + url: https://github.com/aws-powertools/powertools-lambda-python/discussions/new + about: Ask a general question about Lambda Powertools diff --git a/.github/ISSUE_TEMPLATE/documentation-improvements.md b/.github/ISSUE_TEMPLATE/documentation-improvements.md deleted file mode 100644 index 8341ae4e0c5..00000000000 --- a/.github/ISSUE_TEMPLATE/documentation-improvements.md +++ /dev/null @@ -1,17 +0,0 @@ ---- -name: Documentation improvements -about: Suggest a documentation update -title: '' -labels: documentation -assignees: '' - ---- - -**What were you initially searching for in the docs?** - - -**Is this related to an existing part of the documentation? Please share a link** - -**Describe how we could make it clearer** - -**If you have a proposed update, please share it here** diff --git a/.github/ISSUE_TEMPLATE/documentation_improvements.yml b/.github/ISSUE_TEMPLATE/documentation_improvements.yml new file mode 100644 index 00000000000..e750d51923f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/documentation_improvements.yml @@ -0,0 +1,50 @@ +name: Documentation improvements +description: Suggest a documentation update to improve everyone's experience +title: "Docs: TITLE" +labels: ["documentation", "triage"] +body: + - type: markdown + attributes: + value: | + Thank you for helping us improve everyone's experience. We review documentation updates on a case by case basis. + - type: textarea + id: search_area + attributes: + label: What were you searching in the docs? + description: Please help us understand how you looked for information that was either unclear or not available + validations: + required: true + - type: input + id: area + attributes: + label: Is this related to an existing documentation section? + description: Please share a link, if applicable + validations: + required: false + - type: textarea + id: idea + attributes: + label: How can we improve? + description: Please share your thoughts on how we can improve this experience + validations: + required: true + - type: textarea + id: suggestion + attributes: + label: Got a suggestion in mind? + description: Please suggest a proposed update + validations: + required: false + - type: checkboxes + id: acknowledgment + attributes: + label: Acknowledgment + options: + - label: I understand the final update might be different from my proposed suggestion, or refused. + required: true + - type: markdown + attributes: + value: | + --- + + **Disclaimer**: We value your time and bandwidth. As such, any pull requests created on non-triaged issues might not be successful. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md deleted file mode 100644 index b837b7ad5a1..00000000000 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -name: Feature request -about: Suggest an idea for this project -title: '' -labels: feature-request, triage -assignees: '' - ---- - -**Is your feature request related to a problem? Please describe.** - - -**Describe the solution you'd like** - - -**Describe alternatives you've considered** - - -**Additional context** - diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 00000000000..a39fb211ac1 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,48 @@ +name: Feature request +description: Suggest an idea for Powertools for AWS Lambda (Python) +title: "Feature request: TITLE" +labels: ["feature-request", "triage"] +body: + - type: markdown + attributes: + value: | + Thank you for taking the time to suggest an idea to the Powertools for AWS Lambda (Python) project. + + *Future readers*: Please react with 👍 and your use case to help us understand customer demand. + - type: textarea + id: problem + attributes: + label: Use case + description: Please help us understand your use case or problem you're facing + validations: + required: true + - type: textarea + id: suggestion + attributes: + label: Solution/User Experience + description: Please share what a good solution would look like to this use case + validations: + required: true + - type: textarea + id: alternatives + attributes: + label: Alternative solutions + description: Please describe what alternative solutions to this use case, if any + render: markdown + validations: + required: false + - type: checkboxes + id: acknowledgment + attributes: + label: Acknowledgment + options: + - label: This feature request meets [Powertools for AWS Lambda (Python) Tenets](https://docs.powertools.aws.dev/lambda/python/latest/#tenets) + required: true + - label: Should this be considered in other Powertools for AWS Lambda languages? i.e. [Java](https://github.com/aws-powertools/powertools-lambda-java/), [TypeScript](https://github.com/aws-powertools/powertools-lambda-typescript/), and [.NET](https://github.com/aws-powertools/powertools-lambda-dotnet/) + required: false + - type: markdown + attributes: + value: | + --- + + **Disclaimer**: We value your time and bandwidth. As such, any pull requests created on non-triaged issues might not be successful. diff --git a/.github/ISSUE_TEMPLATE/maintenance.yml b/.github/ISSUE_TEMPLATE/maintenance.yml new file mode 100644 index 00000000000..bbc3ff3b3d8 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/maintenance.yml @@ -0,0 +1,65 @@ +name: Maintenance +description: Suggest an activity to help address governance and anything internal +title: "Maintenance: TITLE" +labels: ["internal", "triage"] +body: + - type: markdown + attributes: + value: | + Thank you for taking the time to help us improve operational excellence. + + *Future readers*: Please react with 👍 and your use case to help us understand customer demand. + - type: textarea + id: importance + attributes: + label: Why is this needed? + description: Please help us understand the value so we can prioritize it accordingly + validations: + required: true + - type: dropdown + id: area + attributes: + label: Which area does this relate to? + multiple: true + options: + - Tests + - Static typing + - Tracer + - Logger + - Metrics + - Event Handler - REST API + - Event Handler - GraphQL API + - Middleware factory + - Parameters + - Batch processing + - Validation + - Event Source Data Classes + - Parser + - Idempotency + - Feature flags + - JMESPath functions + - Streaming + - Automation + - Other + - type: textarea + id: suggestion + attributes: + label: Solution + description: If available, please share what a good solution would look like + validations: + required: false + - type: checkboxes + id: acknowledgment + attributes: + label: Acknowledgment + options: + - label: This request meets [Powertools for AWS Lambda (Python) Tenets](https://docs.powertools.aws.dev/lambda/python/latest/#tenets) + required: true + - label: Should this be considered in other Powertools for AWS Lambda languages? i.e. [Java](https://github.com/aws-powertools/powertools-lambda-java/), [TypeScript](https://github.com/aws-powertools/powertools-lambda-typescript/), and [.NET](https://github.com/aws-powertools/powertools-lambda-dotnet/) + required: false + - type: markdown + attributes: + value: | + --- + + **Disclaimer**: We value your time and bandwidth. As such, any pull requests created on non-triaged issues might not be successful. diff --git a/.github/ISSUE_TEMPLATE/rfc.md b/.github/ISSUE_TEMPLATE/rfc.md deleted file mode 100644 index 4b666a16800..00000000000 --- a/.github/ISSUE_TEMPLATE/rfc.md +++ /dev/null @@ -1,52 +0,0 @@ ---- -name: RFC -about: Feature design and proposals -title: 'RFC: ' -labels: RFC, triage -assignees: '' - ---- - -## Key information - -* RFC PR: (leave this empty) -* Related issue(s), if known: -* Area: (i.e. Tracer, Metrics, Logger, etc.) -* Meet [tenets](https://awslabs.github.io/aws-lambda-powertools-python/#tenets): (Yes/no) - -## Summary -[summary]: #summary - -> One paragraph explanation of the feature. - -## Motivation -[motivation]: #motivation - -> Why are we doing this? What use cases does it support? What is the expected outcome? - -## Proposal -[proposal]: #proposal - -> This is the bulk of the RFC. - -> Explain the design in enough detail for somebody familiar with Powertools to understand it, and for somebody familiar with the implementation to implement it. - -> This should get into specifics and corner-cases, and include examples of how the feature is used. Any new terminology should be defined here. - -## Drawbacks -[drawbacks]: #drawbacks - -> Why should we *not* do this? - -> Do we need additional dependencies? Impact performance/package size? - -## Rationale and alternatives -[rationale-and-alternatives]: #rationale-and-alternatives - -* **What other designs have been considered? Why not them?** -* **What is the impact of not doing this?** - -## Unresolved questions -[unresolved-questions]: #unresolved-questions - -> Optional, stash area for topics that need further development e.g. TBD diff --git a/.github/ISSUE_TEMPLATE/rfc.yml b/.github/ISSUE_TEMPLATE/rfc.yml new file mode 100644 index 00000000000..31d8d7fe0c6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/rfc.yml @@ -0,0 +1,109 @@ +name: Request for Comments (RFC) +description: Feature design and detailed proposals +title: "RFC: TITLE" +labels: ["RFC", "triage"] +body: + - type: markdown + attributes: + value: | + Thank you for submitting a RFC. Please add as many details as possible to help further enrich this design. + - type: input + id: relation + attributes: + label: Is this related to an existing feature request or issue? + description: Please share a link, if applicable + - type: dropdown + id: area + attributes: + label: Which Powertools for AWS Lambda (Python) utility does this relate to? + options: + - Tracer + - Logger + - Metrics + - Event Handler - REST API + - Event Handler - GraphQL API + - Middleware factory + - Parameters + - Batch processing + - Typing + - Validation + - Event Source Data Classes + - Parser + - Idempotency + - Feature flags + - JMESPath functions + - Other + validations: + required: true + - type: textarea + id: summary + attributes: + label: Summary + description: Please provide an overview in one or two paragraphs + validations: + required: true + - type: textarea + id: problem + attributes: + label: Use case + description: Please share the use case and motivation behind this proposal + validations: + required: true + - type: textarea + id: proposal + attributes: + label: Proposal + description: Please explain the design in detail, so anyone familiar with the project could implement it + placeholder: What the user experience looks like before and after this design? + validations: + required: true + - type: textarea + id: scope + attributes: + label: Out of scope + description: Please explain what should be considered out of scope in your proposal + validations: + required: true + - type: textarea + id: challenges + attributes: + label: Potential challenges + description: Nothing is perfect. Please share what common challenges, edge cases, unresolved areas, and suggestions on how to mitigate them + validations: + required: true + - type: textarea + id: integrations + attributes: + label: Dependencies and Integrations + description: If applicable, please share whether this feature has additional dependencies, and how it might integrate with other utilities available + validations: + required: false + - type: textarea + id: alternatives + attributes: + label: Alternative solutions + description: Please describe what alternative solutions to this use case, if any + render: markdown + validations: + required: false + - type: checkboxes + id: acknowledgment + attributes: + label: Acknowledgment + options: + - label: This feature request meets [Powertools for AWS Lambda (Python) Tenets](https://docs.powertools.aws.dev/lambda/python/latest/#tenets) + required: true + - label: Should this be considered in other Powertools for AWS Lambda languages? i.e. [Java](https://github.com/aws-powertools/powertools-lambda-java/), [TypeScript](https://github.com/aws-powertools/powertools-lambda-typescript/), and [.NET](https://github.com/aws-powertools/powertools-lambda-dotnet/) + required: false + - type: markdown + attributes: + value: | + --- + + **Disclaimer**: We value your time and bandwidth. As such, any pull requests created on non-triaged issues might not be successful. + + Metadata information for admin purposes, please leave them empty. + + * RFC PR: + * Approved by: '' + * Reviewed by: '' diff --git a/.github/ISSUE_TEMPLATE/share_your_work.yml b/.github/ISSUE_TEMPLATE/share_your_work.yml new file mode 100644 index 00000000000..e4e4ed601be --- /dev/null +++ b/.github/ISSUE_TEMPLATE/share_your_work.yml @@ -0,0 +1,56 @@ +name: I Made This (showcase your work) +description: Share what you did with Powertools for AWS Lambda (Python) 💞💞. Blog post, workshops, presentation, sample apps, etc. +title: "[I Made This]:
record_handler function.
+ ```yaml title="template.yaml" hl_lines="44-45"
+ --8<-- "examples/batch_processing/sam/kinesis_batch_processing.yaml"
+ ```
-=== "app.py"
+=== "DynamoDB Streams"
- ```python
- from aws_lambda_powertools.utilities.batch import sqs_batch_processor
+ ```yaml title="template.yaml" hl_lines="43-44"
+ --8<-- "examples/batch_processing/sam/dynamodb_batch_processing.yaml"
+ ```
- def record_handler(record):
- # This will be called for each individual message from a batch
- # It should raise an exception if the message was not processed successfully
- return_value = do_something_with(record["body"])
- return return_value
+=== "Kafka (MSK)"
- @sqs_batch_processor(record_handler=record_handler)
- def lambda_handler(event, context):
- return {"statusCode": 200}
+ ```yaml title="template.yaml" hl_lines="74-75"
+ --8<-- "examples/batch_processing/sam/kafka_batch_processing.yaml"
```
-### PartialSQSProcessor context manager
+### Processing messages from SQS
-If you require access to the result of processed messages, you can use this context manager.
+Processing batches from SQS works in three stages:
-The result from calling `process()` on the context manager will be a list of all the return values from your `record_handler` function.
+1. Instantiate **`BatchProcessor`** and choose **`EventType.SQS`** for the event type
+2. Define your function to handle each batch record, and use [`SQSRecord`](data_classes.md#sqs){target="_blank"} type annotation for autocompletion
+3. Use **`process_partial_response`** to kick off processing
-=== "app.py"
+!!! info "This code example uses Tracer and Logger for completion."
- ```python
- from aws_lambda_powertools.utilities.batch import PartialSQSProcessor
+=== "Recommended"
+
+ ```python hl_lines="2-7 10 16 24"
+ --8<-- "examples/batch_processing/src/getting_started_sqs.py"
+ ```
+
+ 1. **Step 1**. Creates a partial failure batch processor for SQS queues. See [partial failure mechanics for details](#partial-failure-mechanics)
+ 2. **Step 2**. Defines a function to receive one record at a time from the batch
+ 3. **Step 3**. Kicks off processing
+
+=== "As a context manager"
+
+ ```python hl_lines="4-5 8 14 25-26 29"
+ --8<-- "examples/batch_processing/src/getting_started_sqs_context_manager.py"
+ ```
+
+=== "Sample response"
+
+ The second record failed to be processed, therefore the processor added its message ID in the response.
+
+ ```json
+ --8<-- "examples/batch_processing/src/getting_started_sqs_response.json"
+ ```
+
+=== "Sample event"
+
+ ```json
+ --8<-- "examples/batch_processing/src/getting_started_sqs_event.json"
+ ```
+
+#### FIFO queues
+
+When working with [SQS FIFO queues](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/FIFO-queues.html){target="_blank"}, a batch may include messages from different group IDs.
+
+By default, we will stop processing at the first failure and mark unprocessed messages as failed to preserve ordering. However, this behavior may not be optimal for customers who wish to proceed with processing messages from a different group ID.
+
+Enable the `skip_group_on_error` option for seamless processing of messages from various group IDs. This setup ensures that messages from a failed group ID are sent back to SQS, enabling uninterrupted processing of messages from the subsequent group ID.
+
+=== "Recommended"
+
+ ```python hl_lines="2-6 9 23"
+ --8<-- "examples/batch_processing/src/getting_started_sqs_fifo.py"
+ ```
+
+ 1. **Step 1**. Creates a partial failure batch processor for SQS FIFO queues. See [partial failure mechanics for details](#partial-failure-mechanics)
+
+=== "As a context manager"
+
+ ```python hl_lines="4 8"
+ --8<-- "examples/batch_processing/src/getting_started_sqs_fifo_context_manager.py"
+ ```
+
+=== "Enabling skip_group_on_error flag"
+
+ ```python hl_lines="2-6 9 23"
+ --8<-- "examples/batch_processing/src/getting_started_sqs_fifo_skip_on_error.py"
+ ```
+
+### Processing messages from Kinesis
+
+Processing batches from Kinesis works in three stages:
+
+1. Instantiate **`BatchProcessor`** and choose **`EventType.KinesisDataStreams`** for the event type
+2. Define your function to handle each batch record, and use [`KinesisStreamRecord`](data_classes.md#kinesis-streams){target="_blank"} type annotation for autocompletion
+3. Use **`process_partial_response`** to kick off processing
+
+!!! info "This code example uses Tracer and Logger for completion."
+
+=== "Recommended"
+
+ ```python hl_lines="2-9 12 18 27"
+ --8<-- "examples/batch_processing/src/getting_started_kinesis.py"
+ ```
+
+ 1. **Step 1**. Creates a partial failure batch processor for Kinesis Data Streams. See [partial failure mechanics for details](#partial-failure-mechanics)
+
+=== "As a context manager"
+
+ ```python hl_lines="3-5 8 14 23-25 28"
+ --8<-- "examples/batch_processing/src/getting_started_kinesis_context_manager.py"
+ ```
+
+=== "Sample response"
+
+ The second record failed to be processed, therefore the processor added its sequence number in the response.
+
+ ```json
+ --8<-- "examples/batch_processing/src/getting_started_kinesis_response.json"
+ ```
+
+=== "Sample event"
+
+ ```json
+ --8<-- "examples/batch_processing/src/getting_started_kinesis_event.json"
+ ```
+
+### Processing messages from DynamoDB
+
+Processing batches from DynamoDB Streams works in three stages:
+
+1. Instantiate **`BatchProcessor`** and choose **`EventType.DynamoDBStreams`** for the event type
+2. Define your function to handle each batch record, and use [`DynamoDBRecord`](data_classes.md#dynamodb-streams){target="_blank"} type annotation for autocompletion
+3. Use **`process_partial_response`** to kick off processing
+
+!!! info "This code example uses Tracer and Logger for completion."
+
+=== "Recommended"
+
+ ```python hl_lines="4-11 14 20 32"
+ --8<-- "examples/batch_processing/src/getting_started_dynamodb.py"
+ ```
+
+ 1. **Step 1**. Creates a partial failure batch processor for DynamoDB Streams. See [partial failure mechanics for details](#partial-failure-mechanics)
+
+=== "As a context manager"
+
+ ```python hl_lines="5-7 10 16 28-30 33"
+ --8<-- "examples/batch_processing/src/getting_started_dynamodb_context_manager.py"
+ ```
+
+=== "Sample response"
+
+ The second record failed to be processed, therefore the processor added its sequence number in the response.
+
+ ```json
+ --8<-- "examples/batch_processing/src/getting_started_dynamodb_response.json"
+ ```
+
+=== "Sample event"
+
+ ```json
+ --8<-- "examples/batch_processing/src/getting_started_dynamodb_event.json"
+ ```
+
+### Processing messages from Kafka
+
+Processing batches from Amazon MSK or self-managed Apache Kafka works in three stages:
+
+1. Instantiate **`BatchProcessor`** and choose **`EventType.Kafka`** for the event type
+2. Define your function to handle each batch record, and use [`KafkaEventRecord`](data_classes.md#kafka){target="_blank"} type annotation for autocompletion
+3. Use **`process_partial_response`** to kick off processing
+
+!!! info "This works with both MSK and self-managed Apache Kafka"
+ The batch processor automatically handles the different event structures from MSK and self-managed Kafka clusters.
+
+=== "Recommended"
+
+ ```python hl_lines="2-9 12 18 27"
+ --8<-- "examples/batch_processing/src/getting_started_kafka.py"
+ ```
+
+ 1. **Step 1**. Creates a partial failure batch processor for Kafka. See [partial failure mechanics for details](#partial-failure-mechanics)
+
+=== "Sample response"
- def record_handler(record):
- # This will be called for each individual message from a batch
- # It should raise an exception if the message was not processed successfully
- return_value = do_something_with(record["body"])
- return return_value
+ The second record failed to be processed, therefore the processor added its topic-partition and offset in the response.
+ ```json
+ --8<-- "examples/batch_processing/src/getting_started_kafka_response.json"
+ ```
+
+=== "Sample event"
+
+ ```json
+ --8<-- "examples/batch_processing/src/getting_started_kafka_event.json"
+ ```
+
+!!! tip "Extracting message value"
+ Use `record.json_value` to get the deserialized JSON body from the Kafka record. For raw bytes access, use `record.decoded_value`.
+
+ For advanced deserialization (Avro, Protobuf), see the [Kafka Consumer utility](kafka.md){target="_blank"} which can be used alongside the batch processor.
+
+### Error handling
+
+By default, we catch any exception raised by your record handler function. This allows us to **(1)** continue processing the batch, **(2)** collect each batch item that failed processing, and **(3)** return the appropriate response correctly without failing your Lambda function execution.
+
+=== "Sample error handling with custom exception"
+
+ ```python title="" hl_lines="23"
+ --8<-- "examples/batch_processing/src/getting_started_error_handling.py"
+ ```
- def lambda_handler(event, context):
- records = event["Records"]
+ 1. Any exception works here. See [extending BatchProcessor section, if you want to override this behavior.](#extending-batchprocessor)
- processor = PartialSQSProcessor()
+ 2. Exceptions raised in `record_handler` will propagate to `process_partial_response`.