aws-python-helper 0.30.1__tar.gz → 0.32.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {aws_python_helper-0.30.1 → aws_python_helper-0.32.0}/PKG-INFO +230 -2
- {aws_python_helper-0.30.1 → aws_python_helper-0.32.0}/README.md +229 -1
- {aws_python_helper-0.30.1 → aws_python_helper-0.32.0}/aws_python_helper/__init__.py +9 -0
- {aws_python_helper-0.30.1 → aws_python_helper-0.32.0}/aws_python_helper/api/dispatcher.py +15 -1
- aws_python_helper-0.32.0/aws_python_helper/context/__init__.py +3 -0
- aws_python_helper-0.32.0/aws_python_helper/context/state.py +22 -0
- {aws_python_helper-0.30.1 → aws_python_helper-0.32.0}/aws_python_helper/fargate/executor.py +8 -1
- {aws_python_helper-0.30.1 → aws_python_helper-0.32.0}/aws_python_helper/fargate/task_base.py +5 -0
- {aws_python_helper-0.30.1 → aws_python_helper-0.32.0}/aws_python_helper/lambda_standalone/base.py +7 -1
- aws_python_helper-0.32.0/aws_python_helper/repository/__init__.py +3 -0
- aws_python_helper-0.32.0/aws_python_helper/repository/base.py +239 -0
- {aws_python_helper-0.30.1 → aws_python_helper-0.32.0}/aws_python_helper/sns/publisher.py +11 -1
- {aws_python_helper-0.30.1 → aws_python_helper-0.32.0}/aws_python_helper/sqs/consumer_base.py +104 -48
- {aws_python_helper-0.30.1 → aws_python_helper-0.32.0}/aws_python_helper.egg-info/PKG-INFO +230 -2
- {aws_python_helper-0.30.1 → aws_python_helper-0.32.0}/aws_python_helper.egg-info/SOURCES.txt +4 -0
- {aws_python_helper-0.30.1 → aws_python_helper-0.32.0}/pyproject.toml +1 -1
- {aws_python_helper-0.30.1 → aws_python_helper-0.32.0}/aws_python_helper/api/__init__.py +0 -0
- {aws_python_helper-0.30.1 → aws_python_helper-0.32.0}/aws_python_helper/api/auth_middleware.py +0 -0
- {aws_python_helper-0.30.1 → aws_python_helper-0.32.0}/aws_python_helper/api/auth_validators.py +0 -0
- {aws_python_helper-0.30.1 → aws_python_helper-0.32.0}/aws_python_helper/api/base.py +0 -0
- {aws_python_helper-0.30.1 → aws_python_helper-0.32.0}/aws_python_helper/api/exceptions.py +0 -0
- {aws_python_helper-0.30.1 → aws_python_helper-0.32.0}/aws_python_helper/api/fetcher.py +0 -0
- {aws_python_helper-0.30.1 → aws_python_helper-0.32.0}/aws_python_helper/api/handler.py +0 -0
- {aws_python_helper-0.30.1 → aws_python_helper-0.32.0}/aws_python_helper/database/__init__.py +0 -0
- {aws_python_helper-0.30.1 → aws_python_helper-0.32.0}/aws_python_helper/database/database_proxy.py +0 -0
- {aws_python_helper-0.30.1 → aws_python_helper-0.32.0}/aws_python_helper/database/external_database_proxy.py +0 -0
- {aws_python_helper-0.30.1 → aws_python_helper-0.32.0}/aws_python_helper/database/external_mongo_manager.py +0 -0
- {aws_python_helper-0.30.1 → aws_python_helper-0.32.0}/aws_python_helper/database/mongo_manager.py +0 -0
- {aws_python_helper-0.30.1 → aws_python_helper-0.32.0}/aws_python_helper/fargate/__init__.py +0 -0
- {aws_python_helper-0.30.1 → aws_python_helper-0.32.0}/aws_python_helper/fargate/fetcher.py +0 -0
- {aws_python_helper-0.30.1 → aws_python_helper-0.32.0}/aws_python_helper/fargate/handler.py +0 -0
- {aws_python_helper-0.30.1 → aws_python_helper-0.32.0}/aws_python_helper/lambda_standalone/__init__.py +0 -0
- {aws_python_helper-0.30.1 → aws_python_helper-0.32.0}/aws_python_helper/lambda_standalone/fetcher.py +0 -0
- {aws_python_helper-0.30.1 → aws_python_helper-0.32.0}/aws_python_helper/lambda_standalone/handler.py +0 -0
- {aws_python_helper-0.30.1 → aws_python_helper-0.32.0}/aws_python_helper/sns/__init__.py +0 -0
- {aws_python_helper-0.30.1 → aws_python_helper-0.32.0}/aws_python_helper/sqs/__init__.py +0 -0
- {aws_python_helper-0.30.1 → aws_python_helper-0.32.0}/aws_python_helper/sqs/fetcher.py +0 -0
- {aws_python_helper-0.30.1 → aws_python_helper-0.32.0}/aws_python_helper/sqs/handler.py +0 -0
- {aws_python_helper-0.30.1 → aws_python_helper-0.32.0}/aws_python_helper/utils/__init__.py +0 -0
- {aws_python_helper-0.30.1 → aws_python_helper-0.32.0}/aws_python_helper/utils/json_encoder.py +0 -0
- {aws_python_helper-0.30.1 → aws_python_helper-0.32.0}/aws_python_helper/utils/response.py +0 -0
- {aws_python_helper-0.30.1 → aws_python_helper-0.32.0}/aws_python_helper/utils/serializer.py +0 -0
- {aws_python_helper-0.30.1 → aws_python_helper-0.32.0}/aws_python_helper.egg-info/dependency_links.txt +0 -0
- {aws_python_helper-0.30.1 → aws_python_helper-0.32.0}/aws_python_helper.egg-info/requires.txt +0 -0
- {aws_python_helper-0.30.1 → aws_python_helper-0.32.0}/aws_python_helper.egg-info/top_level.txt +0 -0
- {aws_python_helper-0.30.1 → aws_python_helper-0.32.0}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: aws-python-helper
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.32.0
|
|
4
4
|
Summary: AWS Python Helper Framework
|
|
5
5
|
Author-email: Fabian Claros <neufabiae@gmail.com>
|
|
6
6
|
License: MIT
|
|
@@ -29,6 +29,7 @@ Mini-framework to create REST APIs, SQS Consumers, SNS Publishers, Fargate Tasks
|
|
|
29
29
|
- **OOP structure**: Object-oriented programming for your code
|
|
30
30
|
- **Flexible MongoDB**: Direct access to multiple databases without models
|
|
31
31
|
- **External MongoDB**: Connect to multiple MongoDB clusters simultaneously
|
|
32
|
+
- **Multi-state routing**: Automatic `constitution-state` propagation across the entire call chain for per-state database routing
|
|
32
33
|
- **SQS Consumers**: Same pattern to process SQS messages (single or batch mode)
|
|
33
34
|
- **SNS Publishers**: Same pattern to publish messages to SNS topics
|
|
34
35
|
- **Fargate Tasks**: Same pattern to run tasks in Fargate containers
|
|
@@ -60,6 +61,9 @@ All available classes and functions:
|
|
|
60
61
|
| `FargateTask` | `aws_python_helper.fargate.task_base` | Base class for Fargate tasks |
|
|
61
62
|
| `FargateExecutor` | `aws_python_helper.fargate.executor` | Launches Fargate tasks from Lambda |
|
|
62
63
|
| `fargate_handler` | `aws_python_helper.fargate.handler` | Entry point handler for Fargate |
|
|
64
|
+
| `Repository` | `aws_python_helper.repository.base` | Base class for MongoDB repositories |
|
|
65
|
+
| `get_state` | `aws_python_helper` | Read the current constitution-state from async context |
|
|
66
|
+
| `set_state` | `aws_python_helper` | Set the current constitution-state in async context |
|
|
63
67
|
| `MongoJSONEncoder` | `aws_python_helper.utils.json_encoder` | JSON encoder for MongoDB types |
|
|
64
68
|
| `mongo_json_dumps` | `aws_python_helper.utils.json_encoder` | Helper to serialize MongoDB types |
|
|
65
69
|
| `serialize_mongo_types` | `aws_python_helper.utils.serializer` | Recursively serialize MongoDB types |
|
|
@@ -268,6 +272,7 @@ response = lambda_client.invoke(
|
|
|
268
272
|
FunctionName='GenerateRouteLambda',
|
|
269
273
|
InvocationType='RequestResponse',
|
|
270
274
|
Payload=json.dumps({
|
|
275
|
+
'constitution-state': 'connecticut', # Required
|
|
271
276
|
'data': {
|
|
272
277
|
'shipping_id': '507f1f77bcf86cd799439011'
|
|
273
278
|
}
|
|
@@ -291,6 +296,7 @@ lambda_client.invoke(
|
|
|
291
296
|
FunctionName='GenerateRouteLambda',
|
|
292
297
|
InvocationType='Event', # Asynchronous
|
|
293
298
|
Payload=json.dumps({
|
|
299
|
+
'constitution-state': 'connecticut', # Required
|
|
294
300
|
'data': {
|
|
295
301
|
'shipping_id': '507f1f77bcf86cd799439011'
|
|
296
302
|
}
|
|
@@ -434,6 +440,7 @@ from aws_python_helper.fargate.executor import FargateExecutor
|
|
|
434
440
|
|
|
435
441
|
def handler(event, context):
|
|
436
442
|
executor = FargateExecutor()
|
|
443
|
+
# constitution-state is auto-propagated as CONSTITUTION_STATE env var in the container
|
|
437
444
|
task_arn = executor.run_task(
|
|
438
445
|
'search-tax-by-town',
|
|
439
446
|
envs={'TOWN': 'Norwalk', 'ONLY_TAX': 'true'}
|
|
@@ -489,6 +496,219 @@ class AddressAPI(API):
|
|
|
489
496
|
|
|
490
497
|
`self.external_db` is available in `API`, `SQSConsumer`, `Lambda`, and `FargateTask`.
|
|
491
498
|
|
|
499
|
+
## 🗂️ Repository Pattern
|
|
500
|
+
|
|
501
|
+
The framework provides a `Repository` base class that eliminates repetitive boilerplate in data access layers. Each repository only declares what collection it uses, whether it belongs to an external cluster, and what indexes to create. The base class handles the MongoDB connection and index creation automatically.
|
|
502
|
+
|
|
503
|
+
### Properties to override
|
|
504
|
+
|
|
505
|
+
| Property | Type | Default | Required |
|
|
506
|
+
|----------|------|---------|----------|
|
|
507
|
+
| `collection_name` | `str` | — | **Yes** |
|
|
508
|
+
| `database_key` | `str \| None` | `None` | No — if `None`, uses `constitution-state` from context |
|
|
509
|
+
| `is_external` | `bool` | `False` | No |
|
|
510
|
+
| `cluster_name` | `str` | `None` | Only if `is_external=True` |
|
|
511
|
+
| `indexes` | `list` | `[]` | No |
|
|
512
|
+
|
|
513
|
+
**`database_key` controls how the database is resolved:**
|
|
514
|
+
- `database_key = "core"` (or any string) → always connects to that specific database.
|
|
515
|
+
- `database_key = None` (default) → reads `constitution-state` from the async context automatically. This makes the repository **state-scoped**: it connects to `"connecticut"`, `"new_jersey"`, etc. depending on the current request.
|
|
516
|
+
|
|
517
|
+
Collections are cached per `(database_name, collection_name)` key — state-scoped repositories correctly isolate state between concurrent requests.
|
|
518
|
+
|
|
519
|
+
### Index format
|
|
520
|
+
|
|
521
|
+
```python
|
|
522
|
+
@property
|
|
523
|
+
def indexes(self):
|
|
524
|
+
return [
|
|
525
|
+
{"key": [("field", 1)]}, # simple ASC
|
|
526
|
+
{"key": [("field", -1)]}, # simple DESC
|
|
527
|
+
{"key": [("f1", 1), ("f2", -1)], "unique": True}, # compound + unique
|
|
528
|
+
{"key": [("expires_at", 1)], "expireAfterSeconds": 0}, # TTL index
|
|
529
|
+
]
|
|
530
|
+
```
|
|
531
|
+
|
|
532
|
+
Indexes are created automatically in the background on first collection access — no need to call any initialization method.
|
|
533
|
+
|
|
534
|
+
### Repository with a fixed database
|
|
535
|
+
|
|
536
|
+
```python
|
|
537
|
+
from aws_python_helper import Repository
|
|
538
|
+
|
|
539
|
+
class TownsRepository(Repository):
|
|
540
|
+
|
|
541
|
+
@property
|
|
542
|
+
def collection_name(self):
|
|
543
|
+
return "towns"
|
|
544
|
+
|
|
545
|
+
@property
|
|
546
|
+
def database_key(self):
|
|
547
|
+
return "core" # always connects to the "core" database
|
|
548
|
+
|
|
549
|
+
@property
|
|
550
|
+
def indexes(self):
|
|
551
|
+
return [
|
|
552
|
+
{"key": [("name", 1)]},
|
|
553
|
+
{"key": [("platform", 1)]},
|
|
554
|
+
]
|
|
555
|
+
|
|
556
|
+
async def get_available(self, platforms):
|
|
557
|
+
return await self.collection.find(
|
|
558
|
+
{"platform": {"$in": platforms}},
|
|
559
|
+
{"name": 1, "platform": 1}
|
|
560
|
+
).to_list(length=None)
|
|
561
|
+
|
|
562
|
+
async def find_by_name(self, name):
|
|
563
|
+
return await self.collection.find_one({"name": name})
|
|
564
|
+
```
|
|
565
|
+
|
|
566
|
+
### State-scoped repository (no `database_key`)
|
|
567
|
+
|
|
568
|
+
When `database_key` is not set, the repository reads the current `constitution-state` from context and uses it as the database name. The same repository instance connects to `"connecticut"` for one request and to `"new_jersey"` for another — automatically.
|
|
569
|
+
|
|
570
|
+
```python
|
|
571
|
+
from aws_python_helper import Repository
|
|
572
|
+
|
|
573
|
+
class LandRecordsRepository(Repository):
|
|
574
|
+
|
|
575
|
+
@property
|
|
576
|
+
def collection_name(self):
|
|
577
|
+
return "records"
|
|
578
|
+
|
|
579
|
+
# No database_key → uses get_state() automatically
|
|
580
|
+
# If constitution-state = "connecticut" → connects to DB "connecticut"
|
|
581
|
+
# If constitution-state = "new_jersey" → connects to DB "new_jersey"
|
|
582
|
+
|
|
583
|
+
@property
|
|
584
|
+
def indexes(self):
|
|
585
|
+
return [
|
|
586
|
+
{"key": [("unique_id", 1)]},
|
|
587
|
+
{"key": [("owner", 1), ("town", 1)]},
|
|
588
|
+
]
|
|
589
|
+
|
|
590
|
+
async def bulk_upsert(self, records):
|
|
591
|
+
from pymongo import UpdateOne
|
|
592
|
+
operations = [
|
|
593
|
+
UpdateOne({"unique_id": r["unique_id"]}, {"$set": r}, upsert=True)
|
|
594
|
+
for r in records
|
|
595
|
+
]
|
|
596
|
+
result = await self.collection.bulk_write(operations)
|
|
597
|
+
return {"upserted": result.upserted_count, "modified": result.modified_count}
|
|
598
|
+
```
|
|
599
|
+
|
|
600
|
+
> **Note:** A `ValueError` is raised at runtime if `database_key` is `None` and `constitution-state` has not been set in the context. This is prevented automatically by the framework at every entry point (API, Lambda, SQS, Fargate).
|
|
601
|
+
|
|
602
|
+
### Repository on an external cluster
|
|
603
|
+
|
|
604
|
+
```python
|
|
605
|
+
from aws_python_helper import Repository
|
|
606
|
+
|
|
607
|
+
class AddressRepository(Repository):
|
|
608
|
+
|
|
609
|
+
@property
|
|
610
|
+
def database_key(self):
|
|
611
|
+
return "smart_data"
|
|
612
|
+
|
|
613
|
+
@property
|
|
614
|
+
def collection_name(self):
|
|
615
|
+
return "address"
|
|
616
|
+
|
|
617
|
+
@property
|
|
618
|
+
def is_external(self):
|
|
619
|
+
return True
|
|
620
|
+
|
|
621
|
+
@property
|
|
622
|
+
def cluster_name(self):
|
|
623
|
+
return "ClusterDockets" # Must match a name in EXTERNAL_MONGODB_CONNECTIONS
|
|
624
|
+
|
|
625
|
+
async def find_by_query(self, query, limit=None):
|
|
626
|
+
cursor = self.collection.find(query)
|
|
627
|
+
if limit:
|
|
628
|
+
cursor = cursor.limit(limit)
|
|
629
|
+
return await cursor.to_list(length=None)
|
|
630
|
+
```
|
|
631
|
+
|
|
632
|
+
### Instantiation — no `db` argument needed
|
|
633
|
+
|
|
634
|
+
```python
|
|
635
|
+
class MyAPI(API):
|
|
636
|
+
|
|
637
|
+
@property
|
|
638
|
+
def towns_repository(self):
|
|
639
|
+
if not self._towns_repository:
|
|
640
|
+
self._towns_repository = TownsRepository() # no args!
|
|
641
|
+
return self._towns_repository
|
|
642
|
+
|
|
643
|
+
async def process(self):
|
|
644
|
+
towns = await self.towns_repository.get_available(["platform_a", "platform_b"])
|
|
645
|
+
self.set_body({"towns": towns})
|
|
646
|
+
```
|
|
647
|
+
|
|
648
|
+
The repository connects itself using the already-initialized `MongoManager` singleton — the same one used by `self.db`. No need to pass `self.db` or any connection object.
|
|
649
|
+
|
|
650
|
+
## 🌐 Constitution State
|
|
651
|
+
|
|
652
|
+
The framework uses a `constitution-state` value to support **multi-state database routing** — connecting each request to the correct database based on the state it belongs to (e.g., `"connecticut"`, `"new_jersey"`). This value is propagated automatically across the entire async call chain using Python's `contextvars.ContextVar`, so you never need to pass it manually between layers.
|
|
653
|
+
|
|
654
|
+
### How the framework injects it at each entry point
|
|
655
|
+
|
|
656
|
+
| Entry point | How `constitution-state` is read |
|
|
657
|
+
|-------------|----------------------------------|
|
|
658
|
+
| **API Gateway** | HTTP header `constitution-state` — **required**, returns `400` if missing |
|
|
659
|
+
| **Standalone Lambda** | Field `constitution-state` in the event payload — **required**, raises `ValueError` if missing |
|
|
660
|
+
| **SQS Consumer (single mode)** | Per-record: reads from SNS `MessageAttributes['constitution-state']`, falls back to `body.constitution_state` |
|
|
661
|
+
| **SQS Consumer (batch mode)** | Groups records by state; calls `process_batch()` once per group with the correct state in context |
|
|
662
|
+
| **Fargate Task** | Env var `CONSTITUTION_STATE` — auto-injected by `FargateExecutor` |
|
|
663
|
+
|
|
664
|
+
### How the framework propagates it to downstream services
|
|
665
|
+
|
|
666
|
+
| Downstream service | Propagation mechanism |
|
|
667
|
+
|--------------------|-----------------------|
|
|
668
|
+
| **SNS Publisher** | Auto-injects `constitution-state` as a `MessageAttribute` on every published message |
|
|
669
|
+
| **FargateExecutor** | Auto-injects `CONSTITUTION_STATE` as an env var when launching Fargate containers |
|
|
670
|
+
|
|
671
|
+
This means that an API call with `constitution-state: connecticut` will automatically carry that state through SNS → SQS → Fargate without any code changes in your consumers or tasks.
|
|
672
|
+
|
|
673
|
+
### State-scoped repositories
|
|
674
|
+
|
|
675
|
+
Repositories with no `database_key` (default) read `constitution-state` from context to resolve the target database automatically. See the [Repository Pattern](#️-repository-pattern) section for details.
|
|
676
|
+
|
|
677
|
+
### Manual access
|
|
678
|
+
|
|
679
|
+
If you need to read or set the state manually (e.g., in tests or utility code):
|
|
680
|
+
|
|
681
|
+
```python
|
|
682
|
+
from aws_python_helper import get_state, set_state
|
|
683
|
+
|
|
684
|
+
state = get_state() # e.g. "connecticut", or None if not set
|
|
685
|
+
set_state("new_jersey") # set manually (the framework does this automatically)
|
|
686
|
+
```
|
|
687
|
+
|
|
688
|
+
### API example — `constitution-state` header
|
|
689
|
+
|
|
690
|
+
```
|
|
691
|
+
GET /constitutions HTTP/1.1
|
|
692
|
+
constitution-state: connecticut
|
|
693
|
+
Authorization: Bearer <token>
|
|
694
|
+
```
|
|
695
|
+
|
|
696
|
+
### Lambda invocation example — `constitution-state` in event
|
|
697
|
+
|
|
698
|
+
```python
|
|
699
|
+
import boto3, json
|
|
700
|
+
|
|
701
|
+
lambda_client = boto3.client('lambda')
|
|
702
|
+
lambda_client.invoke(
|
|
703
|
+
FunctionName='MyLambdaFunction',
|
|
704
|
+
InvocationType='RequestResponse',
|
|
705
|
+
Payload=json.dumps({
|
|
706
|
+
'constitution-state': 'connecticut', # Required
|
|
707
|
+
'data': {'key': 'value'}
|
|
708
|
+
})
|
|
709
|
+
)
|
|
710
|
+
```
|
|
711
|
+
|
|
492
712
|
## 🔄 Routing Convention
|
|
493
713
|
|
|
494
714
|
The framework uses convention over configuration for the routing:
|
|
@@ -880,6 +1100,7 @@ environment_variables = {
|
|
|
880
1100
|
| `AUTH_BYPASS_TOKEN` | Optional | Master token to bypass authentication |
|
|
881
1101
|
| `ECS_CLUSTER` | Fargate only | ECS cluster name for `FargateExecutor` |
|
|
882
1102
|
| `ECS_SUBNETS` | Fargate only | Comma-separated subnet IDs for Fargate tasks |
|
|
1103
|
+
| `CONSTITUTION_STATE` | Fargate only (auto) | State injected automatically by `FargateExecutor` — do not set manually |
|
|
883
1104
|
| `AWS_REGION` | Fargate/SNS/SQS | AWS region |
|
|
884
1105
|
| `AWS_ACCOUNT_ID` | SQS `get_queue_url` | AWS account ID |
|
|
885
1106
|
| `SERVICE_NAME` | SQS `get_queue_url` | Service name prefix for queue name |
|
|
@@ -890,7 +1111,11 @@ environment_variables = {
|
|
|
890
1111
|
|
|
891
1112
|
### SQS Consumer - Batch Mode
|
|
892
1113
|
|
|
893
|
-
By default, consumers process messages one by one (`"single"` mode). Use `"batch"` mode when you need to group or bulk-process messages
|
|
1114
|
+
By default, consumers process messages one by one (`"single"` mode). Use `"batch"` mode when you need to group or bulk-process messages.
|
|
1115
|
+
|
|
1116
|
+
**Constitution-state handling in SQS:**
|
|
1117
|
+
- **Single mode**: the framework extracts `constitution-state` from each record automatically (from SNS `MessageAttributes`, then from `body.constitution_state`) and sets it in context before calling `process_record()`. You do not need to extract it yourself.
|
|
1118
|
+
- **Batch mode**: the framework groups the incoming records by `constitution-state` and calls `process_batch()` once per group, with the correct state in context for each group. This ensures that state-scoped repositories resolve to the right database even when a batch contains records from different states.
|
|
894
1119
|
|
|
895
1120
|
```python
|
|
896
1121
|
from aws_python_helper.sqs.consumer_base import SQSConsumer
|
|
@@ -939,10 +1164,13 @@ class OrderConsumer(SQSConsumer):
|
|
|
939
1164
|
|
|
940
1165
|
### SNS Publisher - Batch Publishing
|
|
941
1166
|
|
|
1167
|
+
The `SNSPublisher` automatically injects the current `constitution-state` as a `MessageAttribute` on every published message. SQS consumers built with this framework will then extract it automatically, ensuring the state flows end-to-end through the SNS → SQS chain without any manual code.
|
|
1168
|
+
|
|
942
1169
|
```python
|
|
943
1170
|
topic = TitleIndexedTopic()
|
|
944
1171
|
|
|
945
1172
|
# Publish multiple messages in a single call
|
|
1173
|
+
# constitution-state is auto-injected as a MessageAttribute on each message
|
|
946
1174
|
await topic.publish([
|
|
947
1175
|
{'content': {'id': 'id1', 'title': 'Title 1'}, 'attributes': {'type': 'created'}},
|
|
948
1176
|
{'content': {'id': 'id2', 'title': 'Title 2'}, 'attributes': {'type': 'updated'}},
|
|
@@ -9,6 +9,7 @@ Mini-framework to create REST APIs, SQS Consumers, SNS Publishers, Fargate Tasks
|
|
|
9
9
|
- **OOP structure**: Object-oriented programming for your code
|
|
10
10
|
- **Flexible MongoDB**: Direct access to multiple databases without models
|
|
11
11
|
- **External MongoDB**: Connect to multiple MongoDB clusters simultaneously
|
|
12
|
+
- **Multi-state routing**: Automatic `constitution-state` propagation across the entire call chain for per-state database routing
|
|
12
13
|
- **SQS Consumers**: Same pattern to process SQS messages (single or batch mode)
|
|
13
14
|
- **SNS Publishers**: Same pattern to publish messages to SNS topics
|
|
14
15
|
- **Fargate Tasks**: Same pattern to run tasks in Fargate containers
|
|
@@ -40,6 +41,9 @@ All available classes and functions:
|
|
|
40
41
|
| `FargateTask` | `aws_python_helper.fargate.task_base` | Base class for Fargate tasks |
|
|
41
42
|
| `FargateExecutor` | `aws_python_helper.fargate.executor` | Launches Fargate tasks from Lambda |
|
|
42
43
|
| `fargate_handler` | `aws_python_helper.fargate.handler` | Entry point handler for Fargate |
|
|
44
|
+
| `Repository` | `aws_python_helper.repository.base` | Base class for MongoDB repositories |
|
|
45
|
+
| `get_state` | `aws_python_helper` | Read the current constitution-state from async context |
|
|
46
|
+
| `set_state` | `aws_python_helper` | Set the current constitution-state in async context |
|
|
43
47
|
| `MongoJSONEncoder` | `aws_python_helper.utils.json_encoder` | JSON encoder for MongoDB types |
|
|
44
48
|
| `mongo_json_dumps` | `aws_python_helper.utils.json_encoder` | Helper to serialize MongoDB types |
|
|
45
49
|
| `serialize_mongo_types` | `aws_python_helper.utils.serializer` | Recursively serialize MongoDB types |
|
|
@@ -248,6 +252,7 @@ response = lambda_client.invoke(
|
|
|
248
252
|
FunctionName='GenerateRouteLambda',
|
|
249
253
|
InvocationType='RequestResponse',
|
|
250
254
|
Payload=json.dumps({
|
|
255
|
+
'constitution-state': 'connecticut', # Required
|
|
251
256
|
'data': {
|
|
252
257
|
'shipping_id': '507f1f77bcf86cd799439011'
|
|
253
258
|
}
|
|
@@ -271,6 +276,7 @@ lambda_client.invoke(
|
|
|
271
276
|
FunctionName='GenerateRouteLambda',
|
|
272
277
|
InvocationType='Event', # Asynchronous
|
|
273
278
|
Payload=json.dumps({
|
|
279
|
+
'constitution-state': 'connecticut', # Required
|
|
274
280
|
'data': {
|
|
275
281
|
'shipping_id': '507f1f77bcf86cd799439011'
|
|
276
282
|
}
|
|
@@ -414,6 +420,7 @@ from aws_python_helper.fargate.executor import FargateExecutor
|
|
|
414
420
|
|
|
415
421
|
def handler(event, context):
|
|
416
422
|
executor = FargateExecutor()
|
|
423
|
+
# constitution-state is auto-propagated as CONSTITUTION_STATE env var in the container
|
|
417
424
|
task_arn = executor.run_task(
|
|
418
425
|
'search-tax-by-town',
|
|
419
426
|
envs={'TOWN': 'Norwalk', 'ONLY_TAX': 'true'}
|
|
@@ -469,6 +476,219 @@ class AddressAPI(API):
|
|
|
469
476
|
|
|
470
477
|
`self.external_db` is available in `API`, `SQSConsumer`, `Lambda`, and `FargateTask`.
|
|
471
478
|
|
|
479
|
+
## 🗂️ Repository Pattern
|
|
480
|
+
|
|
481
|
+
The framework provides a `Repository` base class that eliminates repetitive boilerplate in data access layers. Each repository only declares what collection it uses, whether it belongs to an external cluster, and what indexes to create. The base class handles the MongoDB connection and index creation automatically.
|
|
482
|
+
|
|
483
|
+
### Properties to override
|
|
484
|
+
|
|
485
|
+
| Property | Type | Default | Required |
|
|
486
|
+
|----------|------|---------|----------|
|
|
487
|
+
| `collection_name` | `str` | — | **Yes** |
|
|
488
|
+
| `database_key` | `str \| None` | `None` | No — if `None`, uses `constitution-state` from context |
|
|
489
|
+
| `is_external` | `bool` | `False` | No |
|
|
490
|
+
| `cluster_name` | `str` | `None` | Only if `is_external=True` |
|
|
491
|
+
| `indexes` | `list` | `[]` | No |
|
|
492
|
+
|
|
493
|
+
**`database_key` controls how the database is resolved:**
|
|
494
|
+
- `database_key = "core"` (or any string) → always connects to that specific database.
|
|
495
|
+
- `database_key = None` (default) → reads `constitution-state` from the async context automatically. This makes the repository **state-scoped**: it connects to `"connecticut"`, `"new_jersey"`, etc. depending on the current request.
|
|
496
|
+
|
|
497
|
+
Collections are cached per `(database_name, collection_name)` key — state-scoped repositories correctly isolate state between concurrent requests.
|
|
498
|
+
|
|
499
|
+
### Index format
|
|
500
|
+
|
|
501
|
+
```python
|
|
502
|
+
@property
|
|
503
|
+
def indexes(self):
|
|
504
|
+
return [
|
|
505
|
+
{"key": [("field", 1)]}, # simple ASC
|
|
506
|
+
{"key": [("field", -1)]}, # simple DESC
|
|
507
|
+
{"key": [("f1", 1), ("f2", -1)], "unique": True}, # compound + unique
|
|
508
|
+
{"key": [("expires_at", 1)], "expireAfterSeconds": 0}, # TTL index
|
|
509
|
+
]
|
|
510
|
+
```
|
|
511
|
+
|
|
512
|
+
Indexes are created automatically in the background on first collection access — no need to call any initialization method.
|
|
513
|
+
|
|
514
|
+
### Repository with a fixed database
|
|
515
|
+
|
|
516
|
+
```python
|
|
517
|
+
from aws_python_helper import Repository
|
|
518
|
+
|
|
519
|
+
class TownsRepository(Repository):
|
|
520
|
+
|
|
521
|
+
@property
|
|
522
|
+
def collection_name(self):
|
|
523
|
+
return "towns"
|
|
524
|
+
|
|
525
|
+
@property
|
|
526
|
+
def database_key(self):
|
|
527
|
+
return "core" # always connects to the "core" database
|
|
528
|
+
|
|
529
|
+
@property
|
|
530
|
+
def indexes(self):
|
|
531
|
+
return [
|
|
532
|
+
{"key": [("name", 1)]},
|
|
533
|
+
{"key": [("platform", 1)]},
|
|
534
|
+
]
|
|
535
|
+
|
|
536
|
+
async def get_available(self, platforms):
|
|
537
|
+
return await self.collection.find(
|
|
538
|
+
{"platform": {"$in": platforms}},
|
|
539
|
+
{"name": 1, "platform": 1}
|
|
540
|
+
).to_list(length=None)
|
|
541
|
+
|
|
542
|
+
async def find_by_name(self, name):
|
|
543
|
+
return await self.collection.find_one({"name": name})
|
|
544
|
+
```
|
|
545
|
+
|
|
546
|
+
### State-scoped repository (no `database_key`)
|
|
547
|
+
|
|
548
|
+
When `database_key` is not set, the repository reads the current `constitution-state` from context and uses it as the database name. The same repository instance connects to `"connecticut"` for one request and to `"new_jersey"` for another — automatically.
|
|
549
|
+
|
|
550
|
+
```python
|
|
551
|
+
from aws_python_helper import Repository
|
|
552
|
+
|
|
553
|
+
class LandRecordsRepository(Repository):
|
|
554
|
+
|
|
555
|
+
@property
|
|
556
|
+
def collection_name(self):
|
|
557
|
+
return "records"
|
|
558
|
+
|
|
559
|
+
# No database_key → uses get_state() automatically
|
|
560
|
+
# If constitution-state = "connecticut" → connects to DB "connecticut"
|
|
561
|
+
# If constitution-state = "new_jersey" → connects to DB "new_jersey"
|
|
562
|
+
|
|
563
|
+
@property
|
|
564
|
+
def indexes(self):
|
|
565
|
+
return [
|
|
566
|
+
{"key": [("unique_id", 1)]},
|
|
567
|
+
{"key": [("owner", 1), ("town", 1)]},
|
|
568
|
+
]
|
|
569
|
+
|
|
570
|
+
async def bulk_upsert(self, records):
|
|
571
|
+
from pymongo import UpdateOne
|
|
572
|
+
operations = [
|
|
573
|
+
UpdateOne({"unique_id": r["unique_id"]}, {"$set": r}, upsert=True)
|
|
574
|
+
for r in records
|
|
575
|
+
]
|
|
576
|
+
result = await self.collection.bulk_write(operations)
|
|
577
|
+
return {"upserted": result.upserted_count, "modified": result.modified_count}
|
|
578
|
+
```
|
|
579
|
+
|
|
580
|
+
> **Note:** A `ValueError` is raised at runtime if `database_key` is `None` and `constitution-state` has not been set in the context. This is prevented automatically by the framework at every entry point (API, Lambda, SQS, Fargate).
|
|
581
|
+
|
|
582
|
+
### Repository on an external cluster
|
|
583
|
+
|
|
584
|
+
```python
|
|
585
|
+
from aws_python_helper import Repository
|
|
586
|
+
|
|
587
|
+
class AddressRepository(Repository):
|
|
588
|
+
|
|
589
|
+
@property
|
|
590
|
+
def database_key(self):
|
|
591
|
+
return "smart_data"
|
|
592
|
+
|
|
593
|
+
@property
|
|
594
|
+
def collection_name(self):
|
|
595
|
+
return "address"
|
|
596
|
+
|
|
597
|
+
@property
|
|
598
|
+
def is_external(self):
|
|
599
|
+
return True
|
|
600
|
+
|
|
601
|
+
@property
|
|
602
|
+
def cluster_name(self):
|
|
603
|
+
return "ClusterDockets" # Must match a name in EXTERNAL_MONGODB_CONNECTIONS
|
|
604
|
+
|
|
605
|
+
async def find_by_query(self, query, limit=None):
|
|
606
|
+
cursor = self.collection.find(query)
|
|
607
|
+
if limit:
|
|
608
|
+
cursor = cursor.limit(limit)
|
|
609
|
+
return await cursor.to_list(length=None)
|
|
610
|
+
```
|
|
611
|
+
|
|
612
|
+
### Instantiation — no `db` argument needed
|
|
613
|
+
|
|
614
|
+
```python
|
|
615
|
+
class MyAPI(API):
|
|
616
|
+
|
|
617
|
+
@property
|
|
618
|
+
def towns_repository(self):
|
|
619
|
+
if not self._towns_repository:
|
|
620
|
+
self._towns_repository = TownsRepository() # no args!
|
|
621
|
+
return self._towns_repository
|
|
622
|
+
|
|
623
|
+
async def process(self):
|
|
624
|
+
towns = await self.towns_repository.get_available(["platform_a", "platform_b"])
|
|
625
|
+
self.set_body({"towns": towns})
|
|
626
|
+
```
|
|
627
|
+
|
|
628
|
+
The repository connects itself using the already-initialized `MongoManager` singleton — the same one used by `self.db`. No need to pass `self.db` or any connection object.
|
|
629
|
+
|
|
630
|
+
## 🌐 Constitution State
|
|
631
|
+
|
|
632
|
+
The framework uses a `constitution-state` value to support **multi-state database routing** — connecting each request to the correct database based on the state it belongs to (e.g., `"connecticut"`, `"new_jersey"`). This value is propagated automatically across the entire async call chain using Python's `contextvars.ContextVar`, so you never need to pass it manually between layers.
|
|
633
|
+
|
|
634
|
+
### How the framework injects it at each entry point
|
|
635
|
+
|
|
636
|
+
| Entry point | How `constitution-state` is read |
|
|
637
|
+
|-------------|----------------------------------|
|
|
638
|
+
| **API Gateway** | HTTP header `constitution-state` — **required**, returns `400` if missing |
|
|
639
|
+
| **Standalone Lambda** | Field `constitution-state` in the event payload — **required**, raises `ValueError` if missing |
|
|
640
|
+
| **SQS Consumer (single mode)** | Per-record: reads from SNS `MessageAttributes['constitution-state']`, falls back to `body.constitution_state` |
|
|
641
|
+
| **SQS Consumer (batch mode)** | Groups records by state; calls `process_batch()` once per group with the correct state in context |
|
|
642
|
+
| **Fargate Task** | Env var `CONSTITUTION_STATE` — auto-injected by `FargateExecutor` |
|
|
643
|
+
|
|
644
|
+
### How the framework propagates it to downstream services
|
|
645
|
+
|
|
646
|
+
| Downstream service | Propagation mechanism |
|
|
647
|
+
|--------------------|-----------------------|
|
|
648
|
+
| **SNS Publisher** | Auto-injects `constitution-state` as a `MessageAttribute` on every published message |
|
|
649
|
+
| **FargateExecutor** | Auto-injects `CONSTITUTION_STATE` as an env var when launching Fargate containers |
|
|
650
|
+
|
|
651
|
+
This means that an API call with `constitution-state: connecticut` will automatically carry that state through SNS → SQS → Fargate without any code changes in your consumers or tasks.
|
|
652
|
+
|
|
653
|
+
### State-scoped repositories
|
|
654
|
+
|
|
655
|
+
Repositories with no `database_key` (default) read `constitution-state` from context to resolve the target database automatically. See the [Repository Pattern](#️-repository-pattern) section for details.
|
|
656
|
+
|
|
657
|
+
### Manual access
|
|
658
|
+
|
|
659
|
+
If you need to read or set the state manually (e.g., in tests or utility code):
|
|
660
|
+
|
|
661
|
+
```python
|
|
662
|
+
from aws_python_helper import get_state, set_state
|
|
663
|
+
|
|
664
|
+
state = get_state() # e.g. "connecticut", or None if not set
|
|
665
|
+
set_state("new_jersey") # set manually (the framework does this automatically)
|
|
666
|
+
```
|
|
667
|
+
|
|
668
|
+
### API example — `constitution-state` header
|
|
669
|
+
|
|
670
|
+
```
|
|
671
|
+
GET /constitutions HTTP/1.1
|
|
672
|
+
constitution-state: connecticut
|
|
673
|
+
Authorization: Bearer <token>
|
|
674
|
+
```
|
|
675
|
+
|
|
676
|
+
### Lambda invocation example — `constitution-state` in event
|
|
677
|
+
|
|
678
|
+
```python
|
|
679
|
+
import boto3, json
|
|
680
|
+
|
|
681
|
+
lambda_client = boto3.client('lambda')
|
|
682
|
+
lambda_client.invoke(
|
|
683
|
+
FunctionName='MyLambdaFunction',
|
|
684
|
+
InvocationType='RequestResponse',
|
|
685
|
+
Payload=json.dumps({
|
|
686
|
+
'constitution-state': 'connecticut', # Required
|
|
687
|
+
'data': {'key': 'value'}
|
|
688
|
+
})
|
|
689
|
+
)
|
|
690
|
+
```
|
|
691
|
+
|
|
472
692
|
## 🔄 Routing Convention
|
|
473
693
|
|
|
474
694
|
The framework uses convention over configuration for the routing:
|
|
@@ -860,6 +1080,7 @@ environment_variables = {
|
|
|
860
1080
|
| `AUTH_BYPASS_TOKEN` | Optional | Master token to bypass authentication |
|
|
861
1081
|
| `ECS_CLUSTER` | Fargate only | ECS cluster name for `FargateExecutor` |
|
|
862
1082
|
| `ECS_SUBNETS` | Fargate only | Comma-separated subnet IDs for Fargate tasks |
|
|
1083
|
+
| `CONSTITUTION_STATE` | Fargate only (auto) | State injected automatically by `FargateExecutor` — do not set manually |
|
|
863
1084
|
| `AWS_REGION` | Fargate/SNS/SQS | AWS region |
|
|
864
1085
|
| `AWS_ACCOUNT_ID` | SQS `get_queue_url` | AWS account ID |
|
|
865
1086
|
| `SERVICE_NAME` | SQS `get_queue_url` | Service name prefix for queue name |
|
|
@@ -870,7 +1091,11 @@ environment_variables = {
|
|
|
870
1091
|
|
|
871
1092
|
### SQS Consumer - Batch Mode
|
|
872
1093
|
|
|
873
|
-
By default, consumers process messages one by one (`"single"` mode). Use `"batch"` mode when you need to group or bulk-process messages
|
|
1094
|
+
By default, consumers process messages one by one (`"single"` mode). Use `"batch"` mode when you need to group or bulk-process messages.
|
|
1095
|
+
|
|
1096
|
+
**Constitution-state handling in SQS:**
|
|
1097
|
+
- **Single mode**: the framework extracts `constitution-state` from each record automatically (from SNS `MessageAttributes`, then from `body.constitution_state`) and sets it in context before calling `process_record()`. You do not need to extract it yourself.
|
|
1098
|
+
- **Batch mode**: the framework groups the incoming records by `constitution-state` and calls `process_batch()` once per group, with the correct state in context for each group. This ensures that state-scoped repositories resolve to the right database even when a batch contains records from different states.
|
|
874
1099
|
|
|
875
1100
|
```python
|
|
876
1101
|
from aws_python_helper.sqs.consumer_base import SQSConsumer
|
|
@@ -919,10 +1144,13 @@ class OrderConsumer(SQSConsumer):
|
|
|
919
1144
|
|
|
920
1145
|
### SNS Publisher - Batch Publishing
|
|
921
1146
|
|
|
1147
|
+
The `SNSPublisher` automatically injects the current `constitution-state` as a `MessageAttribute` on every published message. SQS consumers built with this framework will then extract it automatically, ensuring the state flows end-to-end through the SNS → SQS chain without any manual code.
|
|
1148
|
+
|
|
922
1149
|
```python
|
|
923
1150
|
topic = TitleIndexedTopic()
|
|
924
1151
|
|
|
925
1152
|
# Publish multiple messages in a single call
|
|
1153
|
+
# constitution-state is auto-injected as a MessageAttribute on each message
|
|
926
1154
|
await topic.publish([
|
|
927
1155
|
{'content': {'id': 'id1', 'title': 'Title 1'}, 'attributes': {'type': 'created'}},
|
|
928
1156
|
{'content': {'id': 'id2', 'title': 'Title 2'}, 'attributes': {'type': 'updated'}},
|
|
@@ -24,6 +24,12 @@ from .lambda_standalone.handler import lambda_handler
|
|
|
24
24
|
from .utils.json_encoder import MongoJSONEncoder, mongo_json_dumps
|
|
25
25
|
from .utils.serializer import serialize_mongo_types
|
|
26
26
|
|
|
27
|
+
# Repository
|
|
28
|
+
from .repository.base import Repository
|
|
29
|
+
|
|
30
|
+
# Context
|
|
31
|
+
from .context.state import get_state, set_state
|
|
32
|
+
|
|
27
33
|
|
|
28
34
|
__all__ = [
|
|
29
35
|
'API',
|
|
@@ -34,6 +40,7 @@ __all__ = [
|
|
|
34
40
|
'FargateTask',
|
|
35
41
|
'FargateExecutor',
|
|
36
42
|
'Lambda',
|
|
43
|
+
'Repository',
|
|
37
44
|
'fargate_handler',
|
|
38
45
|
'api_handler',
|
|
39
46
|
'sqs_handler',
|
|
@@ -41,5 +48,7 @@ __all__ = [
|
|
|
41
48
|
'MongoJSONEncoder',
|
|
42
49
|
'mongo_json_dumps',
|
|
43
50
|
'serialize_mongo_types',
|
|
51
|
+
'get_state',
|
|
52
|
+
'set_state',
|
|
44
53
|
]
|
|
45
54
|
|
|
@@ -12,6 +12,7 @@ from .base import API
|
|
|
12
12
|
from .exceptions import UnauthorizedError, ForbiddenError, AuthenticationError
|
|
13
13
|
from .auth_middleware import AuthMiddleware
|
|
14
14
|
from .auth_validators import TokenValidator
|
|
15
|
+
from ..context.state import set_state
|
|
15
16
|
|
|
16
17
|
logger = logging.getLogger(__name__)
|
|
17
18
|
|
|
@@ -78,7 +79,20 @@ class Dispatcher:
|
|
|
78
79
|
require_auth = os.getenv('REQUIRE_AUTH', 'false').lower() == 'true'
|
|
79
80
|
if require_auth:
|
|
80
81
|
await self._authenticate(api)
|
|
81
|
-
|
|
82
|
+
|
|
83
|
+
# 2.5 Setup constitution-state context
|
|
84
|
+
state = self.headers.get('constitution-state')
|
|
85
|
+
if not state:
|
|
86
|
+
return {
|
|
87
|
+
'code': 400,
|
|
88
|
+
'body': {
|
|
89
|
+
'error': 'Bad Request',
|
|
90
|
+
'message': "Header 'constitution-state' is required"
|
|
91
|
+
},
|
|
92
|
+
'headers': {}
|
|
93
|
+
}
|
|
94
|
+
set_state(state)
|
|
95
|
+
|
|
82
96
|
# 3. Validate
|
|
83
97
|
await api.validate()
|
|
84
98
|
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Constitution State Context - Manages request-scoped state for multi-state database routing.
|
|
3
|
+
|
|
4
|
+
Uses Python's contextvars to propagate the current constitution-state across async call chains.
|
|
5
|
+
Set automatically by the framework at every entry point (API, Lambda, SQS Consumer, Fargate Task).
|
|
6
|
+
Read automatically by state-scoped repositories (database_key = None) to resolve the target database.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from contextvars import ContextVar
|
|
10
|
+
from typing import Optional
|
|
11
|
+
|
|
12
|
+
_constitution_state: ContextVar[Optional[str]] = ContextVar('constitution_state', default=None)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def get_state() -> Optional[str]:
|
|
16
|
+
"""Get the current constitution state from async context."""
|
|
17
|
+
return _constitution_state.get()
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def set_state(state: str) -> None:
|
|
21
|
+
"""Set the current constitution state in async context."""
|
|
22
|
+
_constitution_state.set(state)
|