graphddb-runtime 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- graphddb_runtime-0.1.0/PKG-INFO +160 -0
- graphddb_runtime-0.1.0/README.md +149 -0
- graphddb_runtime-0.1.0/graphddb_runtime/__init__.py +58 -0
- graphddb_runtime-0.1.0/graphddb_runtime/async_runtime.py +110 -0
- graphddb_runtime-0.1.0/graphddb_runtime/batch.py +218 -0
- graphddb_runtime-0.1.0/graphddb_runtime/concurrency.py +87 -0
- graphddb_runtime-0.1.0/graphddb_runtime/cursor.py +49 -0
- graphddb_runtime-0.1.0/graphddb_runtime/errors.py +80 -0
- graphddb_runtime-0.1.0/graphddb_runtime/filters.py +194 -0
- graphddb_runtime-0.1.0/graphddb_runtime/hydration.py +75 -0
- graphddb_runtime-0.1.0/graphddb_runtime/limits.py +20 -0
- graphddb_runtime-0.1.0/graphddb_runtime/per_key_cursor.py +105 -0
- graphddb_runtime-0.1.0/graphddb_runtime/relations.py +199 -0
- graphddb_runtime-0.1.0/graphddb_runtime/runtime.py +1674 -0
- graphddb_runtime-0.1.0/graphddb_runtime/templates.py +131 -0
- graphddb_runtime-0.1.0/graphddb_runtime/transactions.py +440 -0
- graphddb_runtime-0.1.0/graphddb_runtime.egg-info/PKG-INFO +160 -0
- graphddb_runtime-0.1.0/graphddb_runtime.egg-info/SOURCES.txt +35 -0
- graphddb_runtime-0.1.0/graphddb_runtime.egg-info/dependency_links.txt +1 -0
- graphddb_runtime-0.1.0/graphddb_runtime.egg-info/requires.txt +4 -0
- graphddb_runtime-0.1.0/graphddb_runtime.egg-info/top_level.txt +1 -0
- graphddb_runtime-0.1.0/pyproject.toml +24 -0
- graphddb_runtime-0.1.0/setup.cfg +4 -0
- graphddb_runtime-0.1.0/tests/test_concurrency.py +371 -0
- graphddb_runtime-0.1.0/tests/test_contract_runtime.py +413 -0
- graphddb_runtime-0.1.0/tests/test_integration.py +453 -0
- graphddb_runtime-0.1.0/tests/test_integration_command.py +314 -0
- graphddb_runtime-0.1.0/tests/test_integration_compose.py +186 -0
- graphddb_runtime-0.1.0/tests/test_integration_contract.py +374 -0
- graphddb_runtime-0.1.0/tests/test_integration_edge_derive.py +238 -0
- graphddb_runtime-0.1.0/tests/test_integration_edge_write.py +234 -0
- graphddb_runtime-0.1.0/tests/test_integration_events.py +199 -0
- graphddb_runtime-0.1.0/tests/test_integration_referential.py +156 -0
- graphddb_runtime-0.1.0/tests/test_integration_relations.py +311 -0
- graphddb_runtime-0.1.0/tests/test_integration_unique.py +246 -0
- graphddb_runtime-0.1.0/tests/test_relations.py +476 -0
- graphddb_runtime-0.1.0/tests/test_unit.py +1007 -0
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: graphddb-runtime
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Thin DynamoDB executor for GraphDDB-generated Python repositories (single-operation core, issue #44).
|
|
5
|
+
License: MIT
|
|
6
|
+
Requires-Python: >=3.9
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
Requires-Dist: boto3>=1.26
|
|
9
|
+
Provides-Extra: test
|
|
10
|
+
Requires-Dist: pytest>=7.0; extra == "test"
|
|
11
|
+
|
|
12
|
+
# graphddb-runtime
|
|
13
|
+
|
|
14
|
+
Thin DynamoDB executor for [GraphDDB](https://www.npmjs.com/package/graphddb)-generated
|
|
15
|
+
Python repositories.
|
|
16
|
+
|
|
17
|
+
`graphddb-runtime` is the small, hand-written package that generated
|
|
18
|
+
`repositories.py` modules import as `from graphddb_runtime import GraphDDBRuntime`.
|
|
19
|
+
It interprets the `manifest.json` / `operations.json` specifications produced by
|
|
20
|
+
`graphddb generate python` and executes the validated access patterns against
|
|
21
|
+
DynamoDB through boto3 — no scans, no hand-written key logic.
|
|
22
|
+
|
|
23
|
+
## Features
|
|
24
|
+
|
|
25
|
+
- **Single-operation core** — `GetItem` / `Query` reads and
|
|
26
|
+
`PutItem` / `UpdateItem` / `DeleteItem` writes.
|
|
27
|
+
- **Relations & assembly** — relation traversal, multi-operation assembly,
|
|
28
|
+
`BatchGetItem`, result limits, and `explain`.
|
|
29
|
+
- **Conditional & transactional writes** — conditional writes, declarative
|
|
30
|
+
transactions (`execute_transaction`, with `forEach` / `when` expansion and
|
|
31
|
+
`TransactWriteItems` batching up to 25 items).
|
|
32
|
+
- **Async adapter** — `AsyncGraphDDBRuntime` exposes an `await`-able surface
|
|
33
|
+
with behavior identical to the synchronous runtime.
|
|
34
|
+
|
|
35
|
+
## Install
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
pip install graphddb-runtime
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
Requires Python 3.9+ and boto3.
|
|
42
|
+
|
|
43
|
+
> **Versioning.** `graphddb-runtime` tracks the `graphddb` npm package version:
|
|
44
|
+
> a given runtime release matches the `graphddb` CLI of the same version, so the
|
|
45
|
+
> generated `manifest.json` / `operations.json` and the runtime that interprets
|
|
46
|
+
> them always stay in sync. Install the `graphddb-runtime` whose version equals
|
|
47
|
+
> the `graphddb` CLI you generated with.
|
|
48
|
+
|
|
49
|
+
## Usage
|
|
50
|
+
|
|
51
|
+
Point the runtime at the two JSON specs emitted by `graphddb generate python`
|
|
52
|
+
and pass a boto3 DynamoDB client. The generated repositories wrap it with typed
|
|
53
|
+
methods:
|
|
54
|
+
|
|
55
|
+
```python
|
|
56
|
+
import boto3
|
|
57
|
+
from graphddb_runtime import GraphDDBRuntime
|
|
58
|
+
from generated import UserRepository
|
|
59
|
+
|
|
60
|
+
runtime = GraphDDBRuntime(
|
|
61
|
+
dynamodb_client=boto3.client("dynamodb"),
|
|
62
|
+
manifest_path="generated/manifest.json",
|
|
63
|
+
operations_path="generated/operations.json",
|
|
64
|
+
# Map logical table names to deployed physical names when they differ.
|
|
65
|
+
table_mapping={"UserPermissions": "UserPermissions-prod"},
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
users = UserRepository(runtime)
|
|
69
|
+
user = users.get_user_by_email(email="alice@example.com")
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
### Async
|
|
73
|
+
|
|
74
|
+
boto3 is a synchronous SDK, so the runtime core is synchronous.
|
|
75
|
+
`AsyncGraphDDBRuntime` is a thin adapter that runs each blocking call in a worker
|
|
76
|
+
thread via `asyncio.to_thread`, giving an `await`-able surface with identical
|
|
77
|
+
behavior (same params, specs, results, and error types). It does not require
|
|
78
|
+
`aioboto3`.
|
|
79
|
+
|
|
80
|
+
```python
|
|
81
|
+
import boto3
|
|
82
|
+
from graphddb_runtime import GraphDDBRuntime, AsyncGraphDDBRuntime
|
|
83
|
+
|
|
84
|
+
sync = GraphDDBRuntime(
|
|
85
|
+
dynamodb_client=boto3.client("dynamodb"),
|
|
86
|
+
manifest_path="generated/manifest.json",
|
|
87
|
+
operations_path="generated/operations.json",
|
|
88
|
+
)
|
|
89
|
+
runtime = AsyncGraphDDBRuntime(sync)
|
|
90
|
+
|
|
91
|
+
user = await runtime.execute_query("getUser", {"userId": "alice"})
|
|
92
|
+
await runtime.execute_transaction("addManyMembers", {"groupId": "eng", "members": [...]})
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
The wrapped synchronous runtime is available as `runtime.sync` for callers that
|
|
96
|
+
need the blocking API directly.
|
|
97
|
+
|
|
98
|
+
## AWS Lambda
|
|
99
|
+
|
|
100
|
+
The runtime loads the JSON specs from disk and constructs a boto3 client — both
|
|
101
|
+
are cold-start costs you want to pay **once**, in module scope, so they are
|
|
102
|
+
reused across warm invocations (and frozen by SnapStart).
|
|
103
|
+
|
|
104
|
+
```python
|
|
105
|
+
# handler.py — module scope runs once per execution environment (cold start).
|
|
106
|
+
import json
|
|
107
|
+
import boto3
|
|
108
|
+
from graphddb_runtime import GraphDDBRuntime
|
|
109
|
+
from generated import UserRepository
|
|
110
|
+
|
|
111
|
+
_runtime = GraphDDBRuntime(
|
|
112
|
+
dynamodb_client=boto3.client("dynamodb"),
|
|
113
|
+
manifest_path="generated/manifest.json",
|
|
114
|
+
operations_path="generated/operations.json",
|
|
115
|
+
table_mapping={"UserPermissions": "UserPermissions-prod"},
|
|
116
|
+
)
|
|
117
|
+
_users = UserRepository(_runtime)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def handler(event, context):
|
|
121
|
+
user = _users.get_user_by_email(email=event["queryStringParameters"]["email"])
|
|
122
|
+
if user is None:
|
|
123
|
+
return {"statusCode": 404, "body": "not found"}
|
|
124
|
+
return {"statusCode": 200, "body": json.dumps(user)}
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
### SnapStart
|
|
128
|
+
|
|
129
|
+
Lambda SnapStart snapshots the initialized execution environment after the
|
|
130
|
+
module-scope code runs, so the global client + `GraphDDBRuntime(...)` construction
|
|
131
|
+
is captured in the snapshot and skipped on restore.
|
|
132
|
+
|
|
133
|
+
- **Initialize the runtime and repositories in module scope** (as above), never
|
|
134
|
+
inside the handler — that is what gets snapshotted.
|
|
135
|
+
- **Do not cache short-lived state across the snapshot** (credentials/tokens with
|
|
136
|
+
an expiry, random seeds). The DynamoDB client and the loaded specs are safe to
|
|
137
|
+
snapshot; refresh anything time-sensitive inside the handler.
|
|
138
|
+
|
|
139
|
+
### Packaging
|
|
140
|
+
|
|
141
|
+
The deployment artifact needs three things: this runtime package, the generated
|
|
142
|
+
bindings, and the two JSON specs. boto3/botocore are provided by the Lambda Python
|
|
143
|
+
runtime, so they need not be vendored (pin them only if you require a specific
|
|
144
|
+
version).
|
|
145
|
+
|
|
146
|
+
```bash
|
|
147
|
+
mkdir -p build
|
|
148
|
+
pip install graphddb-runtime --target build # the runtime
|
|
149
|
+
cp -r generated build/generated # manifest.json, operations.json, *.py
|
|
150
|
+
cp handler.py build/
|
|
151
|
+
( cd build && zip -r ../function.zip . ) # handler = handler.handler
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
Make sure the `manifest_path` / `operations_path` you pass to `GraphDDBRuntime`
|
|
155
|
+
resolve relative to the deployed working directory (e.g. `generated/...` when the
|
|
156
|
+
specs are zipped under a `generated/` folder at the artifact root).
|
|
157
|
+
|
|
158
|
+
## License
|
|
159
|
+
|
|
160
|
+
MIT
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
# graphddb-runtime
|
|
2
|
+
|
|
3
|
+
Thin DynamoDB executor for [GraphDDB](https://www.npmjs.com/package/graphddb)-generated
|
|
4
|
+
Python repositories.
|
|
5
|
+
|
|
6
|
+
`graphddb-runtime` is the small, hand-written package that generated
|
|
7
|
+
`repositories.py` modules import as `from graphddb_runtime import GraphDDBRuntime`.
|
|
8
|
+
It interprets the `manifest.json` / `operations.json` specifications produced by
|
|
9
|
+
`graphddb generate python` and executes the validated access patterns against
|
|
10
|
+
DynamoDB through boto3 — no scans, no hand-written key logic.
|
|
11
|
+
|
|
12
|
+
## Features
|
|
13
|
+
|
|
14
|
+
- **Single-operation core** — `GetItem` / `Query` reads and
|
|
15
|
+
`PutItem` / `UpdateItem` / `DeleteItem` writes.
|
|
16
|
+
- **Relations & assembly** — relation traversal, multi-operation assembly,
|
|
17
|
+
`BatchGetItem`, result limits, and `explain`.
|
|
18
|
+
- **Conditional & transactional writes** — conditional writes, declarative
|
|
19
|
+
transactions (`execute_transaction`, with `forEach` / `when` expansion and
|
|
20
|
+
`TransactWriteItems` batching up to 25 items).
|
|
21
|
+
- **Async adapter** — `AsyncGraphDDBRuntime` exposes an `await`-able surface
|
|
22
|
+
with behavior identical to the synchronous runtime.
|
|
23
|
+
|
|
24
|
+
## Install
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
pip install graphddb-runtime
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
Requires Python 3.9+ and boto3.
|
|
31
|
+
|
|
32
|
+
> **Versioning.** `graphddb-runtime` tracks the `graphddb` npm package version:
|
|
33
|
+
> a given runtime release matches the `graphddb` CLI of the same version, so the
|
|
34
|
+
> generated `manifest.json` / `operations.json` and the runtime that interprets
|
|
35
|
+
> them always stay in sync. Install the `graphddb-runtime` whose version equals
|
|
36
|
+
> the `graphddb` CLI you generated with.
|
|
37
|
+
|
|
38
|
+
## Usage
|
|
39
|
+
|
|
40
|
+
Point the runtime at the two JSON specs emitted by `graphddb generate python`
|
|
41
|
+
and pass a boto3 DynamoDB client. The generated repositories wrap it with typed
|
|
42
|
+
methods:
|
|
43
|
+
|
|
44
|
+
```python
|
|
45
|
+
import boto3
|
|
46
|
+
from graphddb_runtime import GraphDDBRuntime
|
|
47
|
+
from generated import UserRepository
|
|
48
|
+
|
|
49
|
+
runtime = GraphDDBRuntime(
|
|
50
|
+
dynamodb_client=boto3.client("dynamodb"),
|
|
51
|
+
manifest_path="generated/manifest.json",
|
|
52
|
+
operations_path="generated/operations.json",
|
|
53
|
+
# Map logical table names to deployed physical names when they differ.
|
|
54
|
+
table_mapping={"UserPermissions": "UserPermissions-prod"},
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
users = UserRepository(runtime)
|
|
58
|
+
user = users.get_user_by_email(email="alice@example.com")
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
### Async
|
|
62
|
+
|
|
63
|
+
boto3 is a synchronous SDK, so the runtime core is synchronous.
|
|
64
|
+
`AsyncGraphDDBRuntime` is a thin adapter that runs each blocking call in a worker
|
|
65
|
+
thread via `asyncio.to_thread`, giving an `await`-able surface with identical
|
|
66
|
+
behavior (same params, specs, results, and error types). It does not require
|
|
67
|
+
`aioboto3`.
|
|
68
|
+
|
|
69
|
+
```python
|
|
70
|
+
import boto3
|
|
71
|
+
from graphddb_runtime import GraphDDBRuntime, AsyncGraphDDBRuntime
|
|
72
|
+
|
|
73
|
+
sync = GraphDDBRuntime(
|
|
74
|
+
dynamodb_client=boto3.client("dynamodb"),
|
|
75
|
+
manifest_path="generated/manifest.json",
|
|
76
|
+
operations_path="generated/operations.json",
|
|
77
|
+
)
|
|
78
|
+
runtime = AsyncGraphDDBRuntime(sync)
|
|
79
|
+
|
|
80
|
+
user = await runtime.execute_query("getUser", {"userId": "alice"})
|
|
81
|
+
await runtime.execute_transaction("addManyMembers", {"groupId": "eng", "members": [...]})
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
The wrapped synchronous runtime is available as `runtime.sync` for callers that
|
|
85
|
+
need the blocking API directly.
|
|
86
|
+
|
|
87
|
+
## AWS Lambda
|
|
88
|
+
|
|
89
|
+
The runtime loads the JSON specs from disk and constructs a boto3 client — both
|
|
90
|
+
are cold-start costs you want to pay **once**, in module scope, so they are
|
|
91
|
+
reused across warm invocations (and frozen by SnapStart).
|
|
92
|
+
|
|
93
|
+
```python
|
|
94
|
+
# handler.py — module scope runs once per execution environment (cold start).
|
|
95
|
+
import json
|
|
96
|
+
import boto3
|
|
97
|
+
from graphddb_runtime import GraphDDBRuntime
|
|
98
|
+
from generated import UserRepository
|
|
99
|
+
|
|
100
|
+
_runtime = GraphDDBRuntime(
|
|
101
|
+
dynamodb_client=boto3.client("dynamodb"),
|
|
102
|
+
manifest_path="generated/manifest.json",
|
|
103
|
+
operations_path="generated/operations.json",
|
|
104
|
+
table_mapping={"UserPermissions": "UserPermissions-prod"},
|
|
105
|
+
)
|
|
106
|
+
_users = UserRepository(_runtime)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def handler(event, context):
|
|
110
|
+
user = _users.get_user_by_email(email=event["queryStringParameters"]["email"])
|
|
111
|
+
if user is None:
|
|
112
|
+
return {"statusCode": 404, "body": "not found"}
|
|
113
|
+
return {"statusCode": 200, "body": json.dumps(user)}
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
### SnapStart
|
|
117
|
+
|
|
118
|
+
Lambda SnapStart snapshots the initialized execution environment after the
|
|
119
|
+
module-scope code runs, so the global client + `GraphDDBRuntime(...)` construction
|
|
120
|
+
is captured in the snapshot and skipped on restore.
|
|
121
|
+
|
|
122
|
+
- **Initialize the runtime and repositories in module scope** (as above), never
|
|
123
|
+
inside the handler — that is what gets snapshotted.
|
|
124
|
+
- **Do not cache short-lived state across the snapshot** (credentials/tokens with
|
|
125
|
+
an expiry, random seeds). The DynamoDB client and the loaded specs are safe to
|
|
126
|
+
snapshot; refresh anything time-sensitive inside the handler.
|
|
127
|
+
|
|
128
|
+
### Packaging
|
|
129
|
+
|
|
130
|
+
The deployment artifact needs three things: this runtime package, the generated
|
|
131
|
+
bindings, and the two JSON specs. boto3/botocore are provided by the Lambda Python
|
|
132
|
+
runtime, so they need not be vendored (pin them only if you require a specific
|
|
133
|
+
version).
|
|
134
|
+
|
|
135
|
+
```bash
|
|
136
|
+
mkdir -p build
|
|
137
|
+
pip install graphddb-runtime --target build # the runtime
|
|
138
|
+
cp -r generated build/generated # manifest.json, operations.json, *.py
|
|
139
|
+
cp handler.py build/
|
|
140
|
+
( cd build && zip -r ../function.zip . ) # handler = handler.handler
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
Make sure the `manifest_path` / `operations_path` you pass to `GraphDDBRuntime`
|
|
144
|
+
resolve relative to the deployed working directory (e.g. `generated/...` when the
|
|
145
|
+
specs are zipped under a `generated/` folder at the artifact root).
|
|
146
|
+
|
|
147
|
+
## License
|
|
148
|
+
|
|
149
|
+
MIT
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"""GraphDDB Python runtime (issue #44, single-operation core).
|
|
2
|
+
|
|
3
|
+
Public surface consumed by generated ``repositories.py`` and applications:
|
|
4
|
+
|
|
5
|
+
- :class:`GraphDDBRuntime` — the executor.
|
|
6
|
+
- :class:`RuntimeLimits` — execution-time bounds.
|
|
7
|
+
- the :class:`GraphDDBError` family.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from .cursor import decode_cursor, encode_cursor
|
|
13
|
+
from .errors import (
|
|
14
|
+
CommandNotFoundError,
|
|
15
|
+
ContractArityError,
|
|
16
|
+
ContractNotFoundError,
|
|
17
|
+
GraphDDBError,
|
|
18
|
+
HydrationError,
|
|
19
|
+
LimitExceededError,
|
|
20
|
+
MultiOperationNotSupportedError,
|
|
21
|
+
OperationExecutionError,
|
|
22
|
+
ParameterValidationError,
|
|
23
|
+
QueryNotFoundError,
|
|
24
|
+
TransactionNotFoundError,
|
|
25
|
+
)
|
|
26
|
+
from .async_runtime import AsyncGraphDDBRuntime
|
|
27
|
+
from .concurrency import RELATION_TRAVERSAL_CONCURRENCY, map_with_concurrency
|
|
28
|
+
from .limits import RuntimeLimits
|
|
29
|
+
from .per_key_cursor import (
|
|
30
|
+
decode_per_key_cursor,
|
|
31
|
+
encode_per_key_cursor,
|
|
32
|
+
serialize_contract_key,
|
|
33
|
+
)
|
|
34
|
+
from .runtime import GraphDDBRuntime
|
|
35
|
+
|
|
36
|
+
__all__ = [
|
|
37
|
+
"GraphDDBRuntime",
|
|
38
|
+
"AsyncGraphDDBRuntime",
|
|
39
|
+
"RuntimeLimits",
|
|
40
|
+
"RELATION_TRAVERSAL_CONCURRENCY",
|
|
41
|
+
"map_with_concurrency",
|
|
42
|
+
"GraphDDBError",
|
|
43
|
+
"QueryNotFoundError",
|
|
44
|
+
"CommandNotFoundError",
|
|
45
|
+
"TransactionNotFoundError",
|
|
46
|
+
"ContractNotFoundError",
|
|
47
|
+
"ContractArityError",
|
|
48
|
+
"ParameterValidationError",
|
|
49
|
+
"LimitExceededError",
|
|
50
|
+
"OperationExecutionError",
|
|
51
|
+
"HydrationError",
|
|
52
|
+
"MultiOperationNotSupportedError",
|
|
53
|
+
"encode_cursor",
|
|
54
|
+
"decode_cursor",
|
|
55
|
+
"serialize_contract_key",
|
|
56
|
+
"encode_per_key_cursor",
|
|
57
|
+
"decode_per_key_cursor",
|
|
58
|
+
]
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
"""Async adapter over the sync :class:`GraphDDBRuntime` (issue #46, Phase 4).
|
|
2
|
+
|
|
3
|
+
boto3 is a synchronous SDK, so the runtime core is synchronous. This module
|
|
4
|
+
provides a **thin** async wrapper that runs each blocking call in a worker
|
|
5
|
+
thread via :func:`asyncio.to_thread`, giving an ``await``-able surface with
|
|
6
|
+
**behavior equivalent** to the sync runtime (same params, same specs, same
|
|
7
|
+
results, same error types) without duplicating the executor.
|
|
8
|
+
|
|
9
|
+
It does not require ``aioboto3``; the wrapped sync runtime keeps using boto3.
|
|
10
|
+
For a project that wants a fully non-blocking DynamoDB client, ``aioboto3`` can
|
|
11
|
+
be adopted later behind the same interface — but for the common Lambda / request
|
|
12
|
+
handler case, ``asyncio.to_thread`` is sufficient and keeps a single executor
|
|
13
|
+
implementation (no TS/Python or sync/async divergence to test twice).
|
|
14
|
+
|
|
15
|
+
Usage::
|
|
16
|
+
|
|
17
|
+
import boto3
|
|
18
|
+
from graphddb_runtime import GraphDDBRuntime, AsyncGraphDDBRuntime
|
|
19
|
+
|
|
20
|
+
sync = GraphDDBRuntime(boto3.client("dynamodb"), manifest, operations)
|
|
21
|
+
runtime = AsyncGraphDDBRuntime(sync)
|
|
22
|
+
|
|
23
|
+
user = await runtime.execute_query("getUser", {"userId": "alice"})
|
|
24
|
+
await runtime.execute_transaction("addManyMembers", {...})
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
from __future__ import annotations
|
|
28
|
+
|
|
29
|
+
import asyncio
|
|
30
|
+
from typing import Any, Mapping, Optional
|
|
31
|
+
|
|
32
|
+
from .runtime import GraphDDBRuntime
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class AsyncGraphDDBRuntime:
|
|
36
|
+
"""An ``await``-able adapter delegating to a sync :class:`GraphDDBRuntime`.
|
|
37
|
+
|
|
38
|
+
Every method runs its synchronous counterpart in a thread, so a slow boto3
|
|
39
|
+
round trip does not block the event loop. The wrapped runtime is exposed as
|
|
40
|
+
:attr:`sync` for callers that need the blocking API directly.
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
def __init__(self, runtime: GraphDDBRuntime) -> None:
|
|
44
|
+
self.sync = runtime
|
|
45
|
+
|
|
46
|
+
async def execute_query(
|
|
47
|
+
self,
|
|
48
|
+
query_id: str,
|
|
49
|
+
params: Mapping[str, Any],
|
|
50
|
+
options: Optional[Mapping[str, Any]] = None,
|
|
51
|
+
) -> Optional[dict]:
|
|
52
|
+
return await asyncio.to_thread(
|
|
53
|
+
self.sync.execute_query, query_id, params, options
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
async def execute_query_method(
|
|
57
|
+
self,
|
|
58
|
+
contract_name: str,
|
|
59
|
+
method_name: str,
|
|
60
|
+
key_or_keys: Any,
|
|
61
|
+
params: Optional[Mapping[str, Any]] = None,
|
|
62
|
+
) -> Any:
|
|
63
|
+
"""Async wrapper over :meth:`GraphDDBRuntime.execute_query_method` (#62)."""
|
|
64
|
+
return await asyncio.to_thread(
|
|
65
|
+
self.sync.execute_query_method,
|
|
66
|
+
contract_name,
|
|
67
|
+
method_name,
|
|
68
|
+
key_or_keys,
|
|
69
|
+
params,
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
async def execute_command(
|
|
73
|
+
self,
|
|
74
|
+
command_id: str,
|
|
75
|
+
params: Mapping[str, Any],
|
|
76
|
+
options: Optional[Mapping[str, Any]] = None,
|
|
77
|
+
) -> None:
|
|
78
|
+
await asyncio.to_thread(
|
|
79
|
+
self.sync.execute_command, command_id, params, options
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
async def execute_command_method(
|
|
83
|
+
self,
|
|
84
|
+
contract_name: str,
|
|
85
|
+
method_name: str,
|
|
86
|
+
key_or_keys: Any,
|
|
87
|
+
params: Optional[Mapping[str, Any]] = None,
|
|
88
|
+
) -> None:
|
|
89
|
+
"""Async wrapper over :meth:`GraphDDBRuntime.execute_command_method` (#64)."""
|
|
90
|
+
await asyncio.to_thread(
|
|
91
|
+
self.sync.execute_command_method,
|
|
92
|
+
contract_name,
|
|
93
|
+
method_name,
|
|
94
|
+
key_or_keys,
|
|
95
|
+
params,
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
async def execute_transaction(
|
|
99
|
+
self,
|
|
100
|
+
transaction_id: str,
|
|
101
|
+
params: Mapping[str, Any],
|
|
102
|
+
) -> None:
|
|
103
|
+
await asyncio.to_thread(
|
|
104
|
+
self.sync.execute_transaction, transaction_id, params
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
async def explain(self, query_id: str, params: Mapping[str, Any]) -> dict:
|
|
108
|
+
# `explain` does not touch DynamoDB, but keeping it on the async surface
|
|
109
|
+
# lets call sites use one runtime object uniformly.
|
|
110
|
+
return await asyncio.to_thread(self.sync.explain, query_id, params)
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
"""BatchGetItem chunking + UnprocessedKeys retry (issue #45).
|
|
2
|
+
|
|
3
|
+
Port of the TypeScript batch semantics (``src/operations/batch-retry.ts`` /
|
|
4
|
+
``src/executor/batch-executor.ts``):
|
|
5
|
+
|
|
6
|
+
- keys are **deduped** before the request (``planBatchGetForQueryKeys`` /
|
|
7
|
+
``dedupeDynamoKeys``);
|
|
8
|
+
- split into chunks of at most :data:`BATCH_GET_MAX_KEYS` (100) keys;
|
|
9
|
+
- each chunk retries ``UnprocessedKeys`` with exponential backoff
|
|
10
|
+
(``50 * 2^(attempt-1)`` ms, capped at 1000ms) up to
|
|
11
|
+
:data:`BATCH_MAX_RETRY_ATTEMPTS` (10) attempts, then raises.
|
|
12
|
+
|
|
13
|
+
The sleep is injected so unit tests can observe the backoff schedule without
|
|
14
|
+
real delays.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import time
|
|
20
|
+
from typing import Any, Callable, Dict, List, Mapping, Optional
|
|
21
|
+
|
|
22
|
+
# Mirrors src/operations/batch-retry.ts.
|
|
23
|
+
BATCH_GET_MAX_KEYS = 100
|
|
24
|
+
BATCH_WRITE_MAX_ITEMS = 25
|
|
25
|
+
BATCH_MAX_RETRY_ATTEMPTS = 10
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def compute_backoff_delay(attempt: int) -> float:
|
|
29
|
+
"""Backoff for a 1-based attempt: ``50 * 2^(attempt-1)`` ms, capped at 1000ms."""
|
|
30
|
+
return min(1000.0, 50.0 * (2 ** (attempt - 1))) / 1000.0
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def chunk(items: List[Any], size: int) -> List[List[Any]]:
|
|
34
|
+
return [items[i : i + size] for i in range(0, len(items), size)]
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def serialize_key(key: Mapping[str, Any]) -> str:
|
|
38
|
+
"""Stable string form of a (plain) key for dedup / parent matching.
|
|
39
|
+
|
|
40
|
+
Mirrors the TS ``dedupeDynamoKeys`` / ``serializeQueryKey`` ordering
|
|
41
|
+
(attributes sorted by name).
|
|
42
|
+
"""
|
|
43
|
+
import json
|
|
44
|
+
|
|
45
|
+
return json.dumps(sorted(key.items()), separators=(",", ":"), default=str)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class BatchGetExecutor:
|
|
49
|
+
"""Executes a deduped, chunked, retrying BatchGetItem against a boto3 client.
|
|
50
|
+
|
|
51
|
+
The client is the low-level ``boto3.client("dynamodb")`` (AttributeValue
|
|
52
|
+
shapes). ``request_extra`` carries the optional ``ProjectionExpression`` /
|
|
53
|
+
``ExpressionAttributeNames``. Returns the flat list of raw (AttributeValue)
|
|
54
|
+
items across all chunks/retries.
|
|
55
|
+
"""
|
|
56
|
+
|
|
57
|
+
def __init__(
|
|
58
|
+
self,
|
|
59
|
+
client: Any,
|
|
60
|
+
*,
|
|
61
|
+
boto_errors: tuple = (),
|
|
62
|
+
on_request: Optional[Callable[[int], None]] = None,
|
|
63
|
+
sleep: Callable[[float], None] = time.sleep,
|
|
64
|
+
max_batch_get_items: int = BATCH_GET_MAX_KEYS,
|
|
65
|
+
) -> None:
|
|
66
|
+
self._client = client
|
|
67
|
+
self._boto_errors = boto_errors
|
|
68
|
+
self._on_request = on_request
|
|
69
|
+
self._sleep = sleep
|
|
70
|
+
self._max_batch_get_items = max_batch_get_items
|
|
71
|
+
|
|
72
|
+
def get(
|
|
73
|
+
self,
|
|
74
|
+
physical_table: str,
|
|
75
|
+
serialized_keys: List[Dict[str, Any]],
|
|
76
|
+
request_extra: Optional[Dict[str, Any]] = None,
|
|
77
|
+
) -> List[Dict[str, Any]]:
|
|
78
|
+
if not serialized_keys:
|
|
79
|
+
return []
|
|
80
|
+
|
|
81
|
+
items: List[Dict[str, Any]] = []
|
|
82
|
+
for batch in chunk(serialized_keys, self._max_batch_get_items):
|
|
83
|
+
items.extend(
|
|
84
|
+
self._get_chunk(physical_table, batch, request_extra or {})
|
|
85
|
+
)
|
|
86
|
+
return items
|
|
87
|
+
|
|
88
|
+
def _get_chunk(
|
|
89
|
+
self,
|
|
90
|
+
physical_table: str,
|
|
91
|
+
keys: List[Dict[str, Any]],
|
|
92
|
+
request_extra: Dict[str, Any],
|
|
93
|
+
) -> List[Dict[str, Any]]:
|
|
94
|
+
pending = keys
|
|
95
|
+
attempt = 0
|
|
96
|
+
out: List[Dict[str, Any]] = []
|
|
97
|
+
|
|
98
|
+
while pending:
|
|
99
|
+
table_request = {"Keys": pending, **request_extra}
|
|
100
|
+
if self._on_request is not None:
|
|
101
|
+
self._on_request(len(pending))
|
|
102
|
+
try:
|
|
103
|
+
resp = self._client.batch_get_item(
|
|
104
|
+
RequestItems={physical_table: table_request}
|
|
105
|
+
)
|
|
106
|
+
except self._boto_errors as exc: # type: ignore[misc]
|
|
107
|
+
from .errors import OperationExecutionError
|
|
108
|
+
|
|
109
|
+
raise OperationExecutionError(
|
|
110
|
+
f"BatchGetItem failed for table {physical_table!r}: {exc}",
|
|
111
|
+
original=exc,
|
|
112
|
+
) from exc
|
|
113
|
+
|
|
114
|
+
out.extend(resp.get("Responses", {}).get(physical_table, []))
|
|
115
|
+
|
|
116
|
+
unprocessed = (
|
|
117
|
+
resp.get("UnprocessedKeys", {})
|
|
118
|
+
.get(physical_table, {})
|
|
119
|
+
.get("Keys", [])
|
|
120
|
+
)
|
|
121
|
+
if not unprocessed:
|
|
122
|
+
break
|
|
123
|
+
|
|
124
|
+
if attempt >= BATCH_MAX_RETRY_ATTEMPTS:
|
|
125
|
+
from .errors import OperationExecutionError
|
|
126
|
+
|
|
127
|
+
raise OperationExecutionError(
|
|
128
|
+
f"BatchGet exceeded the maximum of {BATCH_MAX_RETRY_ATTEMPTS} "
|
|
129
|
+
f"retry attempts with {len(unprocessed)} key(s) still "
|
|
130
|
+
f"unprocessed for table {physical_table!r} (likely sustained "
|
|
131
|
+
f"throttling)."
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
pending = unprocessed
|
|
135
|
+
attempt += 1
|
|
136
|
+
self._sleep(compute_backoff_delay(attempt))
|
|
137
|
+
|
|
138
|
+
return out
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
class BatchWriteExecutor:
|
|
142
|
+
"""Executes a chunked, retrying ``BatchWriteItem`` against a boto3 client.
|
|
143
|
+
|
|
144
|
+
Port of the TypeScript batch-write semantics
|
|
145
|
+
(``src/operations/batch-retry.ts`` ``batchWriteChunkWithRetry`` +
|
|
146
|
+
``src/operations/batch.ts`` ``executeBatchWrite``): the per-table write
|
|
147
|
+
requests (``{"PutRequest": {"Item": …}}`` / ``{"DeleteRequest": {"Key": …}}``,
|
|
148
|
+
AttributeValue shapes) are split into chunks of at most
|
|
149
|
+
:data:`BATCH_WRITE_MAX_ITEMS` (25); each chunk retries ``UnprocessedItems``
|
|
150
|
+
with the same exponential backoff as :class:`BatchGetExecutor` up to
|
|
151
|
+
:data:`BATCH_MAX_RETRY_ATTEMPTS` attempts, then raises.
|
|
152
|
+
|
|
153
|
+
``BatchWriteItem`` carries **no conditions** (DynamoDB has no per-request
|
|
154
|
+
``ConditionExpression`` for it) and is **not atomic** — both are properties of
|
|
155
|
+
the command-contract ``'batchWrite'`` mode (issue #64). The sleep is injected
|
|
156
|
+
so unit tests can observe the backoff schedule without real delays.
|
|
157
|
+
"""
|
|
158
|
+
|
|
159
|
+
def __init__(
|
|
160
|
+
self,
|
|
161
|
+
client: Any,
|
|
162
|
+
*,
|
|
163
|
+
boto_errors: tuple = (),
|
|
164
|
+
on_request: Optional[Callable[[int], None]] = None,
|
|
165
|
+
sleep: Callable[[float], None] = time.sleep,
|
|
166
|
+
max_batch_write_items: int = BATCH_WRITE_MAX_ITEMS,
|
|
167
|
+
) -> None:
|
|
168
|
+
self._client = client
|
|
169
|
+
self._boto_errors = boto_errors
|
|
170
|
+
self._on_request = on_request
|
|
171
|
+
self._sleep = sleep
|
|
172
|
+
self._max_batch_write_items = max_batch_write_items
|
|
173
|
+
|
|
174
|
+
def write(self, physical_table: str, requests: List[Dict[str, Any]]) -> None:
|
|
175
|
+
"""Apply every write request to ``physical_table``, chunked + retried."""
|
|
176
|
+
if not requests:
|
|
177
|
+
return
|
|
178
|
+
for batch in chunk(requests, self._max_batch_write_items):
|
|
179
|
+
self._write_chunk(physical_table, batch)
|
|
180
|
+
|
|
181
|
+
def _write_chunk(
|
|
182
|
+
self, physical_table: str, requests: List[Dict[str, Any]]
|
|
183
|
+
) -> None:
|
|
184
|
+
pending = requests
|
|
185
|
+
attempt = 0
|
|
186
|
+
|
|
187
|
+
while pending:
|
|
188
|
+
if self._on_request is not None:
|
|
189
|
+
self._on_request(len(pending))
|
|
190
|
+
try:
|
|
191
|
+
resp = self._client.batch_write_item(
|
|
192
|
+
RequestItems={physical_table: pending}
|
|
193
|
+
)
|
|
194
|
+
except self._boto_errors as exc: # type: ignore[misc]
|
|
195
|
+
from .errors import OperationExecutionError
|
|
196
|
+
|
|
197
|
+
raise OperationExecutionError(
|
|
198
|
+
f"BatchWriteItem failed for table {physical_table!r}: {exc}",
|
|
199
|
+
original=exc,
|
|
200
|
+
) from exc
|
|
201
|
+
|
|
202
|
+
unprocessed = resp.get("UnprocessedItems", {}).get(physical_table, [])
|
|
203
|
+
if not unprocessed:
|
|
204
|
+
break
|
|
205
|
+
|
|
206
|
+
if attempt >= BATCH_MAX_RETRY_ATTEMPTS:
|
|
207
|
+
from .errors import OperationExecutionError
|
|
208
|
+
|
|
209
|
+
raise OperationExecutionError(
|
|
210
|
+
f"BatchWrite exceeded the maximum of {BATCH_MAX_RETRY_ATTEMPTS} "
|
|
211
|
+
f"retry attempts with {len(unprocessed)} item(s) still "
|
|
212
|
+
f"unprocessed for table {physical_table!r} (likely sustained "
|
|
213
|
+
f"throttling)."
|
|
214
|
+
)
|
|
215
|
+
|
|
216
|
+
pending = unprocessed
|
|
217
|
+
attempt += 1
|
|
218
|
+
self._sleep(compute_backoff_delay(attempt))
|