dex-connector-amazon-s3 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dex_connector_amazon_s3-0.1.0/.gitignore +16 -0
- dex_connector_amazon_s3-0.1.0/LICENSE +21 -0
- dex_connector_amazon_s3-0.1.0/PKG-INFO +19 -0
- dex_connector_amazon_s3-0.1.0/README.md +7 -0
- dex_connector_amazon_s3-0.1.0/examples/report_archive/README.md +7 -0
- dex_connector_amazon_s3-0.1.0/examples/report_archive/__init__.py +3 -0
- dex_connector_amazon_s3-0.1.0/examples/report_archive/flow.py +211 -0
- dex_connector_amazon_s3-0.1.0/examples/report_archive/input.example.json +1 -0
- dex_connector_amazon_s3-0.1.0/examples/report_archive/integration_test.py +199 -0
- dex_connector_amazon_s3-0.1.0/examples/report_archive/main.py +27 -0
- dex_connector_amazon_s3-0.1.0/examples/report_archive/start.py +59 -0
- dex_connector_amazon_s3-0.1.0/examples/report_archive/worker.py +26 -0
- dex_connector_amazon_s3-0.1.0/pyproject.toml +38 -0
- dex_connector_amazon_s3-0.1.0/src/dex_connectors/amazon_s3/__init__.py +46 -0
- dex_connector_amazon_s3-0.1.0/src/dex_connectors/amazon_s3/_generated/__init__.py +2 -0
- dex_connector_amazon_s3-0.1.0/src/dex_connectors/amazon_s3/_generated/config.py +32 -0
- dex_connector_amazon_s3-0.1.0/src/dex_connectors/amazon_s3/_generated/connection.py +184 -0
- dex_connector_amazon_s3-0.1.0/src/dex_connectors/amazon_s3/_generated/definitions.py +97 -0
- dex_connector_amazon_s3-0.1.0/src/dex_connectors/amazon_s3/_generated/mock.py +275 -0
- dex_connector_amazon_s3-0.1.0/src/dex_connectors/amazon_s3/_generated/models.py +207 -0
- dex_connector_amazon_s3-0.1.0/src/dex_connectors/amazon_s3/_generated/steps.py +246 -0
- dex_connector_amazon_s3-0.1.0/src/dex_connectors/amazon_s3/client.py +312 -0
- dex_connector_amazon_s3-0.1.0/src/dex_connectors/amazon_s3/connection.py +12 -0
- dex_connector_amazon_s3-0.1.0/src/dex_connectors/amazon_s3/dex-connector.json +168 -0
- dex_connector_amazon_s3-0.1.0/src/dex_connectors/amazon_s3/models.py +27 -0
- dex_connector_amazon_s3-0.1.0/src/dex_connectors/amazon_s3/operations.py +738 -0
- dex_connector_amazon_s3-0.1.0/src/dex_connectors/amazon_s3/py.typed +0 -0
- dex_connector_amazon_s3-0.1.0/src/dex_connectors/amazon_s3/s3mock/__init__.py +6 -0
- dex_connector_amazon_s3-0.1.0/src/dex_connectors/amazon_s3/steps.py +12 -0
- dex_connector_amazon_s3-0.1.0/tests/test_client.py +139 -0
- dex_connector_amazon_s3-0.1.0/tests/test_conformance.py +46 -0
- dex_connector_amazon_s3-0.1.0/tests/test_operations.py +115 -0
- dex_connector_amazon_s3-0.1.0/uv.lock +932 -0
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
.DS_Store
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
node_modules/
|
|
5
|
+
dist/
|
|
6
|
+
sdkpython/dist/dex_connectors_sdk-0.1.0-py3-none-any.whl
|
|
7
|
+
/.bin/dexcli
|
|
8
|
+
/.bin/dex-server-zd
|
|
9
|
+
/sdkpython/.dexcli-data/
|
|
10
|
+
site/.catalog.yaml
|
|
11
|
+
coverage/
|
|
12
|
+
*.out
|
|
13
|
+
*.test
|
|
14
|
+
go.work
|
|
15
|
+
go.work.sum
|
|
16
|
+
.idea/
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Super Durable
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: dex-connector-amazon-s3
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Amazon S3 connector for Dex Flows
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
License-File: LICENSE
|
|
7
|
+
Requires-Python: >=3.11
|
|
8
|
+
Requires-Dist: dex-connectors-sdk==0.1.0
|
|
9
|
+
Requires-Dist: dex-python-sdk>=1.28.0
|
|
10
|
+
Requires-Dist: httpx>=0.27
|
|
11
|
+
Description-Content-Type: text/markdown
|
|
12
|
+
|
|
13
|
+
# Amazon S3 Python connector
|
|
14
|
+
|
|
15
|
+
`dex-connector-amazon-s3` provides signed `listObjects`, `headObject`, `getObjectText`, and retry-safe `putObject` operations for Dex Flows. The generated Step factories and project connection bind the static `amazon-s3-report-archive` connection. See [the connector authorization guide](../README.md#authentication) for IAM, R2, and MinIO setup.
|
|
16
|
+
|
|
17
|
+
The [report archive example](examples/report_archive/README.md) uses all four operations. The Go-recorded conformance scenarios cover provider branches, retries, create-only read-back, and transport loss. Live Amazon S3, R2, and MinIO authorization and object behavior have not been verified for this Python package.
|
|
18
|
+
|
|
19
|
+
Start `dexcli dev` on unused local ports and set `DEX_FLOW_SERVICE_ADDRESS` to its Dex Server address. From `connectors/amazon/s3/python`, run `uv run python -m examples.report_archive.main`. In another terminal in that directory, run `uv run python -m examples.report_archive.start <unique-flow-id> examples/report_archive/input.example.json` to start a report.
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
# Amazon S3 Python connector
|
|
2
|
+
|
|
3
|
+
`dex-connector-amazon-s3` provides signed `listObjects`, `headObject`, `getObjectText`, and retry-safe `putObject` operations for Dex Flows. The generated Step factories and project connection bind the static `amazon-s3-report-archive` connection. See [the connector authorization guide](../README.md#authentication) for IAM, R2, and MinIO setup.
|
|
4
|
+
|
|
5
|
+
The [report archive example](examples/report_archive/README.md) uses all four operations. The Go-recorded conformance scenarios cover provider branches, retries, create-only read-back, and transport loss. Live Amazon S3, R2, and MinIO authorization and object behavior have not been verified for this Python package.
|
|
6
|
+
|
|
7
|
+
Start `dexcli dev` on unused local ports and set `DEX_FLOW_SERVICE_ADDRESS` to its Dex Server address. From `connectors/amazon/s3/python`, run `uv run python -m examples.report_archive.main`. In another terminal in that directory, run `uv run python -m examples.report_archive.start <unique-flow-id> examples/report_archive/input.example.json` to start a report.
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
# Amazon S3 report archive
|
|
2
|
+
|
|
3
|
+
This runnable Flow writes a UTF-8 report with `putObject` in create-only mode, then calls `headObject`, `getObjectText`, and `listObjects`. The static Dex Web connection name is `amazon-s3-report-archive`.
|
|
4
|
+
|
|
5
|
+
Configure that connection in Dex Web **Connections** using the [authorization guide](../../../README.md#authentication). Set `DEX_FLOW_SERVICE_ADDRESS` to the local `dexcli dev` address, then run `uv run python -m examples.report_archive.main` from the Python package directory. Start a `AmazonS3ReportArchivePython` Flow with `bucket`, `key`, and `text`; the sample is [input.example.json](input.example.json).
|
|
6
|
+
|
|
7
|
+
The integration test uses a local S3 stand-in and a real Dex Worker and Client. It does not verify live Amazon S3, Cloudflare R2, or MinIO authorization or object behavior.
|
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
# Copyright (c) 2026 Super Durable
|
|
2
|
+
# SPDX-License-Identifier: MIT
|
|
3
|
+
"""Write, inspect, read, and list one S3 report through Dex Steps."""
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
|
|
7
|
+
import dex
|
|
8
|
+
from dex_connectors.sdk import MutationResult, QueryResult
|
|
9
|
+
|
|
10
|
+
from dex_connectors.amazon_s3 import (
|
|
11
|
+
Connection,
|
|
12
|
+
GetObjectTextInput,
|
|
13
|
+
HeadObjectInput,
|
|
14
|
+
ListObjectsInput,
|
|
15
|
+
ListObjectsOutput,
|
|
16
|
+
ObjectMetadata,
|
|
17
|
+
ObjectText,
|
|
18
|
+
PutObjectInput,
|
|
19
|
+
StoredObject,
|
|
20
|
+
new_get_object_text_step,
|
|
21
|
+
new_head_object_step,
|
|
22
|
+
new_list_objects_step,
|
|
23
|
+
new_put_object_step,
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
CONNECTION_NAME = "amazon-s3-report-archive"
|
|
27
|
+
FLOW_TYPE = "AmazonS3ReportArchivePython"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass(frozen=True, slots=True)
|
|
31
|
+
class ReportArchiveInput:
|
|
32
|
+
"""Nonsecret report content supplied through Dex Web Start Flow."""
|
|
33
|
+
|
|
34
|
+
bucket: str
|
|
35
|
+
key: str
|
|
36
|
+
text: str
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@dataclass(frozen=True, slots=True)
|
|
40
|
+
class ReportArchiveOutput:
|
|
41
|
+
"""Proof of a stored, inspected, read, and listed report."""
|
|
42
|
+
|
|
43
|
+
stored: StoredObject
|
|
44
|
+
metadata: ObjectMetadata
|
|
45
|
+
text: ObjectText
|
|
46
|
+
listing: ListObjectsOutput
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
_REQUEST = dex.Attribute("s3-python-archive-request", ReportArchiveInput)
|
|
50
|
+
_STORED = dex.Attribute("s3-python-archive-stored", StoredObject)
|
|
51
|
+
_METADATA = dex.Attribute("s3-python-archive-metadata", ObjectMetadata)
|
|
52
|
+
_TEXT = dex.Attribute("s3-python-archive-text", ObjectText)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
# dex:group group-id:s3-archive group-label:"Report archive"
|
|
56
|
+
# dex:explanation text:"Validate safe report input and prepare a create-only object write."
|
|
57
|
+
class PrepareArchive(dex.Step[ReportArchiveInput]):
|
|
58
|
+
"""Validate and persist the report before invoking S3."""
|
|
59
|
+
|
|
60
|
+
def get_step_type(self) -> str:
|
|
61
|
+
"""Return the stable preparation Step identity."""
|
|
62
|
+
return "PrepareArchive"
|
|
63
|
+
|
|
64
|
+
def execute(self, context: dex.Context, input: ReportArchiveInput) -> dex.StepDecision:
|
|
65
|
+
"""Continue to the repeat-safe S3 write."""
|
|
66
|
+
if not input.bucket or not input.key or not input.text:
|
|
67
|
+
return dex.force_fail("bucket, key, and text are required")
|
|
68
|
+
_REQUEST.set(context, input)
|
|
69
|
+
return dex.go_to(
|
|
70
|
+
dex.StepRef("StoreReport"),
|
|
71
|
+
PutObjectInput(
|
|
72
|
+
bucket=input.bucket,
|
|
73
|
+
key=input.key,
|
|
74
|
+
content_type="text/plain; charset=utf-8",
|
|
75
|
+
text_content=input.text,
|
|
76
|
+
is_create_only=True,
|
|
77
|
+
),
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
# dex:group group-id:s3-archive group-label:"Report archive"
|
|
82
|
+
# dex:explanation text:"Persist the stored object and inspect its S3 metadata."
|
|
83
|
+
class ArchiveStored(dex.Step[MutationResult[StoredObject]]):
|
|
84
|
+
"""Persist the write result and route to metadata read-back."""
|
|
85
|
+
|
|
86
|
+
def get_step_type(self) -> str:
|
|
87
|
+
"""Return the stable stored Step identity."""
|
|
88
|
+
return "ArchiveStored"
|
|
89
|
+
|
|
90
|
+
def execute(self, context: dex.Context, input: MutationResult[StoredObject]) -> dex.StepDecision:
|
|
91
|
+
"""Continue with the stored object's identity."""
|
|
92
|
+
_STORED.set(context, input.value)
|
|
93
|
+
return dex.go_to(dex.StepRef("InspectReport"), HeadObjectInput(input.value.bucket, input.value.key))
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
# dex:group group-id:s3-archive group-label:"Report archive"
|
|
97
|
+
# dex:explanation text:"Persist S3 metadata and read the report text."
|
|
98
|
+
class ArchiveInspected(dex.Step[QueryResult[ObjectMetadata]]):
|
|
99
|
+
"""Persist metadata and route to text read-back."""
|
|
100
|
+
|
|
101
|
+
def get_step_type(self) -> str:
|
|
102
|
+
"""Return the stable inspection Step identity."""
|
|
103
|
+
return "ArchiveInspected"
|
|
104
|
+
|
|
105
|
+
def execute(self, context: dex.Context, input: QueryResult[ObjectMetadata]) -> dex.StepDecision:
|
|
106
|
+
"""Continue to bounded text retrieval."""
|
|
107
|
+
_METADATA.set(context, input.value)
|
|
108
|
+
return dex.go_to(dex.StepRef("ReadReport"), GetObjectTextInput(input.value.bucket, input.value.key))
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
# dex:group group-id:s3-archive group-label:"Report archive"
|
|
112
|
+
# dex:explanation text:"Persist bounded text and list the matching archive key."
|
|
113
|
+
class ArchiveRead(dex.Step[QueryResult[ObjectText]]):
|
|
114
|
+
"""Persist text and route to one bounded listing page."""
|
|
115
|
+
|
|
116
|
+
def get_step_type(self) -> str:
|
|
117
|
+
"""Return the stable text Step identity."""
|
|
118
|
+
return "ArchiveRead"
|
|
119
|
+
|
|
120
|
+
def execute(self, context: dex.Context, input: QueryResult[ObjectText]) -> dex.StepDecision:
|
|
121
|
+
"""List the exact report key as a prefix."""
|
|
122
|
+
_TEXT.set(context, input.value)
|
|
123
|
+
request = _REQUEST.get(context)
|
|
124
|
+
return dex.go_to(
|
|
125
|
+
dex.StepRef("ListReports"), ListObjectsInput(bucket=request.bucket, prefix=request.key)
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
# dex:group group-id:s3-archive group-label:"Report archive"
|
|
130
|
+
# dex:explanation text:"Complete with the stored identity, metadata, text, and listing."
|
|
131
|
+
class ArchiveListed(dex.Step[QueryResult[ListObjectsOutput]]):
|
|
132
|
+
"""Complete with safe report evidence."""
|
|
133
|
+
|
|
134
|
+
def get_step_type(self) -> str:
|
|
135
|
+
"""Return the stable listing Step identity."""
|
|
136
|
+
return "ArchiveListed"
|
|
137
|
+
|
|
138
|
+
def execute(self, context: dex.Context, input: QueryResult[ListObjectsOutput]) -> dex.StepDecision:
|
|
139
|
+
"""Return all four operation outcomes."""
|
|
140
|
+
return dex.graceful_complete(
|
|
141
|
+
ReportArchiveOutput(_STORED.get(context), _METADATA.get(context), _TEXT.get(context), input.value)
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
class ReportArchiveFlow(dex.Flow[ReportArchiveInput]):
|
|
146
|
+
"""Archive a text report with the four S3 connector operations."""
|
|
147
|
+
|
|
148
|
+
def __init__(self, connection: Connection) -> None:
|
|
149
|
+
"""Bind the static connection before Worker registration."""
|
|
150
|
+
if connection is None or connection.reference.name != CONNECTION_NAME:
|
|
151
|
+
raise ValueError("report archive requires its static S3 connection")
|
|
152
|
+
self.prepare = PrepareArchive()
|
|
153
|
+
self.stored = ArchiveStored()
|
|
154
|
+
self.inspected = ArchiveInspected()
|
|
155
|
+
self.read = ArchiveRead()
|
|
156
|
+
self.listed = ArchiveListed()
|
|
157
|
+
annotations = {
|
|
158
|
+
"groupId": "s3-archive",
|
|
159
|
+
"groupLabel": "Report archive",
|
|
160
|
+
"explanation": "Process one report through its S3 operation.",
|
|
161
|
+
}
|
|
162
|
+
self.put = new_put_object_step(
|
|
163
|
+
step_type="StoreReport",
|
|
164
|
+
step_input_type=PutObjectInput,
|
|
165
|
+
annotations=annotations,
|
|
166
|
+
connection=connection,
|
|
167
|
+
connection_name=CONNECTION_NAME,
|
|
168
|
+
map_to_operation_input=lambda value: value,
|
|
169
|
+
stored=self.stored,
|
|
170
|
+
)
|
|
171
|
+
self.head = new_head_object_step(
|
|
172
|
+
step_type="InspectReport",
|
|
173
|
+
step_input_type=HeadObjectInput,
|
|
174
|
+
annotations=annotations,
|
|
175
|
+
connection=connection,
|
|
176
|
+
connection_name=CONNECTION_NAME,
|
|
177
|
+
map_to_operation_input=lambda value: value,
|
|
178
|
+
found=self.inspected,
|
|
179
|
+
)
|
|
180
|
+
self.get = new_get_object_text_step(
|
|
181
|
+
step_type="ReadReport",
|
|
182
|
+
step_input_type=GetObjectTextInput,
|
|
183
|
+
annotations=annotations,
|
|
184
|
+
connection=connection,
|
|
185
|
+
connection_name=CONNECTION_NAME,
|
|
186
|
+
map_to_operation_input=lambda value: value,
|
|
187
|
+
read=self.read,
|
|
188
|
+
)
|
|
189
|
+
self.list = new_list_objects_step(
|
|
190
|
+
step_type="ListReports",
|
|
191
|
+
step_input_type=ListObjectsInput,
|
|
192
|
+
annotations=annotations,
|
|
193
|
+
connection=connection,
|
|
194
|
+
connection_name=CONNECTION_NAME,
|
|
195
|
+
map_to_operation_input=lambda value: value,
|
|
196
|
+
found=self.listed,
|
|
197
|
+
)
|
|
198
|
+
|
|
199
|
+
def get_flow_type(self) -> str:
|
|
200
|
+
"""Return the stable Flow identity."""
|
|
201
|
+
return FLOW_TYPE
|
|
202
|
+
|
|
203
|
+
def get_steps(self) -> dex.StepList[ReportArchiveInput]:
|
|
204
|
+
"""Register all provider and application Steps."""
|
|
205
|
+
return dex.StepList.start_step(self.prepare).other_steps(
|
|
206
|
+
self.put, self.stored, self.head, self.inspected, self.get, self.read, self.list, self.listed
|
|
207
|
+
)
|
|
208
|
+
|
|
209
|
+
def get_persistence_schema(self) -> dex.PersistenceSchema:
|
|
210
|
+
"""Register every safe value persisted across Steps."""
|
|
211
|
+
return dex.PersistenceSchema.of(_REQUEST, _STORED, _METADATA, _TEXT)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"bucket":"acme-reports","key":"reports/weekly.md","text":"hello\n"}
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
# Copyright (c) 2026 Super Durable
|
|
2
|
+
# SPDX-License-Identifier: MIT
|
|
3
|
+
"""Run the checked-in archive Flow through a real Dex Worker and Client."""
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import hashlib
|
|
7
|
+
import json
|
|
8
|
+
import os
|
|
9
|
+
import tempfile
|
|
10
|
+
import threading
|
|
11
|
+
from datetime import timedelta
|
|
12
|
+
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from urllib.parse import parse_qs, unquote, urlsplit
|
|
15
|
+
from uuid import uuid4
|
|
16
|
+
|
|
17
|
+
import dex
|
|
18
|
+
import pytest
|
|
19
|
+
from dex_connectors.sdk.projectconfig import load_from_environment
|
|
20
|
+
|
|
21
|
+
from dex_connectors.amazon_s3 import new_project_connection
|
|
22
|
+
|
|
23
|
+
from .flow import CONNECTION_NAME, ReportArchiveFlow, ReportArchiveInput, ReportArchiveOutput
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@pytest.mark.asyncio
|
|
27
|
+
async def test_report_archive_real_dex(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
28
|
+
"""Verify retry, create-only read-back, all four Steps, and unwired rejection."""
|
|
29
|
+
dex_address = os.environ["DEX_FLOW_SERVICE_ADDRESS"]
|
|
30
|
+
objects: dict[str, tuple[bytes, str]] = {}
|
|
31
|
+
requests: list[str] = []
|
|
32
|
+
drop_first_put = True
|
|
33
|
+
|
|
34
|
+
class Provider(BaseHTTPRequestHandler):
|
|
35
|
+
def log_message(self, _format: str, *args: object) -> None:
|
|
36
|
+
"""Keep request details out of test logs."""
|
|
37
|
+
|
|
38
|
+
def send(self, status: int, body: bytes = b"", headers: dict[str, str] | None = None) -> None:
|
|
39
|
+
"""Return one bounded S3 fixture response."""
|
|
40
|
+
self.send_response(status)
|
|
41
|
+
for name, value in (headers or {}).items():
|
|
42
|
+
self.send_header(name, value)
|
|
43
|
+
self.end_headers()
|
|
44
|
+
if self.command != "HEAD":
|
|
45
|
+
self.wfile.write(body)
|
|
46
|
+
|
|
47
|
+
def handle_request(self) -> None:
|
|
48
|
+
"""Store or retrieve one report using the S3 REST shape."""
|
|
49
|
+
nonlocal drop_first_put
|
|
50
|
+
location = urlsplit(self.path)
|
|
51
|
+
key = unquote(location.path.removeprefix("/acme-reports/"))
|
|
52
|
+
is_list = "list-type" in parse_qs(location.query)
|
|
53
|
+
if is_list:
|
|
54
|
+
key = parse_qs(location.query).get("prefix", [""])[0]
|
|
55
|
+
requests.append(f"{self.command} {'list' if is_list else key}")
|
|
56
|
+
size = int(self.headers.get("Content-Length", "0"))
|
|
57
|
+
body = self.rfile.read(size) if size else b""
|
|
58
|
+
if not self.headers.get("Authorization", "").startswith(
|
|
59
|
+
"AWS4-HMAC-SHA256 Credential=AKIAS3CONNECTORTEST1/"
|
|
60
|
+
):
|
|
61
|
+
self.send(403)
|
|
62
|
+
return
|
|
63
|
+
if self.headers.get("x-amz-content-sha256") != hashlib.sha256(body).hexdigest():
|
|
64
|
+
self.send(400)
|
|
65
|
+
return
|
|
66
|
+
if key.startswith("denied/"):
|
|
67
|
+
self.send(403, b"<Error><Code>AccessDenied</Code></Error>")
|
|
68
|
+
return
|
|
69
|
+
if self.command == "PUT":
|
|
70
|
+
if key in objects and self.headers.get("If-None-Match") == "*":
|
|
71
|
+
self.send(412)
|
|
72
|
+
return
|
|
73
|
+
objects[key] = (body, self.headers.get("x-amz-meta-dex-idempotency-key", ""))
|
|
74
|
+
if drop_first_put:
|
|
75
|
+
drop_first_put = False
|
|
76
|
+
self.connection.shutdown(2)
|
|
77
|
+
return
|
|
78
|
+
self.send(200, headers={"ETag": '"saved"'})
|
|
79
|
+
return
|
|
80
|
+
if self.command in ("HEAD", "GET") and not is_list:
|
|
81
|
+
if key not in objects:
|
|
82
|
+
self.send(404)
|
|
83
|
+
return
|
|
84
|
+
content, marker = objects[key]
|
|
85
|
+
self.send(
|
|
86
|
+
200,
|
|
87
|
+
content,
|
|
88
|
+
{
|
|
89
|
+
"Content-Length": str(len(content)),
|
|
90
|
+
"Last-Modified": "Sun, 25 Oct 2026 09:00:00 GMT",
|
|
91
|
+
"Content-Type": "text/plain; charset=utf-8",
|
|
92
|
+
"ETag": '"saved"',
|
|
93
|
+
"X-Amz-Meta-Dex-Idempotency-Key": marker,
|
|
94
|
+
},
|
|
95
|
+
)
|
|
96
|
+
return
|
|
97
|
+
if self.command == "GET" and is_list:
|
|
98
|
+
if key in objects:
|
|
99
|
+
contents = f"<Contents><Key>{key}</Key><Size>{len(objects[key][0])}</Size></Contents>"
|
|
100
|
+
else:
|
|
101
|
+
contents = ""
|
|
102
|
+
xml = f"<ListBucketResult><IsTruncated>false</IsTruncated>{contents}</ListBucketResult>"
|
|
103
|
+
self.send(200, xml.encode())
|
|
104
|
+
return
|
|
105
|
+
self.send(400)
|
|
106
|
+
|
|
107
|
+
do_GET = handle_request
|
|
108
|
+
do_HEAD = handle_request
|
|
109
|
+
do_PUT = handle_request
|
|
110
|
+
|
|
111
|
+
provider = ThreadingHTTPServer(("127.0.0.1", 0), Provider)
|
|
112
|
+
provider_thread = threading.Thread(target=provider.serve_forever, daemon=True)
|
|
113
|
+
provider_thread.start()
|
|
114
|
+
try:
|
|
115
|
+
with tempfile.TemporaryDirectory(prefix="s3-python-dex-") as directory:
|
|
116
|
+
config_path = Path(directory) / "connections.json"
|
|
117
|
+
config_path.write_text(
|
|
118
|
+
json.dumps(
|
|
119
|
+
{
|
|
120
|
+
"schemaVersion": "connectors.dex.dev/local-connections/v1alpha1",
|
|
121
|
+
"connections": [
|
|
122
|
+
{
|
|
123
|
+
"connectorId": "amazon-s3",
|
|
124
|
+
"connectionName": CONNECTION_NAME,
|
|
125
|
+
"modulePath": (
|
|
126
|
+
"github.com/superdurable/dex-connectors-library/"
|
|
127
|
+
"connectors/amazon/s3"
|
|
128
|
+
),
|
|
129
|
+
"provider": "amazon-s3",
|
|
130
|
+
"authMethodId": "api-key-only",
|
|
131
|
+
"configuration": {
|
|
132
|
+
"region": "eu-west-1",
|
|
133
|
+
"endpoint": f"http://127.0.0.1:{provider.server_port}",
|
|
134
|
+
"defaultBucket": "acme-reports",
|
|
135
|
+
},
|
|
136
|
+
"credentials": {
|
|
137
|
+
"access_key_id": "AKIAS3CONNECTORTEST1",
|
|
138
|
+
"secret_access_key": "s3-connector-test/secret+value",
|
|
139
|
+
},
|
|
140
|
+
}
|
|
141
|
+
],
|
|
142
|
+
"triggerBindings": [],
|
|
143
|
+
}
|
|
144
|
+
)
|
|
145
|
+
)
|
|
146
|
+
config_path.chmod(0o600)
|
|
147
|
+
monkeypatch.setenv("DEX_CONNECTOR_CONFIG_FILE", str(config_path))
|
|
148
|
+
project = await load_from_environment({"DEX_CONNECTOR_CONFIG_FILE": str(config_path)})
|
|
149
|
+
connection = new_project_connection(project, CONNECTION_NAME)
|
|
150
|
+
flow = ReportArchiveFlow(connection)
|
|
151
|
+
registry = dex.Registry([flow], allow_async_handlers=True)
|
|
152
|
+
cache = dex.open_blob_cache(dex.BlobCacheConfig(str(Path(directory) / "blobs"), 1 << 24))
|
|
153
|
+
worker = dex.AsyncWorker(registry, cache, dex.WorkerOptions(server_address=dex_address))
|
|
154
|
+
worker_task = asyncio.create_task(worker.start())
|
|
155
|
+
try:
|
|
156
|
+
await worker.wait_pull_registered(timeout=15)
|
|
157
|
+
async with dex.AsyncClient(
|
|
158
|
+
registry, cache, dex.ClientOptions(server_address=dex_address)
|
|
159
|
+
) as client:
|
|
160
|
+
success_id = f"s3-python-success-{uuid4().hex}"
|
|
161
|
+
await client.start_flow(
|
|
162
|
+
flow, success_id, ReportArchiveInput("acme-reports", "reports/weekly.md", "hello\n")
|
|
163
|
+
)
|
|
164
|
+
success = await asyncio.wait_for(
|
|
165
|
+
client.wait_for_flow(success_id, timedelta(seconds=45)), 60
|
|
166
|
+
)
|
|
167
|
+
assert success.status == dex.FlowStatus.COMPLETED, success.error_message
|
|
168
|
+
output = success.single_output(ReportArchiveOutput)
|
|
169
|
+
assert output.stored.is_from_earlier_attempt
|
|
170
|
+
assert output.metadata.size_bytes == 6
|
|
171
|
+
assert output.text.text == "hello\n"
|
|
172
|
+
assert output.listing.objects[0].key == "reports/weekly.md"
|
|
173
|
+
assert requests[:6] == [
|
|
174
|
+
"PUT reports/weekly.md",
|
|
175
|
+
"PUT reports/weekly.md",
|
|
176
|
+
"HEAD reports/weekly.md",
|
|
177
|
+
"HEAD reports/weekly.md",
|
|
178
|
+
"GET reports/weekly.md",
|
|
179
|
+
"GET list",
|
|
180
|
+
]
|
|
181
|
+
denied_id = f"s3-python-denied-{uuid4().hex}"
|
|
182
|
+
await client.start_flow(
|
|
183
|
+
flow, denied_id, ReportArchiveInput("acme-reports", "denied/report.md", "blocked")
|
|
184
|
+
)
|
|
185
|
+
denied = await asyncio.wait_for(
|
|
186
|
+
client.wait_for_flow(denied_id, timedelta(seconds=30)), 45
|
|
187
|
+
)
|
|
188
|
+
assert denied.status == dex.FlowStatus.FAILED
|
|
189
|
+
assert denied.error_message and "providerRejected" in denied.error_message
|
|
190
|
+
finally:
|
|
191
|
+
await worker.stop()
|
|
192
|
+
await worker_task
|
|
193
|
+
cache.close()
|
|
194
|
+
await connection.aclose()
|
|
195
|
+
await project.close()
|
|
196
|
+
finally:
|
|
197
|
+
provider.shutdown()
|
|
198
|
+
provider.server_close()
|
|
199
|
+
provider_thread.join(timeout=5)
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# Copyright (c) 2026 Super Durable
|
|
2
|
+
# SPDX-License-Identifier: MIT
|
|
3
|
+
"""Run the S3 report archive Worker with its Dex Web connection."""
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
|
|
7
|
+
from dex_connectors.sdk.projectconfig import load_from_environment
|
|
8
|
+
|
|
9
|
+
from dex_connectors.amazon_s3 import new_project_connection
|
|
10
|
+
|
|
11
|
+
from .flow import CONNECTION_NAME, ReportArchiveFlow
|
|
12
|
+
from .worker import run_worker
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
async def main() -> None:
|
|
16
|
+
"""Load the project connection and serve the report archive Flow."""
|
|
17
|
+
project = await load_from_environment()
|
|
18
|
+
connection = new_project_connection(project, CONNECTION_NAME)
|
|
19
|
+
try:
|
|
20
|
+
await run_worker(ReportArchiveFlow(connection))
|
|
21
|
+
finally:
|
|
22
|
+
await connection.aclose()
|
|
23
|
+
await project.close()
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
if __name__ == "__main__":
|
|
27
|
+
asyncio.run(main())
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# Copyright (c) 2026 Super Durable
|
|
2
|
+
# SPDX-License-Identifier: MIT
|
|
3
|
+
"""Start a report archive Flow using safe JSON input."""
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import asyncio
|
|
7
|
+
import os
|
|
8
|
+
import tempfile
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
import dex
|
|
12
|
+
from dex_connectors.sdk.projectconfig import load_from_environment
|
|
13
|
+
|
|
14
|
+
from dex_connectors.amazon_s3 import new_project_connection
|
|
15
|
+
|
|
16
|
+
from .flow import CONNECTION_NAME, ReportArchiveFlow, ReportArchiveInput
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
async def start_report_archive(flow_id: str, input_file: Path) -> str:
|
|
20
|
+
"""Decode one bounded start value and submit the registered Flow."""
|
|
21
|
+
encoded = input_file.read_bytes()
|
|
22
|
+
if len(encoded) > 65_536:
|
|
23
|
+
raise ValueError("report archive input exceeds 65536 bytes")
|
|
24
|
+
input_value = (
|
|
25
|
+
dex.CodecRegistry()
|
|
26
|
+
.resolve(ReportArchiveInput)
|
|
27
|
+
.decode(dex.Value(dex.WireKind.JSON, encoded.decode("utf-8")))
|
|
28
|
+
)
|
|
29
|
+
project = await load_from_environment()
|
|
30
|
+
connection = new_project_connection(project, CONNECTION_NAME)
|
|
31
|
+
try:
|
|
32
|
+
flow = ReportArchiveFlow(connection)
|
|
33
|
+
registry = dex.Registry([flow], allow_async_handlers=True)
|
|
34
|
+
with tempfile.TemporaryDirectory(prefix="s3-python-start-blobs-") as directory:
|
|
35
|
+
cache = dex.open_blob_cache(dex.BlobCacheConfig(directory, 1 << 24))
|
|
36
|
+
try:
|
|
37
|
+
address = os.environ.get("DEX_FLOW_SERVICE_ADDRESS", "localhost:8801")
|
|
38
|
+
async with dex.AsyncClient(
|
|
39
|
+
registry, cache, dex.ClientOptions(server_address=address)
|
|
40
|
+
) as client:
|
|
41
|
+
return await client.start_flow(flow, flow_id, input_value)
|
|
42
|
+
finally:
|
|
43
|
+
cache.close()
|
|
44
|
+
finally:
|
|
45
|
+
await connection.aclose()
|
|
46
|
+
await project.close()
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def main() -> None:
|
|
50
|
+
"""Accept a unique Flow ID and JSON input path from the command line."""
|
|
51
|
+
parser = argparse.ArgumentParser(description="Start an S3 report archive Flow")
|
|
52
|
+
parser.add_argument("flow_id")
|
|
53
|
+
parser.add_argument("input_file", type=Path)
|
|
54
|
+
args = parser.parse_args()
|
|
55
|
+
print(asyncio.run(start_report_archive(args.flow_id, args.input_file)))
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
if __name__ == "__main__":
|
|
59
|
+
main()
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# Copyright (c) 2026 Super Durable
|
|
2
|
+
# SPDX-License-Identifier: MIT
|
|
3
|
+
"""Host the report archive Flow with the asynchronous Dex Worker."""
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import tempfile
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from dex import AsyncWorker, BlobCacheConfig, Flow, Registry, WorkerOptions, open_blob_cache
|
|
10
|
+
|
|
11
|
+
from .flow import ReportArchiveInput
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
async def run_worker(flow: Flow[ReportArchiveInput]) -> None:
|
|
15
|
+
"""Serve the registered report archive Flow until shutdown."""
|
|
16
|
+
registry = Registry([flow], allow_async_handlers=True)
|
|
17
|
+
cache_directory = os.environ.get(
|
|
18
|
+
"DEX_BLOB_CACHE_DIR", str(Path(tempfile.gettempdir()) / "dex-s3-python-blobs")
|
|
19
|
+
)
|
|
20
|
+
cache = open_blob_cache(BlobCacheConfig(cache_directory, 1 << 30))
|
|
21
|
+
try:
|
|
22
|
+
options = WorkerOptions(server_address=os.environ.get("DEX_FLOW_SERVICE_ADDRESS", "localhost:8801"))
|
|
23
|
+
async with AsyncWorker(registry, cache, options) as worker:
|
|
24
|
+
await worker.start()
|
|
25
|
+
finally:
|
|
26
|
+
cache.close()
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.27"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "dex-connector-amazon-s3"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Amazon S3 connector for Dex Flows"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
dependencies = ["dex-connectors-sdk==0.1.0", "dex-python-sdk>=1.28.0", "httpx>=0.27"]
|
|
14
|
+
|
|
15
|
+
[dependency-groups]
|
|
16
|
+
dev = ["pytest>=8", "pytest-asyncio>=0.24", "mypy>=1.13", "ruff>=0.7", "pyyaml>=6", "types-PyYAML>=6"]
|
|
17
|
+
|
|
18
|
+
[tool.hatch.build.targets.wheel]
|
|
19
|
+
packages = ["src/dex_connectors"]
|
|
20
|
+
|
|
21
|
+
[tool.pytest.ini_options]
|
|
22
|
+
testpaths = ["tests"]
|
|
23
|
+
pythonpath = [".", "src"]
|
|
24
|
+
asyncio_mode = "auto"
|
|
25
|
+
asyncio_default_fixture_loop_scope = "function"
|
|
26
|
+
|
|
27
|
+
[tool.mypy]
|
|
28
|
+
python_version = "3.11"
|
|
29
|
+
strict = true
|
|
30
|
+
mypy_path = "src"
|
|
31
|
+
explicit_package_bases = true
|
|
32
|
+
|
|
33
|
+
[tool.ruff]
|
|
34
|
+
target-version = "py311"
|
|
35
|
+
line-length = 110
|
|
36
|
+
|
|
37
|
+
[tool.ruff.lint]
|
|
38
|
+
select = ["E", "F", "I", "UP", "B"]
|