tigrbl_engine_s3 0.4.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tigrbl_engine_s3-0.4.5/.gitignore +62 -0
- tigrbl_engine_s3-0.4.5/LICENSE +201 -0
- tigrbl_engine_s3-0.4.5/NOTICE +5 -0
- tigrbl_engine_s3-0.4.5/PKG-INFO +162 -0
- tigrbl_engine_s3-0.4.5/README.md +137 -0
- tigrbl_engine_s3-0.4.5/pyproject.toml +41 -0
- tigrbl_engine_s3-0.4.5/src/tigrbl_engine_s3/__init__.py +26 -0
- tigrbl_engine_s3-0.4.5/src/tigrbl_engine_s3/engine.py +221 -0
- tigrbl_engine_s3-0.4.5/src/tigrbl_engine_s3/session.py +160 -0
- tigrbl_engine_s3-0.4.5/tests/test_s3_engine.py +147 -0
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
# Root policy: ignore every new root-level entry unless explicitly allowed.
|
|
2
|
+
/*
|
|
3
|
+
|
|
4
|
+
# Root entries intentionally allowed by repository policy.
|
|
5
|
+
!/.codex/
|
|
6
|
+
!/.github/
|
|
7
|
+
!/.gitignore
|
|
8
|
+
!/.ssot/
|
|
9
|
+
!/CODE_OF_CONDUCT.md
|
|
10
|
+
!/CONTRIBUTING.md
|
|
11
|
+
!/LICENSE
|
|
12
|
+
!/NOTICE
|
|
13
|
+
!/README.md
|
|
14
|
+
!/SECURITY.md
|
|
15
|
+
!/assets/
|
|
16
|
+
!/bindings/
|
|
17
|
+
!/docs/
|
|
18
|
+
!/examples/
|
|
19
|
+
!/pkgs/
|
|
20
|
+
!/pyproject.toml
|
|
21
|
+
!/tests/
|
|
22
|
+
!/tools/
|
|
23
|
+
!/uv.lock
|
|
24
|
+
|
|
25
|
+
# Generated caches and build artifacts.
|
|
26
|
+
__pycache__/
|
|
27
|
+
.pytest_cache/
|
|
28
|
+
.ruff_cache/
|
|
29
|
+
.mypy_cache/
|
|
30
|
+
.tox/
|
|
31
|
+
.nox/
|
|
32
|
+
.venv/
|
|
33
|
+
.coverage
|
|
34
|
+
htmlcov/
|
|
35
|
+
build/
|
|
36
|
+
dist/
|
|
37
|
+
site/
|
|
38
|
+
target/
|
|
39
|
+
*.egg-info/
|
|
40
|
+
*.pyc
|
|
41
|
+
*.pyd
|
|
42
|
+
*.tmp*
|
|
43
|
+
*.zip
|
|
44
|
+
*.body
|
|
45
|
+
perf.sqlite
|
|
46
|
+
node_modules/
|
|
47
|
+
|
|
48
|
+
# Root-local generated caches and temporary package artifacts.
|
|
49
|
+
/.benchmarks/
|
|
50
|
+
/.pip-cache/
|
|
51
|
+
/.tmp
|
|
52
|
+
/.tmp/
|
|
53
|
+
/.tmp*/
|
|
54
|
+
/.uv-cache*/
|
|
55
|
+
/.uv-pytest*/
|
|
56
|
+
/.uv-pytest-tigrbl-tests/
|
|
57
|
+
/.uv-python*/
|
|
58
|
+
/.vendor
|
|
59
|
+
/.venv*/
|
|
60
|
+
/.venv-gitvs*/
|
|
61
|
+
/generation_examples/
|
|
62
|
+
/tmp/
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
Apache License
|
|
2
|
+
Version 2.0, January 2004
|
|
3
|
+
http://www.apache.org/licenses/
|
|
4
|
+
|
|
5
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
6
|
+
|
|
7
|
+
1. Definitions.
|
|
8
|
+
|
|
9
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
10
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
11
|
+
|
|
12
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
13
|
+
the copyright owner that is granting the License.
|
|
14
|
+
|
|
15
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
16
|
+
other entities that control, are controlled by, or are under common
|
|
17
|
+
control with that entity. For the purposes of this definition,
|
|
18
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
19
|
+
direction or management of such entity, whether by contract or
|
|
20
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
21
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
22
|
+
|
|
23
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
24
|
+
exercising permissions granted by this License.
|
|
25
|
+
|
|
26
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
27
|
+
including but not limited to software source code, documentation
|
|
28
|
+
source, and configuration files.
|
|
29
|
+
|
|
30
|
+
"Object" form shall mean any form resulting from mechanical
|
|
31
|
+
transformation or translation of a Source form, including but
|
|
32
|
+
not limited to compiled object code, generated documentation,
|
|
33
|
+
and conversions to other media types.
|
|
34
|
+
|
|
35
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
36
|
+
Object form, made available under the License, as indicated by a
|
|
37
|
+
copyright notice that is included in or attached to the work
|
|
38
|
+
(an example is provided in the Appendix below).
|
|
39
|
+
|
|
40
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
41
|
+
form, that is based on (or derived from) the Work and for which the
|
|
42
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
43
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
44
|
+
of this License, Derivative Works shall not include works that remain
|
|
45
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
46
|
+
the Work and Derivative Works thereof.
|
|
47
|
+
|
|
48
|
+
"Contribution" shall mean any work of authorship, including
|
|
49
|
+
the original version of the Work and any modifications or additions
|
|
50
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
51
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
52
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
53
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
54
|
+
means any form of electronic, verbal, or written communication sent
|
|
55
|
+
to the Licensor or its representatives, including but not limited to
|
|
56
|
+
communication on electronic mailing lists, source code control systems,
|
|
57
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
58
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
59
|
+
excluding communication that is conspicuously marked or otherwise
|
|
60
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
61
|
+
|
|
62
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
63
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
64
|
+
subsequently incorporated within the Work.
|
|
65
|
+
|
|
66
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
67
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
68
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
69
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
70
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
71
|
+
Work and such Derivative Works in Source or Object form.
|
|
72
|
+
|
|
73
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
74
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
75
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
76
|
+
(except as stated in this section) patent license to make, have made,
|
|
77
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
78
|
+
where such license applies only to those patent claims licensable
|
|
79
|
+
by such Contributor that are necessarily infringed by their
|
|
80
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
81
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
82
|
+
institute patent litigation against any entity (including a
|
|
83
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
84
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
85
|
+
or contributory patent infringement, then any patent licenses
|
|
86
|
+
granted to You under this License for that Work shall terminate
|
|
87
|
+
as of the date such litigation is filed.
|
|
88
|
+
|
|
89
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
90
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
91
|
+
modifications, and in Source or Object form, provided that You
|
|
92
|
+
meet the following conditions:
|
|
93
|
+
|
|
94
|
+
(a) You must give any other recipients of the Work or
|
|
95
|
+
Derivative Works a copy of this License; and
|
|
96
|
+
|
|
97
|
+
(b) You must cause any modified files to carry prominent notices
|
|
98
|
+
stating that You changed the files; and
|
|
99
|
+
|
|
100
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
101
|
+
that You distribute, all copyright, patent, trademark, and
|
|
102
|
+
attribution notices from the Source form of the Work,
|
|
103
|
+
excluding those notices that do not pertain to any part of
|
|
104
|
+
the Derivative Works; and
|
|
105
|
+
|
|
106
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
107
|
+
distribution, then any Derivative Works that You distribute must
|
|
108
|
+
include a readable copy of the attribution notices contained
|
|
109
|
+
within such NOTICE file, excluding those notices that do not
|
|
110
|
+
pertain to any part of the Derivative Works, in at least one
|
|
111
|
+
of the following places: within a NOTICE text file distributed
|
|
112
|
+
as part of the Derivative Works; within the Source form or
|
|
113
|
+
documentation, if provided along with the Derivative Works; or,
|
|
114
|
+
within a display generated by the Derivative Works, if and
|
|
115
|
+
wherever such third-party notices normally appear. The contents
|
|
116
|
+
of the NOTICE file are for informational purposes only and
|
|
117
|
+
do not modify the License. You may add Your own attribution
|
|
118
|
+
notices within Derivative Works that You distribute, alongside
|
|
119
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
120
|
+
that such additional attribution notices cannot be construed
|
|
121
|
+
as modifying the License.
|
|
122
|
+
|
|
123
|
+
You may add Your own copyright statement to Your modifications and
|
|
124
|
+
may provide additional or different license terms and conditions
|
|
125
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
126
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
127
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
128
|
+
the conditions stated in this License.
|
|
129
|
+
|
|
130
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
131
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
132
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
133
|
+
this License, without any additional terms or conditions.
|
|
134
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
135
|
+
the terms of any separate license agreement you may have executed
|
|
136
|
+
with Licensor regarding such Contributions.
|
|
137
|
+
|
|
138
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
139
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
140
|
+
except as required for reasonable and customary use in describing the
|
|
141
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
142
|
+
|
|
143
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
144
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
145
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
146
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
147
|
+
implied, including, without limitation, any warranties or conditions
|
|
148
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
149
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
150
|
+
appropriateness of using or redistributing the Work and assume any
|
|
151
|
+
risks associated with Your exercise of permissions under this License.
|
|
152
|
+
|
|
153
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
154
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
155
|
+
unless required by applicable law (such as deliberate and grossly
|
|
156
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
157
|
+
liable to You for damages, including any direct, indirect, special,
|
|
158
|
+
incidental, or consequential damages of any character arising as a
|
|
159
|
+
result of this License or out of the use or inability to use the
|
|
160
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
161
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
162
|
+
other commercial damages or losses), even if such Contributor
|
|
163
|
+
has been advised of the possibility of such damages.
|
|
164
|
+
|
|
165
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
166
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
167
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
168
|
+
or other liability obligations and/or rights consistent with this
|
|
169
|
+
License. However, in accepting such obligations, You may act only
|
|
170
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
171
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
172
|
+
defend, and hold each Contributor harmless for any liability
|
|
173
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
174
|
+
of your accepting any such warranty or additional liability.
|
|
175
|
+
|
|
176
|
+
END OF TERMS AND CONDITIONS
|
|
177
|
+
|
|
178
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
179
|
+
|
|
180
|
+
To apply the Apache License to your work, attach the following
|
|
181
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
182
|
+
replaced with your own identifying information. (Don't include
|
|
183
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
184
|
+
comment syntax for the file format. We also recommend that a
|
|
185
|
+
file or class name and description of purpose be included on the
|
|
186
|
+
same "printed page" as the copyright notice for easier
|
|
187
|
+
identification within third-party archives.
|
|
188
|
+
|
|
189
|
+
Copyright [yyyy] [name of copyright owner]
|
|
190
|
+
|
|
191
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
192
|
+
you may not use this file except in compliance with the License.
|
|
193
|
+
You may obtain a copy of the License at
|
|
194
|
+
|
|
195
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
196
|
+
|
|
197
|
+
Unless required by applicable law or agreed to in writing, software
|
|
198
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
199
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
200
|
+
See the License for the specific language governing permissions and
|
|
201
|
+
limitations under the License.
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: tigrbl_engine_s3
|
|
3
|
+
Version: 0.4.5
|
|
4
|
+
Summary: Direct S3 API-backed object storage engine plugin for Tigrbl with optional content-addressed storage.
|
|
5
|
+
Project-URL: Homepage, https://github.com/tigrbl/tigrbl
|
|
6
|
+
Project-URL: Repository, https://github.com/tigrbl/tigrbl/tree/master/pkgs/90_engines/tigrbl_engine_s3
|
|
7
|
+
Project-URL: Issues, https://github.com/tigrbl/tigrbl/issues
|
|
8
|
+
License-Expression: Apache-2.0
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
License-File: NOTICE
|
|
11
|
+
Keywords: boto3,cas,engine,object-storage,s3,tigrbl
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Framework :: AsyncIO
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
21
|
+
Requires-Python: <3.15,>=3.10
|
|
22
|
+
Requires-Dist: boto3>=1.35.0
|
|
23
|
+
Requires-Dist: tigrbl>=0.3.0.dev4
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
|
|
26
|
+
<div align="center">
|
|
27
|
+
<h1>tigrbl_engine_s3</h1>
|
|
28
|
+
<a href="https://discord.gg/K4YTAPapjR"><img src="https://img.shields.io/badge/Discord-Join%20chat-5865F2?logo=discord&logoColor=white" alt="Discord community for tigrbl_engine_s3"/></a>
|
|
29
|
+
<a href="LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-525252" alt="Apache 2.0 license"/></a>
|
|
30
|
+
<a href="pyproject.toml"><img src="https://img.shields.io/badge/python-3.10%20%7C%203.11%20%7C%203.12%20%7C%203.13%20%7C%203.14-3776ab" alt="Python versions 3.10 | 3.11 | 3.12 | 3.13 | 3.14 for tigrbl_engine_s3"/></a>
|
|
31
|
+
</div>
|
|
32
|
+
|
|
33
|
+
## What is tigrbl_engine_s3?
|
|
34
|
+
|
|
35
|
+
`tigrbl_engine_s3` is a standalone Tigrbl object-storage engine that calls the
|
|
36
|
+
S3 API directly through a boto3-compatible client. It supports AWS S3 and S3-
|
|
37
|
+
compatible services such as MinIO or R2 when configured with an endpoint URL.
|
|
38
|
+
|
|
39
|
+
## Why use tigrbl_engine_s3?
|
|
40
|
+
|
|
41
|
+
Use it when an application needs direct S3 request semantics, credential-chain
|
|
42
|
+
behavior, endpoint configuration, or server-side upload options.
|
|
43
|
+
|
|
44
|
+
## When should I install tigrbl_engine_s3?
|
|
45
|
+
|
|
46
|
+
Install it for object-native Tigrbl workloads that should communicate through a
|
|
47
|
+
boto3-compatible S3 client.
|
|
48
|
+
|
|
49
|
+
## Who is tigrbl_engine_s3 for?
|
|
50
|
+
|
|
51
|
+
It is for application developers and platform engineers integrating Tigrbl with
|
|
52
|
+
AWS S3 or an S3-compatible service.
|
|
53
|
+
|
|
54
|
+
## Where does tigrbl_engine_s3 fit?
|
|
55
|
+
|
|
56
|
+
It is a layer-90 standalone engine package and depends on the public `tigrbl`
|
|
57
|
+
facade plus `boto3`.
|
|
58
|
+
|
|
59
|
+
## How does tigrbl_engine_s3 work?
|
|
60
|
+
|
|
61
|
+
Its plugin builds an `S3Engine` and `S3Session`; sessions translate strict
|
|
62
|
+
logical object keys into direct `put_object`, `get_object`, and related calls.
|
|
63
|
+
|
|
64
|
+
## Install
|
|
65
|
+
|
|
66
|
+
```bash
|
|
67
|
+
uv add tigrbl_engine_s3
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
## Surface Coverage
|
|
71
|
+
|
|
72
|
+
The session API is object-native:
|
|
73
|
+
|
|
74
|
+
- `put_object(key, data)`, `get_object(key)`, `delete_object(key)`
|
|
75
|
+
- `object_exists(key)`, `head_object(key)`, `list_objects(prefix="")`
|
|
76
|
+
- optional `put_cas(data)`, `get_cas(address)`, `cas_exists(address)`
|
|
77
|
+
|
|
78
|
+
## What It Owns
|
|
79
|
+
|
|
80
|
+
This package owns direct S3 object I/O, logical key normalization, local session
|
|
81
|
+
staging, upload options, and optional SHA-256 content-addressed storage. Its
|
|
82
|
+
engine implements the canonical `EngineBase` contract while retaining this
|
|
83
|
+
object-specific surface.
|
|
84
|
+
|
|
85
|
+
## Public API and Import Surface
|
|
86
|
+
|
|
87
|
+
- `S3Engine`
|
|
88
|
+
- `S3Session`
|
|
89
|
+
- `s3_engine()` and `s3_capabilities()`
|
|
90
|
+
- `register()` under the `tigrbl.engine` entry-point group
|
|
91
|
+
|
|
92
|
+
## Usage Examples
|
|
93
|
+
|
|
94
|
+
## Configuration
|
|
95
|
+
|
|
96
|
+
```python
|
|
97
|
+
from tigrbl import EngineSpec
|
|
98
|
+
|
|
99
|
+
engine, make_session = EngineSpec.from_any(
|
|
100
|
+
{
|
|
101
|
+
"kind": "s3",
|
|
102
|
+
"bucket": "documents",
|
|
103
|
+
"prefix": "tenant-a",
|
|
104
|
+
"region_name": "us-east-1",
|
|
105
|
+
"endpoint_url": "http://localhost:9000",
|
|
106
|
+
"cas": True,
|
|
107
|
+
"put_options": {"ServerSideEncryption": "AES256"},
|
|
108
|
+
}
|
|
109
|
+
).build()
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
Credentials follow boto3's normal provider chain. Explicit `access_key`,
|
|
113
|
+
`secret_key`, and `session_token` mappings are also accepted. Tests and custom
|
|
114
|
+
integrations can inject an existing client as `mapping["client"]`.
|
|
115
|
+
|
|
116
|
+
## Content-addressed storage
|
|
117
|
+
|
|
118
|
+
CAS is opt-in with `"cas": True`. The returned address is `sha256:<hex>` and
|
|
119
|
+
the physical object is sharded below `.cas/sha256` by default. Retrieval hashes
|
|
120
|
+
the received bytes and fails if they do not match the requested address.
|
|
121
|
+
|
|
122
|
+
S3 is not a multi-object transactional store. `begin()` provides local staging
|
|
123
|
+
and rollback before upload, but `commit()` is a sequence of remote S3 calls and
|
|
124
|
+
can be partially applied if a later request fails.
|
|
125
|
+
|
|
126
|
+
## How To Choose This Package
|
|
127
|
+
|
|
128
|
+
Choose this package for direct boto3-compatible S3 calls. Choose
|
|
129
|
+
`tigrbl_engine_s3fs` when the fsspec/s3fs filesystem interface is preferable.
|
|
130
|
+
|
|
131
|
+
## Related Packages
|
|
132
|
+
|
|
133
|
+
- `tigrbl`
|
|
134
|
+
- `tigrbl_engine_s3fs`
|
|
135
|
+
|
|
136
|
+
## Documentation Links
|
|
137
|
+
|
|
138
|
+
- [Workspace docs](https://github.com/tigrbl/tigrbl/blob/master/docs/README.md)
|
|
139
|
+
- [Package catalog](https://github.com/tigrbl/tigrbl/blob/master/docs/developer/PACKAGE_CATALOG.md)
|
|
140
|
+
- [Package layout](https://github.com/tigrbl/tigrbl/blob/master/docs/developer/PACKAGE_LAYOUT.md)
|
|
141
|
+
- [Current target](https://github.com/tigrbl/tigrbl/blob/master/docs/conformance/CURRENT_TARGET.md)
|
|
142
|
+
- [SSOT registry](https://github.com/tigrbl/tigrbl/blob/master/.ssot/registry.json)
|
|
143
|
+
|
|
144
|
+
## Support
|
|
145
|
+
|
|
146
|
+
- [Discord](https://discord.gg/K4YTAPapjR)
|
|
147
|
+
- [GitHub Issues](https://github.com/tigrbl/tigrbl/issues)
|
|
148
|
+
|
|
149
|
+
## Package-local Boundary
|
|
150
|
+
|
|
151
|
+
This README is the package-local distribution entry point for
|
|
152
|
+
`tigrbl_engine_s3`. Cross-package claims and release status remain governed by
|
|
153
|
+
the repository docs and `.ssot/registry.json`.
|
|
154
|
+
|
|
155
|
+
## Certification Status
|
|
156
|
+
|
|
157
|
+
The implementation has package-scoped fake-backend tests and build validation.
|
|
158
|
+
It does not claim live AWS or S3-compatible service conformance yet.
|
|
159
|
+
|
|
160
|
+
## License
|
|
161
|
+
|
|
162
|
+
Licensed under the Apache License, Version 2.0. See `LICENSE` and `NOTICE`.
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
<div align="center">
|
|
2
|
+
<h1>tigrbl_engine_s3</h1>
|
|
3
|
+
<a href="https://discord.gg/K4YTAPapjR"><img src="https://img.shields.io/badge/Discord-Join%20chat-5865F2?logo=discord&logoColor=white" alt="Discord community for tigrbl_engine_s3"/></a>
|
|
4
|
+
<a href="LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-525252" alt="Apache 2.0 license"/></a>
|
|
5
|
+
<a href="pyproject.toml"><img src="https://img.shields.io/badge/python-3.10%20%7C%203.11%20%7C%203.12%20%7C%203.13%20%7C%203.14-3776ab" alt="Python versions 3.10 | 3.11 | 3.12 | 3.13 | 3.14 for tigrbl_engine_s3"/></a>
|
|
6
|
+
</div>
|
|
7
|
+
|
|
8
|
+
## What is tigrbl_engine_s3?
|
|
9
|
+
|
|
10
|
+
`tigrbl_engine_s3` is a standalone Tigrbl object-storage engine that calls the
|
|
11
|
+
S3 API directly through a boto3-compatible client. It supports AWS S3 and S3-
|
|
12
|
+
compatible services such as MinIO or R2 when configured with an endpoint URL.
|
|
13
|
+
|
|
14
|
+
## Why use tigrbl_engine_s3?
|
|
15
|
+
|
|
16
|
+
Use it when an application needs direct S3 request semantics, credential-chain
|
|
17
|
+
behavior, endpoint configuration, or server-side upload options.
|
|
18
|
+
|
|
19
|
+
## When should I install tigrbl_engine_s3?
|
|
20
|
+
|
|
21
|
+
Install it for object-native Tigrbl workloads that should communicate through a
|
|
22
|
+
boto3-compatible S3 client.
|
|
23
|
+
|
|
24
|
+
## Who is tigrbl_engine_s3 for?
|
|
25
|
+
|
|
26
|
+
It is for application developers and platform engineers integrating Tigrbl with
|
|
27
|
+
AWS S3 or an S3-compatible service.
|
|
28
|
+
|
|
29
|
+
## Where does tigrbl_engine_s3 fit?
|
|
30
|
+
|
|
31
|
+
It is a layer-90 standalone engine package and depends on the public `tigrbl`
|
|
32
|
+
facade plus `boto3`.
|
|
33
|
+
|
|
34
|
+
## How does tigrbl_engine_s3 work?
|
|
35
|
+
|
|
36
|
+
Its plugin builds an `S3Engine` and `S3Session`; sessions translate strict
|
|
37
|
+
logical object keys into direct `put_object`, `get_object`, and related calls.
|
|
38
|
+
|
|
39
|
+
## Install
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
uv add tigrbl_engine_s3
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
## Surface Coverage
|
|
46
|
+
|
|
47
|
+
The session API is object-native:
|
|
48
|
+
|
|
49
|
+
- `put_object(key, data)`, `get_object(key)`, `delete_object(key)`
|
|
50
|
+
- `object_exists(key)`, `head_object(key)`, `list_objects(prefix="")`
|
|
51
|
+
- optional `put_cas(data)`, `get_cas(address)`, `cas_exists(address)`
|
|
52
|
+
|
|
53
|
+
## What It Owns
|
|
54
|
+
|
|
55
|
+
This package owns direct S3 object I/O, logical key normalization, local session
|
|
56
|
+
staging, upload options, and optional SHA-256 content-addressed storage. Its
|
|
57
|
+
engine implements the canonical `EngineBase` contract while retaining this
|
|
58
|
+
object-specific surface.
|
|
59
|
+
|
|
60
|
+
## Public API and Import Surface
|
|
61
|
+
|
|
62
|
+
- `S3Engine`
|
|
63
|
+
- `S3Session`
|
|
64
|
+
- `s3_engine()` and `s3_capabilities()`
|
|
65
|
+
- `register()` under the `tigrbl.engine` entry-point group
|
|
66
|
+
|
|
67
|
+
## Usage Examples
|
|
68
|
+
|
|
69
|
+
## Configuration
|
|
70
|
+
|
|
71
|
+
```python
|
|
72
|
+
from tigrbl import EngineSpec
|
|
73
|
+
|
|
74
|
+
engine, make_session = EngineSpec.from_any(
|
|
75
|
+
{
|
|
76
|
+
"kind": "s3",
|
|
77
|
+
"bucket": "documents",
|
|
78
|
+
"prefix": "tenant-a",
|
|
79
|
+
"region_name": "us-east-1",
|
|
80
|
+
"endpoint_url": "http://localhost:9000",
|
|
81
|
+
"cas": True,
|
|
82
|
+
"put_options": {"ServerSideEncryption": "AES256"},
|
|
83
|
+
}
|
|
84
|
+
).build()
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
Credentials follow boto3's normal provider chain. Explicit `access_key`,
|
|
88
|
+
`secret_key`, and `session_token` mappings are also accepted. Tests and custom
|
|
89
|
+
integrations can inject an existing client as `mapping["client"]`.
|
|
90
|
+
|
|
91
|
+
## Content-addressed storage
|
|
92
|
+
|
|
93
|
+
CAS is opt-in with `"cas": True`. The returned address is `sha256:<hex>` and
|
|
94
|
+
the physical object is sharded below `.cas/sha256` by default. Retrieval hashes
|
|
95
|
+
the received bytes and fails if they do not match the requested address.
|
|
96
|
+
|
|
97
|
+
S3 is not a multi-object transactional store. `begin()` provides local staging
|
|
98
|
+
and rollback before upload, but `commit()` is a sequence of remote S3 calls and
|
|
99
|
+
can be partially applied if a later request fails.
|
|
100
|
+
|
|
101
|
+
## How To Choose This Package
|
|
102
|
+
|
|
103
|
+
Choose this package for direct boto3-compatible S3 calls. Choose
|
|
104
|
+
`tigrbl_engine_s3fs` when the fsspec/s3fs filesystem interface is preferable.
|
|
105
|
+
|
|
106
|
+
## Related Packages
|
|
107
|
+
|
|
108
|
+
- `tigrbl`
|
|
109
|
+
- `tigrbl_engine_s3fs`
|
|
110
|
+
|
|
111
|
+
## Documentation Links
|
|
112
|
+
|
|
113
|
+
- [Workspace docs](https://github.com/tigrbl/tigrbl/blob/master/docs/README.md)
|
|
114
|
+
- [Package catalog](https://github.com/tigrbl/tigrbl/blob/master/docs/developer/PACKAGE_CATALOG.md)
|
|
115
|
+
- [Package layout](https://github.com/tigrbl/tigrbl/blob/master/docs/developer/PACKAGE_LAYOUT.md)
|
|
116
|
+
- [Current target](https://github.com/tigrbl/tigrbl/blob/master/docs/conformance/CURRENT_TARGET.md)
|
|
117
|
+
- [SSOT registry](https://github.com/tigrbl/tigrbl/blob/master/.ssot/registry.json)
|
|
118
|
+
|
|
119
|
+
## Support
|
|
120
|
+
|
|
121
|
+
- [Discord](https://discord.gg/K4YTAPapjR)
|
|
122
|
+
- [GitHub Issues](https://github.com/tigrbl/tigrbl/issues)
|
|
123
|
+
|
|
124
|
+
## Package-local Boundary
|
|
125
|
+
|
|
126
|
+
This README is the package-local distribution entry point for
|
|
127
|
+
`tigrbl_engine_s3`. Cross-package claims and release status remain governed by
|
|
128
|
+
the repository docs and `.ssot/registry.json`.
|
|
129
|
+
|
|
130
|
+
## Certification Status
|
|
131
|
+
|
|
132
|
+
The implementation has package-scoped fake-backend tests and build validation.
|
|
133
|
+
It does not claim live AWS or S3-compatible service conformance yet.
|
|
134
|
+
|
|
135
|
+
## License
|
|
136
|
+
|
|
137
|
+
Licensed under the Apache License, Version 2.0. See `LICENSE` and `NOTICE`.
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.25.0"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "tigrbl_engine_s3"
|
|
7
|
+
version = "0.4.5"
|
|
8
|
+
description = "Direct S3 API-backed object storage engine plugin for Tigrbl with optional content-addressed storage."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "Apache-2.0"
|
|
11
|
+
requires-python = ">=3.10,<3.15"
|
|
12
|
+
dependencies = [
|
|
13
|
+
"boto3>=1.35.0",
|
|
14
|
+
"tigrbl>=0.3.0.dev4",
|
|
15
|
+
]
|
|
16
|
+
keywords = ["tigrbl", "engine", "s3", "boto3", "object-storage", "cas"]
|
|
17
|
+
classifiers = [
|
|
18
|
+
"Development Status :: 3 - Alpha",
|
|
19
|
+
"Framework :: AsyncIO",
|
|
20
|
+
"Operating System :: OS Independent",
|
|
21
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
22
|
+
"Programming Language :: Python :: 3.10",
|
|
23
|
+
"Programming Language :: Python :: 3.11",
|
|
24
|
+
"Programming Language :: Python :: 3.12",
|
|
25
|
+
"Programming Language :: Python :: 3.13",
|
|
26
|
+
"Programming Language :: Python :: 3.14",
|
|
27
|
+
]
|
|
28
|
+
|
|
29
|
+
[project.urls]
|
|
30
|
+
Homepage = "https://github.com/tigrbl/tigrbl"
|
|
31
|
+
Repository = "https://github.com/tigrbl/tigrbl/tree/master/pkgs/90_engines/tigrbl_engine_s3"
|
|
32
|
+
Issues = "https://github.com/tigrbl/tigrbl/issues"
|
|
33
|
+
|
|
34
|
+
[project.entry-points."tigrbl.engine_plugins"]
|
|
35
|
+
s3 = "tigrbl_engine_s3:register"
|
|
36
|
+
|
|
37
|
+
[dependency-groups]
|
|
38
|
+
dev = ["pytest>=8.0", "pytest-asyncio>=0.24.0", "ruff>=0.11.0"]
|
|
39
|
+
|
|
40
|
+
[tool.uv.sources]
|
|
41
|
+
tigrbl = { workspace = true }
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
from .engine import S3Engine, s3_capabilities, s3_engine
|
|
2
|
+
from .session import S3Session
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class _Registration:
|
|
6
|
+
def build(self, *, mapping, spec, dsn):
|
|
7
|
+
return s3_engine(mapping=mapping, spec=spec, dsn=dsn)
|
|
8
|
+
|
|
9
|
+
def capabilities(self, *, spec=None, mapping=None):
|
|
10
|
+
return s3_capabilities()
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def register() -> None:
|
|
14
|
+
from tigrbl.engine.registry import register_engine
|
|
15
|
+
|
|
16
|
+
register_engine("s3", _Registration())
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
__version__ = "0.4.5.dev4"
|
|
20
|
+
__all__ = [
|
|
21
|
+
"S3Engine",
|
|
22
|
+
"S3Session",
|
|
23
|
+
"s3_engine",
|
|
24
|
+
"s3_capabilities",
|
|
25
|
+
"register",
|
|
26
|
+
]
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
|
+
from hashlib import sha256
|
|
5
|
+
from pathlib import PurePosixPath
|
|
6
|
+
from typing import Any, Callable
|
|
7
|
+
|
|
8
|
+
from tigrbl_base._base import EngineBase
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _clean_fragment(value: str, *, name: str, allow_empty: bool = False) -> str:
|
|
12
|
+
if not isinstance(value, str):
|
|
13
|
+
raise TypeError(f"{name} must be a string")
|
|
14
|
+
if value.startswith("/") or "//" in value:
|
|
15
|
+
raise ValueError(f"{name} must be relative and contain no empty segments")
|
|
16
|
+
candidate = value.strip("/")
|
|
17
|
+
if not candidate:
|
|
18
|
+
if allow_empty:
|
|
19
|
+
return ""
|
|
20
|
+
raise ValueError(f"{name} must not be empty")
|
|
21
|
+
if "\\" in candidate:
|
|
22
|
+
raise ValueError(f"{name} must use POSIX separators")
|
|
23
|
+
parts = PurePosixPath(candidate).parts
|
|
24
|
+
if any(part in {"", ".", ".."} for part in parts):
|
|
25
|
+
raise ValueError(f"{name} must not contain empty or traversal segments")
|
|
26
|
+
return "/".join(parts)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _content_address(data: bytes) -> str:
|
|
30
|
+
return f"sha256:{sha256(data).hexdigest()}"
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _digest_from_address(address: str) -> str:
|
|
34
|
+
if not isinstance(address, str) or not address.startswith("sha256:"):
|
|
35
|
+
raise ValueError("CAS address must use the sha256:<hex> form")
|
|
36
|
+
digest = address.removeprefix("sha256:").lower()
|
|
37
|
+
if len(digest) != 64 or any(ch not in "0123456789abcdef" for ch in digest):
|
|
38
|
+
raise ValueError("CAS address must contain a 64-character SHA-256 digest")
|
|
39
|
+
return digest
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _is_missing(exc: Exception) -> bool:
|
|
43
|
+
response = getattr(exc, "response", None)
|
|
44
|
+
if isinstance(response, dict):
|
|
45
|
+
code = str(response.get("Error", {}).get("Code", ""))
|
|
46
|
+
return code in {"404", "NoSuchKey", "NotFound"}
|
|
47
|
+
return isinstance(exc, (FileNotFoundError, KeyError))
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@dataclass
|
|
51
|
+
class S3Engine(EngineBase):
|
|
52
|
+
client: Any
|
|
53
|
+
bucket: str
|
|
54
|
+
prefix: str = ""
|
|
55
|
+
cas_enabled: bool = False
|
|
56
|
+
cas_prefix: str = ".cas/sha256"
|
|
57
|
+
verify_cas: bool = True
|
|
58
|
+
put_options: dict[str, Any] = field(default_factory=dict)
|
|
59
|
+
spec: Any = None
|
|
60
|
+
|
|
61
|
+
def __post_init__(self) -> None:
|
|
62
|
+
self.bucket = _clean_fragment(self.bucket, name="bucket")
|
|
63
|
+
self.prefix = _clean_fragment(self.prefix, name="prefix", allow_empty=True)
|
|
64
|
+
self.cas_prefix = _clean_fragment(self.cas_prefix, name="cas_prefix")
|
|
65
|
+
|
|
66
|
+
def object_key(self, key: str) -> str:
|
|
67
|
+
clean = _clean_fragment(key, name="key")
|
|
68
|
+
return "/".join(part for part in (self.prefix, clean) if part)
|
|
69
|
+
|
|
70
|
+
def cas_key(self, address: str) -> str:
|
|
71
|
+
self._require_cas()
|
|
72
|
+
digest = _digest_from_address(address)
|
|
73
|
+
return self.object_key(f"{self.cas_prefix}/{digest[:2]}/{digest[2:]}")
|
|
74
|
+
|
|
75
|
+
def put_key(self, key: str, data: bytes) -> None:
|
|
76
|
+
self.client.put_object(
|
|
77
|
+
Bucket=self.bucket, Key=key, Body=data, **self.put_options
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
def get_key(self, key: str) -> bytes:
|
|
81
|
+
try:
|
|
82
|
+
response = self.client.get_object(Bucket=self.bucket, Key=key)
|
|
83
|
+
except Exception as exc:
|
|
84
|
+
if _is_missing(exc):
|
|
85
|
+
raise FileNotFoundError(key) from exc
|
|
86
|
+
raise
|
|
87
|
+
body = response["Body"]
|
|
88
|
+
return bytes(body.read() if hasattr(body, "read") else body)
|
|
89
|
+
|
|
90
|
+
def delete_key(self, key: str) -> bool:
|
|
91
|
+
self.client.delete_object(Bucket=self.bucket, Key=key)
|
|
92
|
+
return True
|
|
93
|
+
|
|
94
|
+
def key_exists(self, key: str) -> bool:
|
|
95
|
+
try:
|
|
96
|
+
self.client.head_object(Bucket=self.bucket, Key=key)
|
|
97
|
+
return True
|
|
98
|
+
except Exception as exc:
|
|
99
|
+
if _is_missing(exc):
|
|
100
|
+
return False
|
|
101
|
+
raise
|
|
102
|
+
|
|
103
|
+
def head_key(self, key: str) -> dict[str, Any]:
|
|
104
|
+
try:
|
|
105
|
+
return dict(self.client.head_object(Bucket=self.bucket, Key=key))
|
|
106
|
+
except Exception as exc:
|
|
107
|
+
if _is_missing(exc):
|
|
108
|
+
raise FileNotFoundError(key) from exc
|
|
109
|
+
raise
|
|
110
|
+
|
|
111
|
+
def list_keys(self, logical_prefix: str = "") -> list[str]:
|
|
112
|
+
request_prefix = f"{self.prefix}/" if self.prefix else ""
|
|
113
|
+
if logical_prefix:
|
|
114
|
+
request_prefix = self.object_key(logical_prefix)
|
|
115
|
+
kwargs: dict[str, Any] = {"Bucket": self.bucket, "Prefix": request_prefix}
|
|
116
|
+
found: list[str] = []
|
|
117
|
+
while True:
|
|
118
|
+
response = self.client.list_objects_v2(**kwargs)
|
|
119
|
+
found.extend(
|
|
120
|
+
str(row["Key"]) for row in response.get("Contents", []) if "Key" in row
|
|
121
|
+
)
|
|
122
|
+
token = response.get("NextContinuationToken")
|
|
123
|
+
if not response.get("IsTruncated") or not token:
|
|
124
|
+
break
|
|
125
|
+
kwargs["ContinuationToken"] = token
|
|
126
|
+
marker = f"{self.prefix}/" if self.prefix else ""
|
|
127
|
+
return sorted(
|
|
128
|
+
key[len(marker) :] if marker and key.startswith(marker) else key
|
|
129
|
+
for key in found
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
def cas_address(self, data: bytes) -> str:
|
|
133
|
+
self._require_cas()
|
|
134
|
+
return _content_address(data)
|
|
135
|
+
|
|
136
|
+
def to_provider(self) -> "S3Engine":
|
|
137
|
+
return self
|
|
138
|
+
|
|
139
|
+
async def _executeloop_impl(self, statements: Any) -> Any:
|
|
140
|
+
del statements
|
|
141
|
+
raise TypeError("S3 object engines do not execute SQL statements")
|
|
142
|
+
|
|
143
|
+
async def _executemany_impl(self, stmt: Any, parameter_sets: Any) -> Any:
|
|
144
|
+
del stmt, parameter_sets
|
|
145
|
+
raise TypeError("S3 object engines do not execute SQL statements")
|
|
146
|
+
|
|
147
|
+
def verify_cas_payload(self, address: str, data: bytes) -> None:
|
|
148
|
+
if self.verify_cas and _content_address(data) != address.lower():
|
|
149
|
+
raise ValueError(f"CAS integrity check failed for {address}")
|
|
150
|
+
|
|
151
|
+
def _require_cas(self) -> None:
|
|
152
|
+
if not self.cas_enabled:
|
|
153
|
+
raise RuntimeError("content-addressed storage is not enabled")
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def s3_engine(
|
|
157
|
+
*, mapping: dict[str, Any] | None = None, spec: Any = None, **_: Any
|
|
158
|
+
) -> tuple[S3Engine, Callable[..., Any]]:
|
|
159
|
+
config = dict(mapping or {})
|
|
160
|
+
bucket = config.get("bucket")
|
|
161
|
+
if not isinstance(bucket, str) or not bucket:
|
|
162
|
+
raise TypeError("mapping['bucket'] must be a non-empty string")
|
|
163
|
+
|
|
164
|
+
client = config.get("client")
|
|
165
|
+
if client is None:
|
|
166
|
+
try:
|
|
167
|
+
import boto3
|
|
168
|
+
except ImportError as exc: # pragma: no cover - packaging guard
|
|
169
|
+
raise RuntimeError(
|
|
170
|
+
"boto3 is required; install the tigrbl_engine_s3 package"
|
|
171
|
+
) from exc
|
|
172
|
+
session_kwargs: dict[str, Any] = {}
|
|
173
|
+
if config.get("profile_name") is not None:
|
|
174
|
+
session_kwargs["profile_name"] = config["profile_name"]
|
|
175
|
+
session = boto3.Session(**session_kwargs)
|
|
176
|
+
client_kwargs: dict[str, Any] = {}
|
|
177
|
+
for source, target in (
|
|
178
|
+
("region_name", "region_name"),
|
|
179
|
+
("endpoint_url", "endpoint_url"),
|
|
180
|
+
("use_ssl", "use_ssl"),
|
|
181
|
+
("verify", "verify"),
|
|
182
|
+
("access_key", "aws_access_key_id"),
|
|
183
|
+
("secret_key", "aws_secret_access_key"),
|
|
184
|
+
("session_token", "aws_session_token"),
|
|
185
|
+
):
|
|
186
|
+
if source in config:
|
|
187
|
+
client_kwargs[target] = config[source]
|
|
188
|
+
client = session.client("s3", **client_kwargs)
|
|
189
|
+
|
|
190
|
+
engine = S3Engine(
|
|
191
|
+
client=client,
|
|
192
|
+
bucket=bucket,
|
|
193
|
+
prefix=str(config.get("prefix") or ""),
|
|
194
|
+
cas_enabled=bool(config.get("cas", False)),
|
|
195
|
+
cas_prefix=str(config.get("cas_prefix") or ".cas/sha256"),
|
|
196
|
+
verify_cas=bool(config.get("verify_cas", True)),
|
|
197
|
+
put_options=dict(config.get("put_options") or {}),
|
|
198
|
+
spec=spec,
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
from .session import S3Session
|
|
202
|
+
|
|
203
|
+
def session_factory(session_spec: Any = None) -> S3Session:
|
|
204
|
+
return S3Session(engine, spec=session_spec)
|
|
205
|
+
|
|
206
|
+
return engine, session_factory
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def s3_capabilities() -> dict[str, object]:
|
|
210
|
+
return {
|
|
211
|
+
"engine": "s3",
|
|
212
|
+
"object_storage": True,
|
|
213
|
+
"s3_compatible": True,
|
|
214
|
+
"direct_s3_api": True,
|
|
215
|
+
"content_addressed_storage": "optional",
|
|
216
|
+
"cas_algorithm": "sha256",
|
|
217
|
+
"transactional": False,
|
|
218
|
+
"staged_writes": True,
|
|
219
|
+
"async_native": False,
|
|
220
|
+
"read_only_enforced": True,
|
|
221
|
+
}
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any, Iterable
|
|
4
|
+
|
|
5
|
+
from tigrbl_base._base import EngineSessionBase
|
|
6
|
+
from tigrbl_core._spec.engine_session_spec import EngineSessionSpec
|
|
7
|
+
|
|
8
|
+
from .engine import S3Engine
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _as_bytes(data: bytes | bytearray | memoryview) -> bytes:
|
|
12
|
+
if not isinstance(data, (bytes, bytearray, memoryview)):
|
|
13
|
+
raise TypeError("object data must be bytes-like")
|
|
14
|
+
return bytes(data)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class S3Session(EngineSessionBase):
|
|
18
|
+
def __init__(self, engine: S3Engine, spec: EngineSessionSpec | None = None) -> None:
|
|
19
|
+
super().__init__(spec)
|
|
20
|
+
self.engine = engine
|
|
21
|
+
self._closed = False
|
|
22
|
+
self._puts: dict[str, bytes] = {}
|
|
23
|
+
self._deletes: set[str] = set()
|
|
24
|
+
|
|
25
|
+
def put_object(self, key: str, data: bytes | bytearray | memoryview) -> str:
|
|
26
|
+
self._require_write()
|
|
27
|
+
physical_key = self.engine.object_key(key)
|
|
28
|
+
self._put_key(physical_key, _as_bytes(data))
|
|
29
|
+
return key
|
|
30
|
+
|
|
31
|
+
def get_object(self, key: str) -> bytes:
|
|
32
|
+
return self._get_key(self.engine.object_key(key))
|
|
33
|
+
|
|
34
|
+
def delete_object(self, key: str) -> bool:
|
|
35
|
+
self._require_write()
|
|
36
|
+
physical_key = self.engine.object_key(key)
|
|
37
|
+
existed = self._key_exists(physical_key)
|
|
38
|
+
self._delete_key(physical_key)
|
|
39
|
+
return existed
|
|
40
|
+
|
|
41
|
+
def object_exists(self, key: str) -> bool:
|
|
42
|
+
return self._key_exists(self.engine.object_key(key))
|
|
43
|
+
|
|
44
|
+
def head_object(self, key: str) -> dict[str, Any]:
|
|
45
|
+
physical_key = self.engine.object_key(key)
|
|
46
|
+
if physical_key in self._puts:
|
|
47
|
+
return {
|
|
48
|
+
"ContentLength": len(self._puts[physical_key]),
|
|
49
|
+
"staged": True,
|
|
50
|
+
}
|
|
51
|
+
if physical_key in self._deletes:
|
|
52
|
+
raise FileNotFoundError(key)
|
|
53
|
+
return self.engine.head_key(physical_key)
|
|
54
|
+
|
|
55
|
+
def list_objects(self, prefix: str = "") -> list[str]:
|
|
56
|
+
keys = set(self.engine.list_keys(prefix))
|
|
57
|
+
marker = f"{self.engine.prefix}/" if self.engine.prefix else ""
|
|
58
|
+
for key in self._puts:
|
|
59
|
+
keys.add(key[len(marker) :] if marker and key.startswith(marker) else key)
|
|
60
|
+
for key in self._deletes:
|
|
61
|
+
logical = key[len(marker) :] if marker and key.startswith(marker) else key
|
|
62
|
+
keys.discard(logical)
|
|
63
|
+
if prefix:
|
|
64
|
+
keys = {key for key in keys if key.startswith(prefix.strip("/"))}
|
|
65
|
+
return sorted(keys)
|
|
66
|
+
|
|
67
|
+
def put_cas(self, data: bytes | bytearray | memoryview) -> str:
|
|
68
|
+
self._require_write()
|
|
69
|
+
payload = _as_bytes(data)
|
|
70
|
+
address = self.engine.cas_address(payload)
|
|
71
|
+
key = self.engine.cas_key(address)
|
|
72
|
+
if not self._key_exists(key):
|
|
73
|
+
self._put_key(key, payload)
|
|
74
|
+
return address
|
|
75
|
+
|
|
76
|
+
def get_cas(self, address: str) -> bytes:
|
|
77
|
+
payload = self._get_key(self.engine.cas_key(address))
|
|
78
|
+
self.engine.verify_cas_payload(address, payload)
|
|
79
|
+
return payload
|
|
80
|
+
|
|
81
|
+
def cas_exists(self, address: str) -> bool:
|
|
82
|
+
return self._key_exists(self.engine.cas_key(address))
|
|
83
|
+
|
|
84
|
+
async def _tx_begin_impl(self) -> None:
|
|
85
|
+
self._require_open_session()
|
|
86
|
+
|
|
87
|
+
async def _tx_commit_impl(self) -> None:
|
|
88
|
+
for key, data in self._puts.items():
|
|
89
|
+
self.engine.put_key(key, data)
|
|
90
|
+
for key in self._deletes:
|
|
91
|
+
self.engine.delete_key(key)
|
|
92
|
+
self._puts.clear()
|
|
93
|
+
self._deletes.clear()
|
|
94
|
+
|
|
95
|
+
async def _tx_rollback_impl(self) -> None:
|
|
96
|
+
self._puts.clear()
|
|
97
|
+
self._deletes.clear()
|
|
98
|
+
|
|
99
|
+
async def _close_impl(self) -> None:
|
|
100
|
+
self._puts.clear()
|
|
101
|
+
self._deletes.clear()
|
|
102
|
+
self._closed = True
|
|
103
|
+
|
|
104
|
+
def _put_key(self, key: str, data: bytes) -> None:
|
|
105
|
+
if self.in_transaction():
|
|
106
|
+
self._puts[key] = data
|
|
107
|
+
self._deletes.discard(key)
|
|
108
|
+
else:
|
|
109
|
+
self.engine.put_key(key, data)
|
|
110
|
+
|
|
111
|
+
def _get_key(self, key: str) -> bytes:
|
|
112
|
+
self._require_open_session()
|
|
113
|
+
if key in self._deletes:
|
|
114
|
+
raise FileNotFoundError(key)
|
|
115
|
+
if key in self._puts:
|
|
116
|
+
return self._puts[key]
|
|
117
|
+
return self.engine.get_key(key)
|
|
118
|
+
|
|
119
|
+
def _delete_key(self, key: str) -> None:
|
|
120
|
+
if self.in_transaction():
|
|
121
|
+
self._puts.pop(key, None)
|
|
122
|
+
self._deletes.add(key)
|
|
123
|
+
else:
|
|
124
|
+
self.engine.delete_key(key)
|
|
125
|
+
|
|
126
|
+
def _key_exists(self, key: str) -> bool:
|
|
127
|
+
self._require_open_session()
|
|
128
|
+
if key in self._deletes:
|
|
129
|
+
return False
|
|
130
|
+
if key in self._puts:
|
|
131
|
+
return True
|
|
132
|
+
return self.engine.key_exists(key)
|
|
133
|
+
|
|
134
|
+
def _require_write(self) -> None:
|
|
135
|
+
self._require_open_session()
|
|
136
|
+
if self.read_only:
|
|
137
|
+
raise RuntimeError("write attempted in read-only engine session")
|
|
138
|
+
self._dirty = True
|
|
139
|
+
|
|
140
|
+
def _require_open_session(self) -> None:
|
|
141
|
+
if self._closed:
|
|
142
|
+
raise RuntimeError("session is closed")
|
|
143
|
+
|
|
144
|
+
def _add_impl(self, obj: Any) -> Any:
|
|
145
|
+
raise TypeError("S3 object sessions do not support ORM add()")
|
|
146
|
+
|
|
147
|
+
async def _delete_impl(self, obj: Any) -> None:
|
|
148
|
+
raise TypeError("S3 object sessions do not support ORM delete()")
|
|
149
|
+
|
|
150
|
+
async def _get_impl(self, model: type, ident: Any) -> Any | None:
|
|
151
|
+
raise TypeError("use get_object() for S3 object sessions")
|
|
152
|
+
|
|
153
|
+
async def _execute_impl(self, stmt: Any) -> Any:
|
|
154
|
+
raise TypeError("S3 object sessions do not execute SQL statements")
|
|
155
|
+
|
|
156
|
+
async def _executeloop_impl(self, statements: Iterable[Any]) -> Any:
|
|
157
|
+
raise TypeError("S3 object sessions do not execute SQL statements")
|
|
158
|
+
|
|
159
|
+
async def _executemany_impl(self, stmt: Any, parameter_sets: Any) -> Any:
|
|
160
|
+
raise TypeError("S3 object sessions do not execute SQL statements")
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from io import BytesIO
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
import pytest
|
|
7
|
+
|
|
8
|
+
from tigrbl_core._spec.engine_session_spec import EngineSessionSpec
|
|
9
|
+
from tigrbl import EngineSpec
|
|
10
|
+
from tigrbl_base._base import EngineBase
|
|
11
|
+
from tigrbl_engine_s3 import register, s3_capabilities, s3_engine
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class FakeS3Client:
|
|
15
|
+
def __init__(self) -> None:
|
|
16
|
+
self.objects: dict[tuple[str, str], bytes] = {}
|
|
17
|
+
self.put_calls: list[dict[str, Any]] = []
|
|
18
|
+
|
|
19
|
+
def put_object(self, **kwargs: Any) -> dict[str, str]:
|
|
20
|
+
self.put_calls.append(kwargs)
|
|
21
|
+
self.objects[(kwargs["Bucket"], kwargs["Key"])] = bytes(kwargs["Body"])
|
|
22
|
+
return {"ETag": "fake"}
|
|
23
|
+
|
|
24
|
+
def get_object(self, *, Bucket: str, Key: str) -> dict[str, Any]:
|
|
25
|
+
return {"Body": BytesIO(self.objects[(Bucket, Key)])}
|
|
26
|
+
|
|
27
|
+
def delete_object(self, *, Bucket: str, Key: str) -> dict[str, Any]:
|
|
28
|
+
self.objects.pop((Bucket, Key), None)
|
|
29
|
+
return {}
|
|
30
|
+
|
|
31
|
+
def head_object(self, *, Bucket: str, Key: str) -> dict[str, Any]:
|
|
32
|
+
payload = self.objects[(Bucket, Key)]
|
|
33
|
+
return {"ContentLength": len(payload), "ETag": "fake"}
|
|
34
|
+
|
|
35
|
+
def list_objects_v2(self, **kwargs: Any) -> dict[str, Any]:
|
|
36
|
+
bucket = kwargs["Bucket"]
|
|
37
|
+
prefix = kwargs.get("Prefix", "")
|
|
38
|
+
return {
|
|
39
|
+
"Contents": [
|
|
40
|
+
{"Key": key}
|
|
41
|
+
for (known_bucket, key), _ in sorted(self.objects.items())
|
|
42
|
+
if known_bucket == bucket and key.startswith(prefix)
|
|
43
|
+
],
|
|
44
|
+
"IsTruncated": False,
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def make_engine(*, cas: bool = True):
|
|
49
|
+
client = FakeS3Client()
|
|
50
|
+
engine, make_session = s3_engine(
|
|
51
|
+
mapping={
|
|
52
|
+
"bucket": "assets",
|
|
53
|
+
"prefix": "tenant-a",
|
|
54
|
+
"cas": cas,
|
|
55
|
+
"client": client,
|
|
56
|
+
"put_options": {"ServerSideEncryption": "AES256"},
|
|
57
|
+
}
|
|
58
|
+
)
|
|
59
|
+
return client, engine, make_session
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def test_object_crud_listing_options_and_capabilities() -> None:
|
|
63
|
+
client, engine, make_session = make_engine()
|
|
64
|
+
session = make_session()
|
|
65
|
+
|
|
66
|
+
assert session.put_object("docs/readme.txt", b"hello") == "docs/readme.txt"
|
|
67
|
+
assert session.get_object("docs/readme.txt") == b"hello"
|
|
68
|
+
assert session.object_exists("docs/readme.txt")
|
|
69
|
+
assert session.head_object("docs/readme.txt")["ContentLength"] == 5
|
|
70
|
+
assert session.list_objects("docs") == ["docs/readme.txt"]
|
|
71
|
+
assert engine.object_key("docs/readme.txt") == "tenant-a/docs/readme.txt"
|
|
72
|
+
assert client.put_calls[0]["ServerSideEncryption"] == "AES256"
|
|
73
|
+
assert session.delete_object("docs/readme.txt")
|
|
74
|
+
assert not session.object_exists("docs/readme.txt")
|
|
75
|
+
assert s3_capabilities()["direct_s3_api"] is True
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def test_registered_engine_builds_from_engine_spec() -> None:
|
|
79
|
+
client = FakeS3Client()
|
|
80
|
+
register()
|
|
81
|
+
engine, make_session = EngineSpec.from_any(
|
|
82
|
+
{"kind": "s3", "bucket": "assets", "client": client, "cas": True}
|
|
83
|
+
).build()
|
|
84
|
+
assert isinstance(engine, EngineBase)
|
|
85
|
+
assert engine.to_provider() is engine
|
|
86
|
+
assert engine.spec is not None
|
|
87
|
+
assert engine.bucket == "assets"
|
|
88
|
+
assert make_session().put_cas(b"payload").startswith("sha256:")
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
@pytest.mark.asyncio
|
|
92
|
+
async def test_engine_base_batch_surface_fails_closed() -> None:
|
|
93
|
+
_, engine, _ = make_engine()
|
|
94
|
+
with pytest.raises(TypeError, match="do not execute SQL"):
|
|
95
|
+
await engine.executeloop([])
|
|
96
|
+
with pytest.raises(TypeError, match="do not execute SQL"):
|
|
97
|
+
await engine.executemany("INSERT", [])
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def test_cas_is_deterministic_deduplicated_and_verified() -> None:
|
|
101
|
+
client, engine, make_session = make_engine()
|
|
102
|
+
session = make_session()
|
|
103
|
+
|
|
104
|
+
address = session.put_cas(b"same payload")
|
|
105
|
+
assert address == session.put_cas(b"same payload")
|
|
106
|
+
assert len(client.put_calls) == 1
|
|
107
|
+
assert session.cas_exists(address)
|
|
108
|
+
assert session.get_cas(address) == b"same payload"
|
|
109
|
+
|
|
110
|
+
client.objects[(engine.bucket, engine.cas_key(address))] = b"corrupt"
|
|
111
|
+
with pytest.raises(ValueError, match="integrity check failed"):
|
|
112
|
+
session.get_cas(address)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
@pytest.mark.asyncio
|
|
116
|
+
async def test_transaction_stages_commit_and_rollback() -> None:
|
|
117
|
+
client, _, make_session = make_engine()
|
|
118
|
+
session = make_session()
|
|
119
|
+
await session.begin()
|
|
120
|
+
session.put_object("pending.bin", b"pending")
|
|
121
|
+
assert session.get_object("pending.bin") == b"pending"
|
|
122
|
+
assert client.objects == {}
|
|
123
|
+
await session.rollback()
|
|
124
|
+
assert not session.object_exists("pending.bin")
|
|
125
|
+
|
|
126
|
+
await session.begin()
|
|
127
|
+
session.put_object("committed.bin", b"committed")
|
|
128
|
+
await session.commit()
|
|
129
|
+
assert any(key.endswith("committed.bin") for _, key in client.objects)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
@pytest.mark.asyncio
|
|
133
|
+
async def test_read_only_invalid_keys_disabled_cas_and_closed_session() -> None:
|
|
134
|
+
_, engine, make_session = make_engine(cas=False)
|
|
135
|
+
readonly_session = make_session(EngineSessionSpec(read_only=True))
|
|
136
|
+
with pytest.raises(RuntimeError, match="read-only"):
|
|
137
|
+
readonly_session.put_object("blocked", b"data")
|
|
138
|
+
with pytest.raises(RuntimeError, match="not enabled"):
|
|
139
|
+
engine.cas_address(b"data")
|
|
140
|
+
for key in ("/absolute", "a//b", "../escape", "a/../escape", r"a\b"):
|
|
141
|
+
with pytest.raises(ValueError):
|
|
142
|
+
engine.object_key(key)
|
|
143
|
+
|
|
144
|
+
session = make_session()
|
|
145
|
+
await session.close()
|
|
146
|
+
with pytest.raises(RuntimeError, match="closed"):
|
|
147
|
+
session.object_exists("anything")
|