dsw-storage 4.27.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dsw/storage/__init__.py +4 -0
- dsw/storage/build_info.py +17 -0
- dsw/storage/py.typed +0 -0
- dsw/storage/s3storage.py +183 -0
- dsw_storage-4.27.0.dist-info/METADATA +44 -0
- dsw_storage-4.27.0.dist-info/RECORD +7 -0
- dsw_storage-4.27.0.dist-info/WHEEL +4 -0
dsw/storage/__init__.py
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# Generated file
|
|
2
|
+
# - do not overwrite
|
|
3
|
+
# - do not include in git
|
|
4
|
+
from collections import namedtuple
|
|
5
|
+
|
|
6
|
+
BuildInfo = namedtuple(
|
|
7
|
+
'BuildInfo',
|
|
8
|
+
['version', 'built_at', 'sha', 'branch', 'tag'],
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
BUILD_INFO = BuildInfo(
|
|
12
|
+
version='v4.27.0~8ec71bd',
|
|
13
|
+
built_at='2026-02-03 08:44:25Z',
|
|
14
|
+
sha='8ec71bd85dfbea66adedb6590f7d76ae5143bbaa',
|
|
15
|
+
branch='HEAD',
|
|
16
|
+
tag='v4.27.0',
|
|
17
|
+
)
|
dsw/storage/py.typed
ADDED
|
File without changes
|
dsw/storage/s3storage.py
ADDED
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
import contextlib
|
|
2
|
+
import io
|
|
3
|
+
import logging
|
|
4
|
+
import pathlib
|
|
5
|
+
import tempfile
|
|
6
|
+
|
|
7
|
+
import minio
|
|
8
|
+
import minio.error
|
|
9
|
+
import tenacity
|
|
10
|
+
|
|
11
|
+
from dsw.config.model import S3Config
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
LOG = logging.getLogger(__name__)
|
|
15
|
+
|
|
16
|
+
DOCUMENTS_DIR = 'documents'
|
|
17
|
+
|
|
18
|
+
RETRY_S3_MULTIPLIER = 0.5
|
|
19
|
+
RETRY_S3_TRIES = 3
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@contextlib.contextmanager
|
|
23
|
+
def temp_binary_file(data: bytes):
|
|
24
|
+
with tempfile.TemporaryFile() as file:
|
|
25
|
+
file.write(data)
|
|
26
|
+
file.seek(0)
|
|
27
|
+
yield file
|
|
28
|
+
file.close()
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class S3Storage:
|
|
32
|
+
|
|
33
|
+
@staticmethod
|
|
34
|
+
def _get_endpoint(url: str):
|
|
35
|
+
parts = url.split('://', maxsplit=1)
|
|
36
|
+
return parts[0] if len(parts) == 1 else parts[1]
|
|
37
|
+
|
|
38
|
+
def __init__(self, *, cfg: S3Config, multi_tenant: bool):
|
|
39
|
+
self.cfg = cfg
|
|
40
|
+
self.multi_tenant = multi_tenant
|
|
41
|
+
self.client = minio.Minio(
|
|
42
|
+
endpoint=self._get_endpoint(self.cfg.url),
|
|
43
|
+
access_key=self.cfg.username,
|
|
44
|
+
secret_key=self.cfg.password,
|
|
45
|
+
secure=self.cfg.url.startswith('https://'),
|
|
46
|
+
region=self.cfg.region,
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
@property
|
|
50
|
+
def identification(self) -> str:
|
|
51
|
+
return f'{self.cfg.url}/{self.cfg.bucket}'
|
|
52
|
+
|
|
53
|
+
@tenacity.retry(
|
|
54
|
+
reraise=True,
|
|
55
|
+
wait=tenacity.wait_exponential(multiplier=RETRY_S3_MULTIPLIER),
|
|
56
|
+
stop=tenacity.stop_after_attempt(RETRY_S3_TRIES),
|
|
57
|
+
before=tenacity.before_log(LOG, logging.DEBUG),
|
|
58
|
+
after=tenacity.after_log(LOG, logging.DEBUG),
|
|
59
|
+
)
|
|
60
|
+
def ensure_bucket(self):
|
|
61
|
+
found = self.client.bucket_exists(self.cfg.bucket)
|
|
62
|
+
if not found:
|
|
63
|
+
self.client.make_bucket(self.cfg.bucket)
|
|
64
|
+
|
|
65
|
+
@tenacity.retry(
|
|
66
|
+
reraise=True,
|
|
67
|
+
wait=tenacity.wait_exponential(multiplier=RETRY_S3_MULTIPLIER),
|
|
68
|
+
stop=tenacity.stop_after_attempt(RETRY_S3_TRIES),
|
|
69
|
+
before=tenacity.before_log(LOG, logging.DEBUG),
|
|
70
|
+
after=tenacity.after_log(LOG, logging.DEBUG),
|
|
71
|
+
)
|
|
72
|
+
def store_document(self, *, tenant_uuid: str, file_name: str,
|
|
73
|
+
content_type: str, data: bytes,
|
|
74
|
+
metadata: dict | None = None):
|
|
75
|
+
object_name = f'{DOCUMENTS_DIR}/{file_name}'
|
|
76
|
+
if self.multi_tenant:
|
|
77
|
+
object_name = f'{tenant_uuid}/{object_name}'
|
|
78
|
+
with temp_binary_file(data=data) as file:
|
|
79
|
+
self.client.put_object(
|
|
80
|
+
bucket_name=self.cfg.bucket,
|
|
81
|
+
object_name=object_name,
|
|
82
|
+
data=file,
|
|
83
|
+
length=len(data),
|
|
84
|
+
content_type=content_type,
|
|
85
|
+
metadata=metadata,
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
@tenacity.retry(
|
|
89
|
+
reraise=True,
|
|
90
|
+
wait=tenacity.wait_exponential(multiplier=RETRY_S3_MULTIPLIER),
|
|
91
|
+
stop=tenacity.stop_after_attempt(RETRY_S3_TRIES),
|
|
92
|
+
before=tenacity.before_log(LOG, logging.DEBUG),
|
|
93
|
+
after=tenacity.after_log(LOG, logging.DEBUG),
|
|
94
|
+
)
|
|
95
|
+
def download_project_file(self, *, tenant_uuid: str, project_uuid: str,
|
|
96
|
+
file_uuid: str, target_path: pathlib.Path) -> bool:
|
|
97
|
+
return self._download_file(
|
|
98
|
+
tenant_uuid=tenant_uuid,
|
|
99
|
+
file_name=f'project-files/{project_uuid}/{file_uuid}',
|
|
100
|
+
target_path=target_path,
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
@tenacity.retry(
|
|
104
|
+
reraise=True,
|
|
105
|
+
wait=tenacity.wait_exponential(multiplier=RETRY_S3_MULTIPLIER),
|
|
106
|
+
stop=tenacity.stop_after_attempt(RETRY_S3_TRIES),
|
|
107
|
+
before=tenacity.before_log(LOG, logging.DEBUG),
|
|
108
|
+
after=tenacity.after_log(LOG, logging.DEBUG),
|
|
109
|
+
)
|
|
110
|
+
def download_template_asset(self, *, tenant_uuid: str, template_id: str,
|
|
111
|
+
file_name: str, target_path: pathlib.Path) -> bool:
|
|
112
|
+
return self._download_file(
|
|
113
|
+
tenant_uuid=tenant_uuid,
|
|
114
|
+
file_name=f'templates/{template_id}/{file_name}',
|
|
115
|
+
target_path=target_path,
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
@tenacity.retry(
|
|
119
|
+
reraise=True,
|
|
120
|
+
wait=tenacity.wait_exponential(multiplier=RETRY_S3_MULTIPLIER),
|
|
121
|
+
stop=tenacity.stop_after_attempt(RETRY_S3_TRIES),
|
|
122
|
+
before=tenacity.before_log(LOG, logging.DEBUG),
|
|
123
|
+
after=tenacity.after_log(LOG, logging.DEBUG),
|
|
124
|
+
)
|
|
125
|
+
def download_locale(self, *, tenant_uuid: str, locale_uuid: str,
|
|
126
|
+
file_name: str, target_path: pathlib.Path) -> bool:
|
|
127
|
+
return self._download_file(
|
|
128
|
+
tenant_uuid=tenant_uuid,
|
|
129
|
+
file_name=f'locales/{locale_uuid}/{file_name}',
|
|
130
|
+
target_path=target_path,
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
@tenacity.retry(
|
|
134
|
+
reraise=True,
|
|
135
|
+
wait=tenacity.wait_exponential(multiplier=RETRY_S3_MULTIPLIER),
|
|
136
|
+
stop=tenacity.stop_after_attempt(RETRY_S3_TRIES),
|
|
137
|
+
before=tenacity.before_log(LOG, logging.DEBUG),
|
|
138
|
+
after=tenacity.after_log(LOG, logging.DEBUG),
|
|
139
|
+
)
|
|
140
|
+
def _download_file(self, *, tenant_uuid: str, file_name: str,
|
|
141
|
+
target_path: pathlib.Path) -> bool:
|
|
142
|
+
if self.multi_tenant:
|
|
143
|
+
file_name = f'{tenant_uuid}/{file_name}'
|
|
144
|
+
try:
|
|
145
|
+
self.client.fget_object(
|
|
146
|
+
bucket_name=self.cfg.bucket,
|
|
147
|
+
object_name=file_name,
|
|
148
|
+
file_path=str(target_path),
|
|
149
|
+
)
|
|
150
|
+
except minio.error.S3Error as e:
|
|
151
|
+
if e.code != 'NoSuchKey':
|
|
152
|
+
raise e
|
|
153
|
+
return False
|
|
154
|
+
return True
|
|
155
|
+
|
|
156
|
+
@tenacity.retry(
|
|
157
|
+
reraise=True,
|
|
158
|
+
wait=tenacity.wait_exponential(multiplier=RETRY_S3_MULTIPLIER),
|
|
159
|
+
stop=tenacity.stop_after_attempt(RETRY_S3_TRIES),
|
|
160
|
+
before=tenacity.before_log(LOG, logging.DEBUG),
|
|
161
|
+
after=tenacity.after_log(LOG, logging.DEBUG),
|
|
162
|
+
)
|
|
163
|
+
def store_object(self, *, tenant_uuid: str, object_name: str,
|
|
164
|
+
content_type: str, data: bytes,
|
|
165
|
+
metadata: dict | None = None):
|
|
166
|
+
if self.multi_tenant:
|
|
167
|
+
object_name = f'{tenant_uuid}/{object_name}'
|
|
168
|
+
with io.BytesIO(data) as file:
|
|
169
|
+
self.client.put_object(
|
|
170
|
+
bucket_name=self.cfg.bucket,
|
|
171
|
+
object_name=object_name,
|
|
172
|
+
data=file,
|
|
173
|
+
length=len(data),
|
|
174
|
+
content_type=content_type,
|
|
175
|
+
metadata=metadata,
|
|
176
|
+
)
|
|
177
|
+
|
|
178
|
+
def make_path(self, fragments: list[str], tenant_uuid: str) -> str:
|
|
179
|
+
lst = []
|
|
180
|
+
if self.multi_tenant:
|
|
181
|
+
lst.append(tenant_uuid)
|
|
182
|
+
lst.extend(fragments)
|
|
183
|
+
return '/'.join(lst)
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: dsw-storage
|
|
3
|
+
Version: 4.27.0
|
|
4
|
+
Summary: Library for managing DSW S3 storage
|
|
5
|
+
Keywords: dsw,s3,bucket,storage
|
|
6
|
+
Author: Marek Suchánek
|
|
7
|
+
Author-email: Marek Suchánek <marek.suchanek@ds-wizard.org>
|
|
8
|
+
License: Apache License 2.0
|
|
9
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
10
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
11
|
+
Classifier: Programming Language :: Python
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
14
|
+
Classifier: Topic :: Communications :: File Sharing
|
|
15
|
+
Classifier: Topic :: Utilities
|
|
16
|
+
Requires-Dist: minio
|
|
17
|
+
Requires-Dist: tenacity
|
|
18
|
+
Requires-Dist: dsw-config==4.27.0
|
|
19
|
+
Requires-Python: >=3.12, <4
|
|
20
|
+
Project-URL: Homepage, https://ds-wizard.org
|
|
21
|
+
Project-URL: Repository, https://github.com/ds-wizard/engine-tools
|
|
22
|
+
Project-URL: Documentation, https://guide.ds-wizard.org
|
|
23
|
+
Project-URL: Issues, https://github.com/ds-wizard/ds-wizard/issues
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
|
|
26
|
+
# Data Stewardship Wizard: Storage
|
|
27
|
+
|
|
28
|
+
[](https://github.com/ds-wizard/engine-tools/releases)
|
|
29
|
+
[](https://pypi.org/project/dsw-storage/)
|
|
30
|
+
[](LICENSE)
|
|
31
|
+
[](https://bestpractices.coreinfrastructure.org/projects/4975)
|
|
32
|
+
[](https://python.org)
|
|
33
|
+
|
|
34
|
+
*Library for working with object storage of DSW*
|
|
35
|
+
|
|
36
|
+
## Usage
|
|
37
|
+
|
|
38
|
+
Currently, this library is intended for internal use of DSW tooling only.
|
|
39
|
+
Enhancements for use in custom scripts are planned for future development.
|
|
40
|
+
|
|
41
|
+
## License
|
|
42
|
+
|
|
43
|
+
This project is licensed under the Apache License v2.0 - see the
|
|
44
|
+
[LICENSE](LICENSE) file for more details.
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
dsw/storage/__init__.py,sha256=4aD999C5EEm9TJ0Ew5FfPcu_zY550B5ThcDxn_6sU_U,59
|
|
2
|
+
dsw/storage/build_info.py,sha256=gKufnDA6bBPf9hTSXt7kXE_nO5_DgqwnpvuNUFUT8Dg,381
|
|
3
|
+
dsw/storage/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
4
|
+
dsw/storage/s3storage.py,sha256=tgewTEwJVDufXyRBT_K5Gfs506-2fX54a5FgZVqNxEI,6326
|
|
5
|
+
dsw_storage-4.27.0.dist-info/WHEEL,sha256=fAguSjoiATBe7TNBkJwOjyL1Tt4wwiaQGtNtjRPNMQA,80
|
|
6
|
+
dsw_storage-4.27.0.dist-info/METADATA,sha256=kn_GR633_ehezFucLBNWZmni32_K_8AYSMq9UVGtTZk,1919
|
|
7
|
+
dsw_storage-4.27.0.dist-info/RECORD,,
|