python3-commons 0.24.6__py3-none-any.whl → 0.25.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- python3_commons/api_client.py +15 -3
- python3_commons/audit.py +17 -5
- python3_commons/conf.py +22 -8
- python3_commons/object_storage.py +147 -113
- python3_commons-0.25.0.dist-info/METADATA +434 -0
- {python3_commons-0.24.6.dist-info → python3_commons-0.25.0.dist-info}/RECORD +10 -10
- python3_commons-0.24.6.dist-info/METADATA +0 -310
- {python3_commons-0.24.6.dist-info → python3_commons-0.25.0.dist-info}/WHEEL +0 -0
- {python3_commons-0.24.6.dist-info → python3_commons-0.25.0.dist-info}/licenses/AUTHORS.rst +0 -0
- {python3_commons-0.24.6.dist-info → python3_commons-0.25.0.dist-info}/licenses/LICENSE +0 -0
- {python3_commons-0.24.6.dist-info → python3_commons-0.25.0.dist-info}/top_level.txt +0 -0
python3_commons/api_client.py
CHANGED
|
@@ -17,7 +17,7 @@ except ImportError as e:
|
|
|
17
17
|
raise RuntimeError(msg) from e
|
|
18
18
|
|
|
19
19
|
from python3_commons import audit
|
|
20
|
-
from python3_commons.conf import s3_settings
|
|
20
|
+
from python3_commons.conf import api_client_settings, s3_settings
|
|
21
21
|
from python3_commons.helpers import request_to_curl
|
|
22
22
|
from python3_commons.serializers.json import CustomJSONEncoder
|
|
23
23
|
|
|
@@ -34,9 +34,15 @@ async def _store_response_for_audit(
|
|
|
34
34
|
date_path = now.strftime('%Y/%m/%d')
|
|
35
35
|
timestamp = now.strftime('%H%M%S_%f')
|
|
36
36
|
|
|
37
|
+
if api_client_settings.audit_path:
|
|
38
|
+
audit_root = api_client_settings.audit_path.strip('/')
|
|
39
|
+
path = f'{audit_root}/{date_path}/{audit_name}/{uri_path}/{method}_{timestamp}_{request_id}_response.txt'
|
|
40
|
+
else:
|
|
41
|
+
path = f'{date_path}/{audit_name}/{uri_path}/{method}_{timestamp}_{request_id}_response.txt'
|
|
42
|
+
|
|
37
43
|
await audit.write_audit_data(
|
|
38
44
|
s3_settings,
|
|
39
|
-
|
|
45
|
+
path,
|
|
40
46
|
response_text.encode('utf-8'),
|
|
41
47
|
)
|
|
42
48
|
|
|
@@ -75,9 +81,15 @@ async def request(
|
|
|
75
81
|
)
|
|
76
82
|
|
|
77
83
|
if curl_request:
|
|
84
|
+
if api_client_settings.audit_path:
|
|
85
|
+
audit_root = api_client_settings.audit_path.strip('/')
|
|
86
|
+
path = f'{audit_root}/{date_path}/{audit_name}/{uri_path}/{method}_{timestamp}_{request_id}_request.txt'
|
|
87
|
+
else:
|
|
88
|
+
path = f'{date_path}/{audit_name}/{uri_path}/{method}_{timestamp}_{request_id}_request.txt'
|
|
89
|
+
|
|
78
90
|
await audit.write_audit_data(
|
|
79
91
|
s3_settings,
|
|
80
|
-
|
|
92
|
+
path,
|
|
81
93
|
curl_request.encode('utf-8'),
|
|
82
94
|
)
|
|
83
95
|
|
python3_commons/audit.py
CHANGED
|
@@ -14,7 +14,7 @@ except ImportError as e:
|
|
|
14
14
|
raise RuntimeError(msg) from e
|
|
15
15
|
|
|
16
16
|
from python3_commons import object_storage
|
|
17
|
-
from python3_commons.conf import s3_settings
|
|
17
|
+
from python3_commons.conf import api_client_settings, object_storage_settings, s3_settings
|
|
18
18
|
|
|
19
19
|
if TYPE_CHECKING:
|
|
20
20
|
from zeep import AsyncClient
|
|
@@ -25,16 +25,28 @@ if TYPE_CHECKING:
|
|
|
25
25
|
logger = logging.getLogger(__name__)
|
|
26
26
|
|
|
27
27
|
|
|
28
|
-
async def write_audit_data(settings: S3Settings, key: str, data: bytes) -> None:
|
|
29
|
-
if settings.
|
|
28
|
+
async def write_audit_data(settings: S3Settings = s3_settings, key: str = '', data: bytes = b'') -> None:
|
|
29
|
+
if settings.secret_access_key or api_client_settings.audit_path or object_storage_settings.work_path:
|
|
30
30
|
try:
|
|
31
|
-
|
|
31
|
+
if api_client_settings.audit_path and key.startswith(api_client_settings.audit_path.strip('/')):
|
|
32
|
+
work_path = api_client_settings.audit_path
|
|
33
|
+
path = key
|
|
34
|
+
elif api_client_settings.audit_path:
|
|
35
|
+
work_path = api_client_settings.audit_path
|
|
36
|
+
path = key.lstrip('/')
|
|
37
|
+
elif object_storage_settings.work_path:
|
|
38
|
+
work_path = object_storage_settings.work_path
|
|
39
|
+
path = f'audit/{key.lstrip("/")}'
|
|
40
|
+
else:
|
|
41
|
+
work_path = 'audit'
|
|
42
|
+
path = key.lstrip('/')
|
|
43
|
+
await object_storage.put_object(work_path=work_path, path=path, data=io.BytesIO(data), length=len(data))
|
|
32
44
|
except Exception:
|
|
33
45
|
logger.exception('Failed storing object in storage.')
|
|
34
46
|
else:
|
|
35
47
|
logger.debug('Stored object in storage: %s', key)
|
|
36
48
|
else:
|
|
37
|
-
logger.debug('
|
|
49
|
+
logger.debug('Storage is not configured, not storing object in storage: %s', key)
|
|
38
50
|
|
|
39
51
|
|
|
40
52
|
class ZeepAuditPlugin(Plugin):
|
python3_commons/conf.py
CHANGED
|
@@ -86,17 +86,31 @@ class DBSettings(BaseSettings):
|
|
|
86
86
|
|
|
87
87
|
|
|
88
88
|
class S3Settings(BaseSettings):
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
89
|
+
model_config = SettingsConfigDict(env_prefix='S3_')
|
|
90
|
+
|
|
91
|
+
access_key_id: SecretStr | None = None
|
|
92
|
+
addressing_style: Literal['path', 'virtual'] = 'virtual'
|
|
93
|
+
allow_http: bool = False
|
|
94
|
+
cert_verify: bool = True
|
|
95
|
+
region: str | None = None
|
|
96
|
+
secret_access_key: SecretStr | None = None
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class ObjectStorageSettings(BaseSettings):
|
|
100
|
+
model_config = SettingsConfigDict(env_prefix='OBJECT_STORAGE_')
|
|
101
|
+
|
|
102
|
+
work_path: str | None = None
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
class ApiClientSettings(BaseSettings):
|
|
106
|
+
model_config = SettingsConfigDict(env_prefix='API_CLIENT_')
|
|
107
|
+
|
|
108
|
+
audit_path: str | None = None
|
|
97
109
|
|
|
98
110
|
|
|
99
111
|
settings = CommonSettings()
|
|
100
112
|
valkey_settings = ValkeySettings()
|
|
101
113
|
db_settings = DBSettings()
|
|
114
|
+
api_client_settings = ApiClientSettings()
|
|
102
115
|
s3_settings = S3Settings()
|
|
116
|
+
object_storage_settings = ObjectStorageSettings()
|
|
@@ -1,28 +1,22 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
import io
|
|
3
4
|
import logging
|
|
4
5
|
import threading
|
|
5
6
|
from contextlib import asynccontextmanager
|
|
7
|
+
from datetime import datetime
|
|
6
8
|
from typing import TYPE_CHECKING
|
|
9
|
+
from urllib.parse import urlparse
|
|
7
10
|
|
|
8
11
|
try:
|
|
9
|
-
import
|
|
10
|
-
from botocore.config import Config
|
|
11
|
-
from object_storage_client import ObjectStorageClient
|
|
12
|
+
from object_storage_client import ByteStream, ObjectStorageClient
|
|
12
13
|
except ImportError as e:
|
|
13
14
|
msg = 'Install python3-commons[object-storage] to use this feature'
|
|
14
15
|
raise RuntimeError(msg) from e
|
|
15
16
|
|
|
16
17
|
if TYPE_CHECKING:
|
|
17
|
-
import
|
|
18
|
-
from collections.abc import AsyncGenerator, Iterable, Mapping, Sequence
|
|
19
|
-
from datetime import datetime
|
|
18
|
+
from collections.abc import AsyncGenerator, AsyncIterator, Iterable, Mapping, Sequence
|
|
20
19
|
|
|
21
|
-
from aiobotocore.response import StreamingBody
|
|
22
|
-
from types_aiobotocore_s3.client import S3Client
|
|
23
|
-
|
|
24
|
-
from python3_commons.conf import S3Settings, s3_settings
|
|
25
|
-
from python3_commons.helpers import SingletonMeta
|
|
26
20
|
|
|
27
21
|
logger = logging.getLogger(__name__)
|
|
28
22
|
_CLIENT: ObjectStorageClient | None = None
|
|
@@ -40,174 +34,214 @@ def get_client() -> ObjectStorageClient:
|
|
|
40
34
|
return _CLIENT
|
|
41
35
|
|
|
42
36
|
|
|
43
|
-
class
|
|
44
|
-
def __init__(self,
|
|
45
|
-
self.
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
37
|
+
class ObjectStorageStream:
|
|
38
|
+
def __init__(self, stream: ByteStream) -> None:
|
|
39
|
+
self._stream = stream
|
|
40
|
+
|
|
41
|
+
async def read(self) -> bytes:
|
|
42
|
+
chunks = []
|
|
43
|
+
|
|
44
|
+
while chunk := await self._stream.next():
|
|
45
|
+
chunks.append(chunk)
|
|
46
|
+
|
|
47
|
+
return b''.join(chunks)
|
|
48
|
+
|
|
49
|
+
async def next(self) -> bytes:
|
|
50
|
+
return await self._stream.next()
|
|
51
|
+
|
|
52
|
+
def __aiter__(self) -> AsyncIterator[bytes]:
|
|
53
|
+
return self
|
|
54
|
+
|
|
55
|
+
async def __anext__(self) -> bytes:
|
|
56
|
+
chunk = await self._stream.next()
|
|
57
|
+
|
|
58
|
+
if not chunk:
|
|
59
|
+
raise StopAsyncIteration
|
|
60
|
+
|
|
61
|
+
return chunk
|
|
52
62
|
|
|
53
|
-
if s3_access_key_id := settings.s3_access_key_id:
|
|
54
|
-
config['aws_access_key_id'] = s3_access_key_id.get_secret_value()
|
|
55
63
|
|
|
56
|
-
|
|
57
|
-
|
|
64
|
+
def build_url(work_path: str, path: str = '') -> str:
|
|
65
|
+
path = path.lstrip('/')
|
|
58
66
|
|
|
59
|
-
|
|
67
|
+
if path.startswith(('s3://', 'file://', 'gs://', 'az://')):
|
|
68
|
+
return path
|
|
60
69
|
|
|
61
|
-
|
|
62
|
-
async def get_client(self) -> AsyncGenerator[S3Client]:
|
|
63
|
-
async with self._session.create_client('s3', **self._config) as client:
|
|
64
|
-
yield client
|
|
70
|
+
base = work_path.rstrip('/')
|
|
65
71
|
|
|
72
|
+
if not base.startswith(('s3://', 'file://', 'gs://', 'az://', '/')):
|
|
73
|
+
base = f's3://{base}'
|
|
66
74
|
|
|
67
|
-
|
|
68
|
-
|
|
75
|
+
if not path:
|
|
76
|
+
return base
|
|
69
77
|
|
|
70
|
-
|
|
71
|
-
path = f'{bucket_root[:1] if bucket_root.startswith("/") else bucket_root}/{path}'
|
|
78
|
+
base_no_scheme = base.split('://', 1)[-1].lstrip('/')
|
|
72
79
|
|
|
73
|
-
|
|
80
|
+
if path.startswith(f'{base_no_scheme}/') or path == base_no_scheme:
|
|
81
|
+
scheme = base.split('://', 1)[0] + '://' if '://' in base else ''
|
|
74
82
|
|
|
83
|
+
return f'{scheme}{path}'
|
|
75
84
|
|
|
76
|
-
|
|
85
|
+
parsed = urlparse(base)
|
|
86
|
+
base_path = parsed.path.lstrip('/')
|
|
87
|
+
|
|
88
|
+
if base_path and (path.startswith(f'{base_path}/') or path == base_path):
|
|
89
|
+
prefix_url = f'{parsed.scheme}://{parsed.netloc}' if parsed.scheme else ''
|
|
90
|
+
|
|
91
|
+
return f'{prefix_url}/{path}'
|
|
92
|
+
|
|
93
|
+
return f'{base}/{path}'
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
async def put_object(
|
|
97
|
+
work_path: str,
|
|
98
|
+
path: str = '',
|
|
99
|
+
data: io.BytesIO | bytes | None = None,
|
|
100
|
+
length: int = 0,
|
|
101
|
+
part_size: int = 0,
|
|
102
|
+
) -> str | None:
|
|
77
103
|
try:
|
|
78
104
|
client = get_client()
|
|
79
|
-
data.seek(0)
|
|
80
|
-
absolute_path = get_absolute_path(path)
|
|
81
|
-
url = f's3://{bucket_name}/{absolute_path}'
|
|
82
105
|
|
|
83
|
-
|
|
106
|
+
if isinstance(data, io.BytesIO):
|
|
107
|
+
content = data.getvalue()
|
|
108
|
+
data.seek(0)
|
|
109
|
+
elif isinstance(data, bytes):
|
|
110
|
+
content = data
|
|
111
|
+
elif data is None:
|
|
112
|
+
content = b''
|
|
113
|
+
else:
|
|
114
|
+
content = bytes(data)
|
|
84
115
|
|
|
85
|
-
|
|
116
|
+
url = build_url(work_path, path)
|
|
117
|
+
await client.put_object(url, content)
|
|
118
|
+
logger.debug('Stored object into object storage: %s', url)
|
|
86
119
|
except Exception as e:
|
|
87
|
-
logger.exception('Failed to put object to object storage: %s:%s',
|
|
88
|
-
|
|
120
|
+
logger.exception('Failed to put object to object storage: %s:%s', work_path, path, exc_info=e)
|
|
89
121
|
raise
|
|
90
122
|
|
|
91
|
-
return
|
|
123
|
+
return url
|
|
92
124
|
|
|
93
125
|
|
|
94
126
|
@asynccontextmanager
|
|
95
|
-
async def get_object_stream(
|
|
96
|
-
storage
|
|
97
|
-
|
|
98
|
-
async with storage.get_client() as s3_client:
|
|
99
|
-
logger.debug('Getting object from object storage: %s:%s', bucket_name, path)
|
|
100
|
-
|
|
101
|
-
try:
|
|
102
|
-
response = await s3_client.get_object(Bucket=bucket_name, Key=path)
|
|
127
|
+
async def get_object_stream(work_path: str, path: str = '') -> AsyncGenerator[ObjectStorageStream]:
|
|
128
|
+
logger.debug('Getting object stream from object storage: %s:%s', work_path, path)
|
|
103
129
|
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
130
|
+
try:
|
|
131
|
+
client = get_client()
|
|
132
|
+
url = build_url(work_path, path)
|
|
133
|
+
stream = await client.get_object_stream(url)
|
|
134
|
+
yield ObjectStorageStream(stream)
|
|
135
|
+
except Exception as e:
|
|
136
|
+
logger.exception('Failed getting object from object storage: %s:%s', work_path, path, exc_info=e)
|
|
137
|
+
raise
|
|
110
138
|
|
|
111
139
|
|
|
112
|
-
async def get_object(
|
|
113
|
-
logger.debug('Getting object from object storage: %s:%s',
|
|
140
|
+
async def get_object(work_path: str, path: str = '') -> bytes:
|
|
141
|
+
logger.debug('Getting object from object storage: %s:%s', work_path, path)
|
|
114
142
|
|
|
115
143
|
try:
|
|
116
144
|
client = get_client()
|
|
117
|
-
|
|
118
|
-
url = f's3://{bucket_name}/{absolute_path}'
|
|
145
|
+
url = build_url(work_path, path)
|
|
119
146
|
body = await client.get_object(url)
|
|
120
147
|
except Exception as e:
|
|
121
|
-
logger.exception('Failed getting object from object storage: %s:%s',
|
|
122
|
-
|
|
148
|
+
logger.exception('Failed getting object from object storage: %s:%s', work_path, path, exc_info=e)
|
|
123
149
|
raise
|
|
124
150
|
|
|
125
151
|
return body
|
|
126
152
|
|
|
127
153
|
|
|
128
|
-
async def list_objects(
|
|
129
|
-
|
|
154
|
+
async def list_objects(work_path: str, prefix: str = '', *, recursive: bool = True) -> AsyncGenerator[Mapping]:
|
|
155
|
+
client = get_client()
|
|
156
|
+
url = build_url(work_path, prefix)
|
|
157
|
+
items = await client.list_objects(url)
|
|
158
|
+
prefix_clean = prefix.strip('/')
|
|
130
159
|
|
|
131
|
-
|
|
132
|
-
|
|
160
|
+
for item in items:
|
|
161
|
+
if not recursive and '/' in item:
|
|
162
|
+
continue
|
|
133
163
|
|
|
134
|
-
|
|
164
|
+
rel_key = f'{prefix_clean}/{item}' if prefix_clean else item
|
|
165
|
+
obj_url = build_url(work_path, rel_key)
|
|
135
166
|
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
167
|
+
try:
|
|
168
|
+
meta = await client.get_object_metadata(obj_url)
|
|
169
|
+
last_modified = (
|
|
170
|
+
datetime.fromisoformat(meta['last_modified']) if meta.get('last_modified') else datetime.now()
|
|
171
|
+
)
|
|
172
|
+
size = meta.get('size_bytes', 0)
|
|
173
|
+
etag = meta.get('e_tag')
|
|
174
|
+
except Exception:
|
|
175
|
+
last_modified = datetime.now()
|
|
176
|
+
size = 0
|
|
177
|
+
etag = None
|
|
178
|
+
|
|
179
|
+
yield {
|
|
180
|
+
'Key': rel_key,
|
|
181
|
+
'LastModified': last_modified,
|
|
182
|
+
'Size': size,
|
|
183
|
+
'ETag': etag,
|
|
184
|
+
}
|
|
140
185
|
|
|
141
186
|
|
|
142
187
|
async def get_object_streams(
|
|
143
|
-
|
|
144
|
-
) -> AsyncGenerator[tuple[str, datetime,
|
|
145
|
-
async for obj in list_objects(
|
|
188
|
+
work_path: str, path: str = '', *, recursive: bool = True
|
|
189
|
+
) -> AsyncGenerator[tuple[str, datetime, ObjectStorageStream]]:
|
|
190
|
+
async for obj in list_objects(work_path, path, recursive=recursive):
|
|
146
191
|
object_name = obj['Key']
|
|
147
192
|
last_modified = obj['LastModified']
|
|
148
193
|
|
|
149
|
-
async with get_object_stream(
|
|
194
|
+
async with get_object_stream(work_path, object_name) as stream:
|
|
150
195
|
yield object_name, last_modified, stream
|
|
151
196
|
|
|
152
197
|
|
|
153
198
|
async def get_objects(
|
|
154
|
-
|
|
199
|
+
work_path: str, path: str = '', *, recursive: bool = True
|
|
155
200
|
) -> AsyncGenerator[tuple[str, datetime, bytes]]:
|
|
156
|
-
async for object_name, last_modified, stream in get_object_streams(
|
|
201
|
+
async for object_name, last_modified, stream in get_object_streams(work_path, path, recursive=recursive):
|
|
157
202
|
data = await stream.read()
|
|
158
203
|
|
|
159
204
|
yield object_name, last_modified, data
|
|
160
205
|
|
|
161
206
|
|
|
162
|
-
async def remove_object(
|
|
163
|
-
logger.debug('Removing object from object storage: %s:%s',
|
|
207
|
+
async def remove_object(work_path: str, path: str = '') -> None:
|
|
208
|
+
logger.debug('Removing object from object storage: %s:%s', work_path, path)
|
|
164
209
|
|
|
165
210
|
try:
|
|
166
211
|
client = get_client()
|
|
167
|
-
|
|
168
|
-
url = f's3://{bucket_name}/{absolute_path}'
|
|
212
|
+
url = build_url(work_path, path)
|
|
169
213
|
await client.delete_object(url)
|
|
170
214
|
except Exception as e:
|
|
171
|
-
logger.exception('Failed to remove object from object storage: %s:%s',
|
|
172
|
-
|
|
215
|
+
logger.exception('Failed to remove object from object storage: %s:%s', work_path, path, exc_info=e)
|
|
173
216
|
raise
|
|
174
217
|
|
|
175
218
|
|
|
176
219
|
async def remove_objects(
|
|
177
|
-
|
|
220
|
+
work_path: str, prefix: str | None = None, object_names: Iterable[str] | None = None
|
|
178
221
|
) -> Sequence[Mapping] | None:
|
|
179
|
-
|
|
222
|
+
if not prefix and not object_names:
|
|
223
|
+
return None
|
|
180
224
|
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
objects_to_delete = tuple(
|
|
184
|
-
{'Key': obj['Key']} async for obj in list_objects(bucket_name, prefix, recursive=True)
|
|
185
|
-
)
|
|
186
|
-
elif object_names:
|
|
187
|
-
objects_to_delete = tuple({'Key': name} for name in object_names)
|
|
188
|
-
else:
|
|
189
|
-
return None
|
|
225
|
+
client = get_client()
|
|
226
|
+
errors: list[Mapping] = []
|
|
190
227
|
|
|
191
|
-
|
|
192
|
-
|
|
228
|
+
if prefix:
|
|
229
|
+
url = build_url(work_path, prefix)
|
|
193
230
|
|
|
194
231
|
try:
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
chunk_size = 1000
|
|
198
|
-
|
|
199
|
-
for i in range(0, len(objects_to_delete), chunk_size):
|
|
200
|
-
chunk = objects_to_delete[i : i + chunk_size]
|
|
201
|
-
|
|
202
|
-
response = await s3_client.delete_objects(Bucket=bucket_name, Delete={'Objects': chunk})
|
|
203
|
-
|
|
204
|
-
if 'Errors' in response:
|
|
205
|
-
errors.extend(response['Errors'])
|
|
206
|
-
|
|
207
|
-
logger.debug('Removed %d objects from object storage: %s', len(objects_to_delete), bucket_name)
|
|
232
|
+
await client.delete_objects_by_prefix(url)
|
|
233
|
+
logger.debug('Removed objects by prefix from object storage: %s', url)
|
|
208
234
|
except Exception as e:
|
|
209
|
-
logger.exception('Failed to remove objects
|
|
210
|
-
|
|
235
|
+
logger.exception('Failed to remove objects by prefix: %s', url, exc_info=e)
|
|
211
236
|
raise
|
|
237
|
+
elif object_names:
|
|
238
|
+
for name in object_names:
|
|
239
|
+
url = build_url(work_path, name)
|
|
240
|
+
|
|
241
|
+
try:
|
|
242
|
+
await client.delete_object(url)
|
|
243
|
+
except Exception as e:
|
|
244
|
+
logger.exception('Failed to remove object: %s', url, exc_info=e)
|
|
245
|
+
errors.append({'Key': name, 'Error': str(e)})
|
|
212
246
|
|
|
213
|
-
|
|
247
|
+
return errors or None
|
|
@@ -0,0 +1,434 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: python3-commons
|
|
3
|
+
Version: 0.25.0
|
|
4
|
+
Summary: Re-usable Python3 code
|
|
5
|
+
Author-email: Oleg Korsak <kamikaze.is.waiting.you@gmail.com>
|
|
6
|
+
License-Expression: GPL-3.0-only
|
|
7
|
+
Project-URL: Homepage, https://github.com/kamikaze/python3-commons
|
|
8
|
+
Project-URL: Documentation, https://github.com/kamikaze/python3-commons/wiki
|
|
9
|
+
Classifier: Development Status :: 4 - Beta
|
|
10
|
+
Classifier: Programming Language :: Python
|
|
11
|
+
Requires-Python: <3.15.0,>=3.14.0
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
License-File: AUTHORS.rst
|
|
15
|
+
Requires-Dist: msgpack~=1.2.2
|
|
16
|
+
Requires-Dist: msgspec~=0.21.1
|
|
17
|
+
Requires-Dist: pydantic-settings~=2.15.0
|
|
18
|
+
Provides-Extra: all
|
|
19
|
+
Requires-Dist: python3_commons[api-client,audit,authn,authz,cache,database,object-storage,soap-client]; extra == "all"
|
|
20
|
+
Provides-Extra: api-client
|
|
21
|
+
Requires-Dist: aiohttp[speedups]<3.15.0,>=3.14.3; extra == "api-client"
|
|
22
|
+
Requires-Dist: certifi~=2026.7.22; extra == "api-client"
|
|
23
|
+
Requires-Dist: python3_commons[audit,object-storage]; extra == "api-client"
|
|
24
|
+
Provides-Extra: audit
|
|
25
|
+
Requires-Dist: lxml~=6.1.3; extra == "audit"
|
|
26
|
+
Requires-Dist: zeep[async]~=4.3.3; extra == "audit"
|
|
27
|
+
Requires-Dist: python3_commons[object-storage]; extra == "audit"
|
|
28
|
+
Provides-Extra: authn
|
|
29
|
+
Requires-Dist: aiohttp[speedups]<3.15.0,>=3.14.3; extra == "authn"
|
|
30
|
+
Requires-Dist: certifi~=2026.7.22; extra == "authn"
|
|
31
|
+
Requires-Dist: python3_commons[api-client]; extra == "authn"
|
|
32
|
+
Provides-Extra: authz
|
|
33
|
+
Requires-Dist: python3_commons[database]; extra == "authz"
|
|
34
|
+
Requires-Dist: python3_commons[api-client]; extra == "authz"
|
|
35
|
+
Provides-Extra: cache
|
|
36
|
+
Requires-Dist: valkey[libvalkey]~=6.1.1; extra == "cache"
|
|
37
|
+
Provides-Extra: database
|
|
38
|
+
Requires-Dist: asyncpg~=0.31.0; extra == "database"
|
|
39
|
+
Requires-Dist: SQLAlchemy[asyncio]~=2.1.1; extra == "database"
|
|
40
|
+
Provides-Extra: object-storage
|
|
41
|
+
Requires-Dist: object-storage-client~=0.1.12; extra == "object-storage"
|
|
42
|
+
Provides-Extra: soap-client
|
|
43
|
+
Requires-Dist: aiohttp[speedups]<3.15.0,>=3.14.3; extra == "soap-client"
|
|
44
|
+
Requires-Dist: certifi~=2026.7.22; extra == "soap-client"
|
|
45
|
+
Requires-Dist: lxml~=6.1.3; extra == "soap-client"
|
|
46
|
+
Requires-Dist: requests~=2.34.2; extra == "soap-client"
|
|
47
|
+
Requires-Dist: zeep[async]~=4.3.3; extra == "soap-client"
|
|
48
|
+
Dynamic: license-file
|
|
49
|
+
|
|
50
|
+
# python3-commons
|
|
51
|
+
|
|
52
|
+
Re-usable code for Python 3 projects.
|
|
53
|
+
|
|
54
|
+
## Installation
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
uv add python3-commons
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
### Optional Dependencies
|
|
61
|
+
|
|
62
|
+
Some features require extra dependencies. You can install them individually or all at once:
|
|
63
|
+
|
|
64
|
+
- `api-client`: For `python3_commons.api_client`
|
|
65
|
+
- `audit`: For `python3_commons.audit`
|
|
66
|
+
- `authn`: For `python3_commons.auth`
|
|
67
|
+
- `authz`: For `python3_commons.permissions`
|
|
68
|
+
- `cache`: For `python3_commons.cache`
|
|
69
|
+
- `database`: For `python3_commons.db`
|
|
70
|
+
- `object-storage`: For `python3_commons.object_storage`
|
|
71
|
+
- `soap-client`: For `python3_commons.soap_client`
|
|
72
|
+
- `all`: Install all optional dependencies
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
uv add "python3-commons[all]"
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
## Features
|
|
79
|
+
|
|
80
|
+
### Async LRU Cache
|
|
81
|
+
|
|
82
|
+
An LRU cache decorator for asynchronous functions that handles `await` and prevents the dogpile effect (mitigating multiple concurrent calls for the same key by waiting for the first one to finish).
|
|
83
|
+
|
|
84
|
+
```python
|
|
85
|
+
from python3_commons.async_functools import async_lru_cache
|
|
86
|
+
|
|
87
|
+
@async_lru_cache(maxsize=128)
|
|
88
|
+
async def fetch_expensive_data(item_id: int):
|
|
89
|
+
# This function will only be called once for a given item_id
|
|
90
|
+
# even if multiple tasks await it simultaneously.
|
|
91
|
+
return await db.get(item_id)
|
|
92
|
+
|
|
93
|
+
# Usage
|
|
94
|
+
result = await fetch_expensive_data(42)
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
### API Client
|
|
98
|
+
|
|
99
|
+
A context manager for `aiohttp` requests with built-in audit logging to S3 and standardized error mapping to Python exceptions.
|
|
100
|
+
|
|
101
|
+
#### Standardized Exception Mapping
|
|
102
|
+
|
|
103
|
+
HTTP error status codes and network failures are mapped to built-in Python exceptions:
|
|
104
|
+
- `400 Bad Request` -> `ValueError`
|
|
105
|
+
- `401 Unauthorized`, `403 Forbidden` -> `PermissionError`
|
|
106
|
+
- `404 Not Found` -> `LookupError`
|
|
107
|
+
- `429 Too Many Requests` -> `InterruptedError`
|
|
108
|
+
- Other 4xx/5xx responses -> `aiohttp.ClientResponseError`
|
|
109
|
+
- Connection errors -> `ConnectionRefusedError` / `ConnectionResetError`
|
|
110
|
+
|
|
111
|
+
#### Enabling Audit Logging
|
|
112
|
+
|
|
113
|
+
Audit logging captures the outgoing request (formatted as an executable `curl` command) and the incoming response body, writing them asynchronously to Object Storage.
|
|
114
|
+
|
|
115
|
+
The destination path is resolved as follows:
|
|
116
|
+
- If `ApiClientSettings.audit_path` is configured (via `API_CLIENT_AUDIT_PATH`), it is used directly as the audit root for `api_client` (without prepending `"audit/"`):
|
|
117
|
+
`<audit_path>/<YYYY/MM/DD>/<audit_name>/<uri>/<method>_<timestamp>_<request_id>_{request,response}.txt`
|
|
118
|
+
- Otherwise, if `ObjectStorageSettings.work_path` is configured (via `OBJECT_STORAGE_WORK_PATH`), it is used as the root with `"audit/"` prepended:
|
|
119
|
+
`<work_path>/audit/<YYYY/MM/DD>/<audit_name>/<uri>/<method>_<timestamp>_<request_id>_{request,response}.txt`
|
|
120
|
+
- If neither is configured, the logs default to:
|
|
121
|
+
`audit/<YYYY/MM/DD>/<audit_name>/<uri>/<method>_<timestamp>_<request_id>_{request,response}.txt`
|
|
122
|
+
|
|
123
|
+
To enable audit logging:
|
|
124
|
+
1. **Configure Object Storage & S3 credentials**:
|
|
125
|
+
```bash
|
|
126
|
+
# API client audit path or Object storage root
|
|
127
|
+
export API_CLIENT_AUDIT_PATH="my-audit-bucket/audit-logs"
|
|
128
|
+
# or export OBJECT_STORAGE_WORK_PATH="my-audit-bucket"
|
|
129
|
+
|
|
130
|
+
# S3 credentials
|
|
131
|
+
export S3_ACCESS_KEY_ID="your-access-key-id"
|
|
132
|
+
export S3_SECRET_ACCESS_KEY="your-secret-access-key"
|
|
133
|
+
export S3_REGION="us-east-1"
|
|
134
|
+
# Optional configurations:
|
|
135
|
+
# export S3_ALLOW_HTTP="false"
|
|
136
|
+
# export S3_ADDRESSING_STYLE="virtual" # or "path"
|
|
137
|
+
# export S3_CERT_VERIFY="true"
|
|
138
|
+
```
|
|
139
|
+
Alternatively, you can configure them directly in Python:
|
|
140
|
+
```python
|
|
141
|
+
from python3_commons.conf import api_client_settings, object_storage_settings, s3_settings
|
|
142
|
+
from pydantic import SecretStr
|
|
143
|
+
|
|
144
|
+
api_client_settings.audit_path = "my-audit-bucket/audit-logs"
|
|
145
|
+
# or object_storage_settings.work_path = "my-audit-bucket"
|
|
146
|
+
|
|
147
|
+
s3_settings.access_key_id = SecretStr("your-access-key-id")
|
|
148
|
+
s3_settings.secret_access_key = SecretStr("your-secret-access-key")
|
|
149
|
+
s3_settings.region = "us-east-1"
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
2. **Pass `audit_name` to `request()`**: Provide a non-empty string identifier (such as the target service or operation name) in the `audit_name` argument. If `audit_name` is omitted (default `None`), audit logging is disabled for that call. If S3 credentials are not configured, audit logging is safely bypassed.
|
|
153
|
+
|
|
154
|
+
#### Usage Example
|
|
155
|
+
|
|
156
|
+
```python
|
|
157
|
+
from aiohttp import ClientSession
|
|
158
|
+
from python3_commons.api_client import request
|
|
159
|
+
|
|
160
|
+
async with ClientSession() as session:
|
|
161
|
+
# 1. GET request with audit logging enabled
|
|
162
|
+
async with request(
|
|
163
|
+
session,
|
|
164
|
+
base_url="https://api.example.com",
|
|
165
|
+
uri="/users/123",
|
|
166
|
+
method="get",
|
|
167
|
+
headers={"Accept": "application/json"},
|
|
168
|
+
audit_name="user_service",
|
|
169
|
+
) as response:
|
|
170
|
+
user = await response.json()
|
|
171
|
+
|
|
172
|
+
# 2. POST request with JSON body and audit logging
|
|
173
|
+
async with request(
|
|
174
|
+
session,
|
|
175
|
+
base_url="https://api.example.com",
|
|
176
|
+
uri="/users",
|
|
177
|
+
method="post",
|
|
178
|
+
json={"name": "Alice", "role": "admin"},
|
|
179
|
+
audit_name="user_service",
|
|
180
|
+
) as response:
|
|
181
|
+
result = await response.json()
|
|
182
|
+
|
|
183
|
+
# 3. Request without audit logging (audit_name omitted)
|
|
184
|
+
async with request(
|
|
185
|
+
session,
|
|
186
|
+
base_url="https://api.example.com",
|
|
187
|
+
uri="/health",
|
|
188
|
+
method="get",
|
|
189
|
+
) as response:
|
|
190
|
+
status = await response.json()
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
### SOAP Client
|
|
194
|
+
|
|
195
|
+
Async SOAP client support for `zeep` with S3 auditing capabilities.
|
|
196
|
+
|
|
197
|
+
```python
|
|
198
|
+
from python3_commons.soap_client import soap_client
|
|
199
|
+
|
|
200
|
+
async with soap_client("https://example.com/service?wsdl") as client:
|
|
201
|
+
result = await client.service.GetData(id=42)
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
### Database Management
|
|
205
|
+
|
|
206
|
+
SQLAlchemy async engine and session management with pool tuning, health checks, dynamic query builders, and declarative base models.
|
|
207
|
+
|
|
208
|
+
```python
|
|
209
|
+
from python3_commons.db import AsyncSessionManager, is_healthy
|
|
210
|
+
from python3_commons.db.helpers import get_query
|
|
211
|
+
from python3_commons.conf import DBSettings
|
|
212
|
+
|
|
213
|
+
# Configuration
|
|
214
|
+
configs = {
|
|
215
|
+
"default": DBSettings(
|
|
216
|
+
dsn="postgresql+asyncpg://user:pass@localhost/db",
|
|
217
|
+
pool_size=20,
|
|
218
|
+
pool_timeout=30,
|
|
219
|
+
statement_timeout=30,
|
|
220
|
+
)
|
|
221
|
+
}
|
|
222
|
+
manager = AsyncSessionManager(configs)
|
|
223
|
+
|
|
224
|
+
# Usage
|
|
225
|
+
async with manager.get_session_context("default") as session:
|
|
226
|
+
result = await session.execute(...)
|
|
227
|
+
|
|
228
|
+
# Engine health check (executes SELECT 1 with timeout)
|
|
229
|
+
healthy = await is_healthy(manager.get_engine("default"))
|
|
230
|
+
|
|
231
|
+
# Dynamic query filtering and sorting
|
|
232
|
+
where_clause, order_clauses = get_query(
|
|
233
|
+
search={"name": "Alice"},
|
|
234
|
+
order_by="-created_at,name",
|
|
235
|
+
columns={
|
|
236
|
+
# name: (column, case_insensitive, cast_func, exact_match)
|
|
237
|
+
"name": (User.name, True, str, False),
|
|
238
|
+
"created_at": (User.created_at, False, str, True),
|
|
239
|
+
},
|
|
240
|
+
)
|
|
241
|
+
```
|
|
242
|
+
|
|
243
|
+
### Object Storage
|
|
244
|
+
|
|
245
|
+
Thin wrapper for `object-storage-client` supporting S3, GCS, Azure, and local file storage, configured via `object_storage_settings` (`OBJECT_STORAGE_WORK_PATH`) and `s3_settings` (`S3_ACCESS_KEY_ID`, `S3_SECRET_ACCESS_KEY`, `S3_REGION`). Storage functions accept a required `work_path` parameter to define the base storage location.
|
|
246
|
+
|
|
247
|
+
```python
|
|
248
|
+
from python3_commons import object_storage
|
|
249
|
+
import io
|
|
250
|
+
|
|
251
|
+
# Upload an object
|
|
252
|
+
await object_storage.put_object(
|
|
253
|
+
work_path="my-bucket",
|
|
254
|
+
path="uploads/file.txt",
|
|
255
|
+
data=io.BytesIO(b"Hello World"),
|
|
256
|
+
length=11
|
|
257
|
+
)
|
|
258
|
+
|
|
259
|
+
# Download an object as bytes
|
|
260
|
+
content = await object_storage.get_object("my-bucket", "uploads/file.txt")
|
|
261
|
+
|
|
262
|
+
# Stream an object
|
|
263
|
+
async with object_storage.get_object_stream("my-bucket", "uploads/file.txt") as stream:
|
|
264
|
+
data = await stream.read()
|
|
265
|
+
|
|
266
|
+
# List objects with a prefix
|
|
267
|
+
async for obj in object_storage.list_objects("my-bucket", "uploads/"):
|
|
268
|
+
print(obj['Key'])
|
|
269
|
+
|
|
270
|
+
# Delete single or multiple objects
|
|
271
|
+
await object_storage.remove_object("my-bucket", "uploads/file.txt")
|
|
272
|
+
await object_storage.remove_objects("my-bucket", object_names=["uploads/file1.txt", "uploads/file2.txt"])
|
|
273
|
+
```
|
|
274
|
+
|
|
275
|
+
### OIDC Authentication
|
|
276
|
+
|
|
277
|
+
Client for OpenID Connect authentication, supporting configuration fetching, JWKS, and token acquisition.
|
|
278
|
+
|
|
279
|
+
```python
|
|
280
|
+
from python3_commons.auth import OIDCClient
|
|
281
|
+
from pydantic import HttpUrl
|
|
282
|
+
|
|
283
|
+
client = OIDCClient(
|
|
284
|
+
authority_url=HttpUrl("https://auth.example.com/realms/myrealm"),
|
|
285
|
+
client_id="my-app-client",
|
|
286
|
+
client_secret="secret"
|
|
287
|
+
)
|
|
288
|
+
|
|
289
|
+
async with client:
|
|
290
|
+
token_response = await client.fetch_token(username="user", password="password")
|
|
291
|
+
print(token_response.access_token)
|
|
292
|
+
```
|
|
293
|
+
|
|
294
|
+
### Valkey Cache
|
|
295
|
+
|
|
296
|
+
Async caching using Valkey (Redis-compatible) with automatic Msgpack serialization for complex types.
|
|
297
|
+
|
|
298
|
+
```python
|
|
299
|
+
from python3_commons import cache
|
|
300
|
+
|
|
301
|
+
# Store a dictionary
|
|
302
|
+
await cache.store("user:123", {"name": "Alice", "role": "admin"}, ttl=3600)
|
|
303
|
+
|
|
304
|
+
# Retrieve it
|
|
305
|
+
user_data = await cache.get("user:123")
|
|
306
|
+
|
|
307
|
+
# Set operations
|
|
308
|
+
await cache.add_set_item("active_users", "user:123")
|
|
309
|
+
is_active = await cache.has_set_item("active_users", "user:123")
|
|
310
|
+
```
|
|
311
|
+
|
|
312
|
+
### Structured Logging
|
|
313
|
+
|
|
314
|
+
A `JSONFormatter` for structured JSON logging and a `filter_maker` utility for level-based log filtering, compatible with standard Python `logging`.
|
|
315
|
+
|
|
316
|
+
```python
|
|
317
|
+
import logging
|
|
318
|
+
from python3_commons.log.formatters import JSONFormatter
|
|
319
|
+
from python3_commons.log.filters import filter_maker
|
|
320
|
+
|
|
321
|
+
handler = logging.StreamHandler()
|
|
322
|
+
handler.setFormatter(JSONFormatter())
|
|
323
|
+
# Filter to process logs up to INFO level only
|
|
324
|
+
handler.addFilter(filter_maker("INFO"))
|
|
325
|
+
logging.getLogger().addHandler(handler)
|
|
326
|
+
|
|
327
|
+
logger = logging.getLogger("app")
|
|
328
|
+
logger.info("User logged in", extra={"user_id": "abc-123", "ip": "1.2.3.4"})
|
|
329
|
+
```
|
|
330
|
+
|
|
331
|
+
### Serialization
|
|
332
|
+
|
|
333
|
+
Enhanced JSON and Msgpack serialization for types not handled by default (Decimal, datetime, date, dataclasses, Pydantic models).
|
|
334
|
+
|
|
335
|
+
```python
|
|
336
|
+
from python3_commons.serializers.msgspec import serialize_msgpack, deserialize_msgpack
|
|
337
|
+
from decimal import Decimal
|
|
338
|
+
from datetime import datetime
|
|
339
|
+
|
|
340
|
+
data = {
|
|
341
|
+
"amount": Decimal("150.75"),
|
|
342
|
+
"timestamp": datetime.now(),
|
|
343
|
+
"tags": {"finance", "internal"}
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
# Serialize to Msgpack
|
|
347
|
+
binary = serialize_msgpack(data)
|
|
348
|
+
|
|
349
|
+
# Deserialize back
|
|
350
|
+
restored = deserialize_msgpack(binary)
|
|
351
|
+
```
|
|
352
|
+
|
|
353
|
+
### RBAC Permissions
|
|
354
|
+
|
|
355
|
+
Database-backed Role-Based Access Control (RBAC) permission checking for users and API keys.
|
|
356
|
+
|
|
357
|
+
```python
|
|
358
|
+
from python3_commons.permissions import has_user_permission, has_api_key_permission
|
|
359
|
+
from uuid import UUID
|
|
360
|
+
|
|
361
|
+
user_uuid = UUID("...")
|
|
362
|
+
allowed = await has_user_permission(session, user_uuid, "reports.view")
|
|
363
|
+
|
|
364
|
+
api_key_uuid = UUID("...")
|
|
365
|
+
key_allowed = await has_api_key_permission(session, api_key_uuid, "reports.view")
|
|
366
|
+
```
|
|
367
|
+
|
|
368
|
+
### Configuration Management
|
|
369
|
+
|
|
370
|
+
Typed application settings powered by Pydantic Settings:
|
|
371
|
+
|
|
372
|
+
- `CommonSettings` / `settings`: General logging level and format.
|
|
373
|
+
- `DBSettings` / `db_settings`: PostgreSQL database settings (`DB_` environment prefix, with `DB_PASS` alias for password).
|
|
374
|
+
- `S3Settings` / `s3_settings`: S3 credentials and connection configuration (`S3_` environment prefix: `ACCESS_KEY_ID`, `SECRET_ACCESS_KEY`, `REGION`).
|
|
375
|
+
- `ObjectStorageSettings` / `object_storage_settings`: Object storage root configuration (`OBJECT_STORAGE_` environment prefix: `WORK_PATH`).
|
|
376
|
+
- `ApiClientSettings` / `api_client_settings`: API client configuration (`API_CLIENT_` environment prefix: `AUDIT_PATH`).
|
|
377
|
+
- `OIDCSettings`: OpenID Connect configuration (`OIDC_` environment prefix).
|
|
378
|
+
- `ValkeySettings` / `valkey_settings`: Valkey/Redis cache configuration (`VALKEY_` environment prefix).
|
|
379
|
+
|
|
380
|
+
### File System Utilities
|
|
381
|
+
|
|
382
|
+
File system traversal helpers:
|
|
383
|
+
|
|
384
|
+
```python
|
|
385
|
+
from python3_commons.fs import iter_files
|
|
386
|
+
from pathlib import Path
|
|
387
|
+
|
|
388
|
+
# Recursively iterate files, ignoring hidden directories
|
|
389
|
+
for file_path in iter_files(Path("./data"), recursive=True):
|
|
390
|
+
print(file_path)
|
|
391
|
+
```
|
|
392
|
+
|
|
393
|
+
### General Helpers
|
|
394
|
+
|
|
395
|
+
A collection of useful utility functions:
|
|
396
|
+
|
|
397
|
+
- `to_snake_case(text)`: Converts strings to snake_case.
|
|
398
|
+
- `round_decimal(value, places)`: Rounds `Decimal` values.
|
|
399
|
+
- `tries(n)`: An async retry decorator.
|
|
400
|
+
- `log_execution_time`: An async decorator to log how long a function takes.
|
|
401
|
+
- `date_from_string` / `datetime_from_string`: Flexible date/time parsing.
|
|
402
|
+
- `date_range(start, end)`: Generator yielding dates in a range.
|
|
403
|
+
- `request_to_curl`: Converts request parameters to a `curl` command string.
|
|
404
|
+
- `SingletonMeta`: Thread-safe singleton metaclass.
|
|
405
|
+
- `replace_origin(url, host_url)`: Replaces scheme, host, and port of a URL.
|
|
406
|
+
- `parse_string_list`: Parses comma-separated string lists into tuples.
|
|
407
|
+
|
|
408
|
+
```python
|
|
409
|
+
from python3_commons.helpers import tries, log_execution_time, SingletonMeta
|
|
410
|
+
|
|
411
|
+
@tries(3)
|
|
412
|
+
@log_execution_time
|
|
413
|
+
async def flaky_network_call():
|
|
414
|
+
...
|
|
415
|
+
|
|
416
|
+
class AppService(metaclass=SingletonMeta):
|
|
417
|
+
pass
|
|
418
|
+
```
|
|
419
|
+
|
|
420
|
+
### Async CSV Stream
|
|
421
|
+
|
|
422
|
+
Efficiently generate CSV data as a byte stream from an async generator of tuples.
|
|
423
|
+
|
|
424
|
+
```python
|
|
425
|
+
from python3_commons.generators import tuple_csv_stream
|
|
426
|
+
|
|
427
|
+
async def generate_rows():
|
|
428
|
+
for i in range(1000):
|
|
429
|
+
yield (i, f"Name {i}", 10.5 * i)
|
|
430
|
+
|
|
431
|
+
async for chunk in tuple_csv_stream(generate_rows(), header=("ID", "Name", "Value")):
|
|
432
|
+
# Send chunk to HTTP response or write to file
|
|
433
|
+
pass
|
|
434
|
+
```
|
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
python3_commons/__init__.py,sha256=0KgaYU46H_IMKn-BuasoRN3C4Hi45KlkHHoPbU9cwiA,189
|
|
2
|
-
python3_commons/api_client.py,sha256=
|
|
2
|
+
python3_commons/api_client.py,sha256=kQCTUEkMtNkoHexSEa3qQCRJyxzLKEsbP-i1g7msPxw,6106
|
|
3
3
|
python3_commons/async_functools.py,sha256=A2HvwFzZHxOWTp4IQM5UiBY2yg1S_0U1CWra5BWK0gk,9101
|
|
4
|
-
python3_commons/audit.py,sha256=
|
|
4
|
+
python3_commons/audit.py,sha256=y80A6cE2h3uGzjT1eg-6fiZ-IT-kKDmEtWNrXSQa5sY,3358
|
|
5
5
|
python3_commons/auth.py,sha256=s928lYwpyX03BDTo1R6LpOgHiUSrjVrd1OVgTjvIiQU,8279
|
|
6
6
|
python3_commons/cache.py,sha256=lowiXJqFgFy1Yg86wi9IhuoNqIUGP6nc5eNibmf0dfY,8018
|
|
7
|
-
python3_commons/conf.py,sha256=
|
|
7
|
+
python3_commons/conf.py,sha256=t9jE2vnsetg-GCxOqsx6pPIAFzAKwO0w0g9kqSnhv1Q,3488
|
|
8
8
|
python3_commons/exceptions.py,sha256=EGjHZVBnsM6CeBfPMqhL0IPMKjDJ_2-Z-aSPXwq91LE,36
|
|
9
9
|
python3_commons/fs.py,sha256=dn8ZcwsQf9xcAEg6neoxLN6IzJbWpprfm8wV8S55BL0,337
|
|
10
10
|
python3_commons/generators.py,sha256=P6sKdCFhHT79-DZgzPU-qmwZvHg3wU8a75UFIwrycOY,1163
|
|
11
11
|
python3_commons/helpers.py,sha256=WTPu_jIOGYtxt1GYnH6sBMNDHry99AKv6ZaggGW2l1A,4986
|
|
12
|
-
python3_commons/object_storage.py,sha256=
|
|
12
|
+
python3_commons/object_storage.py,sha256=MBZdOZg8-BWoZ_8_i4-nkjlPRYV6pDxSR7fhoQmdFX4,7321
|
|
13
13
|
python3_commons/permissions.py,sha256=gaMKSWg0MgPQTdP1voll4ItXcblXku9BlD0Lq3Xv64U,1724
|
|
14
14
|
python3_commons/soap_client.py,sha256=w2lOyhhtkhwiNnHjvir8DfjGBO51XVg5rlDHyuudU2A,7169
|
|
15
15
|
python3_commons/db/__init__.py,sha256=u-ZArdz5NQFvDLz2cjKT3_VCW3aPvEqIg1-bQ6rIDoQ,10196
|
|
@@ -27,9 +27,9 @@ python3_commons/serializers/common.py,sha256=VkA7C6wODvHk0QBXVX_x2JieDstihx3U__U
|
|
|
27
27
|
python3_commons/serializers/json.py,sha256=UPkC3ps13x2C_NxwVV-K7Ewp4VjkVHSSUkJVw5k7Wiw,712
|
|
28
28
|
python3_commons/serializers/msgpack.py,sha256=zESFBX34GsZ8rDu6Zk5V6CLT6P0mPilU0r04Ka6TblI,1474
|
|
29
29
|
python3_commons/serializers/msgspec.py,sha256=upy5CBmK66-8hYnK5bAM_sZvZY5CAqZmzCw9GIF346I,2988
|
|
30
|
-
python3_commons-0.
|
|
31
|
-
python3_commons-0.
|
|
32
|
-
python3_commons-0.
|
|
33
|
-
python3_commons-0.
|
|
34
|
-
python3_commons-0.
|
|
35
|
-
python3_commons-0.
|
|
30
|
+
python3_commons-0.25.0.dist-info/licenses/AUTHORS.rst,sha256=3R9JnfjfjH5RoPWOeqKFJgxVShSSfzQPIrEr1nxIo9Q,90
|
|
31
|
+
python3_commons-0.25.0.dist-info/licenses/LICENSE,sha256=xxILuojHm4fKQOrMHPSslbyy6WuKAN2RiG74HbrYfzM,34575
|
|
32
|
+
python3_commons-0.25.0.dist-info/METADATA,sha256=vNf600tOvWqIn98Kc86RteZ5UhVwx0p_S2t6ZhHpsWA,15092
|
|
33
|
+
python3_commons-0.25.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
34
|
+
python3_commons-0.25.0.dist-info/top_level.txt,sha256=lJI6sCBf68eUHzupCnn2dzG10lH3jJKTWM_hrN1cQ7M,16
|
|
35
|
+
python3_commons-0.25.0.dist-info/RECORD,,
|
|
@@ -1,310 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: python3-commons
|
|
3
|
-
Version: 0.24.6
|
|
4
|
-
Summary: Re-usable Python3 code
|
|
5
|
-
Author-email: Oleg Korsak <kamikaze.is.waiting.you@gmail.com>
|
|
6
|
-
License-Expression: GPL-3.0
|
|
7
|
-
Project-URL: Homepage, https://github.com/kamikaze/python3-commons
|
|
8
|
-
Project-URL: Documentation, https://github.com/kamikaze/python3-commons/wiki
|
|
9
|
-
Classifier: Development Status :: 4 - Beta
|
|
10
|
-
Classifier: Programming Language :: Python
|
|
11
|
-
Requires-Python: <3.15.0,>=3.14.0
|
|
12
|
-
Description-Content-Type: text/x-rst
|
|
13
|
-
License-File: LICENSE
|
|
14
|
-
License-File: AUTHORS.rst
|
|
15
|
-
Requires-Dist: msgpack~=1.2.1
|
|
16
|
-
Requires-Dist: msgspec~=0.21.1
|
|
17
|
-
Requires-Dist: pydantic-settings~=2.15.0
|
|
18
|
-
Provides-Extra: all
|
|
19
|
-
Requires-Dist: python3_commons[api-client,audit,authn,authz,cache,database,object-storage,soap-client]; extra == "all"
|
|
20
|
-
Provides-Extra: api-client
|
|
21
|
-
Requires-Dist: aiohttp[speedups]<3.15.0,>=3.14.3; extra == "api-client"
|
|
22
|
-
Requires-Dist: certifi~=2026.7.22; extra == "api-client"
|
|
23
|
-
Requires-Dist: python3_commons[audit,object-storage]; extra == "api-client"
|
|
24
|
-
Provides-Extra: audit
|
|
25
|
-
Requires-Dist: lxml~=6.1.2; extra == "audit"
|
|
26
|
-
Requires-Dist: zeep[async]~=4.3.3; extra == "audit"
|
|
27
|
-
Requires-Dist: python3_commons[object-storage]; extra == "audit"
|
|
28
|
-
Provides-Extra: authn
|
|
29
|
-
Requires-Dist: aiohttp[speedups]<3.15.0,>=3.14.3; extra == "authn"
|
|
30
|
-
Requires-Dist: certifi~=2026.7.22; extra == "authn"
|
|
31
|
-
Requires-Dist: python3_commons[api-client]; extra == "authn"
|
|
32
|
-
Provides-Extra: authz
|
|
33
|
-
Requires-Dist: python3_commons[database]; extra == "authz"
|
|
34
|
-
Requires-Dist: python3_commons[api-client]; extra == "authz"
|
|
35
|
-
Provides-Extra: cache
|
|
36
|
-
Requires-Dist: valkey[libvalkey]~=6.1.1; extra == "cache"
|
|
37
|
-
Provides-Extra: database
|
|
38
|
-
Requires-Dist: asyncpg~=0.31.0; extra == "database"
|
|
39
|
-
Requires-Dist: SQLAlchemy[asyncio]~=2.0.52; extra == "database"
|
|
40
|
-
Provides-Extra: object-storage
|
|
41
|
-
Requires-Dist: aiobotocore==3.9.0; extra == "object-storage"
|
|
42
|
-
Requires-Dist: object-storage-client~=0.1.8; extra == "object-storage"
|
|
43
|
-
Provides-Extra: soap-client
|
|
44
|
-
Requires-Dist: aiohttp[speedups]<3.15.0,>=3.14.3; extra == "soap-client"
|
|
45
|
-
Requires-Dist: certifi~=2026.7.22; extra == "soap-client"
|
|
46
|
-
Requires-Dist: lxml~=6.1.3; extra == "soap-client"
|
|
47
|
-
Requires-Dist: requests~=2.34.2; extra == "soap-client"
|
|
48
|
-
Requires-Dist: zeep[async]~=4.3.3; extra == "soap-client"
|
|
49
|
-
Dynamic: license-file
|
|
50
|
-
|
|
51
|
-
Re-usable Python3 code
|
|
52
|
-
======================
|
|
53
|
-
|
|
54
|
-
Re-usable code for Python 3 projects.
|
|
55
|
-
|
|
56
|
-
Installation
|
|
57
|
-
------------
|
|
58
|
-
|
|
59
|
-
.. code-block:: bash
|
|
60
|
-
|
|
61
|
-
uv add python3-commons
|
|
62
|
-
|
|
63
|
-
Optional Dependencies
|
|
64
|
-
~~~~~~~~~~~~~~~~~~~~~
|
|
65
|
-
|
|
66
|
-
Some features require extra dependencies. You can install them individually or all at once:
|
|
67
|
-
|
|
68
|
-
- ``api-client``: For ``python3_commons.api_client``
|
|
69
|
-
- ``audit``: For ``python3_commons.audit``
|
|
70
|
-
- ``authn``: For ``python3_commons.auth``
|
|
71
|
-
- ``authz``: For ``python3_commons.permissions``
|
|
72
|
-
- ``cache``: For ``python3_commons.cache``
|
|
73
|
-
- ``database``: For ``python3_commons.db``
|
|
74
|
-
- ``object-storage``: For ``python3_commons.object_storage``
|
|
75
|
-
- ``soap-client``: For ``python3_commons.soap_client``
|
|
76
|
-
- ``all``: Install all optional dependencies
|
|
77
|
-
|
|
78
|
-
.. code-block:: bash
|
|
79
|
-
|
|
80
|
-
uv add "python3-commons[all]"
|
|
81
|
-
|
|
82
|
-
Features
|
|
83
|
-
--------
|
|
84
|
-
|
|
85
|
-
Async LRU Cache
|
|
86
|
-
~~~~~~~~~~~~~~~
|
|
87
|
-
|
|
88
|
-
An LRU cache decorator for asynchronous functions that handles ``await`` and prevents the dogpile effect (mitigating multiple concurrent calls for the same key by waiting for the first one to finish).
|
|
89
|
-
|
|
90
|
-
.. code-block:: python
|
|
91
|
-
|
|
92
|
-
from python3_commons.async_functools import async_lru_cache
|
|
93
|
-
|
|
94
|
-
@async_lru_cache(maxsize=128)
|
|
95
|
-
async def fetch_expensive_data(item_id: int):
|
|
96
|
-
# This function will only be called once for a given item_id
|
|
97
|
-
# even if multiple tasks await it simultaneously.
|
|
98
|
-
return await db.get(item_id)
|
|
99
|
-
|
|
100
|
-
# Usage
|
|
101
|
-
result = await fetch_expensive_data(42)
|
|
102
|
-
|
|
103
|
-
API Client
|
|
104
|
-
~~~~~~~~~~
|
|
105
|
-
|
|
106
|
-
A context manager for ``aiohttp`` requests with built-in audit logging to S3 and standardized error mapping to Python exceptions.
|
|
107
|
-
|
|
108
|
-
.. code-block:: python
|
|
109
|
-
|
|
110
|
-
from python3_commons.api_client import request
|
|
111
|
-
from aiohttp import ClientSession
|
|
112
|
-
|
|
113
|
-
async with ClientSession() as session:
|
|
114
|
-
async with request(
|
|
115
|
-
session,
|
|
116
|
-
base_url="https://api.example.com",
|
|
117
|
-
uri="/data",
|
|
118
|
-
method="get",
|
|
119
|
-
audit_name="my_service_audit"
|
|
120
|
-
) as response:
|
|
121
|
-
data = await response.json()
|
|
122
|
-
|
|
123
|
-
SOAP Client
|
|
124
|
-
~~~~~~~~~~~
|
|
125
|
-
|
|
126
|
-
Async SOAP client support for ``zeep`` with S3 auditing capabilities.
|
|
127
|
-
|
|
128
|
-
.. code-block:: python
|
|
129
|
-
|
|
130
|
-
from python3_commons.soap_client import soap_client
|
|
131
|
-
|
|
132
|
-
async with soap_client("https://example.com/service?wsdl") as client:
|
|
133
|
-
result = await client.service.GetData(id=42)
|
|
134
|
-
|
|
135
|
-
Database Management
|
|
136
|
-
~~~~~~~~~~~~~~~~~~~
|
|
137
|
-
|
|
138
|
-
SQLAlchemy async engine and session management with pool tuning, health checks, and dynamic query builders.
|
|
139
|
-
|
|
140
|
-
.. code-block:: python
|
|
141
|
-
|
|
142
|
-
from python3_commons.db import AsyncSessionManager
|
|
143
|
-
from python3_commons.conf import DBSettings
|
|
144
|
-
|
|
145
|
-
# Configuration
|
|
146
|
-
configs = {"default": DBSettings(dsn="postgresql+asyncpg://user:pass@localhost/db")}
|
|
147
|
-
manager = AsyncSessionManager(configs)
|
|
148
|
-
|
|
149
|
-
# Usage
|
|
150
|
-
async with manager.get_session_context("default") as session:
|
|
151
|
-
result = await session.execute(...)
|
|
152
|
-
|
|
153
|
-
# Health check
|
|
154
|
-
from python3_commons.db import is_healthy
|
|
155
|
-
await is_healthy(manager.get_engine("default"))
|
|
156
|
-
|
|
157
|
-
Object Storage
|
|
158
|
-
~~~~~~~~~~~~~~
|
|
159
|
-
|
|
160
|
-
Utilities for async S3 operations using ``aiobotocore``.
|
|
161
|
-
|
|
162
|
-
.. code-block:: python
|
|
163
|
-
|
|
164
|
-
from python3_commons import object_storage
|
|
165
|
-
import io
|
|
166
|
-
|
|
167
|
-
# Upload an object
|
|
168
|
-
await object_storage.put_object(
|
|
169
|
-
bucket_name="my-bucket",
|
|
170
|
-
path="uploads/file.txt",
|
|
171
|
-
data=io.BytesIO(b"Hello World"),
|
|
172
|
-
length=11
|
|
173
|
-
)
|
|
174
|
-
|
|
175
|
-
# Download an object
|
|
176
|
-
content = await object_storage.get_object("my-bucket", "uploads/file.txt")
|
|
177
|
-
|
|
178
|
-
# List objects
|
|
179
|
-
async for obj in object_storage.list_objects("my-bucket", "uploads/"):
|
|
180
|
-
print(obj['Key'])
|
|
181
|
-
|
|
182
|
-
OIDC Authentication
|
|
183
|
-
~~~~~~~~~~~~~~~~~~~
|
|
184
|
-
|
|
185
|
-
Client for OpenID Connect authentication, supporting configuration fetching, JWKS, and token acquisition.
|
|
186
|
-
|
|
187
|
-
.. code-block:: python
|
|
188
|
-
|
|
189
|
-
from python3_commons.auth import OIDCClient
|
|
190
|
-
from pydantic import HttpUrl
|
|
191
|
-
|
|
192
|
-
client = OIDCClient(
|
|
193
|
-
authority_url=HttpUrl("https://auth.example.com/realms/myrealm"),
|
|
194
|
-
client_id="my-app-client",
|
|
195
|
-
client_secret="secret"
|
|
196
|
-
)
|
|
197
|
-
|
|
198
|
-
async with client:
|
|
199
|
-
token_response = await client.fetch_token(username="user", password="password")
|
|
200
|
-
print(token_response.access_token)
|
|
201
|
-
|
|
202
|
-
Valkey Cache
|
|
203
|
-
~~~~~~~~~~~~
|
|
204
|
-
|
|
205
|
-
Async caching using Valkey (Redis-compatible) with automatic Msgpack serialization for complex types.
|
|
206
|
-
|
|
207
|
-
.. code-block:: python
|
|
208
|
-
|
|
209
|
-
from python3_commons import cache
|
|
210
|
-
|
|
211
|
-
# Store a dictionary
|
|
212
|
-
await cache.store("user:123", {"name": "Alice", "role": "admin"}, ttl=3600)
|
|
213
|
-
|
|
214
|
-
# Retrieve it
|
|
215
|
-
user_data = await cache.get("user:123")
|
|
216
|
-
|
|
217
|
-
# Set operations
|
|
218
|
-
await cache.add_set_item("active_users", "user:123")
|
|
219
|
-
is_active = await cache.has_set_item("active_users", "user:123")
|
|
220
|
-
|
|
221
|
-
Structured Logging
|
|
222
|
-
~~~~~~~~~~~~~~~~~~
|
|
223
|
-
|
|
224
|
-
A ``JSONFormatter`` for structured logging, compatible with standard Python ``logging``.
|
|
225
|
-
|
|
226
|
-
.. code-block:: python
|
|
227
|
-
|
|
228
|
-
import logging
|
|
229
|
-
from python3_commons.log.formatters import JSONFormatter
|
|
230
|
-
|
|
231
|
-
handler = logging.StreamHandler()
|
|
232
|
-
handler.setFormatter(JSONFormatter())
|
|
233
|
-
logging.getLogger().addHandler(handler)
|
|
234
|
-
|
|
235
|
-
logger = logging.getLogger("app")
|
|
236
|
-
logger.info("User logged in", extra={"user_id": "abc-123", "ip": "1.2.3.4"})
|
|
237
|
-
|
|
238
|
-
Serialization
|
|
239
|
-
~~~~~~~~~~~~~
|
|
240
|
-
|
|
241
|
-
Enhanced JSON and Msgpack serialization for types not handled by default (Decimal, datetime, date, dataclasses, Pydantic models).
|
|
242
|
-
|
|
243
|
-
.. code-block:: python
|
|
244
|
-
|
|
245
|
-
from python3_commons.serializers.msgspec import serialize_msgpack, deserialize_msgpack
|
|
246
|
-
from decimal import Decimal
|
|
247
|
-
from datetime import datetime
|
|
248
|
-
|
|
249
|
-
data = {
|
|
250
|
-
"amount": Decimal("150.75"),
|
|
251
|
-
"timestamp": datetime.now(),
|
|
252
|
-
"tags": {"finance", "internal"}
|
|
253
|
-
}
|
|
254
|
-
|
|
255
|
-
# Serialize to Msgpack
|
|
256
|
-
binary = serialize_msgpack(data)
|
|
257
|
-
|
|
258
|
-
# Deserialize back
|
|
259
|
-
restored = deserialize_msgpack(binary)
|
|
260
|
-
|
|
261
|
-
RBAC Permissions
|
|
262
|
-
~~~~~~~~~~~~~~~~
|
|
263
|
-
|
|
264
|
-
Database-backed Role-Based Access Control (RBAC) permission checking.
|
|
265
|
-
|
|
266
|
-
.. code-block:: python
|
|
267
|
-
|
|
268
|
-
from python3_commons.permissions import has_user_permission
|
|
269
|
-
from uuid import UUID
|
|
270
|
-
|
|
271
|
-
user_uuid = UUID("...")
|
|
272
|
-
allowed = await has_user_permission(session, user_uuid, "reports.view")
|
|
273
|
-
|
|
274
|
-
General Helpers
|
|
275
|
-
~~~~~~~~~~~~~~~
|
|
276
|
-
|
|
277
|
-
A collection of useful utility functions:
|
|
278
|
-
|
|
279
|
-
- ``to_snake_case(text)``: Converts strings to snake_case.
|
|
280
|
-
- ``round_decimal(value, places)``: Rounds ``Decimal`` values.
|
|
281
|
-
- ``tries(n)``: An async retry decorator.
|
|
282
|
-
- ``log_execution_time``: An async decorator to log how long a function takes.
|
|
283
|
-
- ``date_from_string`` / ``datetime_from_string``: Flexible date/time parsing.
|
|
284
|
-
- ``request_to_curl``: Converts request parameters to a ``curl`` command string.
|
|
285
|
-
|
|
286
|
-
.. code-block:: python
|
|
287
|
-
|
|
288
|
-
from python3_commons.helpers import tries, log_execution_time
|
|
289
|
-
|
|
290
|
-
@tries(3)
|
|
291
|
-
@log_execution_time
|
|
292
|
-
async def flaky_network_call():
|
|
293
|
-
...
|
|
294
|
-
|
|
295
|
-
Async CSV Stream
|
|
296
|
-
~~~~~~~~~~~~~~~~
|
|
297
|
-
|
|
298
|
-
Efficiently generate CSV data as a byte stream from an async generator of tuples.
|
|
299
|
-
|
|
300
|
-
.. code-block:: python
|
|
301
|
-
|
|
302
|
-
from python3_commons.generators import tuple_csv_stream
|
|
303
|
-
|
|
304
|
-
async def generate_rows():
|
|
305
|
-
for i in range(1000):
|
|
306
|
-
yield (i, f"Name {i}", 10.5 * i)
|
|
307
|
-
|
|
308
|
-
async for chunk in tuple_csv_stream(generate_rows(), header=("ID", "Name", "Value")):
|
|
309
|
-
# Send chunk to HTTP response or write to file
|
|
310
|
-
pass
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|