python3-commons 0.24.6__py3-none-any.whl → 0.25.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -17,7 +17,7 @@ except ImportError as e:
17
17
  raise RuntimeError(msg) from e
18
18
 
19
19
  from python3_commons import audit
20
- from python3_commons.conf import s3_settings
20
+ from python3_commons.conf import api_client_settings, s3_settings
21
21
  from python3_commons.helpers import request_to_curl
22
22
  from python3_commons.serializers.json import CustomJSONEncoder
23
23
 
@@ -34,9 +34,15 @@ async def _store_response_for_audit(
34
34
  date_path = now.strftime('%Y/%m/%d')
35
35
  timestamp = now.strftime('%H%M%S_%f')
36
36
 
37
+ if api_client_settings.audit_path:
38
+ audit_root = api_client_settings.audit_path.strip('/')
39
+ path = f'{audit_root}/{date_path}/{audit_name}/{uri_path}/{method}_{timestamp}_{request_id}_response.txt'
40
+ else:
41
+ path = f'{date_path}/{audit_name}/{uri_path}/{method}_{timestamp}_{request_id}_response.txt'
42
+
37
43
  await audit.write_audit_data(
38
44
  s3_settings,
39
- f'{date_path}/{audit_name}/{uri_path}/{method}_{timestamp}_{request_id}_response.txt',
45
+ path,
40
46
  response_text.encode('utf-8'),
41
47
  )
42
48
 
@@ -75,9 +81,15 @@ async def request(
75
81
  )
76
82
 
77
83
  if curl_request:
84
+ if api_client_settings.audit_path:
85
+ audit_root = api_client_settings.audit_path.strip('/')
86
+ path = f'{audit_root}/{date_path}/{audit_name}/{uri_path}/{method}_{timestamp}_{request_id}_request.txt'
87
+ else:
88
+ path = f'{date_path}/{audit_name}/{uri_path}/{method}_{timestamp}_{request_id}_request.txt'
89
+
78
90
  await audit.write_audit_data(
79
91
  s3_settings,
80
- f'{date_path}/{audit_name}/{uri_path}/{method}_{timestamp}_{request_id}_request.txt',
92
+ path,
81
93
  curl_request.encode('utf-8'),
82
94
  )
83
95
 
python3_commons/audit.py CHANGED
@@ -14,7 +14,7 @@ except ImportError as e:
14
14
  raise RuntimeError(msg) from e
15
15
 
16
16
  from python3_commons import object_storage
17
- from python3_commons.conf import s3_settings
17
+ from python3_commons.conf import api_client_settings, object_storage_settings, s3_settings
18
18
 
19
19
  if TYPE_CHECKING:
20
20
  from zeep import AsyncClient
@@ -25,16 +25,28 @@ if TYPE_CHECKING:
25
25
  logger = logging.getLogger(__name__)
26
26
 
27
27
 
28
- async def write_audit_data(settings: S3Settings, key: str, data: bytes) -> None:
29
- if settings.s3_secret_access_key:
28
+ async def write_audit_data(settings: S3Settings = s3_settings, key: str = '', data: bytes = b'') -> None:
29
+ if settings.secret_access_key or api_client_settings.audit_path or object_storage_settings.work_path:
30
30
  try:
31
- await object_storage.put_object(settings.s3_bucket, f'audit/{key}', io.BytesIO(data), len(data))
31
+ if api_client_settings.audit_path and key.startswith(api_client_settings.audit_path.strip('/')):
32
+ work_path = api_client_settings.audit_path
33
+ path = key
34
+ elif api_client_settings.audit_path:
35
+ work_path = api_client_settings.audit_path
36
+ path = key.lstrip('/')
37
+ elif object_storage_settings.work_path:
38
+ work_path = object_storage_settings.work_path
39
+ path = f'audit/{key.lstrip("/")}'
40
+ else:
41
+ work_path = 'audit'
42
+ path = key.lstrip('/')
43
+ await object_storage.put_object(work_path=work_path, path=path, data=io.BytesIO(data), length=len(data))
32
44
  except Exception:
33
45
  logger.exception('Failed storing object in storage.')
34
46
  else:
35
47
  logger.debug('Stored object in storage: %s', key)
36
48
  else:
37
- logger.debug('S3 is not configured, not storing object in storage: %s', key)
49
+ logger.debug('Storage is not configured, not storing object in storage: %s', key)
38
50
 
39
51
 
40
52
  class ZeepAuditPlugin(Plugin):
python3_commons/conf.py CHANGED
@@ -86,17 +86,31 @@ class DBSettings(BaseSettings):
86
86
 
87
87
 
88
88
  class S3Settings(BaseSettings):
89
- s3_access_key_id: SecretStr | None = None
90
- s3_addressing_style: Literal['path', 'virtual'] = 'virtual'
91
- s3_allow_http: bool = False
92
- s3_bucket: str | None = None
93
- s3_bucket_root: str | None = None
94
- s3_cert_verify: bool = True
95
- s3_region: str | None = None
96
- s3_secret_access_key: SecretStr | None = None
89
+ model_config = SettingsConfigDict(env_prefix='S3_')
90
+
91
+ access_key_id: SecretStr | None = None
92
+ addressing_style: Literal['path', 'virtual'] = 'virtual'
93
+ allow_http: bool = False
94
+ cert_verify: bool = True
95
+ region: str | None = None
96
+ secret_access_key: SecretStr | None = None
97
+
98
+
99
+ class ObjectStorageSettings(BaseSettings):
100
+ model_config = SettingsConfigDict(env_prefix='OBJECT_STORAGE_')
101
+
102
+ work_path: str | None = None
103
+
104
+
105
+ class ApiClientSettings(BaseSettings):
106
+ model_config = SettingsConfigDict(env_prefix='API_CLIENT_')
107
+
108
+ audit_path: str | None = None
97
109
 
98
110
 
99
111
  settings = CommonSettings()
100
112
  valkey_settings = ValkeySettings()
101
113
  db_settings = DBSettings()
114
+ api_client_settings = ApiClientSettings()
102
115
  s3_settings = S3Settings()
116
+ object_storage_settings = ObjectStorageSettings()
@@ -1,28 +1,22 @@
1
1
  from __future__ import annotations
2
2
 
3
+ import io
3
4
  import logging
4
5
  import threading
5
6
  from contextlib import asynccontextmanager
7
+ from datetime import datetime
6
8
  from typing import TYPE_CHECKING
9
+ from urllib.parse import urlparse
7
10
 
8
11
  try:
9
- import aiobotocore.session
10
- from botocore.config import Config
11
- from object_storage_client import ObjectStorageClient
12
+ from object_storage_client import ByteStream, ObjectStorageClient
12
13
  except ImportError as e:
13
14
  msg = 'Install python3-commons[object-storage] to use this feature'
14
15
  raise RuntimeError(msg) from e
15
16
 
16
17
  if TYPE_CHECKING:
17
- import io
18
- from collections.abc import AsyncGenerator, Iterable, Mapping, Sequence
19
- from datetime import datetime
18
+ from collections.abc import AsyncGenerator, AsyncIterator, Iterable, Mapping, Sequence
20
19
 
21
- from aiobotocore.response import StreamingBody
22
- from types_aiobotocore_s3.client import S3Client
23
-
24
- from python3_commons.conf import S3Settings, s3_settings
25
- from python3_commons.helpers import SingletonMeta
26
20
 
27
21
  logger = logging.getLogger(__name__)
28
22
  _CLIENT: ObjectStorageClient | None = None
@@ -40,174 +34,214 @@ def get_client() -> ObjectStorageClient:
40
34
  return _CLIENT
41
35
 
42
36
 
43
- class ObjectStorage(metaclass=SingletonMeta):
44
- def __init__(self, settings: S3Settings) -> None:
45
- self._session = aiobotocore.session.get_session()
46
- config = {
47
- 'region_name': settings.s3_region,
48
- 'use_ssl': not settings.s3_allow_http,
49
- 'verify': settings.s3_cert_verify,
50
- 'config': Config(s3={'addressing_style': settings.s3_addressing_style}, signature_version='s3v4'),
51
- }
37
+ class ObjectStorageStream:
38
+ def __init__(self, stream: ByteStream) -> None:
39
+ self._stream = stream
40
+
41
+ async def read(self) -> bytes:
42
+ chunks = []
43
+
44
+ while chunk := await self._stream.next():
45
+ chunks.append(chunk)
46
+
47
+ return b''.join(chunks)
48
+
49
+ async def next(self) -> bytes:
50
+ return await self._stream.next()
51
+
52
+ def __aiter__(self) -> AsyncIterator[bytes]:
53
+ return self
54
+
55
+ async def __anext__(self) -> bytes:
56
+ chunk = await self._stream.next()
57
+
58
+ if not chunk:
59
+ raise StopAsyncIteration
60
+
61
+ return chunk
52
62
 
53
- if s3_access_key_id := settings.s3_access_key_id:
54
- config['aws_access_key_id'] = s3_access_key_id.get_secret_value()
55
63
 
56
- if s3_secret_access_key := settings.s3_secret_access_key:
57
- config['aws_secret_access_key'] = s3_secret_access_key.get_secret_value()
64
+ def build_url(work_path: str, path: str = '') -> str:
65
+ path = path.lstrip('/')
58
66
 
59
- self._config = config
67
+ if path.startswith(('s3://', 'file://', 'gs://', 'az://')):
68
+ return path
60
69
 
61
- @asynccontextmanager
62
- async def get_client(self) -> AsyncGenerator[S3Client]:
63
- async with self._session.create_client('s3', **self._config) as client:
64
- yield client
70
+ base = work_path.rstrip('/')
65
71
 
72
+ if not base.startswith(('s3://', 'file://', 'gs://', 'az://', '/')):
73
+ base = f's3://{base}'
66
74
 
67
- def get_absolute_path(path: str) -> str:
68
- path = path.removeprefix('/')
75
+ if not path:
76
+ return base
69
77
 
70
- if bucket_root := s3_settings.s3_bucket_root:
71
- path = f'{bucket_root[:1] if bucket_root.startswith("/") else bucket_root}/{path}'
78
+ base_no_scheme = base.split('://', 1)[-1].lstrip('/')
72
79
 
73
- return path
80
+ if path.startswith(f'{base_no_scheme}/') or path == base_no_scheme:
81
+ scheme = base.split('://', 1)[0] + '://' if '://' in base else ''
74
82
 
83
+ return f'{scheme}{path}'
75
84
 
76
- async def put_object(bucket_name: str, path: str, data: io.BytesIO, length: int, part_size: int = 0) -> str | None:
85
+ parsed = urlparse(base)
86
+ base_path = parsed.path.lstrip('/')
87
+
88
+ if base_path and (path.startswith(f'{base_path}/') or path == base_path):
89
+ prefix_url = f'{parsed.scheme}://{parsed.netloc}' if parsed.scheme else ''
90
+
91
+ return f'{prefix_url}/{path}'
92
+
93
+ return f'{base}/{path}'
94
+
95
+
96
+ async def put_object(
97
+ work_path: str,
98
+ path: str = '',
99
+ data: io.BytesIO | bytes | None = None,
100
+ length: int = 0,
101
+ part_size: int = 0,
102
+ ) -> str | None:
77
103
  try:
78
104
  client = get_client()
79
- data.seek(0)
80
- absolute_path = get_absolute_path(path)
81
- url = f's3://{bucket_name}/{absolute_path}'
82
105
 
83
- await client.put_object(url, data.getvalue())
106
+ if isinstance(data, io.BytesIO):
107
+ content = data.getvalue()
108
+ data.seek(0)
109
+ elif isinstance(data, bytes):
110
+ content = data
111
+ elif data is None:
112
+ content = b''
113
+ else:
114
+ content = bytes(data)
84
115
 
85
- logger.debug('Stored object into object storage: %s:%s', bucket_name, path)
116
+ url = build_url(work_path, path)
117
+ await client.put_object(url, content)
118
+ logger.debug('Stored object into object storage: %s', url)
86
119
  except Exception as e:
87
- logger.exception('Failed to put object to object storage: %s:%s', bucket_name, path, exc_info=e)
88
-
120
+ logger.exception('Failed to put object to object storage: %s:%s', work_path, path, exc_info=e)
89
121
  raise
90
122
 
91
- return f's3://{bucket_name}/{path}'
123
+ return url
92
124
 
93
125
 
94
126
  @asynccontextmanager
95
- async def get_object_stream(bucket_name: str, path: str) -> AsyncGenerator[StreamingBody]:
96
- storage = ObjectStorage(s3_settings)
97
-
98
- async with storage.get_client() as s3_client:
99
- logger.debug('Getting object from object storage: %s:%s', bucket_name, path)
100
-
101
- try:
102
- response = await s3_client.get_object(Bucket=bucket_name, Key=path)
127
+ async def get_object_stream(work_path: str, path: str = '') -> AsyncGenerator[ObjectStorageStream]:
128
+ logger.debug('Getting object stream from object storage: %s:%s', work_path, path)
103
129
 
104
- async with response['Body'] as stream:
105
- yield stream
106
- except Exception as e:
107
- logger.exception('Failed getting object from object storage: %s:%s', bucket_name, path, exc_info=e)
108
-
109
- raise
130
+ try:
131
+ client = get_client()
132
+ url = build_url(work_path, path)
133
+ stream = await client.get_object_stream(url)
134
+ yield ObjectStorageStream(stream)
135
+ except Exception as e:
136
+ logger.exception('Failed getting object from object storage: %s:%s', work_path, path, exc_info=e)
137
+ raise
110
138
 
111
139
 
112
- async def get_object(bucket_name: str, path: str) -> bytes:
113
- logger.debug('Getting object from object storage: %s:%s', bucket_name, path)
140
+ async def get_object(work_path: str, path: str = '') -> bytes:
141
+ logger.debug('Getting object from object storage: %s:%s', work_path, path)
114
142
 
115
143
  try:
116
144
  client = get_client()
117
- absolute_path = get_absolute_path(path)
118
- url = f's3://{bucket_name}/{absolute_path}'
145
+ url = build_url(work_path, path)
119
146
  body = await client.get_object(url)
120
147
  except Exception as e:
121
- logger.exception('Failed getting object from object storage: %s:%s', bucket_name, path, exc_info=e)
122
-
148
+ logger.exception('Failed getting object from object storage: %s:%s', work_path, path, exc_info=e)
123
149
  raise
124
150
 
125
151
  return body
126
152
 
127
153
 
128
- async def list_objects(bucket_name: str, prefix: str, *, recursive: bool = True) -> AsyncGenerator[Mapping]:
129
- storage = ObjectStorage(s3_settings)
154
+ async def list_objects(work_path: str, prefix: str = '', *, recursive: bool = True) -> AsyncGenerator[Mapping]:
155
+ client = get_client()
156
+ url = build_url(work_path, prefix)
157
+ items = await client.list_objects(url)
158
+ prefix_clean = prefix.strip('/')
130
159
 
131
- async with storage.get_client() as s3_client:
132
- paginator = s3_client.get_paginator('list_objects_v2')
160
+ for item in items:
161
+ if not recursive and '/' in item:
162
+ continue
133
163
 
134
- page_iterator = paginator.paginate(Bucket=bucket_name, Prefix=prefix, Delimiter='' if recursive else '/')
164
+ rel_key = f'{prefix_clean}/{item}' if prefix_clean else item
165
+ obj_url = build_url(work_path, rel_key)
135
166
 
136
- async for page in page_iterator:
137
- if 'Contents' in page:
138
- for obj in page['Contents']:
139
- yield dict(obj)
167
+ try:
168
+ meta = await client.get_object_metadata(obj_url)
169
+ last_modified = (
170
+ datetime.fromisoformat(meta['last_modified']) if meta.get('last_modified') else datetime.now()
171
+ )
172
+ size = meta.get('size_bytes', 0)
173
+ etag = meta.get('e_tag')
174
+ except Exception:
175
+ last_modified = datetime.now()
176
+ size = 0
177
+ etag = None
178
+
179
+ yield {
180
+ 'Key': rel_key,
181
+ 'LastModified': last_modified,
182
+ 'Size': size,
183
+ 'ETag': etag,
184
+ }
140
185
 
141
186
 
142
187
  async def get_object_streams(
143
- bucket_name: str, path: str, *, recursive: bool = True
144
- ) -> AsyncGenerator[tuple[str, datetime, StreamingBody]]:
145
- async for obj in list_objects(bucket_name, path, recursive=recursive):
188
+ work_path: str, path: str = '', *, recursive: bool = True
189
+ ) -> AsyncGenerator[tuple[str, datetime, ObjectStorageStream]]:
190
+ async for obj in list_objects(work_path, path, recursive=recursive):
146
191
  object_name = obj['Key']
147
192
  last_modified = obj['LastModified']
148
193
 
149
- async with get_object_stream(bucket_name, path) as stream:
194
+ async with get_object_stream(work_path, object_name) as stream:
150
195
  yield object_name, last_modified, stream
151
196
 
152
197
 
153
198
  async def get_objects(
154
- bucket_name: str, path: str, *, recursive: bool = True
199
+ work_path: str, path: str = '', *, recursive: bool = True
155
200
  ) -> AsyncGenerator[tuple[str, datetime, bytes]]:
156
- async for object_name, last_modified, stream in get_object_streams(bucket_name, path, recursive=recursive):
201
+ async for object_name, last_modified, stream in get_object_streams(work_path, path, recursive=recursive):
157
202
  data = await stream.read()
158
203
 
159
204
  yield object_name, last_modified, data
160
205
 
161
206
 
162
- async def remove_object(bucket_name: str, path: str) -> None:
163
- logger.debug('Removing object from object storage: %s:%s', bucket_name, path)
207
+ async def remove_object(work_path: str, path: str = '') -> None:
208
+ logger.debug('Removing object from object storage: %s:%s', work_path, path)
164
209
 
165
210
  try:
166
211
  client = get_client()
167
- absolute_path = get_absolute_path(path)
168
- url = f's3://{bucket_name}/{absolute_path}'
212
+ url = build_url(work_path, path)
169
213
  await client.delete_object(url)
170
214
  except Exception as e:
171
- logger.exception('Failed to remove object from object storage: %s:%s', bucket_name, path, exc_info=e)
172
-
215
+ logger.exception('Failed to remove object from object storage: %s:%s', work_path, path, exc_info=e)
173
216
  raise
174
217
 
175
218
 
176
219
  async def remove_objects(
177
- bucket_name: str, prefix: str | None = None, object_names: Iterable[str] | None = None
220
+ work_path: str, prefix: str | None = None, object_names: Iterable[str] | None = None
178
221
  ) -> Sequence[Mapping] | None:
179
- storage = ObjectStorage(s3_settings)
222
+ if not prefix and not object_names:
223
+ return None
180
224
 
181
- async with storage.get_client() as s3_client:
182
- if prefix:
183
- objects_to_delete = tuple(
184
- {'Key': obj['Key']} async for obj in list_objects(bucket_name, prefix, recursive=True)
185
- )
186
- elif object_names:
187
- objects_to_delete = tuple({'Key': name} for name in object_names)
188
- else:
189
- return None
225
+ client = get_client()
226
+ errors: list[Mapping] = []
190
227
 
191
- if not objects_to_delete:
192
- return None
228
+ if prefix:
229
+ url = build_url(work_path, prefix)
193
230
 
194
231
  try:
195
- errors = []
196
- # S3 delete_objects can handle up to 1000 objects at once
197
- chunk_size = 1000
198
-
199
- for i in range(0, len(objects_to_delete), chunk_size):
200
- chunk = objects_to_delete[i : i + chunk_size]
201
-
202
- response = await s3_client.delete_objects(Bucket=bucket_name, Delete={'Objects': chunk})
203
-
204
- if 'Errors' in response:
205
- errors.extend(response['Errors'])
206
-
207
- logger.debug('Removed %d objects from object storage: %s', len(objects_to_delete), bucket_name)
232
+ await client.delete_objects_by_prefix(url)
233
+ logger.debug('Removed objects by prefix from object storage: %s', url)
208
234
  except Exception as e:
209
- logger.exception('Failed to remove objects from object storage: %s', bucket_name, exc_info=e)
210
-
235
+ logger.exception('Failed to remove objects by prefix: %s', url, exc_info=e)
211
236
  raise
237
+ elif object_names:
238
+ for name in object_names:
239
+ url = build_url(work_path, name)
240
+
241
+ try:
242
+ await client.delete_object(url)
243
+ except Exception as e:
244
+ logger.exception('Failed to remove object: %s', url, exc_info=e)
245
+ errors.append({'Key': name, 'Error': str(e)})
212
246
 
213
- return errors or None
247
+ return errors or None
@@ -0,0 +1,434 @@
1
+ Metadata-Version: 2.4
2
+ Name: python3-commons
3
+ Version: 0.25.0
4
+ Summary: Re-usable Python3 code
5
+ Author-email: Oleg Korsak <kamikaze.is.waiting.you@gmail.com>
6
+ License-Expression: GPL-3.0-only
7
+ Project-URL: Homepage, https://github.com/kamikaze/python3-commons
8
+ Project-URL: Documentation, https://github.com/kamikaze/python3-commons/wiki
9
+ Classifier: Development Status :: 4 - Beta
10
+ Classifier: Programming Language :: Python
11
+ Requires-Python: <3.15.0,>=3.14.0
12
+ Description-Content-Type: text/markdown
13
+ License-File: LICENSE
14
+ License-File: AUTHORS.rst
15
+ Requires-Dist: msgpack~=1.2.2
16
+ Requires-Dist: msgspec~=0.21.1
17
+ Requires-Dist: pydantic-settings~=2.15.0
18
+ Provides-Extra: all
19
+ Requires-Dist: python3_commons[api-client,audit,authn,authz,cache,database,object-storage,soap-client]; extra == "all"
20
+ Provides-Extra: api-client
21
+ Requires-Dist: aiohttp[speedups]<3.15.0,>=3.14.3; extra == "api-client"
22
+ Requires-Dist: certifi~=2026.7.22; extra == "api-client"
23
+ Requires-Dist: python3_commons[audit,object-storage]; extra == "api-client"
24
+ Provides-Extra: audit
25
+ Requires-Dist: lxml~=6.1.3; extra == "audit"
26
+ Requires-Dist: zeep[async]~=4.3.3; extra == "audit"
27
+ Requires-Dist: python3_commons[object-storage]; extra == "audit"
28
+ Provides-Extra: authn
29
+ Requires-Dist: aiohttp[speedups]<3.15.0,>=3.14.3; extra == "authn"
30
+ Requires-Dist: certifi~=2026.7.22; extra == "authn"
31
+ Requires-Dist: python3_commons[api-client]; extra == "authn"
32
+ Provides-Extra: authz
33
+ Requires-Dist: python3_commons[database]; extra == "authz"
34
+ Requires-Dist: python3_commons[api-client]; extra == "authz"
35
+ Provides-Extra: cache
36
+ Requires-Dist: valkey[libvalkey]~=6.1.1; extra == "cache"
37
+ Provides-Extra: database
38
+ Requires-Dist: asyncpg~=0.31.0; extra == "database"
39
+ Requires-Dist: SQLAlchemy[asyncio]~=2.1.1; extra == "database"
40
+ Provides-Extra: object-storage
41
+ Requires-Dist: object-storage-client~=0.1.12; extra == "object-storage"
42
+ Provides-Extra: soap-client
43
+ Requires-Dist: aiohttp[speedups]<3.15.0,>=3.14.3; extra == "soap-client"
44
+ Requires-Dist: certifi~=2026.7.22; extra == "soap-client"
45
+ Requires-Dist: lxml~=6.1.3; extra == "soap-client"
46
+ Requires-Dist: requests~=2.34.2; extra == "soap-client"
47
+ Requires-Dist: zeep[async]~=4.3.3; extra == "soap-client"
48
+ Dynamic: license-file
49
+
50
+ # python3-commons
51
+
52
+ Re-usable code for Python 3 projects.
53
+
54
+ ## Installation
55
+
56
+ ```bash
57
+ uv add python3-commons
58
+ ```
59
+
60
+ ### Optional Dependencies
61
+
62
+ Some features require extra dependencies. You can install them individually or all at once:
63
+
64
+ - `api-client`: For `python3_commons.api_client`
65
+ - `audit`: For `python3_commons.audit`
66
+ - `authn`: For `python3_commons.auth`
67
+ - `authz`: For `python3_commons.permissions`
68
+ - `cache`: For `python3_commons.cache`
69
+ - `database`: For `python3_commons.db`
70
+ - `object-storage`: For `python3_commons.object_storage`
71
+ - `soap-client`: For `python3_commons.soap_client`
72
+ - `all`: Install all optional dependencies
73
+
74
+ ```bash
75
+ uv add "python3-commons[all]"
76
+ ```
77
+
78
+ ## Features
79
+
80
+ ### Async LRU Cache
81
+
82
+ An LRU cache decorator for asynchronous functions that handles `await` and prevents the dogpile effect (mitigating multiple concurrent calls for the same key by waiting for the first one to finish).
83
+
84
+ ```python
85
+ from python3_commons.async_functools import async_lru_cache
86
+
87
+ @async_lru_cache(maxsize=128)
88
+ async def fetch_expensive_data(item_id: int):
89
+ # This function will only be called once for a given item_id
90
+ # even if multiple tasks await it simultaneously.
91
+ return await db.get(item_id)
92
+
93
+ # Usage
94
+ result = await fetch_expensive_data(42)
95
+ ```
96
+
97
+ ### API Client
98
+
99
+ A context manager for `aiohttp` requests with built-in audit logging to S3 and standardized error mapping to Python exceptions.
100
+
101
+ #### Standardized Exception Mapping
102
+
103
+ HTTP error status codes and network failures are mapped to built-in Python exceptions:
104
+ - `400 Bad Request` -> `ValueError`
105
+ - `401 Unauthorized`, `403 Forbidden` -> `PermissionError`
106
+ - `404 Not Found` -> `LookupError`
107
+ - `429 Too Many Requests` -> `InterruptedError`
108
+ - Other 4xx/5xx responses -> `aiohttp.ClientResponseError`
109
+ - Connection errors -> `ConnectionRefusedError` / `ConnectionResetError`
110
+
111
+ #### Enabling Audit Logging
112
+
113
+ Audit logging captures the outgoing request (formatted as an executable `curl` command) and the incoming response body, writing them asynchronously to Object Storage.
114
+
115
+ The destination path is resolved as follows:
116
+ - If `ApiClientSettings.audit_path` is configured (via `API_CLIENT_AUDIT_PATH`), it is used directly as the audit root for `api_client` (without prepending `"audit/"`):
117
+ `<audit_path>/<YYYY/MM/DD>/<audit_name>/<uri>/<method>_<timestamp>_<request_id>_{request,response}.txt`
118
+ - Otherwise, if `ObjectStorageSettings.work_path` is configured (via `OBJECT_STORAGE_WORK_PATH`), it is used as the root with `"audit/"` prepended:
119
+ `<work_path>/audit/<YYYY/MM/DD>/<audit_name>/<uri>/<method>_<timestamp>_<request_id>_{request,response}.txt`
120
+ - If neither is configured, the logs default to:
121
+ `audit/<YYYY/MM/DD>/<audit_name>/<uri>/<method>_<timestamp>_<request_id>_{request,response}.txt`
122
+
123
+ To enable audit logging:
124
+ 1. **Configure Object Storage & S3 credentials**:
125
+ ```bash
126
+ # API client audit path or Object storage root
127
+ export API_CLIENT_AUDIT_PATH="my-audit-bucket/audit-logs"
128
+ # or export OBJECT_STORAGE_WORK_PATH="my-audit-bucket"
129
+
130
+ # S3 credentials
131
+ export S3_ACCESS_KEY_ID="your-access-key-id"
132
+ export S3_SECRET_ACCESS_KEY="your-secret-access-key"
133
+ export S3_REGION="us-east-1"
134
+ # Optional configurations:
135
+ # export S3_ALLOW_HTTP="false"
136
+ # export S3_ADDRESSING_STYLE="virtual" # or "path"
137
+ # export S3_CERT_VERIFY="true"
138
+ ```
139
+ Alternatively, you can configure them directly in Python:
140
+ ```python
141
+ from python3_commons.conf import api_client_settings, object_storage_settings, s3_settings
142
+ from pydantic import SecretStr
143
+
144
+ api_client_settings.audit_path = "my-audit-bucket/audit-logs"
145
+ # or object_storage_settings.work_path = "my-audit-bucket"
146
+
147
+ s3_settings.access_key_id = SecretStr("your-access-key-id")
148
+ s3_settings.secret_access_key = SecretStr("your-secret-access-key")
149
+ s3_settings.region = "us-east-1"
150
+ ```
151
+
152
+ 2. **Pass `audit_name` to `request()`**: Provide a non-empty string identifier (such as the target service or operation name) in the `audit_name` argument. If `audit_name` is omitted (default `None`), audit logging is disabled for that call. If S3 credentials are not configured, audit logging is safely bypassed.
153
+
154
+ #### Usage Example
155
+
156
+ ```python
157
+ from aiohttp import ClientSession
158
+ from python3_commons.api_client import request
159
+
160
+ async with ClientSession() as session:
161
+ # 1. GET request with audit logging enabled
162
+ async with request(
163
+ session,
164
+ base_url="https://api.example.com",
165
+ uri="/users/123",
166
+ method="get",
167
+ headers={"Accept": "application/json"},
168
+ audit_name="user_service",
169
+ ) as response:
170
+ user = await response.json()
171
+
172
+ # 2. POST request with JSON body and audit logging
173
+ async with request(
174
+ session,
175
+ base_url="https://api.example.com",
176
+ uri="/users",
177
+ method="post",
178
+ json={"name": "Alice", "role": "admin"},
179
+ audit_name="user_service",
180
+ ) as response:
181
+ result = await response.json()
182
+
183
+ # 3. Request without audit logging (audit_name omitted)
184
+ async with request(
185
+ session,
186
+ base_url="https://api.example.com",
187
+ uri="/health",
188
+ method="get",
189
+ ) as response:
190
+ status = await response.json()
191
+ ```
192
+
193
+ ### SOAP Client
194
+
195
+ Async SOAP client support for `zeep` with S3 auditing capabilities.
196
+
197
+ ```python
198
+ from python3_commons.soap_client import soap_client
199
+
200
+ async with soap_client("https://example.com/service?wsdl") as client:
201
+ result = await client.service.GetData(id=42)
202
+ ```
203
+
204
+ ### Database Management
205
+
206
+ SQLAlchemy async engine and session management with pool tuning, health checks, dynamic query builders, and declarative base models.
207
+
208
+ ```python
209
+ from python3_commons.db import AsyncSessionManager, is_healthy
210
+ from python3_commons.db.helpers import get_query
211
+ from python3_commons.conf import DBSettings
212
+
213
+ # Configuration
214
+ configs = {
215
+ "default": DBSettings(
216
+ dsn="postgresql+asyncpg://user:pass@localhost/db",
217
+ pool_size=20,
218
+ pool_timeout=30,
219
+ statement_timeout=30,
220
+ )
221
+ }
222
+ manager = AsyncSessionManager(configs)
223
+
224
+ # Usage
225
+ async with manager.get_session_context("default") as session:
226
+ result = await session.execute(...)
227
+
228
+ # Engine health check (executes SELECT 1 with timeout)
229
+ healthy = await is_healthy(manager.get_engine("default"))
230
+
231
+ # Dynamic query filtering and sorting
232
+ where_clause, order_clauses = get_query(
233
+ search={"name": "Alice"},
234
+ order_by="-created_at,name",
235
+ columns={
236
+ # name: (column, case_insensitive, cast_func, exact_match)
237
+ "name": (User.name, True, str, False),
238
+ "created_at": (User.created_at, False, str, True),
239
+ },
240
+ )
241
+ ```
242
+
243
+ ### Object Storage
244
+
245
+ Thin wrapper for `object-storage-client` supporting S3, GCS, Azure, and local file storage, configured via `object_storage_settings` (`OBJECT_STORAGE_WORK_PATH`) and `s3_settings` (`S3_ACCESS_KEY_ID`, `S3_SECRET_ACCESS_KEY`, `S3_REGION`). Storage functions accept a required `work_path` parameter to define the base storage location.
246
+
247
+ ```python
248
+ from python3_commons import object_storage
249
+ import io
250
+
251
+ # Upload an object
252
+ await object_storage.put_object(
253
+ work_path="my-bucket",
254
+ path="uploads/file.txt",
255
+ data=io.BytesIO(b"Hello World"),
256
+ length=11
257
+ )
258
+
259
+ # Download an object as bytes
260
+ content = await object_storage.get_object("my-bucket", "uploads/file.txt")
261
+
262
+ # Stream an object
263
+ async with object_storage.get_object_stream("my-bucket", "uploads/file.txt") as stream:
264
+ data = await stream.read()
265
+
266
+ # List objects with a prefix
267
+ async for obj in object_storage.list_objects("my-bucket", "uploads/"):
268
+ print(obj['Key'])
269
+
270
+ # Delete single or multiple objects
271
+ await object_storage.remove_object("my-bucket", "uploads/file.txt")
272
+ await object_storage.remove_objects("my-bucket", object_names=["uploads/file1.txt", "uploads/file2.txt"])
273
+ ```
274
+
275
+ ### OIDC Authentication
276
+
277
+ Client for OpenID Connect authentication, supporting configuration fetching, JWKS, and token acquisition.
278
+
279
+ ```python
280
+ from python3_commons.auth import OIDCClient
281
+ from pydantic import HttpUrl
282
+
283
+ client = OIDCClient(
284
+ authority_url=HttpUrl("https://auth.example.com/realms/myrealm"),
285
+ client_id="my-app-client",
286
+ client_secret="secret"
287
+ )
288
+
289
+ async with client:
290
+ token_response = await client.fetch_token(username="user", password="password")
291
+ print(token_response.access_token)
292
+ ```
293
+
294
+ ### Valkey Cache
295
+
296
+ Async caching using Valkey (Redis-compatible) with automatic Msgpack serialization for complex types.
297
+
298
+ ```python
299
+ from python3_commons import cache
300
+
301
+ # Store a dictionary
302
+ await cache.store("user:123", {"name": "Alice", "role": "admin"}, ttl=3600)
303
+
304
+ # Retrieve it
305
+ user_data = await cache.get("user:123")
306
+
307
+ # Set operations
308
+ await cache.add_set_item("active_users", "user:123")
309
+ is_active = await cache.has_set_item("active_users", "user:123")
310
+ ```
311
+
312
+ ### Structured Logging
313
+
314
+ A `JSONFormatter` for structured JSON logging and a `filter_maker` utility for level-based log filtering, compatible with standard Python `logging`.
315
+
316
+ ```python
317
+ import logging
318
+ from python3_commons.log.formatters import JSONFormatter
319
+ from python3_commons.log.filters import filter_maker
320
+
321
+ handler = logging.StreamHandler()
322
+ handler.setFormatter(JSONFormatter())
323
+ # Filter to process logs up to INFO level only
324
+ handler.addFilter(filter_maker("INFO"))
325
+ logging.getLogger().addHandler(handler)
326
+
327
+ logger = logging.getLogger("app")
328
+ logger.info("User logged in", extra={"user_id": "abc-123", "ip": "1.2.3.4"})
329
+ ```
330
+
331
+ ### Serialization
332
+
333
+ Enhanced JSON and Msgpack serialization for types not handled by default (Decimal, datetime, date, dataclasses, Pydantic models).
334
+
335
+ ```python
336
+ from python3_commons.serializers.msgspec import serialize_msgpack, deserialize_msgpack
337
+ from decimal import Decimal
338
+ from datetime import datetime
339
+
340
+ data = {
341
+ "amount": Decimal("150.75"),
342
+ "timestamp": datetime.now(),
343
+ "tags": {"finance", "internal"}
344
+ }
345
+
346
+ # Serialize to Msgpack
347
+ binary = serialize_msgpack(data)
348
+
349
+ # Deserialize back
350
+ restored = deserialize_msgpack(binary)
351
+ ```
352
+
353
+ ### RBAC Permissions
354
+
355
+ Database-backed Role-Based Access Control (RBAC) permission checking for users and API keys.
356
+
357
+ ```python
358
+ from python3_commons.permissions import has_user_permission, has_api_key_permission
359
+ from uuid import UUID
360
+
361
+ user_uuid = UUID("...")
362
+ allowed = await has_user_permission(session, user_uuid, "reports.view")
363
+
364
+ api_key_uuid = UUID("...")
365
+ key_allowed = await has_api_key_permission(session, api_key_uuid, "reports.view")
366
+ ```
367
+
368
+ ### Configuration Management
369
+
370
+ Typed application settings powered by Pydantic Settings:
371
+
372
+ - `CommonSettings` / `settings`: General logging level and format.
373
+ - `DBSettings` / `db_settings`: PostgreSQL database settings (`DB_` environment prefix, with `DB_PASS` alias for password).
374
+ - `S3Settings` / `s3_settings`: S3 credentials and connection configuration (`S3_` environment prefix: `ACCESS_KEY_ID`, `SECRET_ACCESS_KEY`, `REGION`).
375
+ - `ObjectStorageSettings` / `object_storage_settings`: Object storage root configuration (`OBJECT_STORAGE_` environment prefix: `WORK_PATH`).
376
+ - `ApiClientSettings` / `api_client_settings`: API client configuration (`API_CLIENT_` environment prefix: `AUDIT_PATH`).
377
+ - `OIDCSettings`: OpenID Connect configuration (`OIDC_` environment prefix).
378
+ - `ValkeySettings` / `valkey_settings`: Valkey/Redis cache configuration (`VALKEY_` environment prefix).
379
+
380
+ ### File System Utilities
381
+
382
+ File system traversal helpers:
383
+
384
+ ```python
385
+ from python3_commons.fs import iter_files
386
+ from pathlib import Path
387
+
388
+ # Recursively iterate files, ignoring hidden directories
389
+ for file_path in iter_files(Path("./data"), recursive=True):
390
+ print(file_path)
391
+ ```
392
+
393
+ ### General Helpers
394
+
395
+ A collection of useful utility functions:
396
+
397
+ - `to_snake_case(text)`: Converts strings to snake_case.
398
+ - `round_decimal(value, places)`: Rounds `Decimal` values.
399
+ - `tries(n)`: An async retry decorator.
400
+ - `log_execution_time`: An async decorator to log how long a function takes.
401
+ - `date_from_string` / `datetime_from_string`: Flexible date/time parsing.
402
+ - `date_range(start, end)`: Generator yielding dates in a range.
403
+ - `request_to_curl`: Converts request parameters to a `curl` command string.
404
+ - `SingletonMeta`: Thread-safe singleton metaclass.
405
+ - `replace_origin(url, host_url)`: Replaces scheme, host, and port of a URL.
406
+ - `parse_string_list`: Parses comma-separated string lists into tuples.
407
+
408
+ ```python
409
+ from python3_commons.helpers import tries, log_execution_time, SingletonMeta
410
+
411
+ @tries(3)
412
+ @log_execution_time
413
+ async def flaky_network_call():
414
+ ...
415
+
416
+ class AppService(metaclass=SingletonMeta):
417
+ pass
418
+ ```
419
+
420
+ ### Async CSV Stream
421
+
422
+ Efficiently generate CSV data as a byte stream from an async generator of tuples.
423
+
424
+ ```python
425
+ from python3_commons.generators import tuple_csv_stream
426
+
427
+ async def generate_rows():
428
+ for i in range(1000):
429
+ yield (i, f"Name {i}", 10.5 * i)
430
+
431
+ async for chunk in tuple_csv_stream(generate_rows(), header=("ID", "Name", "Value")):
432
+ # Send chunk to HTTP response or write to file
433
+ pass
434
+ ```
@@ -1,15 +1,15 @@
1
1
  python3_commons/__init__.py,sha256=0KgaYU46H_IMKn-BuasoRN3C4Hi45KlkHHoPbU9cwiA,189
2
- python3_commons/api_client.py,sha256=SThSzLhZJxVsKP8n1xDMCczA7SBFbxGFgrhKy9k-9_g,5532
2
+ python3_commons/api_client.py,sha256=kQCTUEkMtNkoHexSEa3qQCRJyxzLKEsbP-i1g7msPxw,6106
3
3
  python3_commons/async_functools.py,sha256=A2HvwFzZHxOWTp4IQM5UiBY2yg1S_0U1CWra5BWK0gk,9101
4
- python3_commons/audit.py,sha256=yMhNFHYO77efNjMi_Th0SEnIrTAqI0ZBnuiDzT4lEPE,2607
4
+ python3_commons/audit.py,sha256=y80A6cE2h3uGzjT1eg-6fiZ-IT-kKDmEtWNrXSQa5sY,3358
5
5
  python3_commons/auth.py,sha256=s928lYwpyX03BDTo1R6LpOgHiUSrjVrd1OVgTjvIiQU,8279
6
6
  python3_commons/cache.py,sha256=lowiXJqFgFy1Yg86wi9IhuoNqIUGP6nc5eNibmf0dfY,8018
7
- python3_commons/conf.py,sha256=p-El9tvxo4jkld3hzCH2U5ryX8sARexfv8Mgy5Cx5q8,3141
7
+ python3_commons/conf.py,sha256=t9jE2vnsetg-GCxOqsx6pPIAFzAKwO0w0g9kqSnhv1Q,3488
8
8
  python3_commons/exceptions.py,sha256=EGjHZVBnsM6CeBfPMqhL0IPMKjDJ_2-Z-aSPXwq91LE,36
9
9
  python3_commons/fs.py,sha256=dn8ZcwsQf9xcAEg6neoxLN6IzJbWpprfm8wV8S55BL0,337
10
10
  python3_commons/generators.py,sha256=P6sKdCFhHT79-DZgzPU-qmwZvHg3wU8a75UFIwrycOY,1163
11
11
  python3_commons/helpers.py,sha256=WTPu_jIOGYtxt1GYnH6sBMNDHry99AKv6ZaggGW2l1A,4986
12
- python3_commons/object_storage.py,sha256=EkDPTw373UdWp6h3klOYygizx-0uyXT8Sim-zRCbB3g,7176
12
+ python3_commons/object_storage.py,sha256=MBZdOZg8-BWoZ_8_i4-nkjlPRYV6pDxSR7fhoQmdFX4,7321
13
13
  python3_commons/permissions.py,sha256=gaMKSWg0MgPQTdP1voll4ItXcblXku9BlD0Lq3Xv64U,1724
14
14
  python3_commons/soap_client.py,sha256=w2lOyhhtkhwiNnHjvir8DfjGBO51XVg5rlDHyuudU2A,7169
15
15
  python3_commons/db/__init__.py,sha256=u-ZArdz5NQFvDLz2cjKT3_VCW3aPvEqIg1-bQ6rIDoQ,10196
@@ -27,9 +27,9 @@ python3_commons/serializers/common.py,sha256=VkA7C6wODvHk0QBXVX_x2JieDstihx3U__U
27
27
  python3_commons/serializers/json.py,sha256=UPkC3ps13x2C_NxwVV-K7Ewp4VjkVHSSUkJVw5k7Wiw,712
28
28
  python3_commons/serializers/msgpack.py,sha256=zESFBX34GsZ8rDu6Zk5V6CLT6P0mPilU0r04Ka6TblI,1474
29
29
  python3_commons/serializers/msgspec.py,sha256=upy5CBmK66-8hYnK5bAM_sZvZY5CAqZmzCw9GIF346I,2988
30
- python3_commons-0.24.6.dist-info/licenses/AUTHORS.rst,sha256=3R9JnfjfjH5RoPWOeqKFJgxVShSSfzQPIrEr1nxIo9Q,90
31
- python3_commons-0.24.6.dist-info/licenses/LICENSE,sha256=xxILuojHm4fKQOrMHPSslbyy6WuKAN2RiG74HbrYfzM,34575
32
- python3_commons-0.24.6.dist-info/METADATA,sha256=BlwCl4RT6crukvxjVm5VTE5GdhaetiOR-chlBg3-Qm8,9471
33
- python3_commons-0.24.6.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
34
- python3_commons-0.24.6.dist-info/top_level.txt,sha256=lJI6sCBf68eUHzupCnn2dzG10lH3jJKTWM_hrN1cQ7M,16
35
- python3_commons-0.24.6.dist-info/RECORD,,
30
+ python3_commons-0.25.0.dist-info/licenses/AUTHORS.rst,sha256=3R9JnfjfjH5RoPWOeqKFJgxVShSSfzQPIrEr1nxIo9Q,90
31
+ python3_commons-0.25.0.dist-info/licenses/LICENSE,sha256=xxILuojHm4fKQOrMHPSslbyy6WuKAN2RiG74HbrYfzM,34575
32
+ python3_commons-0.25.0.dist-info/METADATA,sha256=vNf600tOvWqIn98Kc86RteZ5UhVwx0p_S2t6ZhHpsWA,15092
33
+ python3_commons-0.25.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
34
+ python3_commons-0.25.0.dist-info/top_level.txt,sha256=lJI6sCBf68eUHzupCnn2dzG10lH3jJKTWM_hrN1cQ7M,16
35
+ python3_commons-0.25.0.dist-info/RECORD,,
@@ -1,310 +0,0 @@
1
- Metadata-Version: 2.4
2
- Name: python3-commons
3
- Version: 0.24.6
4
- Summary: Re-usable Python3 code
5
- Author-email: Oleg Korsak <kamikaze.is.waiting.you@gmail.com>
6
- License-Expression: GPL-3.0
7
- Project-URL: Homepage, https://github.com/kamikaze/python3-commons
8
- Project-URL: Documentation, https://github.com/kamikaze/python3-commons/wiki
9
- Classifier: Development Status :: 4 - Beta
10
- Classifier: Programming Language :: Python
11
- Requires-Python: <3.15.0,>=3.14.0
12
- Description-Content-Type: text/x-rst
13
- License-File: LICENSE
14
- License-File: AUTHORS.rst
15
- Requires-Dist: msgpack~=1.2.1
16
- Requires-Dist: msgspec~=0.21.1
17
- Requires-Dist: pydantic-settings~=2.15.0
18
- Provides-Extra: all
19
- Requires-Dist: python3_commons[api-client,audit,authn,authz,cache,database,object-storage,soap-client]; extra == "all"
20
- Provides-Extra: api-client
21
- Requires-Dist: aiohttp[speedups]<3.15.0,>=3.14.3; extra == "api-client"
22
- Requires-Dist: certifi~=2026.7.22; extra == "api-client"
23
- Requires-Dist: python3_commons[audit,object-storage]; extra == "api-client"
24
- Provides-Extra: audit
25
- Requires-Dist: lxml~=6.1.2; extra == "audit"
26
- Requires-Dist: zeep[async]~=4.3.3; extra == "audit"
27
- Requires-Dist: python3_commons[object-storage]; extra == "audit"
28
- Provides-Extra: authn
29
- Requires-Dist: aiohttp[speedups]<3.15.0,>=3.14.3; extra == "authn"
30
- Requires-Dist: certifi~=2026.7.22; extra == "authn"
31
- Requires-Dist: python3_commons[api-client]; extra == "authn"
32
- Provides-Extra: authz
33
- Requires-Dist: python3_commons[database]; extra == "authz"
34
- Requires-Dist: python3_commons[api-client]; extra == "authz"
35
- Provides-Extra: cache
36
- Requires-Dist: valkey[libvalkey]~=6.1.1; extra == "cache"
37
- Provides-Extra: database
38
- Requires-Dist: asyncpg~=0.31.0; extra == "database"
39
- Requires-Dist: SQLAlchemy[asyncio]~=2.0.52; extra == "database"
40
- Provides-Extra: object-storage
41
- Requires-Dist: aiobotocore==3.9.0; extra == "object-storage"
42
- Requires-Dist: object-storage-client~=0.1.8; extra == "object-storage"
43
- Provides-Extra: soap-client
44
- Requires-Dist: aiohttp[speedups]<3.15.0,>=3.14.3; extra == "soap-client"
45
- Requires-Dist: certifi~=2026.7.22; extra == "soap-client"
46
- Requires-Dist: lxml~=6.1.3; extra == "soap-client"
47
- Requires-Dist: requests~=2.34.2; extra == "soap-client"
48
- Requires-Dist: zeep[async]~=4.3.3; extra == "soap-client"
49
- Dynamic: license-file
50
-
51
- Re-usable Python3 code
52
- ======================
53
-
54
- Re-usable code for Python 3 projects.
55
-
56
- Installation
57
- ------------
58
-
59
- .. code-block:: bash
60
-
61
- uv add python3-commons
62
-
63
- Optional Dependencies
64
- ~~~~~~~~~~~~~~~~~~~~~
65
-
66
- Some features require extra dependencies. You can install them individually or all at once:
67
-
68
- - ``api-client``: For ``python3_commons.api_client``
69
- - ``audit``: For ``python3_commons.audit``
70
- - ``authn``: For ``python3_commons.auth``
71
- - ``authz``: For ``python3_commons.permissions``
72
- - ``cache``: For ``python3_commons.cache``
73
- - ``database``: For ``python3_commons.db``
74
- - ``object-storage``: For ``python3_commons.object_storage``
75
- - ``soap-client``: For ``python3_commons.soap_client``
76
- - ``all``: Install all optional dependencies
77
-
78
- .. code-block:: bash
79
-
80
- uv add "python3-commons[all]"
81
-
82
- Features
83
- --------
84
-
85
- Async LRU Cache
86
- ~~~~~~~~~~~~~~~
87
-
88
- An LRU cache decorator for asynchronous functions that handles ``await`` and prevents the dogpile effect (mitigating multiple concurrent calls for the same key by waiting for the first one to finish).
89
-
90
- .. code-block:: python
91
-
92
- from python3_commons.async_functools import async_lru_cache
93
-
94
- @async_lru_cache(maxsize=128)
95
- async def fetch_expensive_data(item_id: int):
96
- # This function will only be called once for a given item_id
97
- # even if multiple tasks await it simultaneously.
98
- return await db.get(item_id)
99
-
100
- # Usage
101
- result = await fetch_expensive_data(42)
102
-
103
- API Client
104
- ~~~~~~~~~~
105
-
106
- A context manager for ``aiohttp`` requests with built-in audit logging to S3 and standardized error mapping to Python exceptions.
107
-
108
- .. code-block:: python
109
-
110
- from python3_commons.api_client import request
111
- from aiohttp import ClientSession
112
-
113
- async with ClientSession() as session:
114
- async with request(
115
- session,
116
- base_url="https://api.example.com",
117
- uri="/data",
118
- method="get",
119
- audit_name="my_service_audit"
120
- ) as response:
121
- data = await response.json()
122
-
123
- SOAP Client
124
- ~~~~~~~~~~~
125
-
126
- Async SOAP client support for ``zeep`` with S3 auditing capabilities.
127
-
128
- .. code-block:: python
129
-
130
- from python3_commons.soap_client import soap_client
131
-
132
- async with soap_client("https://example.com/service?wsdl") as client:
133
- result = await client.service.GetData(id=42)
134
-
135
- Database Management
136
- ~~~~~~~~~~~~~~~~~~~
137
-
138
- SQLAlchemy async engine and session management with pool tuning, health checks, and dynamic query builders.
139
-
140
- .. code-block:: python
141
-
142
- from python3_commons.db import AsyncSessionManager
143
- from python3_commons.conf import DBSettings
144
-
145
- # Configuration
146
- configs = {"default": DBSettings(dsn="postgresql+asyncpg://user:pass@localhost/db")}
147
- manager = AsyncSessionManager(configs)
148
-
149
- # Usage
150
- async with manager.get_session_context("default") as session:
151
- result = await session.execute(...)
152
-
153
- # Health check
154
- from python3_commons.db import is_healthy
155
- await is_healthy(manager.get_engine("default"))
156
-
157
- Object Storage
158
- ~~~~~~~~~~~~~~
159
-
160
- Utilities for async S3 operations using ``aiobotocore``.
161
-
162
- .. code-block:: python
163
-
164
- from python3_commons import object_storage
165
- import io
166
-
167
- # Upload an object
168
- await object_storage.put_object(
169
- bucket_name="my-bucket",
170
- path="uploads/file.txt",
171
- data=io.BytesIO(b"Hello World"),
172
- length=11
173
- )
174
-
175
- # Download an object
176
- content = await object_storage.get_object("my-bucket", "uploads/file.txt")
177
-
178
- # List objects
179
- async for obj in object_storage.list_objects("my-bucket", "uploads/"):
180
- print(obj['Key'])
181
-
182
- OIDC Authentication
183
- ~~~~~~~~~~~~~~~~~~~
184
-
185
- Client for OpenID Connect authentication, supporting configuration fetching, JWKS, and token acquisition.
186
-
187
- .. code-block:: python
188
-
189
- from python3_commons.auth import OIDCClient
190
- from pydantic import HttpUrl
191
-
192
- client = OIDCClient(
193
- authority_url=HttpUrl("https://auth.example.com/realms/myrealm"),
194
- client_id="my-app-client",
195
- client_secret="secret"
196
- )
197
-
198
- async with client:
199
- token_response = await client.fetch_token(username="user", password="password")
200
- print(token_response.access_token)
201
-
202
- Valkey Cache
203
- ~~~~~~~~~~~~
204
-
205
- Async caching using Valkey (Redis-compatible) with automatic Msgpack serialization for complex types.
206
-
207
- .. code-block:: python
208
-
209
- from python3_commons import cache
210
-
211
- # Store a dictionary
212
- await cache.store("user:123", {"name": "Alice", "role": "admin"}, ttl=3600)
213
-
214
- # Retrieve it
215
- user_data = await cache.get("user:123")
216
-
217
- # Set operations
218
- await cache.add_set_item("active_users", "user:123")
219
- is_active = await cache.has_set_item("active_users", "user:123")
220
-
221
- Structured Logging
222
- ~~~~~~~~~~~~~~~~~~
223
-
224
- A ``JSONFormatter`` for structured logging, compatible with standard Python ``logging``.
225
-
226
- .. code-block:: python
227
-
228
- import logging
229
- from python3_commons.log.formatters import JSONFormatter
230
-
231
- handler = logging.StreamHandler()
232
- handler.setFormatter(JSONFormatter())
233
- logging.getLogger().addHandler(handler)
234
-
235
- logger = logging.getLogger("app")
236
- logger.info("User logged in", extra={"user_id": "abc-123", "ip": "1.2.3.4"})
237
-
238
- Serialization
239
- ~~~~~~~~~~~~~
240
-
241
- Enhanced JSON and Msgpack serialization for types not handled by default (Decimal, datetime, date, dataclasses, Pydantic models).
242
-
243
- .. code-block:: python
244
-
245
- from python3_commons.serializers.msgspec import serialize_msgpack, deserialize_msgpack
246
- from decimal import Decimal
247
- from datetime import datetime
248
-
249
- data = {
250
- "amount": Decimal("150.75"),
251
- "timestamp": datetime.now(),
252
- "tags": {"finance", "internal"}
253
- }
254
-
255
- # Serialize to Msgpack
256
- binary = serialize_msgpack(data)
257
-
258
- # Deserialize back
259
- restored = deserialize_msgpack(binary)
260
-
261
- RBAC Permissions
262
- ~~~~~~~~~~~~~~~~
263
-
264
- Database-backed Role-Based Access Control (RBAC) permission checking.
265
-
266
- .. code-block:: python
267
-
268
- from python3_commons.permissions import has_user_permission
269
- from uuid import UUID
270
-
271
- user_uuid = UUID("...")
272
- allowed = await has_user_permission(session, user_uuid, "reports.view")
273
-
274
- General Helpers
275
- ~~~~~~~~~~~~~~~
276
-
277
- A collection of useful utility functions:
278
-
279
- - ``to_snake_case(text)``: Converts strings to snake_case.
280
- - ``round_decimal(value, places)``: Rounds ``Decimal`` values.
281
- - ``tries(n)``: An async retry decorator.
282
- - ``log_execution_time``: An async decorator to log how long a function takes.
283
- - ``date_from_string`` / ``datetime_from_string``: Flexible date/time parsing.
284
- - ``request_to_curl``: Converts request parameters to a ``curl`` command string.
285
-
286
- .. code-block:: python
287
-
288
- from python3_commons.helpers import tries, log_execution_time
289
-
290
- @tries(3)
291
- @log_execution_time
292
- async def flaky_network_call():
293
- ...
294
-
295
- Async CSV Stream
296
- ~~~~~~~~~~~~~~~~
297
-
298
- Efficiently generate CSV data as a byte stream from an async generator of tuples.
299
-
300
- .. code-block:: python
301
-
302
- from python3_commons.generators import tuple_csv_stream
303
-
304
- async def generate_rows():
305
- for i in range(1000):
306
- yield (i, f"Name {i}", 10.5 * i)
307
-
308
- async for chunk in tuple_csv_stream(generate_rows(), header=("ID", "Name", "Value")):
309
- # Send chunk to HTTP response or write to file
310
- pass