python3-commons 0.8.36__py3-none-any.whl → 0.9.1__py3-none-any.whl

python3_commons/audit.py

@@ -5,23 +5,21 @@ import tarfile
 from bz2 import BZ2Compressor
 from collections import deque
 from datetime import UTC, datetime, timedelta
-from typing import Generator, Iterable
+from typing import AsyncGenerator
 from uuid import uuid4
 
 from lxml import etree
-from minio import S3Error
 from zeep.plugins import Plugin
 from zeep.wsdl.definitions import AbstractOperation
 
 from python3_commons import object_storage
 from python3_commons.conf import S3Settings, s3_settings
-from python3_commons.object_storage import ObjectStorage
 
 logger = logging.getLogger(__name__)
 
 
 class GeneratedStream(io.BytesIO):
-    def __init__(self, generator: Generator[bytes, None, None], *args, **kwargs):
+    def __init__(self, generator: AsyncGenerator[bytes], *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.generator = generator
 
@@ -29,7 +27,7 @@ class GeneratedStream(io.BytesIO):
         if size < 0:
             while True:
                 try:
-                    chunk = next(self.generator)
+                    chunk = anext(self.generator)
                 except StopIteration:
                     break
                 else:
@@ -39,7 +37,7 @@ class GeneratedStream(io.BytesIO):
 
             while total_written_size < size:
                 try:
-                    chunk = next(self.generator)
+                    chunk = anext(self.generator)
                 except StopIteration:
                     break
                 else:
@@ -67,17 +65,17 @@ class GeneratedStream(io.BytesIO):
         return True
 
 
-def generate_archive(
-    objects: Iterable[tuple[str, datetime, bytes]], chunk_size: int = 4096
-) -> Generator[bytes, None, None]:
+async def generate_archive(
+    objects: AsyncGenerator[tuple[str, datetime, bytes]], chunk_size: int = 4096
+) -> AsyncGenerator[bytes]:
     buffer = deque()
 
     with tarfile.open(fileobj=buffer, mode='w') as archive:
-        for name, last_modified, content in objects:
+        async for name, last_modified, content in objects:
             logger.info(f'Adding {name} to archive')
             info = tarfile.TarInfo(name)
             info.size = len(content)
-            info.mtime = last_modified.timestamp()
+            info.mtime = int(last_modified.timestamp())
             archive.addfile(info, io.BytesIO(content))
 
             buffer_length = buffer.tell()
@@ -109,10 +107,10 @@ def generate_archive(
     buffer.truncate(0)
 
 
-def generate_bzip2(chunks: Generator[bytes, None, None]) -> Generator[bytes, None, None]:
+async def generate_bzip2(chunks: AsyncGenerator[bytes]) -> AsyncGenerator[bytes]:
     compressor = BZ2Compressor()
 
-    for chunk in chunks:
+    async for chunk in chunks:
         if compressed_chunk := compressor.compress(chunk):
             yield compressed_chunk
 
@@ -120,14 +118,13 @@ def generate_bzip2(chunks: Generator[bytes, None, None]) -> Generator[bytes, Non
         yield compressed_chunk
 
 
-def write_audit_data_sync(settings: S3Settings, key: str, data: bytes):
+async def write_audit_data(settings: S3Settings, key: str, data: bytes):
     if settings.s3_secret_access_key:
         try:
-            client = ObjectStorage(settings).get_client()
             absolute_path = object_storage.get_absolute_path(f'audit/{key}')
 
-            client.put_object(settings.s3_bucket, absolute_path, io.BytesIO(data), len(data))
-        except S3Error as e:
+            await object_storage.put_object(settings.s3_bucket, absolute_path, io.BytesIO(data), len(data))
+        except Exception as e:
             logger.error(f'Failed storing object in storage: {e}')
         else:
             logger.debug(f'Stored object in storage: {key}')
@@ -135,10 +132,6 @@ def write_audit_data_sync(settings: S3Settings, key: str, data: bytes):
         logger.debug(f'S3 is not configured, not storing object in storage: {key}')
 
 
-async def write_audit_data(settings: S3Settings, key: str, data: bytes):
-    await asyncio.to_thread(write_audit_data_sync, settings, key, data)
-
-
 async def archive_audit_data(root_path: str = 'audit'):
     now = datetime.now(tz=UTC) - timedelta(days=1)
     year = now.year
@@ -155,9 +148,9 @@ async def archive_audit_data(root_path: str = 'audit'):
         archive_stream = GeneratedStream(bzip2_generator)
 
         archive_path = object_storage.get_absolute_path(f'audit/.archive/{year}_{month:02}_{day:02}.tar.bz2')
-        object_storage.put_object(bucket_name, archive_path, archive_stream, -1, part_size=5 * 1024 * 1024)
+        await object_storage.put_object(bucket_name, archive_path, archive_stream, -1, part_size=5 * 1024 * 1024)
 
-        if errors := object_storage.remove_objects(bucket_name, date_path):
+        if errors := await object_storage.remove_objects(bucket_name, date_path):
             for error in errors:
                 logger.error(f'Failed to delete object in {bucket_name=}: {error}')
 

python3_commons/conf.py

@@ -64,8 +64,8 @@ class DBSettings(BaseSettings):
 class S3Settings(BaseSettings):
     s3_endpoint_url: str | None = None
     s3_region_name: str | None = None
-    s3_access_key_id: SecretStr = ''
-    s3_secret_access_key: SecretStr = ''
+    s3_access_key_id: SecretStr = SecretStr('')
+    s3_secret_access_key: SecretStr = SecretStr('')
     s3_secure: bool = True
     s3_bucket: str | None = None
     s3_bucket_root: str | None = None

python3_commons/object_storage.py

@@ -65,7 +65,7 @@ async def put_object(bucket_name: str, path: str, data: io.BytesIO, length: int,
         except Exception as e:
             logger.error(f'Failed to put object to object storage: {bucket_name}:{path}', exc_info=e)
 
-            return None
+            raise
 
 
 @asynccontextmanager

{python3_commons-0.8.36.dist-info → python3_commons-0.9.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python3-commons
-Version: 0.8.36
+Version: 0.9.1
 Summary: Re-usable Python3 code
 Author-email: Oleg Korsak <kamikaze.is.waiting.you@gmail.com>
 License-Expression: GPL-3.0

{python3_commons-0.8.36.dist-info → python3_commons-0.9.1.dist-info}/RECORD

@@ -1,12 +1,12 @@
 python3_commons/__init__.py,sha256=0KgaYU46H_IMKn-BuasoRN3C4Hi45KlkHHoPbU9cwiA,189
 python3_commons/api_client.py,sha256=LT7_YmnYVHK2ucKxIhUJCZrmxgfy-lfOxx08-R0WvW0,4505
-python3_commons/audit.py,sha256=osx2ywZXf-V0zOkrhlNgSyzCBvojXQwSYBQ4-ze1xiM,6249
+python3_commons/audit.py,sha256=-jYGjkQ2r8rg3gj-C-5uTQ1lXhK3dRXkktonZxOs1PM,5994
 python3_commons/auth.py,sha256=vVaiJ5MHUMSbiLF6TIxe4dqVPhBlLttf940jjODL3a4,2934
 python3_commons/cache.py,sha256=lf27LTD4Z9Iqi5GaK8jH8UC0cL9sHH8wicZ88YDp6Mg,7725
-python3_commons/conf.py,sha256=JenspXyTqTlYeEb8X9Njfje1AiLCee23nm0k8zhYLfs,2283
+python3_commons/conf.py,sha256=K3GuXDp7iopHms_E-rNR4OzTBIoWSEZmMwRjxKYltTk,2305
 python3_commons/fs.py,sha256=wfLjybXndwLqNlOxTpm_HRJnuTcC4wbrHEOaEeCo9Wc,337
 python3_commons/helpers.py,sha256=qSBQ1A02BEmIY8UPJkCoO28aZq0BPsMg6AGW9k-CThM,3166
-python3_commons/object_storage.py,sha256=eF0EOB29yFn3oVr8y0SegEVJLo1W5gJT3IYcINEtLOU,6301
+python3_commons/object_storage.py,sha256=FET3R5gI_v1UB0z9a0K-aC-mhvA6AuOEd-Wav6kgjCk,6295
 python3_commons/permissions.py,sha256=bhjTp-tq-oaTGFMHNnSBlcVX5XQCTL0nWcu6SdPEAB4,1555
 python3_commons/db/__init__.py,sha256=5nArsGm17e-pelpOwAeBKy2n_Py20XqklZsNgkcJ-DQ,2947
 python3_commons/db/helpers.py,sha256=PY0h08aLiGx-J54wmP3GHPCgGCcLd60rayAUnR3aWdI,1742
@@ -21,9 +21,9 @@ python3_commons/serializers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMp
 python3_commons/serializers/json.py,sha256=91UaXLGKGj0yPyrnuMeNrkG2GuPUgcgAsmIokUgEwpU,808
 python3_commons/serializers/msgpack.py,sha256=WrvaPE187shSK8zkH4UHHMimEZNMv9RaDSwsBE2HlCw,1269
 python3_commons/serializers/msgspec.py,sha256=0AliXlEl5sewi0UENjI8St5ZScXE5DNRERKzqWKy2Ps,2674
-python3_commons-0.8.36.dist-info/licenses/AUTHORS.rst,sha256=3R9JnfjfjH5RoPWOeqKFJgxVShSSfzQPIrEr1nxIo9Q,90
-python3_commons-0.8.36.dist-info/licenses/LICENSE,sha256=xxILuojHm4fKQOrMHPSslbyy6WuKAN2RiG74HbrYfzM,34575
-python3_commons-0.8.36.dist-info/METADATA,sha256=SRAoOrNJjXa2lxTv7ncHdEq6yMyPCVhsoggAHQp_rwE,1134
-python3_commons-0.8.36.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-python3_commons-0.8.36.dist-info/top_level.txt,sha256=lJI6sCBf68eUHzupCnn2dzG10lH3jJKTWM_hrN1cQ7M,16
-python3_commons-0.8.36.dist-info/RECORD,,
+python3_commons-0.9.1.dist-info/licenses/AUTHORS.rst,sha256=3R9JnfjfjH5RoPWOeqKFJgxVShSSfzQPIrEr1nxIo9Q,90
+python3_commons-0.9.1.dist-info/licenses/LICENSE,sha256=xxILuojHm4fKQOrMHPSslbyy6WuKAN2RiG74HbrYfzM,34575
+python3_commons-0.9.1.dist-info/METADATA,sha256=dv7fEarrTDewpnOVFIHk_Mud2nWcanfkMyQa4GWjGNk,1133
+python3_commons-0.9.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+python3_commons-0.9.1.dist-info/top_level.txt,sha256=lJI6sCBf68eUHzupCnn2dzG10lH3jJKTWM_hrN1cQ7M,16
+python3_commons-0.9.1.dist-info/RECORD,,