python3-commons 0.5.14__tar.gz → 0.5.16__tar.gz

{python3_commons-0.5.14/src/python3_commons.egg-info → python3_commons-0.5.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: python3-commons
-Version: 0.5.14
+Version: 0.5.16
 Summary: Re-usable Python3 code
 Home-page: https://github.com/kamikaze/python3-commons
 Author: Oleg Korsak
@@ -19,8 +19,8 @@ Requires-Dist: lxml==5.2.2
 Requires-Dist: minio==7.2.7
 Requires-Dist: msgpack==1.0.8
 Requires-Dist: msgspec==0.18.6
-Requires-Dist: pydantic[email]==2.7.2
-Requires-Dist: pydantic-settings==2.2.1
+Requires-Dist: pydantic[email]==2.8.2
+Requires-Dist: pydantic-settings==2.3.4
 Requires-Dist: zeep==4.2.1
 Provides-Extra: testing
 Requires-Dist: pytest; extra == "testing"

{python3_commons-0.5.14 → python3_commons-0.5.16}/requirements.txt

@@ -3,6 +3,6 @@ lxml==5.2.2
 minio==7.2.7
 msgpack==1.0.8
 msgspec==0.18.6
-pydantic[email]==2.7.2
-pydantic-settings==2.2.1
+pydantic[email]==2.8.2
+pydantic-settings==2.3.4
 zeep==4.2.1

{python3_commons-0.5.14 → python3_commons-0.5.16}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = python3-commons
-version = 0.5.14
+version = 0.5.16
 description = Re-usable Python3 code
 author = Oleg Korsak
 author_email = kamikaze.is.waiting.you@gmail.com
@@ -28,8 +28,8 @@ install_requires =
 	minio==7.2.7
 	msgpack==1.0.8
 	msgspec==0.18.6
-	pydantic[email]==2.7.2
-	pydantic-settings==2.2.1
+	pydantic[email]==2.8.2
+	pydantic-settings==2.3.4
 	zeep==4.2.1
 python_requires = >=3.12
 

{python3_commons-0.5.14 → python3_commons-0.5.16}/src/python3_commons/audit.py

@@ -3,8 +3,7 @@ import io
 import logging
 import tarfile
 from datetime import datetime, timedelta, UTC
-from io import BytesIO
-from typing import Generator
+from typing import Generator, Iterable
 from uuid import uuid4
 
 from lxml import etree
@@ -19,7 +18,7 @@ from python3_commons.object_storage import get_s3_client
 logger = logging.getLogger(__name__)
 
 
-class BytesIOStream(io.BytesIO):
+class GeneratedStream(io.BytesIO):
     def __init__(self, generator: Generator[bytes, None, None], *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.generator = generator
@@ -54,33 +53,30 @@ class BytesIOStream(io.BytesIO):
         return True
 
 
-def generate_archive(bucket_name: str, date_path: str, chunk_size: int = 4096) -> Generator[bytes, None, None]:
+def generate_archive(objects: Iterable[tuple[str, datetime, bytes]],
+                     chunk_size: int = 4096) -> Generator[bytes, None, None]:
     buffer = io.BytesIO()
 
-    with tarfile.open(fileobj=buffer, mode='w|bz2') as archive:
-        objects = object_storage.get_objects(bucket_name, date_path, recursive=True)
+    with tarfile.open(fileobj=buffer, mode='w') as archive:
+        for name, last_modified, content in objects:
+            logger.info(f'Adding {name} to archive')
+            info = tarfile.TarInfo(name)
+            info.size = len(content)
+            info.mtime = last_modified.timestamp()
+            archive.addfile(info, io.BytesIO(content))
 
-        if objects:
-            logger.info(f'Compacting files in: {date_path}')
+            buffer.seek(0)
 
-            for name, last_modified, content in objects:
-                logger.info(f'Adding {name} to archive')
-                info = tarfile.TarInfo(name)
-                info.size = len(content)
-                info.mtime = last_modified.timestamp()
-                archive.addfile(info, io.BytesIO(content))
-                buffer.seek(0)
+            while True:
+                chunk = buffer.read(chunk_size)
 
-                while True:
-                    chunk = buffer.read(chunk_size)
+                if not chunk:
+                    break
 
-                    if not chunk:
-                        break
+                yield chunk
 
-                    yield chunk
-
-                buffer.seek(0)
-                buffer.truncate(0)
+            buffer.seek(0)
+            buffer.truncate(0)
 
 
 def write_audit_data_sync(settings: S3Settings, key: str, data: bytes):
@@ -110,15 +106,18 @@ async def archive_audit_data(root_path: str = 'audit'):
     bucket_name = s3_settings.s3_bucket
     date_path = object_storage.get_absolute_path(f'{root_path}/{year}/{month:02}/{day:02}')
 
-    generator = generate_archive(bucket_name, date_path, chunk_size=5*1024*1024)
-    archive_stream = BytesIOStream(generator)
+    if objects := object_storage.get_objects(bucket_name, date_path, recursive=True):
+        logger.info(f'Compacting files in: {date_path}')
+
+        generator = generate_archive(objects, chunk_size=5*1024*1024)
+        archive_stream = GeneratedStream(generator)
 
-    archive_path = object_storage.get_absolute_path(f'audit/.archive/{year}_{month:02}_{day:02}.tar.bz2')
-    object_storage.put_object(bucket_name, archive_path, archive_stream, -1, part_size=5*1024*1024)
+        archive_path = object_storage.get_absolute_path(f'audit/.archive/{year}_{month:02}_{day:02}.tar.bz2')
+        object_storage.put_object(bucket_name, archive_path, archive_stream, -1, part_size=5*1024*1024)
 
-    if errors := object_storage.remove_objects(bucket_name, date_path):
-        for error in errors:
-            logger.error(f'Failed to delete object in {bucket_name=}: {error}')
+        if errors := object_storage.remove_objects(bucket_name, date_path):
+            for error in errors:
+                logger.error(f'Failed to delete object in {bucket_name=}: {error}')
 
 
 class ZeepAuditPlugin(Plugin):

{python3_commons-0.5.14 → python3_commons-0.5.16}/src/python3_commons/helpers.py

@@ -1,8 +1,9 @@
 import datetime
 import logging
+import shlex
 
 from decimal import Decimal, ROUND_HALF_UP
-
+from typing import Mapping
 
 logger = logging.getLogger(__name__)
 
@@ -47,3 +48,18 @@ def round_decimal(value: Decimal, decimal_places=2, rounding_mode=ROUND_HALF_UP)
         return value.quantize(Decimal(10) ** -decimal_places, rounding=rounding_mode)
     except AttributeError:
         return value
+
+
+def request_to_curl(url: str, method: str, headers: Mapping, body: bytes | None = None) -> str:
+    curl_cmd = ['curl', '-i', '-X', method, shlex.quote(url)]
+
+    for key, value in headers.items():
+        header_line = f'{key}: {value}'
+        curl_cmd.append('-H')
+        curl_cmd.append(shlex.quote(header_line))
+
+    if body is not None:
+        curl_cmd.append('--data')
+        curl_cmd.append(shlex.quote(body.decode('utf-8')))
+
+    return ' '.join(curl_cmd)

{python3_commons-0.5.14 → python3_commons-0.5.16/src/python3_commons.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: python3-commons
-Version: 0.5.14
+Version: 0.5.16
 Summary: Re-usable Python3 code
 Home-page: https://github.com/kamikaze/python3-commons
 Author: Oleg Korsak
@@ -19,8 +19,8 @@ Requires-Dist: lxml==5.2.2
 Requires-Dist: minio==7.2.7
 Requires-Dist: msgpack==1.0.8
 Requires-Dist: msgspec==0.18.6
-Requires-Dist: pydantic[email]==2.7.2
-Requires-Dist: pydantic-settings==2.2.1
+Requires-Dist: pydantic[email]==2.8.2
+Requires-Dist: pydantic-settings==2.3.4
 Requires-Dist: zeep==4.2.1
 Provides-Extra: testing
 Requires-Dist: pytest; extra == "testing"

{python3_commons-0.5.14 → python3_commons-0.5.16}/src/python3_commons.egg-info/SOURCES.txt

@@ -40,5 +40,6 @@ src/python3_commons/serializers/json.py
 src/python3_commons/serializers/msgpack.py
 src/python3_commons/serializers/msgspec.py
 tests/conftest.py
+tests/test_audit.py
 tests/test_msgpack.py
 tests/test_msgspec.py

{python3_commons-0.5.14 → python3_commons-0.5.16}/src/python3_commons.egg-info/requires.txt

@@ -3,8 +3,8 @@ lxml==5.2.2
 minio==7.2.7
 msgpack==1.0.8
 msgspec==0.18.6
-pydantic[email]==2.7.2
-pydantic-settings==2.2.1
+pydantic[email]==2.8.2
+pydantic-settings==2.3.4
 zeep==4.2.1
 
 [testing]

{python3_commons-0.5.14 → python3_commons-0.5.16}/tests/conftest.py

@@ -59,3 +59,13 @@ def data_struct():
         e=date(2023, 7, 24),
         f=Decimal('1.23')
     )
+
+
+@pytest.fixture
+def s3_file_objects() -> tuple:
+    return (
+        ('file_a.txt', datetime(2024, 1, 1), b'ABCDE', ),
+        ('file_b.txt', datetime(2024, 1, 2), b'FGHIJ', ),
+        ('file_c.txt', datetime(2024, 1, 3), b'KLMNO', ),
+        ('file_d.txt', datetime(2024, 1, 4), b'PQRST', ),
+    )

python3_commons-0.5.16/tests/test_audit.py

@@ -0,0 +1,10 @@
+from python3_commons.audit import GeneratedStream, generate_archive
+
+
+def test_generated_stream(s3_file_objects):
+    expected_data = b''
+    generator = generate_archive(s3_file_objects, chunk_size=5 * 1024 * 1024)
+    archive_stream = GeneratedStream(generator)
+    archived_data = archive_stream.read()
+
+    assert archived_data == expected_data