python3-commons 0.5.8__tar.gz → 0.5.10__tar.gz

{python3_commons-0.5.8/src/python3_commons.egg-info → python3_commons-0.5.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: python3-commons
-Version: 0.5.8
+Version: 0.5.10
 Summary: Re-usable Python3 code
 Home-page: https://github.com/kamikaze/python3-commons
 Author: Oleg Korsak

{python3_commons-0.5.8 → python3_commons-0.5.10}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = python3-commons
-version = 0.5.8
+version = 0.5.10
 description = Re-usable Python3 code
 author = Oleg Korsak
 author_email = kamikaze.is.waiting.you@gmail.com

{python3_commons-0.5.8 → python3_commons-0.5.10}/src/python3_commons/audit.py

@@ -4,6 +4,7 @@ import logging
 import tarfile
 from datetime import datetime, timedelta, UTC
 from io import BytesIO
+from typing import Generator
 from uuid import uuid4
 
 from lxml import etree
@@ -18,6 +19,70 @@ from python3_commons.object_storage import get_s3_client
 logger = logging.getLogger(__name__)
 
 
+class BytesIOStream(io.BytesIO):
+    def __init__(self, generator: Generator[bytes, None, None], *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.generator = generator
+
+    def read(self, size: int = -1):
+        if size == -1:
+            size = 4096
+
+        while self.tell() < size:
+            try:
+                chunk = next(self.generator)
+            except StopIteration:
+                break
+
+            self.write(chunk)
+
+        if chunk := self.read(size):
+            pos = self.tell()
+
+            buf = self.getbuffer()
+            unread_data_size = len(buf) - pos
+
+            if unread_data_size > 0:
+                buf[:unread_data_size] = buf[pos:pos+unread_data_size]
+
+            self.seek(0)
+            self.truncate(unread_data_size)
+
+        return chunk
+
+    def readable(self):
+        return True
+
+
+def generate_archive(bucket_name: str, date_path: str, chunk_size: int = 4096) -> Generator[bytes, None, None]:
+    buffer = io.BytesIO()
+
+    with tarfile.open(fileobj=buffer, mode='w|bz2') as archive:
+        objects = object_storage.get_objects(bucket_name, date_path, recursive=True)
+
+        if objects:
+            logger.info(f'Compacting files in: {date_path}')
+
+            for name, last_modified, content in objects:
+                logger.info(f'Adding {name} to archive')
+                info = tarfile.TarInfo(name)
+                info.size = len(content)
+                info.mtime = last_modified.timestamp()
+                archive.addfile(info, io.BytesIO(content))
+                buffer.seek(0)
+
+                while True:
+                    chunk = buffer.read(chunk_size)
+
+                    if not chunk:
+                        break
+
+                    yield chunk
+
+                buffer.seek(0)
+                buffer.truncate(0)
+
+
 def write_audit_data_sync(settings: S3Settings, key: str, data: bytes):
     if settings.s3_secret_access_key:
         try:
@@ -43,27 +108,15 @@ async def archive_audit_data(root_path: str = 'audit'):
     month = now.month
     day = now.day
     bucket_name = s3_settings.s3_bucket
-    fo = BytesIO()
     object_names = []
     date_path = object_storage.get_absolute_path(f'{root_path}/{year}/{month:02}/{day:02}')
 
-    with tarfile.open(fileobj=fo, mode='w|bz2') as archive:
-        if objects := object_storage.get_objects(bucket_name, date_path, recursive=True):
-            logger.info(f'Compacting files in: {date_path}')
-
-            for name, last_modified, content in objects:
-                info = tarfile.TarInfo(name)
-                info.size = len(content)
-                info.mtime = last_modified.timestamp()
-                archive.addfile(info, BytesIO(content))
-                object_names.append(name)
-
-    fo.seek(0)
+    generator = generate_archive(bucket_name, date_path, chunk_size=4096)
+    archive_stream = BytesIOStream(generator)
 
     if object_names:
-        archive_path = object_storage.get_absolute_path(
-            f'audit/.archive/{year}_{month:02}_{day:02}.tar.bz2')
-        object_storage.put_object(bucket_name, archive_path, fo, fo.getbuffer().nbytes)
+        archive_path = object_storage.get_absolute_path(f'audit/.archive/{year}_{month:02}_{day:02}.tar.bz2')
+        object_storage.put_object(bucket_name, archive_path, archive_stream, -1, part_size=4096)
 
         if errors := object_storage.remove_objects(bucket_name, object_names=object_names):
             for error in errors:
@@ -82,10 +135,8 @@ class ZeepAuditPlugin(Plugin):
         now = datetime.now(tz=UTC)
         date_path = now.strftime('%Y/%m/%d')
         timestamp = now.strftime('%H%M%S')
-        coro = write_audit_data(
-            s3_settings,
-            f'{date_path}/{self.audit_name}/{operation.name}/{timestamp}_{str(uuid4())[-12:]}_{direction}.xml', xml
-        )
+        path = f'{date_path}/{self.audit_name}/{operation.name}/{timestamp}_{str(uuid4())[-12:]}_{direction}.xml'
+        coro = write_audit_data(s3_settings, path, xml)
 
         try:
             loop = asyncio.get_running_loop()

{python3_commons-0.5.8 → python3_commons-0.5.10}/src/python3_commons/object_storage.py

@@ -39,11 +39,11 @@ def get_absolute_path(path: str) -> str:
     return path
 
 
-def put_object(bucket_name: str, path: str, data: io.BytesIO, length: int) -> str:
+def put_object(bucket_name: str, path: str, data: io.BytesIO, length: int, part_size: int = 0) -> str:
     s3_client = get_s3_client(s3_settings)
 
     if s3_client:
-        result = s3_client.put_object(bucket_name, path, data, length)
+        result = s3_client.put_object(bucket_name, path, data, length, part_size=part_size)
 
         logger.debug(f'Stored object into object storage: {bucket_name}:{path}')
 

{python3_commons-0.5.8 → python3_commons-0.5.10/src/python3_commons.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: python3-commons
-Version: 0.5.8
+Version: 0.5.10
 Summary: Re-usable Python3 code
 Home-page: https://github.com/kamikaze/python3-commons
 Author: Oleg Korsak