python3-commons 0.5.7__py2.py3-none-any.whl → 0.5.9__py2.py3-none-any.whl

python3_commons/audit.py

@@ -4,6 +4,7 @@ import logging
 import tarfile
 from datetime import datetime, timedelta, UTC
 from io import BytesIO
+from typing import Generator
 from uuid import uuid4
 
 from lxml import etree
@@ -18,6 +19,69 @@ from python3_commons.object_storage import get_s3_client
 logger = logging.getLogger(__name__)
 
 
+class BytesIOStream(io.BytesIO):
+    def __init__(self, generator: Generator[bytes, None, None], *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.generator = generator
+
+    def read(self, size: int = -1):
+        if size == -1:
+            size = 4096
+
+        while self.tell() < size:
+            try:
+                chunk = next(self.generator)
+            except StopIteration:
+                break
+
+            self.write(chunk)
+
+        if chunk := self.read(size):
+            pos = self.tell()
+
+            buf = self.getbuffer()
+            unread_data_size = len(buf) - pos
+
+            if unread_data_size > 0:
+                buf[:unread_data_size] = buf[pos:pos+unread_data_size]
+
+            self.truncate(unread_data_size)
+            self.seek(0)
+
+        return chunk
+
+    def readable(self):
+        return True
+
+
+def generate_archive(bucket_name: str, date_path: str, chunk_size: int = 4096) -> Generator[bytes, None, None]:
+    buffer = io.BytesIO()
+
+    with tarfile.open(fileobj=buffer, mode='w|bz2') as archive:
+        objects = object_storage.get_objects(bucket_name, date_path, recursive=True)
+
+        if objects:
+            logger.info(f'Compacting files in: {date_path}')
+
+            for name, last_modified, content in objects:
+                info = tarfile.TarInfo(name)
+                info.size = len(content)
+                info.mtime = last_modified.timestamp()
+                archive.addfile(info, io.BytesIO(content))
+                buffer.seek(0)
+
+                while True:
+                    chunk = buffer.read(chunk_size)
+
+                    if not chunk:
+                        break
+
+                    yield chunk
+
+                buffer.seek(0)
+                buffer.truncate(0)
+
+
 def write_audit_data_sync(settings: S3Settings, key: str, data: bytes):
     if settings.s3_secret_access_key:
         try:
@@ -45,25 +109,14 @@ async def archive_audit_data(root_path: str = 'audit'):
     bucket_name = s3_settings.s3_bucket
     fo = BytesIO()
     object_names = []
-    date_path = f'{root_path}/{year}/{month:02}/{day:02}'
-
-    with tarfile.open(fileobj=fo, mode='w|bz2') as archive:
-        if objects := object_storage.get_objects(bucket_name, date_path, recursive=True):
-            logger.info(f'Compacting files in: {date_path}')
-
-            for name, last_modified, content in objects:
-                info = tarfile.TarInfo(name)
-                info.size = len(content)
-                info.mtime = last_modified.timestamp()
-                archive.addfile(info, BytesIO(content))
-                object_names.append(name)
+    date_path = object_storage.get_absolute_path(f'{root_path}/{year}/{month:02}/{day:02}')
 
-    fo.seek(0)
+    generator = generate_archive(bucket_name, date_path, chunk_size=4096)
+    archive_stream = BytesIOStream(generator)
 
     if object_names:
-        archive_path = object_storage.get_absolute_path(
-            f'audit/.archive/{year}_{month:02}_{day:02}.tar.bz2')
-        object_storage.put_object(bucket_name, archive_path, fo, fo.getbuffer().nbytes)
+        archive_path = object_storage.get_absolute_path(f'audit/.archive/{year}_{month:02}_{day:02}.tar.bz2')
+        object_storage.put_object(bucket_name, archive_path, archive_stream, -1, part_size=4096)
 
         if errors := object_storage.remove_objects(bucket_name, object_names=object_names):
             for error in errors:
@@ -82,10 +135,8 @@ class ZeepAuditPlugin(Plugin):
         now = datetime.now(tz=UTC)
         date_path = now.strftime('%Y/%m/%d')
         timestamp = now.strftime('%H%M%S')
-        coro = write_audit_data(
-            s3_settings,
-            f'{date_path}/{self.audit_name}/{operation.name}/{timestamp}_{str(uuid4())[-12:]}_{direction}.xml', xml
-        )
+        path = f'{date_path}/{self.audit_name}/{operation.name}/{timestamp}_{str(uuid4())[-12:]}_{direction}.xml'
+        coro = write_audit_data(s3_settings, path, xml)
 
         try:
             loop = asyncio.get_running_loop()

python3_commons/object_storage.py

@@ -39,12 +39,11 @@ def get_absolute_path(path: str) -> str:
     return path
 
 
-def put_object(bucket_name: str, path: str, data: io.BytesIO, length: int) -> str:
+def put_object(bucket_name: str, path: str, data: io.BytesIO, length: int, part_size: int = 0) -> str:
     s3_client = get_s3_client(s3_settings)
 
     if s3_client:
-        path = get_absolute_path(path)
-        result = s3_client.put_object(bucket_name, path, data, length)
+        result = s3_client.put_object(bucket_name, path, data, length, part_size=part_size)
 
         logger.debug(f'Stored object into object storage: {bucket_name}:{path}')
 
@@ -57,7 +56,6 @@ def get_object_stream(bucket_name: str, path: str):
     s3_client = get_s3_client(s3_settings)
 
     if s3_client:
-        path = get_absolute_path(path)
         logger.debug(f'Getting object from object storage: {bucket_name}:{path}')
 
         try:
@@ -94,8 +92,6 @@ def list_objects(bucket_name: str, prefix: str, recursive: bool = True) -> Gener
 
 def get_objects(bucket_name: str, path: str,
                 recursive: bool = True) -> Generator[tuple[str, datetime, bytes], None, None]:
-    path = get_absolute_path(path)
-
     for obj in list_objects(bucket_name, path, recursive):
         object_name = obj.object_name
 
@@ -117,7 +113,6 @@ def remove_objects(bucket_name: str, prefix: str = None,
     s3_client = get_s3_client(s3_settings)
 
     if prefix:
-        prefix = get_absolute_path(prefix)
         delete_object_list = map(
             lambda obj: DeleteObject(obj.object_name), s3_client.list_objects(bucket_name, prefix=prefix,
                                                                               recursive=True)

{python3_commons-0.5.7.dist-info → python3_commons-0.5.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: python3-commons
-Version: 0.5.7
+Version: 0.5.9
 Summary: Re-usable Python3 code
 Home-page: https://github.com/kamikaze/python3-commons
 Author: Oleg Korsak

{python3_commons-0.5.7.dist-info → python3_commons-0.5.9.dist-info}/RECORD

@@ -1,10 +1,10 @@
 python3_commons/__init__.py,sha256=h-KTJUaQ50E3RmkTn_GO88IRunmDTEpNc3ylpFvCTOc,339
-python3_commons/audit.py,sha256=O9-Qf8TC1CJEDbWlTMdx-ap3UIoxbncpkEhSszrf_N4,3771
+python3_commons/audit.py,sha256=M8vAZOUsif4EhHMca9ahM-LvfbUrE0GvOLARJVrtjms,5177
 python3_commons/conf.py,sha256=vSXyFwXx2wb1uy8IffeeI-RoTqhUZs0RLSSG2OLc2ss,598
 python3_commons/db.py,sha256=qhaDIdzBWgFyeP_XPKfHZlYVlwS2bpBPYMv84yV6820,738
 python3_commons/fs.py,sha256=wfLjybXndwLqNlOxTpm_HRJnuTcC4wbrHEOaEeCo9Wc,337
 python3_commons/helpers.py,sha256=wI7afc-8o-SBpLBHQnWXiudw456EAVKguJaeZiSNXwE,1377
-python3_commons/object_storage.py,sha256=6iXf5GyYKE40X4E0LbabpcseLzNYMJzGTVkyTdse64c,3946
+python3_commons/object_storage.py,sha256=oTA56qS-m_1OSWLCTAKdDgcxjcBhXgbttIhNyKY0WrY,3830
 python3_commons/logging/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 python3_commons/logging/filters.py,sha256=fuyjXZAUm-i2MNrxvFYag8F8Rr27x8W8MdV3ke6miSs,175
 python3_commons/logging/formatter.py,sha256=UXmmh1yd5Kc2dpvSHn6uCWLDWE2LMjlYAaH8cg3siV4,720
@@ -12,9 +12,9 @@ python3_commons/serializers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMp
 python3_commons/serializers/json.py,sha256=P288wWz9ic38QWEMrpp_uwKPYkQiOgvE1cI4WZn6ZCg,808
 python3_commons/serializers/msgpack.py,sha256=P7CZoRTBeDtgALT5GTOZVCNM_3snOfCnfh5S3zcCIEY,1679
 python3_commons/serializers/msgspec.py,sha256=ZrfQWBz_67t3yjU_S6avnRJZzYN-HFzef2qGB5W21KI,2023
-python3_commons-0.5.7.dist-info/AUTHORS.rst,sha256=3R9JnfjfjH5RoPWOeqKFJgxVShSSfzQPIrEr1nxIo9Q,90
-python3_commons-0.5.7.dist-info/LICENSE,sha256=xxILuojHm4fKQOrMHPSslbyy6WuKAN2RiG74HbrYfzM,34575
-python3_commons-0.5.7.dist-info/METADATA,sha256=II9MLcbrYmnS8RmEUZ8JDcyZa44K2Zrx5kYP1lS8onE,974
-python3_commons-0.5.7.dist-info/WHEEL,sha256=DZajD4pwLWue70CAfc7YaxT1wLUciNBvN_TTcvXpltE,110
-python3_commons-0.5.7.dist-info/top_level.txt,sha256=lJI6sCBf68eUHzupCnn2dzG10lH3jJKTWM_hrN1cQ7M,16
-python3_commons-0.5.7.dist-info/RECORD,,
+python3_commons-0.5.9.dist-info/AUTHORS.rst,sha256=3R9JnfjfjH5RoPWOeqKFJgxVShSSfzQPIrEr1nxIo9Q,90
+python3_commons-0.5.9.dist-info/LICENSE,sha256=xxILuojHm4fKQOrMHPSslbyy6WuKAN2RiG74HbrYfzM,34575
+python3_commons-0.5.9.dist-info/METADATA,sha256=f4jClE1oP4nICSJvizStlkBotry6tCx7LTl818aiABU,974
+python3_commons-0.5.9.dist-info/WHEEL,sha256=DZajD4pwLWue70CAfc7YaxT1wLUciNBvN_TTcvXpltE,110
+python3_commons-0.5.9.dist-info/top_level.txt,sha256=lJI6sCBf68eUHzupCnn2dzG10lH3jJKTWM_hrN1cQ7M,16
+python3_commons-0.5.9.dist-info/RECORD,,