python3-commons 0.5.19__py2.py3-none-any.whl → 0.5.20__py2.py3-none-any.whl

python3_commons/audit.py

@@ -2,6 +2,7 @@ import asyncio
 import io
 import logging
 import tarfile
+from bz2 import BZ2Compressor
 from datetime import datetime, timedelta, UTC
 from typing import Generator, Iterable
 from uuid import uuid4
@@ -92,6 +93,17 @@ def generate_archive(objects: Iterable[tuple[str, datetime, bytes]],
             buffer.truncate(0)
 
 
+def generate_bzip2(chunks: Generator[bytes, None, None]) -> Generator[bytes, None, None]:
+    compressor = BZ2Compressor()
+
+    for chunk in chunks:
+        if compressed_chunk := compressor.compress(chunk):
+            yield compressed_chunk
+
+    if compressed_chunk := compressor.flush():
+        yield compressed_chunk
+
+
 def write_audit_data_sync(settings: S3Settings, key: str, data: bytes):
     if settings.s3_secret_access_key:
         try:
@@ -123,7 +135,8 @@ async def archive_audit_data(root_path: str = 'audit'):
         logger.info(f'Compacting files in: {date_path}')
 
         generator = generate_archive(objects, chunk_size=5*1024*1024)
-        archive_stream = GeneratedStream(generator)
+        bzip2_generator = generate_bzip2(generator)
+        archive_stream = GeneratedStream(bzip2_generator)
 
         archive_path = object_storage.get_absolute_path(f'audit/.archive/{year}_{month:02}_{day:02}.tar.bz2')
         object_storage.put_object(bucket_name, archive_path, archive_stream, -1, part_size=5*1024*1024)

{python3_commons-0.5.19.dist-info → python3_commons-0.5.20.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: python3-commons
-Version: 0.5.19
+Version: 0.5.20
 Summary: Re-usable Python3 code
 Home-page: https://github.com/kamikaze/python3-commons
 Author: Oleg Korsak
@@ -14,17 +14,17 @@ Requires-Python: >=3.12
 Description-Content-Type: text/x-rst; charset=UTF-8
 License-File: LICENSE
 License-File: AUTHORS.rst
-Requires-Dist: asyncpg ==0.29.0
-Requires-Dist: lxml ==5.2.2
-Requires-Dist: minio ==7.2.7
-Requires-Dist: msgpack ==1.0.8
-Requires-Dist: msgspec ==0.18.6
-Requires-Dist: pydantic[email] ==2.8.2
-Requires-Dist: pydantic-settings ==2.4.0
-Requires-Dist: zeep ==4.2.1
+Requires-Dist: asyncpg==0.29.0
+Requires-Dist: lxml==5.2.2
+Requires-Dist: minio==7.2.7
+Requires-Dist: msgpack==1.0.8
+Requires-Dist: msgspec==0.18.6
+Requires-Dist: pydantic[email]==2.8.2
+Requires-Dist: pydantic-settings==2.4.0
+Requires-Dist: zeep==4.2.1
 Provides-Extra: testing
-Requires-Dist: pytest ; extra == 'testing'
-Requires-Dist: pytest-cov ; extra == 'testing'
+Requires-Dist: pytest; extra == "testing"
+Requires-Dist: pytest-cov; extra == "testing"
 
 Re-usable Python3 code
 ======================

{python3_commons-0.5.19.dist-info → python3_commons-0.5.20.dist-info}/RECORD

@@ -1,11 +1,10 @@
 python3_commons/__init__.py,sha256=h-KTJUaQ50E3RmkTn_GO88IRunmDTEpNc3ylpFvCTOc,339
-python3_commons/audit.py,sha256=lQbgThInSSGtEGQDLhrb0XrxDWPteTUqKf8Kx0Oywtk,5453
+python3_commons/audit.py,sha256=fokdHYthtZWc1HQA2lZ74mQVkAh57oFzH-N97RS3tSc,5865
 python3_commons/conf.py,sha256=vSXyFwXx2wb1uy8IffeeI-RoTqhUZs0RLSSG2OLc2ss,598
 python3_commons/db.py,sha256=qhaDIdzBWgFyeP_XPKfHZlYVlwS2bpBPYMv84yV6820,738
 python3_commons/fs.py,sha256=wfLjybXndwLqNlOxTpm_HRJnuTcC4wbrHEOaEeCo9Wc,337
 python3_commons/helpers.py,sha256=hZG8M-mltBC8I9yx5ZuAM7bABFNuOsqX6FzSaQz4y9U,2480
 python3_commons/object_storage.py,sha256=pk2J14RL9FLTwaks-IS4EJX9TBMLzid35CroGftLNhU,4067
-python3_commons/stream_tar.py,sha256=1aA_IIM4oLdh2EZx_5tSeSm6flgCJ_QWbXLFAjg6hEo,2442
 python3_commons/logging/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 python3_commons/logging/filters.py,sha256=fuyjXZAUm-i2MNrxvFYag8F8Rr27x8W8MdV3ke6miSs,175
 python3_commons/logging/formatter.py,sha256=UXmmh1yd5Kc2dpvSHn6uCWLDWE2LMjlYAaH8cg3siV4,720
@@ -13,9 +12,9 @@ python3_commons/serializers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMp
 python3_commons/serializers/json.py,sha256=P288wWz9ic38QWEMrpp_uwKPYkQiOgvE1cI4WZn6ZCg,808
 python3_commons/serializers/msgpack.py,sha256=tzIGGyDL3UpZnnouCtnxuYDx6InKM_C3PP1N4PN8wd4,1269
 python3_commons/serializers/msgspec.py,sha256=EknuMpxi_kU25Iv_m10E9rk8b31AkjVumuzyjp7WgrU,1699
-python3_commons-0.5.19.dist-info/AUTHORS.rst,sha256=3R9JnfjfjH5RoPWOeqKFJgxVShSSfzQPIrEr1nxIo9Q,90
-python3_commons-0.5.19.dist-info/LICENSE,sha256=xxILuojHm4fKQOrMHPSslbyy6WuKAN2RiG74HbrYfzM,34575
-python3_commons-0.5.19.dist-info/METADATA,sha256=uQN3EfXdlIgIHQJMcfpK6a0Arju4Vx4xuGuAe-uCiHk,975
-python3_commons-0.5.19.dist-info/WHEEL,sha256=XRxW4r1PNiVhMpP4bT9oWtu3HyndxpJ84SkubFgzp_Y,109
-python3_commons-0.5.19.dist-info/top_level.txt,sha256=lJI6sCBf68eUHzupCnn2dzG10lH3jJKTWM_hrN1cQ7M,16
-python3_commons-0.5.19.dist-info/RECORD,,
+python3_commons-0.5.20.dist-info/AUTHORS.rst,sha256=3R9JnfjfjH5RoPWOeqKFJgxVShSSfzQPIrEr1nxIo9Q,90
+python3_commons-0.5.20.dist-info/LICENSE,sha256=xxILuojHm4fKQOrMHPSslbyy6WuKAN2RiG74HbrYfzM,34575
+python3_commons-0.5.20.dist-info/METADATA,sha256=q_9B5XQRV-h0giivNG1AhyY09xUW1g2rnTEVLc9rYwc,965
+python3_commons-0.5.20.dist-info/WHEEL,sha256=XRxW4r1PNiVhMpP4bT9oWtu3HyndxpJ84SkubFgzp_Y,109
+python3_commons-0.5.20.dist-info/top_level.txt,sha256=lJI6sCBf68eUHzupCnn2dzG10lH3jJKTWM_hrN1cQ7M,16
+python3_commons-0.5.20.dist-info/RECORD,,

python3_commons/stream_tar.py

@@ -1,105 +0,0 @@
-import logging
-import os
-import sys
-import tarfile
-from io import BytesIO
-
-
-logger = logging.getLogger(__name__)
-BLOCK_SIZE = 4096
-
-
-class FileStream(object):
-    def __init__(self):
-        self.buffer = BytesIO()
-        self.offset = 0
-
-    def write(self, chunk):
-        self.buffer.write(chunk)
-        self.offset += len(chunk)
-
-    def tell(self):
-        return self.offset
-
-    def close(self):
-        self.buffer.close()
-
-    def pop(self):
-        value = self.buffer.getvalue()
-        self.buffer.seek(0)
-        self.buffer.truncate()
-
-        return value
-
-
-def stream_build_tar(in_filename, streaming_fp):
-    tar = tarfile.TarFile.open('test.tar.gz', 'w|gz', streaming_fp)
-
-    stat = os.stat(in_filename)
-
-    tar_info = tarfile.TarInfo(in_filename)
-
-    # Note that you can get this information from the storage backend,
-    # but it's valid for either to raise a NotImplementedError, so it's
-    # important to check.
-    #
-    # Things like the mode or ownership won't be available.
-    tar_info.mtime = stat.st_mtime
-    tar_info.size = stat.st_size
-
-    # Note that we don't pass a fileobj, so we don't write any data
-    # through addfile. We'll do this ourselves.
-    tar.addfile(tar_info)
-
-    yield
-
-    with open(in_filename, 'rb') as in_fp:
-        total_size = 0
-
-        while True:
-            s = in_fp.read(BLOCK_SIZE)
-
-            if len(s) > 0:
-                tar.fileobj.write(s)
-
-                yield
-
-            if len(s) < BLOCK_SIZE:
-                blocks, remainder = divmod(tar_info.size, tarfile.BLOCKSIZE)
-
-                if remainder > 0:
-                    tar.fileobj.write(tarfile.NUL *
-                                      (tarfile.BLOCKSIZE - remainder))
-
-                    yield
-
-                    blocks += 1
-
-                tar.offset += blocks * tarfile.BLOCKSIZE
-                break
-
-    tar.close()
-
-    yield
-
-
-def main():
-    if len(sys.argv) != 3:
-        print('Usage: %s in_filename out_filename' % sys.argv[0])
-        sys.exit(1)
-
-    in_filename = sys.argv[1]
-    out_filename = sys.argv[2]
-
-    streaming_fp = FileStream()
-
-    with open(out_filename, 'wb') as out_fp:
-        for i in stream_build_tar(in_filename, streaming_fp):
-            s = streaming_fp.pop()
-
-            if len(s) > 0:
-                print('Writing %d bytes...' % len(s))
-                out_fp.write(s)
-                out_fp.flush()
-
-    print('Wrote tar file to %s' % out_filename)