python3-commons 0.5.19__tar.gz → 0.5.21__tar.gz

{python3_commons-0.5.19/src/python3_commons.egg-info → python3_commons-0.5.21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: python3-commons
-Version: 0.5.19
+Version: 0.5.21
 Summary: Re-usable Python3 code
 Home-page: https://github.com/kamikaze/python3-commons
 Author: Oleg Korsak
@@ -15,7 +15,7 @@ Description-Content-Type: text/x-rst; charset=UTF-8
 License-File: LICENSE
 License-File: AUTHORS.rst
 Requires-Dist: asyncpg==0.29.0
-Requires-Dist: lxml==5.2.2
+Requires-Dist: lxml==5.3.0
 Requires-Dist: minio==7.2.7
 Requires-Dist: msgpack==1.0.8
 Requires-Dist: msgspec==0.18.6

{python3_commons-0.5.19 → python3_commons-0.5.21}/requirements.txt

@@ -1,5 +1,5 @@
 asyncpg==0.29.0
-lxml==5.2.2
+lxml==5.3.0
 minio==7.2.7
 msgpack==1.0.8
 msgspec==0.18.6

{python3_commons-0.5.19 → python3_commons-0.5.21}/setup.cfg

@@ -24,7 +24,7 @@ package_dir =
 setup_requires = pyscaffold>=3.2a0,<3.3a0
 install_requires = 
 	asyncpg==0.29.0
-	lxml==5.2.2
+	lxml==5.3.0
 	minio==7.2.7
 	msgpack==1.0.8
 	msgspec==0.18.6

{python3_commons-0.5.19 → python3_commons-0.5.21}/src/python3_commons/audit.py

@@ -2,6 +2,7 @@ import asyncio
 import io
 import logging
 import tarfile
+from bz2 import BZ2Compressor
 from datetime import datetime, timedelta, UTC
 from typing import Generator, Iterable
 from uuid import uuid4
@@ -92,6 +93,17 @@ def generate_archive(objects: Iterable[tuple[str, datetime, bytes]],
             buffer.truncate(0)
 
 
+def generate_bzip2(chunks: Generator[bytes, None, None]) -> Generator[bytes, None, None]:
+    compressor = BZ2Compressor()
+
+    for chunk in chunks:
+        if compressed_chunk := compressor.compress(chunk):
+            yield compressed_chunk
+
+    if compressed_chunk := compressor.flush():
+        yield compressed_chunk
+
+
 def write_audit_data_sync(settings: S3Settings, key: str, data: bytes):
     if settings.s3_secret_access_key:
         try:
@@ -123,7 +135,8 @@ async def archive_audit_data(root_path: str = 'audit'):
         logger.info(f'Compacting files in: {date_path}')
 
         generator = generate_archive(objects, chunk_size=5*1024*1024)
-        archive_stream = GeneratedStream(generator)
+        bzip2_generator = generate_bzip2(generator)
+        archive_stream = GeneratedStream(bzip2_generator)
 
         archive_path = object_storage.get_absolute_path(f'audit/.archive/{year}_{month:02}_{day:02}.tar.bz2')
         object_storage.put_object(bucket_name, archive_path, archive_stream, -1, part_size=5*1024*1024)

{python3_commons-0.5.19 → python3_commons-0.5.21/src/python3_commons.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: python3-commons
-Version: 0.5.19
+Version: 0.5.21
 Summary: Re-usable Python3 code
 Home-page: https://github.com/kamikaze/python3-commons
 Author: Oleg Korsak
@@ -15,7 +15,7 @@ Description-Content-Type: text/x-rst; charset=UTF-8
 License-File: LICENSE
 License-File: AUTHORS.rst
 Requires-Dist: asyncpg==0.29.0
-Requires-Dist: lxml==5.2.2
+Requires-Dist: lxml==5.3.0
 Requires-Dist: minio==7.2.7
 Requires-Dist: msgpack==1.0.8
 Requires-Dist: msgspec==0.18.6

{python3_commons-0.5.19 → python3_commons-0.5.21}/src/python3_commons.egg-info/SOURCES.txt

@@ -26,7 +26,6 @@ src/python3_commons/db.py
 src/python3_commons/fs.py
 src/python3_commons/helpers.py
 src/python3_commons/object_storage.py
-src/python3_commons/stream_tar.py
 src/python3_commons.egg-info/PKG-INFO
 src/python3_commons.egg-info/SOURCES.txt
 src/python3_commons.egg-info/dependency_links.txt
@@ -42,5 +41,6 @@ src/python3_commons/serializers/msgpack.py
 src/python3_commons/serializers/msgspec.py
 tests/conftest.py
 tests/test_audit.py
+tests/test_helpers.py
 tests/test_msgpack.py
 tests/test_msgspec.py

{python3_commons-0.5.19 → python3_commons-0.5.21}/src/python3_commons.egg-info/requires.txt

@@ -1,5 +1,5 @@
 asyncpg==0.29.0
-lxml==5.2.2
+lxml==5.3.0
 minio==7.2.7
 msgpack==1.0.8
 msgspec==0.18.6

{python3_commons-0.5.19 → python3_commons-0.5.21}/tests/test_audit.py

@@ -1,15 +1,16 @@
 from io import BytesIO
 
-from python3_commons.audit import GeneratedStream, generate_archive
+from python3_commons.audit import GeneratedStream, generate_archive, generate_bzip2
 
 
 def test_generated_stream(s3_file_objects):
     expected_data = b''
     generator = generate_archive(s3_file_objects, chunk_size=5 * 1024 * 1024)
-    archive_stream = GeneratedStream(generator)
+    bzip2_generator = generate_bzip2(generator)
+    archive_stream = GeneratedStream(bzip2_generator)
     archived_data = archive_stream.read()
 
-    with open('/tmp/test.tar', 'wb') as f:
+    with open('/tmp/test.tar.bz2', 'wb') as f:
         f.write(archived_data)
 
     assert archived_data == expected_data
@@ -18,10 +19,11 @@ def test_generated_stream(s3_file_objects):
 def test_generated_stream_by_chunks(s3_file_objects):
     expected_data = b''
     generator = generate_archive(s3_file_objects, chunk_size=2)
-    archive_stream = GeneratedStream(generator)
+    bzip2_generator = generate_bzip2(generator)
+    archive_stream = GeneratedStream(bzip2_generator)
     archived_data = BytesIO()
 
-    with open('/tmp/test_chunked.tar', 'wb') as f:
+    with open('/tmp/test_chunked.tar.bz2', 'wb') as f:
         while chunk := archive_stream.read(2):
             f.write(chunk)
             archived_data.write(chunk)

python3_commons-0.5.21/tests/test_helpers.py

@@ -0,0 +1,10 @@
+from datetime import date
+
+from python3_commons.helpers import date_range
+
+
+def test_date_range():
+    expected_dates = (date(2024, 1, 1), date(2024, 1, 2), date(2024, 1, 3), date(2024, 1, 4), date(2024, 1, 5))
+    dates = tuple(date_range(date(2024, 1, 1), date(2024, 1, 5)))
+
+    assert dates == expected_dates

python3_commons-0.5.19/src/python3_commons/stream_tar.py

@@ -1,105 +0,0 @@
-import logging
-import os
-import sys
-import tarfile
-from io import BytesIO
-
-
-logger = logging.getLogger(__name__)
-BLOCK_SIZE = 4096
-
-
-class FileStream(object):
-    def __init__(self):
-        self.buffer = BytesIO()
-        self.offset = 0
-
-    def write(self, chunk):
-        self.buffer.write(chunk)
-        self.offset += len(chunk)
-
-    def tell(self):
-        return self.offset
-
-    def close(self):
-        self.buffer.close()
-
-    def pop(self):
-        value = self.buffer.getvalue()
-        self.buffer.seek(0)
-        self.buffer.truncate()
-
-        return value
-
-
-def stream_build_tar(in_filename, streaming_fp):
-    tar = tarfile.TarFile.open('test.tar.gz', 'w|gz', streaming_fp)
-
-    stat = os.stat(in_filename)
-
-    tar_info = tarfile.TarInfo(in_filename)
-
-    # Note that you can get this information from the storage backend,
-    # but it's valid for either to raise a NotImplementedError, so it's
-    # important to check.
-    #
-    # Things like the mode or ownership won't be available.
-    tar_info.mtime = stat.st_mtime
-    tar_info.size = stat.st_size
-
-    # Note that we don't pass a fileobj, so we don't write any data
-    # through addfile. We'll do this ourselves.
-    tar.addfile(tar_info)
-
-    yield
-
-    with open(in_filename, 'rb') as in_fp:
-        total_size = 0
-
-        while True:
-            s = in_fp.read(BLOCK_SIZE)
-
-            if len(s) > 0:
-                tar.fileobj.write(s)
-
-                yield
-
-            if len(s) < BLOCK_SIZE:
-                blocks, remainder = divmod(tar_info.size, tarfile.BLOCKSIZE)
-
-                if remainder > 0:
-                    tar.fileobj.write(tarfile.NUL *
-                                      (tarfile.BLOCKSIZE - remainder))
-
-                    yield
-
-                    blocks += 1
-
-                tar.offset += blocks * tarfile.BLOCKSIZE
-                break
-
-    tar.close()
-
-    yield
-
-
-def main():
-    if len(sys.argv) != 3:
-        print('Usage: %s in_filename out_filename' % sys.argv[0])
-        sys.exit(1)
-
-    in_filename = sys.argv[1]
-    out_filename = sys.argv[2]
-
-    streaming_fp = FileStream()
-
-    with open(out_filename, 'wb') as out_fp:
-        for i in stream_build_tar(in_filename, streaming_fp):
-            s = streaming_fp.pop()
-
-            if len(s) > 0:
-                print('Writing %d bytes...' % len(s))
-                out_fp.write(s)
-                out_fp.flush()
-
-    print('Wrote tar file to %s' % out_filename)