python3-commons 0.5.17__tar.gz → 0.5.19__tar.gz

{python3_commons-0.5.17/src/python3_commons.egg-info → python3_commons-0.5.19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: python3-commons
-Version: 0.5.17
+Version: 0.5.19
 Summary: Re-usable Python3 code
 Home-page: https://github.com/kamikaze/python3-commons
 Author: Oleg Korsak
@@ -20,7 +20,7 @@ Requires-Dist: minio==7.2.7
 Requires-Dist: msgpack==1.0.8
 Requires-Dist: msgspec==0.18.6
 Requires-Dist: pydantic[email]==2.8.2
-Requires-Dist: pydantic-settings==2.3.4
+Requires-Dist: pydantic-settings==2.4.0
 Requires-Dist: zeep==4.2.1
 Provides-Extra: testing
 Requires-Dist: pytest; extra == "testing"

{python3_commons-0.5.17 → python3_commons-0.5.19}/requirements.txt

@@ -4,5 +4,5 @@ minio==7.2.7
 msgpack==1.0.8
 msgspec==0.18.6
 pydantic[email]==2.8.2
-pydantic-settings==2.3.4
+pydantic-settings==2.4.0
 zeep==4.2.1

{python3_commons-0.5.17 → python3_commons-0.5.19}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = python3-commons
-version = 0.5.16
+version = 0.5.19
 description = Re-usable Python3 code
 author = Oleg Korsak
 author_email = kamikaze.is.waiting.you@gmail.com
@@ -29,7 +29,7 @@ install_requires =
 	msgpack==1.0.8
 	msgspec==0.18.6
 	pydantic[email]==2.8.2
-	pydantic-settings==2.3.4
+	pydantic-settings==2.4.0
 	zeep==4.2.1
 python_requires = >=3.12
 

{python3_commons-0.5.17 → python3_commons-0.5.19}/src/python3_commons/audit.py

@@ -24,16 +24,25 @@ class GeneratedStream(io.BytesIO):
         self.generator = generator
 
     def read(self, size: int = -1):
-        if size == -1:
-            size = 4096
-
-        while self.tell() < size:
-            try:
-                chunk = next(self.generator)
-            except StopIteration:
-                break
+        if size < 0:
+            while True:
+                try:
+                    chunk = next(self.generator)
+                except StopIteration:
+                    break
+                else:
+                    self.write(chunk)
+        else:
+            total_written_size = 0
 
-            self.write(chunk)
+            while total_written_size < size:
+                try:
+                    chunk = next(self.generator)
+                except StopIteration:
+                    break
+                else:
+                    self.write(chunk)
+                    total_written_size += len(chunk)
 
         self.seek(0)
 
@@ -46,6 +55,8 @@ class GeneratedStream(io.BytesIO):
             if unread_data_size > 0:
                 buf[:unread_data_size] = buf[pos:pos+unread_data_size]
 
+            del buf
+
             self.seek(0)
             self.truncate(unread_data_size)
 
@@ -66,7 +77,6 @@ def generate_archive(objects: Iterable[tuple[str, datetime, bytes]],
             info.size = len(content)
             info.mtime = last_modified.timestamp()
             archive.addfile(info, io.BytesIO(content))
-            archive.fileobj.flush()
 
             buffer.seek(0)
 

{python3_commons-0.5.17 → python3_commons-0.5.19}/src/python3_commons/object_storage.py

@@ -1,5 +1,6 @@
 import io
 import logging
+from contextlib import contextmanager
 from datetime import datetime
 from typing import Generator, Iterable
 
@@ -42,9 +43,7 @@ def get_absolute_path(path: str) -> str:
 
 
 def put_object(bucket_name: str, path: str, data: io.BytesIO, length: int, part_size: int = 0) -> str:
-    s3_client = ObjectStorage(s3_settings).get_client()
-
-    if s3_client:
+    if s3_client := ObjectStorage(s3_settings).get_client():
         result = s3_client.put_object(bucket_name, path, data, length, part_size=part_size)
 
         logger.debug(f'Stored object into object storage: {bucket_name}:{path}')
@@ -54,10 +53,9 @@ def put_object(bucket_name: str, path: str, data: io.BytesIO, length: int, part_
         logger.warning(f'No S3 client available, skipping object put')
 
 
+@contextmanager
 def get_object_stream(bucket_name: str, path: str):
-    s3_client = ObjectStorage(s3_settings).get_client()
-
-    if s3_client:
+    if s3_client := ObjectStorage(s3_settings).get_client():
         logger.debug(f'Getting object from object storage: {bucket_name}:{path}')
 
         try:
@@ -67,19 +65,17 @@ def get_object_stream(bucket_name: str, path: str):
 
             raise
 
-        return response
+        yield response
+
+        response.close()
+        response.release_conn()
     else:
         logger.warning(f'No S3 client available, skipping object put')
 
 
 def get_object(bucket_name: str, path: str) -> bytes:
-    response = get_object_stream(bucket_name, path)
-
-    try:
-        body = response.read()
-    finally:
-        response.close()
-        response.release_conn()
+    with get_object_stream(bucket_name, path) as stream:
+        body = stream.read()
 
     logger.debug(f'Loaded object from object storage: {bucket_name}:{path}')
 

{python3_commons-0.5.17 → python3_commons-0.5.19}/src/python3_commons/stream_tar.py

@@ -1,9 +1,14 @@
+import logging
 import os
 import sys
 import tarfile
 from io import BytesIO
 
 
+logger = logging.getLogger(__name__)
+BLOCK_SIZE = 4096
+
+
 class FileStream(object):
     def __init__(self):
         self.buffer = BytesIO()
@@ -28,7 +33,7 @@ class FileStream(object):
 
 
 def stream_build_tar(in_filename, streaming_fp):
-    tar = tarfile.TarFile.open(out_filename, 'w|gz', streaming_fp)
+    tar = tarfile.TarFile.open('test.tar.gz', 'w|gz', streaming_fp)
 
     stat = os.stat(in_filename)
 
@@ -78,25 +83,23 @@ def stream_build_tar(in_filename, streaming_fp):
     yield
 
 
-BLOCK_SIZE = 4096
+def main():
+    if len(sys.argv) != 3:
+        print('Usage: %s in_filename out_filename' % sys.argv[0])
+        sys.exit(1)
 
+    in_filename = sys.argv[1]
+    out_filename = sys.argv[2]
 
-if len(sys.argv) != 3:
-    print('Usage: %s in_filename out_filename' % sys.argv[0])
-    sys.exit(1)
+    streaming_fp = FileStream()
 
-in_filename = sys.argv[1]
-out_filename = sys.argv[2]
+    with open(out_filename, 'wb') as out_fp:
+        for i in stream_build_tar(in_filename, streaming_fp):
+            s = streaming_fp.pop()
 
-streaming_fp = FileStream()
-
-with open(out_filename, 'wb') as out_fp:
-    for i in stream_build_tar(in_filename, streaming_fp):
-        s = streaming_fp.pop()
-
-        if len(s) > 0:
-            print('Writing %d bytes...' % len(s))
-            out_fp.write(s)
-            out_fp.flush()
+            if len(s) > 0:
+                print('Writing %d bytes...' % len(s))
+                out_fp.write(s)
+                out_fp.flush()
 
-print('Wrote tar file to %s' % out_filename)
+    print('Wrote tar file to %s' % out_filename)

{python3_commons-0.5.17 → python3_commons-0.5.19/src/python3_commons.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: python3-commons
-Version: 0.5.17
+Version: 0.5.19
 Summary: Re-usable Python3 code
 Home-page: https://github.com/kamikaze/python3-commons
 Author: Oleg Korsak
@@ -20,7 +20,7 @@ Requires-Dist: minio==7.2.7
 Requires-Dist: msgpack==1.0.8
 Requires-Dist: msgspec==0.18.6
 Requires-Dist: pydantic[email]==2.8.2
-Requires-Dist: pydantic-settings==2.3.4
+Requires-Dist: pydantic-settings==2.4.0
 Requires-Dist: zeep==4.2.1
 Provides-Extra: testing
 Requires-Dist: pytest; extra == "testing"

{python3_commons-0.5.17 → python3_commons-0.5.19}/src/python3_commons.egg-info/requires.txt

@@ -4,7 +4,7 @@ minio==7.2.7
 msgpack==1.0.8
 msgspec==0.18.6
 pydantic[email]==2.8.2
-pydantic-settings==2.3.4
+pydantic-settings==2.4.0
 zeep==4.2.1
 
 [testing]

python3_commons-0.5.19/tests/test_audit.py

@@ -0,0 +1,29 @@
+from io import BytesIO
+
+from python3_commons.audit import GeneratedStream, generate_archive
+
+
+def test_generated_stream(s3_file_objects):
+    expected_data = b''
+    generator = generate_archive(s3_file_objects, chunk_size=5 * 1024 * 1024)
+    archive_stream = GeneratedStream(generator)
+    archived_data = archive_stream.read()
+
+    with open('/tmp/test.tar', 'wb') as f:
+        f.write(archived_data)
+
+    assert archived_data == expected_data
+
+
+def test_generated_stream_by_chunks(s3_file_objects):
+    expected_data = b''
+    generator = generate_archive(s3_file_objects, chunk_size=2)
+    archive_stream = GeneratedStream(generator)
+    archived_data = BytesIO()
+
+    with open('/tmp/test_chunked.tar', 'wb') as f:
+        while chunk := archive_stream.read(2):
+            f.write(chunk)
+            archived_data.write(chunk)
+
+    assert archived_data.getvalue() == expected_data

python3_commons-0.5.17/tests/test_audit.py

@@ -1,10 +0,0 @@
-from python3_commons.audit import GeneratedStream, generate_archive
-
-
-def test_generated_stream(s3_file_objects):
-    expected_data = b''
-    generator = generate_archive(s3_file_objects, chunk_size=5 * 1024 * 1024)
-    archive_stream = GeneratedStream(generator)
-    archived_data = archive_stream.read()
-
-    assert archived_data == expected_data