python3-commons 0.5.17__py2.py3-none-any.whl → 0.5.19__py2.py3-none-any.whl

python3_commons/audit.py

@@ -24,16 +24,25 @@ class GeneratedStream(io.BytesIO):
         self.generator = generator
 
     def read(self, size: int = -1):
-        if size == -1:
-            size = 4096
-
-        while self.tell() < size:
-            try:
-                chunk = next(self.generator)
-            except StopIteration:
-                break
+        if size < 0:
+            while True:
+                try:
+                    chunk = next(self.generator)
+                except StopIteration:
+                    break
+                else:
+                    self.write(chunk)
+        else:
+            total_written_size = 0
 
-            self.write(chunk)
+            while total_written_size < size:
+                try:
+                    chunk = next(self.generator)
+                except StopIteration:
+                    break
+                else:
+                    self.write(chunk)
+                    total_written_size += len(chunk)
 
         self.seek(0)
 
@@ -46,6 +55,8 @@ class GeneratedStream(io.BytesIO):
             if unread_data_size > 0:
                 buf[:unread_data_size] = buf[pos:pos+unread_data_size]
 
+            del buf
+
             self.seek(0)
             self.truncate(unread_data_size)
 
@@ -66,7 +77,6 @@ def generate_archive(objects: Iterable[tuple[str, datetime, bytes]],
             info.size = len(content)
             info.mtime = last_modified.timestamp()
             archive.addfile(info, io.BytesIO(content))
-            archive.fileobj.flush()
 
             buffer.seek(0)
 

python3_commons/object_storage.py

@@ -1,5 +1,6 @@
 import io
 import logging
+from contextlib import contextmanager
 from datetime import datetime
 from typing import Generator, Iterable
 
@@ -42,9 +43,7 @@ def get_absolute_path(path: str) -> str:
 
 
 def put_object(bucket_name: str, path: str, data: io.BytesIO, length: int, part_size: int = 0) -> str:
-    s3_client = ObjectStorage(s3_settings).get_client()
-
-    if s3_client:
+    if s3_client := ObjectStorage(s3_settings).get_client():
         result = s3_client.put_object(bucket_name, path, data, length, part_size=part_size)
 
         logger.debug(f'Stored object into object storage: {bucket_name}:{path}')
@@ -54,10 +53,9 @@ def put_object(bucket_name: str, path: str, data: io.BytesIO, length: int, part_
         logger.warning(f'No S3 client available, skipping object put')
 
 
+@contextmanager
 def get_object_stream(bucket_name: str, path: str):
-    s3_client = ObjectStorage(s3_settings).get_client()
-
-    if s3_client:
+    if s3_client := ObjectStorage(s3_settings).get_client():
         logger.debug(f'Getting object from object storage: {bucket_name}:{path}')
 
         try:
@@ -67,19 +65,17 @@ def get_object_stream(bucket_name: str, path: str):
 
             raise
 
-        return response
+        yield response
+
+        response.close()
+        response.release_conn()
     else:
         logger.warning(f'No S3 client available, skipping object put')
 
 
 def get_object(bucket_name: str, path: str) -> bytes:
-    response = get_object_stream(bucket_name, path)
-
-    try:
-        body = response.read()
-    finally:
-        response.close()
-        response.release_conn()
+    with get_object_stream(bucket_name, path) as stream:
+        body = stream.read()
 
     logger.debug(f'Loaded object from object storage: {bucket_name}:{path}')
 

python3_commons/stream_tar.py

@@ -1,9 +1,14 @@
+import logging
 import os
 import sys
 import tarfile
 from io import BytesIO
 
 
+logger = logging.getLogger(__name__)
+BLOCK_SIZE = 4096
+
+
 class FileStream(object):
     def __init__(self):
         self.buffer = BytesIO()
@@ -28,7 +33,7 @@ class FileStream(object):
 
 
 def stream_build_tar(in_filename, streaming_fp):
-    tar = tarfile.TarFile.open(out_filename, 'w|gz', streaming_fp)
+    tar = tarfile.TarFile.open('test.tar.gz', 'w|gz', streaming_fp)
 
     stat = os.stat(in_filename)
 
@@ -78,25 +83,23 @@ def stream_build_tar(in_filename, streaming_fp):
     yield
 
 
-BLOCK_SIZE = 4096
+def main():
+    if len(sys.argv) != 3:
+        print('Usage: %s in_filename out_filename' % sys.argv[0])
+        sys.exit(1)
 
+    in_filename = sys.argv[1]
+    out_filename = sys.argv[2]
 
-if len(sys.argv) != 3:
-    print('Usage: %s in_filename out_filename' % sys.argv[0])
-    sys.exit(1)
+    streaming_fp = FileStream()
 
-in_filename = sys.argv[1]
-out_filename = sys.argv[2]
+    with open(out_filename, 'wb') as out_fp:
+        for i in stream_build_tar(in_filename, streaming_fp):
+            s = streaming_fp.pop()
 
-streaming_fp = FileStream()
-
-with open(out_filename, 'wb') as out_fp:
-    for i in stream_build_tar(in_filename, streaming_fp):
-        s = streaming_fp.pop()
-
-        if len(s) > 0:
-            print('Writing %d bytes...' % len(s))
-            out_fp.write(s)
-            out_fp.flush()
+            if len(s) > 0:
+                print('Writing %d bytes...' % len(s))
+                out_fp.write(s)
+                out_fp.flush()
 
-print('Wrote tar file to %s' % out_filename)
+    print('Wrote tar file to %s' % out_filename)

{python3_commons-0.5.17.dist-info → python3_commons-0.5.19.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: python3-commons
-Version: 0.5.17
+Version: 0.5.19
 Summary: Re-usable Python3 code
 Home-page: https://github.com/kamikaze/python3-commons
 Author: Oleg Korsak
@@ -20,7 +20,7 @@ Requires-Dist: minio ==7.2.7
 Requires-Dist: msgpack ==1.0.8
 Requires-Dist: msgspec ==0.18.6
 Requires-Dist: pydantic[email] ==2.8.2
-Requires-Dist: pydantic-settings ==2.3.4
+Requires-Dist: pydantic-settings ==2.4.0
 Requires-Dist: zeep ==4.2.1
 Provides-Extra: testing
 Requires-Dist: pytest ; extra == 'testing'

{python3_commons-0.5.17.dist-info → python3_commons-0.5.19.dist-info}/RECORD

@@ -1,11 +1,11 @@
 python3_commons/__init__.py,sha256=h-KTJUaQ50E3RmkTn_GO88IRunmDTEpNc3ylpFvCTOc,339
-python3_commons/audit.py,sha256=1CHNFcz87F_QvaiI0CUCc2IYpIGsxT718oHjIl4Wqs4,5118
+python3_commons/audit.py,sha256=lQbgThInSSGtEGQDLhrb0XrxDWPteTUqKf8Kx0Oywtk,5453
 python3_commons/conf.py,sha256=vSXyFwXx2wb1uy8IffeeI-RoTqhUZs0RLSSG2OLc2ss,598
 python3_commons/db.py,sha256=qhaDIdzBWgFyeP_XPKfHZlYVlwS2bpBPYMv84yV6820,738
 python3_commons/fs.py,sha256=wfLjybXndwLqNlOxTpm_HRJnuTcC4wbrHEOaEeCo9Wc,337
 python3_commons/helpers.py,sha256=hZG8M-mltBC8I9yx5ZuAM7bABFNuOsqX6FzSaQz4y9U,2480
-python3_commons/object_storage.py,sha256=GEy3G-Y6QNY3NTp5N7QfaKYIYqBbUAtDDo1JWvUgNtw,4061
-python3_commons/stream_tar.py,sha256=JPmiHzPokow8CsEG7bel8egn7tMuAsAKFnYfeDkmz34,2321
+python3_commons/object_storage.py,sha256=pk2J14RL9FLTwaks-IS4EJX9TBMLzid35CroGftLNhU,4067
+python3_commons/stream_tar.py,sha256=1aA_IIM4oLdh2EZx_5tSeSm6flgCJ_QWbXLFAjg6hEo,2442
 python3_commons/logging/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 python3_commons/logging/filters.py,sha256=fuyjXZAUm-i2MNrxvFYag8F8Rr27x8W8MdV3ke6miSs,175
 python3_commons/logging/formatter.py,sha256=UXmmh1yd5Kc2dpvSHn6uCWLDWE2LMjlYAaH8cg3siV4,720
@@ -13,9 +13,9 @@ python3_commons/serializers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMp
 python3_commons/serializers/json.py,sha256=P288wWz9ic38QWEMrpp_uwKPYkQiOgvE1cI4WZn6ZCg,808
 python3_commons/serializers/msgpack.py,sha256=tzIGGyDL3UpZnnouCtnxuYDx6InKM_C3PP1N4PN8wd4,1269
 python3_commons/serializers/msgspec.py,sha256=EknuMpxi_kU25Iv_m10E9rk8b31AkjVumuzyjp7WgrU,1699
-python3_commons-0.5.17.dist-info/AUTHORS.rst,sha256=3R9JnfjfjH5RoPWOeqKFJgxVShSSfzQPIrEr1nxIo9Q,90
-python3_commons-0.5.17.dist-info/LICENSE,sha256=xxILuojHm4fKQOrMHPSslbyy6WuKAN2RiG74HbrYfzM,34575
-python3_commons-0.5.17.dist-info/METADATA,sha256=TY7PX7pUGLrFc0Iua413-DfQhYgm0y4Ur7SnSa8H3uQ,975
-python3_commons-0.5.17.dist-info/WHEEL,sha256=pWvVuNuBTVmNV7Lp2jMAgt1NplTICeFdl1SW8U3MWN4,109
-python3_commons-0.5.17.dist-info/top_level.txt,sha256=lJI6sCBf68eUHzupCnn2dzG10lH3jJKTWM_hrN1cQ7M,16
-python3_commons-0.5.17.dist-info/RECORD,,
+python3_commons-0.5.19.dist-info/AUTHORS.rst,sha256=3R9JnfjfjH5RoPWOeqKFJgxVShSSfzQPIrEr1nxIo9Q,90
+python3_commons-0.5.19.dist-info/LICENSE,sha256=xxILuojHm4fKQOrMHPSslbyy6WuKAN2RiG74HbrYfzM,34575
+python3_commons-0.5.19.dist-info/METADATA,sha256=uQN3EfXdlIgIHQJMcfpK6a0Arju4Vx4xuGuAe-uCiHk,975
+python3_commons-0.5.19.dist-info/WHEEL,sha256=XRxW4r1PNiVhMpP4bT9oWtu3HyndxpJ84SkubFgzp_Y,109
+python3_commons-0.5.19.dist-info/top_level.txt,sha256=lJI6sCBf68eUHzupCnn2dzG10lH3jJKTWM_hrN1cQ7M,16
+python3_commons-0.5.19.dist-info/RECORD,,

{python3_commons-0.5.17.dist-info → python3_commons-0.5.19.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (70.3.0)
+Generator: setuptools (72.1.0)
 Root-Is-Purelib: true
 Tag: py2-none-any
 Tag: py3-none-any