python3-commons 0.5.16__tar.gz → 0.5.18__tar.gz

{python3_commons-0.5.16/src/python3_commons.egg-info → python3_commons-0.5.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: python3-commons
-Version: 0.5.16
+Version: 0.5.18
 Summary: Re-usable Python3 code
 Home-page: https://github.com/kamikaze/python3-commons
 Author: Oleg Korsak

{python3_commons-0.5.16 → python3_commons-0.5.18}/src/python3_commons/audit.py

@@ -13,7 +13,7 @@ from zeep.wsdl.definitions import AbstractOperation
 
 from python3_commons import object_storage
 from python3_commons.conf import S3Settings, s3_settings
-from python3_commons.object_storage import get_s3_client
+from python3_commons.object_storage import ObjectStorage
 
 logger = logging.getLogger(__name__)
 
@@ -24,16 +24,27 @@ class GeneratedStream(io.BytesIO):
         self.generator = generator
 
     def read(self, size: int = -1):
-        if size == -1:
-            size = 4096
+        if size < 0:
+            while True:
+                try:
+                    chunk = next(self.generator)
+                except StopIteration:
+                    break
+                else:
+                    self.write(chunk)
+        else:
+            total_written_size = 0
 
-        while self.tell() < size:
-            try:
-                chunk = next(self.generator)
-            except StopIteration:
-                break
+            while total_written_size < size:
+                try:
+                    chunk = next(self.generator)
+                except StopIteration:
+                    break
+                else:
+                    self.write(chunk)
+                    total_written_size += len(chunk)
 
-            self.write(chunk)
+        self.seek(0)
 
         if chunk := super().read(size):
             pos = self.tell()
@@ -44,6 +55,8 @@ class GeneratedStream(io.BytesIO):
             if unread_data_size > 0:
                 buf[:unread_data_size] = buf[pos:pos+unread_data_size]
 
+            del buf
+
             self.seek(0)
             self.truncate(unread_data_size)
 
@@ -82,7 +95,7 @@ def generate_archive(objects: Iterable[tuple[str, datetime, bytes]],
 def write_audit_data_sync(settings: S3Settings, key: str, data: bytes):
     if settings.s3_secret_access_key:
         try:
-            client = get_s3_client(settings)
+            client = ObjectStorage(settings).get_client()
             absolute_path = object_storage.get_absolute_path(f'audit/{key}')
 
             client.put_object(settings.s3_bucket, absolute_path, io.BytesIO(data), len(data))

{python3_commons-0.5.16 → python3_commons-0.5.18}/src/python3_commons/helpers.py

@@ -1,6 +1,7 @@
 import datetime
 import logging
 import shlex
+import threading
 
 from decimal import Decimal, ROUND_HALF_UP
 from typing import Mapping
@@ -8,6 +9,27 @@ from typing import Mapping
 logger = logging.getLogger(__name__)
 
 
+class SingletonMeta(type):
+    """
+    A metaclass that creates a Singleton base class when called.
+    """
+    _instances = {}
+    _lock = threading.Lock()
+
+    def __call__(cls, *args, **kwargs):
+        try:
+            return cls._instances[cls]
+        except KeyError:
+            with cls._lock:
+                try:
+                    return cls._instances[cls]
+                except KeyError:
+                    instance = super(SingletonMeta, cls).__call__(*args, **kwargs)
+                    cls._instances[cls] = instance
+
+                    return instance
+
+
 def date_from_string(string: str, fmt: str = '%d.%m.%Y') -> datetime.date:
     try:
         return datetime.datetime.strptime(string, fmt).date()

{python3_commons-0.5.16 → python3_commons-0.5.18}/src/python3_commons/object_storage.py

@@ -1,5 +1,6 @@
 import io
 import logging
+from contextlib import contextmanager
 from datetime import datetime
 from typing import Generator, Iterable
 
@@ -8,16 +9,17 @@ from minio.datatypes import Object
 from minio.deleteobjects import DeleteObject, DeleteError
 
 from python3_commons.conf import s3_settings, S3Settings
+from python3_commons.helpers import SingletonMeta
 
 logger = logging.getLogger(__name__)
-__CLIENT = None
 
 
-def get_s3_client(settings: S3Settings) -> Minio:
-    global __CLIENT
+class ObjectStorage(metaclass=SingletonMeta):
+    def __init__(self, settings: S3Settings):
+        if not s3_settings.s3_endpoint_url:
+            raise ValueError('s3_settings.s3_endpoint_url must be set')
 
-    if not __CLIENT and s3_settings.s3_endpoint_url:
-        __CLIENT = Minio(
+        self._client = Minio(
             settings.s3_endpoint_url,
             region=settings.s3_region_name,
             access_key=settings.s3_access_key_id.get_secret_value(),
@@ -26,7 +28,8 @@ def get_s3_client(settings: S3Settings) -> Minio:
             cert_check=settings.s3_cert_verify
         )
 
-    return __CLIENT
+    def get_client(self) -> Minio:
+        return self._client
 
 
 def get_absolute_path(path: str) -> str:
@@ -40,9 +43,7 @@ def get_absolute_path(path: str) -> str:
 
 
 def put_object(bucket_name: str, path: str, data: io.BytesIO, length: int, part_size: int = 0) -> str:
-    s3_client = get_s3_client(s3_settings)
-
-    if s3_client:
+    if s3_client := ObjectStorage(s3_settings).get_client():
         result = s3_client.put_object(bucket_name, path, data, length, part_size=part_size)
 
         logger.debug(f'Stored object into object storage: {bucket_name}:{path}')
@@ -52,10 +53,9 @@ def put_object(bucket_name: str, path: str, data: io.BytesIO, length: int, part_
         logger.warning(f'No S3 client available, skipping object put')
 
 
+@contextmanager
 def get_object_stream(bucket_name: str, path: str):
-    s3_client = get_s3_client(s3_settings)
-
-    if s3_client:
+    if s3_client := ObjectStorage(s3_settings).get_client():
         logger.debug(f'Getting object from object storage: {bucket_name}:{path}')
 
         try:
@@ -65,19 +65,17 @@ def get_object_stream(bucket_name: str, path: str):
 
             raise
 
-        return response
+        yield response
+
+        response.close()
+        response.release_conn()
     else:
         logger.warning(f'No S3 client available, skipping object put')
 
 
 def get_object(bucket_name: str, path: str) -> bytes:
-    response = get_object_stream(bucket_name, path)
-
-    try:
-        body = response.read()
-    finally:
-        response.close()
-        response.release_conn()
+    with get_object_stream(bucket_name, path) as stream:
+        body = stream.read()
 
     logger.debug(f'Loaded object from object storage: {bucket_name}:{path}')
 
@@ -85,7 +83,7 @@ def get_object(bucket_name: str, path: str) -> bytes:
 
 
 def list_objects(bucket_name: str, prefix: str, recursive: bool = True) -> Generator[Object, None, None]:
-    s3_client = get_s3_client(s3_settings)
+    s3_client = ObjectStorage(s3_settings).get_client()
 
     yield from s3_client.list_objects(bucket_name, prefix=prefix, recursive=recursive)
 
@@ -104,13 +102,13 @@ def get_objects(bucket_name: str, path: str,
 
 
 def remove_object(bucket_name: str, object_name: str):
-    s3_client = get_s3_client(s3_settings)
+    s3_client = ObjectStorage(s3_settings).get_client()
     s3_client.remove_object(bucket_name, object_name)
 
 
 def remove_objects(bucket_name: str, prefix: str = None,
                    object_names: Iterable[str] = None) -> Iterable[DeleteError] | None:
-    s3_client = get_s3_client(s3_settings)
+    s3_client = ObjectStorage(s3_settings).get_client()
 
     if prefix:
         delete_object_list = map(

{python3_commons-0.5.16 → python3_commons-0.5.18}/src/python3_commons/serializers/msgspec.py

@@ -40,8 +40,12 @@ def ext_hook(code: int, data: memoryview) -> Any:
     raise NotImplementedError(f'Extension type code {code} is not supported')
 
 
+MSGPACK_ENCODER = msgpack.Encoder(enc_hook=enc_hook)
+MSGPACK_DECODER = msgpack.Decoder(ext_hook=ext_hook)
+
+
 def serialize_msgpack(data) -> bytes:
-    result = msgpack.encode(data, enc_hook=enc_hook)
+    result = MSGPACK_ENCODER.encode(data)
 
     return result
 
@@ -50,6 +54,6 @@ def deserialize_msgpack(data: bytes, data_type=None):
     if data_type:
         result = msgpack.decode(data, type=data_type)
     else:
-        result = msgpack.decode(data, ext_hook=ext_hook)
+        result = MSGPACK_DECODER.decode(data)
 
     return result

python3_commons-0.5.18/src/python3_commons/stream_tar.py

@@ -0,0 +1,105 @@
+import logging
+import os
+import sys
+import tarfile
+from io import BytesIO
+
+
+logger = logging.getLogger(__name__)
+BLOCK_SIZE = 4096
+
+
+class FileStream(object):
+    def __init__(self):
+        self.buffer = BytesIO()
+        self.offset = 0
+
+    def write(self, chunk):
+        self.buffer.write(chunk)
+        self.offset += len(chunk)
+
+    def tell(self):
+        return self.offset
+
+    def close(self):
+        self.buffer.close()
+
+    def pop(self):
+        value = self.buffer.getvalue()
+        self.buffer.seek(0)
+        self.buffer.truncate()
+
+        return value
+
+
+def stream_build_tar(in_filename, streaming_fp):
+    tar = tarfile.TarFile.open('test.tar.gz', 'w|gz', streaming_fp)
+
+    stat = os.stat(in_filename)
+
+    tar_info = tarfile.TarInfo(in_filename)
+
+    # Note that you can get this information from the storage backend,
+    # but it's valid for either to raise a NotImplementedError, so it's
+    # important to check.
+    #
+    # Things like the mode or ownership won't be available.
+    tar_info.mtime = stat.st_mtime
+    tar_info.size = stat.st_size
+
+    # Note that we don't pass a fileobj, so we don't write any data
+    # through addfile. We'll do this ourselves.
+    tar.addfile(tar_info)
+
+    yield
+
+    with open(in_filename, 'rb') as in_fp:
+        total_size = 0
+
+        while True:
+            s = in_fp.read(BLOCK_SIZE)
+
+            if len(s) > 0:
+                tar.fileobj.write(s)
+
+                yield
+
+            if len(s) < BLOCK_SIZE:
+                blocks, remainder = divmod(tar_info.size, tarfile.BLOCKSIZE)
+
+                if remainder > 0:
+                    tar.fileobj.write(tarfile.NUL *
+                                      (tarfile.BLOCKSIZE - remainder))
+
+                    yield
+
+                    blocks += 1
+
+                tar.offset += blocks * tarfile.BLOCKSIZE
+                break
+
+    tar.close()
+
+    yield
+
+
+def main():
+    if len(sys.argv) != 3:
+        print('Usage: %s in_filename out_filename' % sys.argv[0])
+        sys.exit(1)
+
+    in_filename = sys.argv[1]
+    out_filename = sys.argv[2]
+
+    streaming_fp = FileStream()
+
+    with open(out_filename, 'wb') as out_fp:
+        for i in stream_build_tar(in_filename, streaming_fp):
+            s = streaming_fp.pop()
+
+            if len(s) > 0:
+                print('Writing %d bytes...' % len(s))
+                out_fp.write(s)
+                out_fp.flush()
+
+    print('Wrote tar file to %s' % out_filename)

{python3_commons-0.5.16 → python3_commons-0.5.18/src/python3_commons.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: python3-commons
-Version: 0.5.16
+Version: 0.5.18
 Summary: Re-usable Python3 code
 Home-page: https://github.com/kamikaze/python3-commons
 Author: Oleg Korsak

{python3_commons-0.5.16 → python3_commons-0.5.18}/src/python3_commons.egg-info/SOURCES.txt

@@ -26,6 +26,7 @@ src/python3_commons/db.py
 src/python3_commons/fs.py
 src/python3_commons/helpers.py
 src/python3_commons/object_storage.py
+src/python3_commons/stream_tar.py
 src/python3_commons.egg-info/PKG-INFO
 src/python3_commons.egg-info/SOURCES.txt
 src/python3_commons.egg-info/dependency_links.txt

python3_commons-0.5.18/tests/test_audit.py

@@ -0,0 +1,29 @@
+from io import BytesIO
+
+from python3_commons.audit import GeneratedStream, generate_archive
+
+
+def test_generated_stream(s3_file_objects):
+    expected_data = b''
+    generator = generate_archive(s3_file_objects, chunk_size=5 * 1024 * 1024)
+    archive_stream = GeneratedStream(generator)
+    archived_data = archive_stream.read()
+
+    with open('/tmp/test.tar', 'wb') as f:
+        f.write(archived_data)
+
+    assert archived_data == expected_data
+
+
+def test_generated_stream_by_chunks(s3_file_objects):
+    expected_data = b''
+    generator = generate_archive(s3_file_objects, chunk_size=2)
+    archive_stream = GeneratedStream(generator)
+    archived_data = BytesIO()
+
+    with open('/tmp/test_chunked.tar', 'wb') as f:
+        while chunk := archive_stream.read(2):
+            f.write(chunk)
+            archived_data.write(chunk)
+
+    assert archived_data.getvalue() == expected_data

python3_commons-0.5.16/tests/test_audit.py

@@ -1,10 +0,0 @@
-from python3_commons.audit import GeneratedStream, generate_archive
-
-
-def test_generated_stream(s3_file_objects):
-    expected_data = b''
-    generator = generate_archive(s3_file_objects, chunk_size=5 * 1024 * 1024)
-    archive_stream = GeneratedStream(generator)
-    archived_data = archive_stream.read()
-
-    assert archived_data == expected_data