python3-commons 0.5.16__tar.gz → 0.5.17__tar.gz

{python3_commons-0.5.16/src/python3_commons.egg-info → python3_commons-0.5.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: python3-commons
-Version: 0.5.16
+Version: 0.5.17
 Summary: Re-usable Python3 code
 Home-page: https://github.com/kamikaze/python3-commons
 Author: Oleg Korsak

{python3_commons-0.5.16 → python3_commons-0.5.17}/src/python3_commons/audit.py

@@ -13,7 +13,7 @@ from zeep.wsdl.definitions import AbstractOperation
 
 from python3_commons import object_storage
 from python3_commons.conf import S3Settings, s3_settings
-from python3_commons.object_storage import get_s3_client
+from python3_commons.object_storage import ObjectStorage
 
 logger = logging.getLogger(__name__)
 
@@ -35,6 +35,8 @@ class GeneratedStream(io.BytesIO):
 
             self.write(chunk)
 
+        self.seek(0)
+
         if chunk := super().read(size):
             pos = self.tell()
 
@@ -64,6 +66,7 @@ def generate_archive(objects: Iterable[tuple[str, datetime, bytes]],
             info.size = len(content)
             info.mtime = last_modified.timestamp()
             archive.addfile(info, io.BytesIO(content))
+            archive.fileobj.flush()
 
             buffer.seek(0)
 
@@ -82,7 +85,7 @@ def generate_archive(objects: Iterable[tuple[str, datetime, bytes]],
 def write_audit_data_sync(settings: S3Settings, key: str, data: bytes):
     if settings.s3_secret_access_key:
         try:
-            client = get_s3_client(settings)
+            client = ObjectStorage(settings).get_client()
             absolute_path = object_storage.get_absolute_path(f'audit/{key}')
 
             client.put_object(settings.s3_bucket, absolute_path, io.BytesIO(data), len(data))

{python3_commons-0.5.16 → python3_commons-0.5.17}/src/python3_commons/helpers.py

@@ -1,6 +1,7 @@
 import datetime
 import logging
 import shlex
+import threading
 
 from decimal import Decimal, ROUND_HALF_UP
 from typing import Mapping
@@ -8,6 +9,27 @@ from typing import Mapping
 logger = logging.getLogger(__name__)
 
 
+class SingletonMeta(type):
+    """
+    A metaclass that creates a Singleton base class when called.
+    """
+    _instances = {}
+    _lock = threading.Lock()
+
+    def __call__(cls, *args, **kwargs):
+        try:
+            return cls._instances[cls]
+        except KeyError:
+            with cls._lock:
+                try:
+                    return cls._instances[cls]
+                except KeyError:
+                    instance = super(SingletonMeta, cls).__call__(*args, **kwargs)
+                    cls._instances[cls] = instance
+
+                    return instance
+
+
 def date_from_string(string: str, fmt: str = '%d.%m.%Y') -> datetime.date:
     try:
         return datetime.datetime.strptime(string, fmt).date()

{python3_commons-0.5.16 → python3_commons-0.5.17}/src/python3_commons/object_storage.py

@@ -8,16 +8,17 @@ from minio.datatypes import Object
 from minio.deleteobjects import DeleteObject, DeleteError
 
 from python3_commons.conf import s3_settings, S3Settings
+from python3_commons.helpers import SingletonMeta
 
 logger = logging.getLogger(__name__)
-__CLIENT = None
 
 
-def get_s3_client(settings: S3Settings) -> Minio:
-    global __CLIENT
+class ObjectStorage(metaclass=SingletonMeta):
+    def __init__(self, settings: S3Settings):
+        if not s3_settings.s3_endpoint_url:
+            raise ValueError('s3_settings.s3_endpoint_url must be set')
 
-    if not __CLIENT and s3_settings.s3_endpoint_url:
-        __CLIENT = Minio(
+        self._client = Minio(
             settings.s3_endpoint_url,
             region=settings.s3_region_name,
             access_key=settings.s3_access_key_id.get_secret_value(),
@@ -26,7 +27,8 @@ def get_s3_client(settings: S3Settings) -> Minio:
             cert_check=settings.s3_cert_verify
         )
 
-    return __CLIENT
+    def get_client(self) -> Minio:
+        return self._client
 
 
 def get_absolute_path(path: str) -> str:
@@ -40,7 +42,7 @@ def get_absolute_path(path: str) -> str:
 
 
 def put_object(bucket_name: str, path: str, data: io.BytesIO, length: int, part_size: int = 0) -> str:
-    s3_client = get_s3_client(s3_settings)
+    s3_client = ObjectStorage(s3_settings).get_client()
 
     if s3_client:
         result = s3_client.put_object(bucket_name, path, data, length, part_size=part_size)
@@ -53,7 +55,7 @@ def put_object(bucket_name: str, path: str, data: io.BytesIO, length: int, part_
 
 
 def get_object_stream(bucket_name: str, path: str):
-    s3_client = get_s3_client(s3_settings)
+    s3_client = ObjectStorage(s3_settings).get_client()
 
     if s3_client:
         logger.debug(f'Getting object from object storage: {bucket_name}:{path}')
@@ -85,7 +87,7 @@ def get_object(bucket_name: str, path: str) -> bytes:
 
 
 def list_objects(bucket_name: str, prefix: str, recursive: bool = True) -> Generator[Object, None, None]:
-    s3_client = get_s3_client(s3_settings)
+    s3_client = ObjectStorage(s3_settings).get_client()
 
     yield from s3_client.list_objects(bucket_name, prefix=prefix, recursive=recursive)
 
@@ -104,13 +106,13 @@ def get_objects(bucket_name: str, path: str,
 
 
 def remove_object(bucket_name: str, object_name: str):
-    s3_client = get_s3_client(s3_settings)
+    s3_client = ObjectStorage(s3_settings).get_client()
     s3_client.remove_object(bucket_name, object_name)
 
 
 def remove_objects(bucket_name: str, prefix: str = None,
                    object_names: Iterable[str] = None) -> Iterable[DeleteError] | None:
-    s3_client = get_s3_client(s3_settings)
+    s3_client = ObjectStorage(s3_settings).get_client()
 
     if prefix:
         delete_object_list = map(

{python3_commons-0.5.16 → python3_commons-0.5.17}/src/python3_commons/serializers/msgspec.py

@@ -40,8 +40,12 @@ def ext_hook(code: int, data: memoryview) -> Any:
     raise NotImplementedError(f'Extension type code {code} is not supported')
 
 
+MSGPACK_ENCODER = msgpack.Encoder(enc_hook=enc_hook)
+MSGPACK_DECODER = msgpack.Decoder(ext_hook=ext_hook)
+
+
 def serialize_msgpack(data) -> bytes:
-    result = msgpack.encode(data, enc_hook=enc_hook)
+    result = MSGPACK_ENCODER.encode(data)
 
     return result
 
@@ -50,6 +54,6 @@ def deserialize_msgpack(data: bytes, data_type=None):
     if data_type:
         result = msgpack.decode(data, type=data_type)
     else:
-        result = msgpack.decode(data, ext_hook=ext_hook)
+        result = MSGPACK_DECODER.decode(data)
 
     return result

python3_commons-0.5.17/src/python3_commons/stream_tar.py

@@ -0,0 +1,102 @@
+import os
+import sys
+import tarfile
+from io import BytesIO
+
+
+class FileStream(object):
+    def __init__(self):
+        self.buffer = BytesIO()
+        self.offset = 0
+
+    def write(self, chunk):
+        self.buffer.write(chunk)
+        self.offset += len(chunk)
+
+    def tell(self):
+        return self.offset
+
+    def close(self):
+        self.buffer.close()
+
+    def pop(self):
+        value = self.buffer.getvalue()
+        self.buffer.seek(0)
+        self.buffer.truncate()
+
+        return value
+
+
+def stream_build_tar(in_filename, streaming_fp):
+    tar = tarfile.TarFile.open(out_filename, 'w|gz', streaming_fp)
+
+    stat = os.stat(in_filename)
+
+    tar_info = tarfile.TarInfo(in_filename)
+
+    # Note that you can get this information from the storage backend,
+    # but it's valid for either to raise a NotImplementedError, so it's
+    # important to check.
+    #
+    # Things like the mode or ownership won't be available.
+    tar_info.mtime = stat.st_mtime
+    tar_info.size = stat.st_size
+
+    # Note that we don't pass a fileobj, so we don't write any data
+    # through addfile. We'll do this ourselves.
+    tar.addfile(tar_info)
+
+    yield
+
+    with open(in_filename, 'rb') as in_fp:
+        total_size = 0
+
+        while True:
+            s = in_fp.read(BLOCK_SIZE)
+
+            if len(s) > 0:
+                tar.fileobj.write(s)
+
+                yield
+
+            if len(s) < BLOCK_SIZE:
+                blocks, remainder = divmod(tar_info.size, tarfile.BLOCKSIZE)
+
+                if remainder > 0:
+                    tar.fileobj.write(tarfile.NUL *
+                                      (tarfile.BLOCKSIZE - remainder))
+
+                    yield
+
+                    blocks += 1
+
+                tar.offset += blocks * tarfile.BLOCKSIZE
+                break
+
+    tar.close()
+
+    yield
+
+
+BLOCK_SIZE = 4096
+
+
+if len(sys.argv) != 3:
+    print('Usage: %s in_filename out_filename' % sys.argv[0])
+    sys.exit(1)
+
+in_filename = sys.argv[1]
+out_filename = sys.argv[2]
+
+streaming_fp = FileStream()
+
+with open(out_filename, 'wb') as out_fp:
+    for i in stream_build_tar(in_filename, streaming_fp):
+        s = streaming_fp.pop()
+
+        if len(s) > 0:
+            print('Writing %d bytes...' % len(s))
+            out_fp.write(s)
+            out_fp.flush()
+
+print('Wrote tar file to %s' % out_filename)

{python3_commons-0.5.16 → python3_commons-0.5.17/src/python3_commons.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: python3-commons
-Version: 0.5.16
+Version: 0.5.17
 Summary: Re-usable Python3 code
 Home-page: https://github.com/kamikaze/python3-commons
 Author: Oleg Korsak

{python3_commons-0.5.16 → python3_commons-0.5.17}/src/python3_commons.egg-info/SOURCES.txt

@@ -26,6 +26,7 @@ src/python3_commons/db.py
 src/python3_commons/fs.py
 src/python3_commons/helpers.py
 src/python3_commons/object_storage.py
+src/python3_commons/stream_tar.py
 src/python3_commons.egg-info/PKG-INFO
 src/python3_commons.egg-info/SOURCES.txt
 src/python3_commons.egg-info/dependency_links.txt