python3-commons 0.8.36__py3-none-any.whl → 0.8.38__py3-none-any.whl

python3_commons/db/models/__init__.py

@@ -1,8 +1,2 @@
-from python3_commons.db.models.auth import ApiKey as ApiKey
-from python3_commons.db.models.auth import User as User
-from python3_commons.db.models.auth import UserGroup as UserGroup
-from python3_commons.db.models.rbac import RBACApiKeyRole as RBACApiKeyRole
-from python3_commons.db.models.rbac import RBACPermission as RBACPermission
-from python3_commons.db.models.rbac import RBACRole as RBACRole
-from python3_commons.db.models.rbac import RBACRolePermission as RBACRolePermission
-from python3_commons.db.models.rbac import RBACUserRole as RBACUserRole
+from python3_commons.db.models.auth import ApiKey, User, UserGroup
+from python3_commons.db.models.rbac import RBACApiKeyRole, RBACPermission, RBACRole, RBACRolePermission, RBACUserRole

python3_commons/helpers.py

@@ -1,8 +1,6 @@
 import logging
 import shlex
 import threading
-from abc import ABCMeta
-from collections import defaultdict
 from datetime import date, datetime, timedelta
 from decimal import ROUND_HALF_UP, Decimal
 from json import dumps
@@ -14,24 +12,24 @@ from python3_commons.serializers.json import CustomJSONEncoder
 logger = logging.getLogger(__name__)
 
 
-class SingletonMeta(ABCMeta):
+class SingletonMeta(type):
     """
     A metaclass that creates a Singleton base class when called.
     """
 
-    __instances = {}
-    __locks = defaultdict(threading.Lock)
+    _instances = {}
+    _lock = threading.Lock()
 
     def __call__(cls, *args, **kwargs):
         try:
-            return cls.__instances[cls]
+            return cls._instances[cls]
         except KeyError:
-            with cls.__locks[cls]:
+            with cls._lock:
                 try:
-                    return cls.__instances[cls]
+                    return cls._instances[cls]
                 except KeyError:
                     instance = super(SingletonMeta, cls).__call__(*args, **kwargs)
-                    cls.__instances[cls] = instance
+                    cls._instances[cls] = instance
 
                     return instance
 

python3_commons/object_storage.py

@@ -1,16 +1,12 @@
-from __future__ import annotations
-
 import io
 import logging
-from contextlib import asynccontextmanager
+from contextlib import contextmanager
 from datetime import datetime
-from typing import TYPE_CHECKING, AsyncGenerator, Iterable, Mapping, Sequence
-
-import aiobotocore.session
-from aiobotocore.response import StreamingBody
+from typing import Generator, Iterable
 
-if TYPE_CHECKING:
-    from types_aiobotocore_s3.client import S3Client
+from minio import Minio
+from minio.datatypes import Object
+from minio.deleteobjects import DeleteError, DeleteObject
 
 from python3_commons.conf import S3Settings, s3_settings
 from python3_commons.helpers import SingletonMeta
@@ -20,23 +16,20 @@ logger = logging.getLogger(__name__)
 
 class ObjectStorage(metaclass=SingletonMeta):
     def __init__(self, settings: S3Settings):
-        if not settings.s3_endpoint_url:
-            raise ValueError('s3_endpoint_url must be set')
+        if not s3_settings.s3_endpoint_url:
+            raise ValueError('s3_settings.s3_endpoint_url must be set')
 
-        self._session = aiobotocore.session.get_session()
-        self._config = {
-            'endpoint_url': settings.s3_endpoint_url,
-            'region_name': settings.s3_region_name,
-            'aws_access_key_id': settings.s3_access_key_id.get_secret_value(),
-            'aws_secret_access_key': settings.s3_secret_access_key.get_secret_value(),
-            'use_ssl': settings.s3_secure,
-            'verify': settings.s3_cert_verify,
-        }
+        self._client = Minio(
+            settings.s3_endpoint_url,
+            region=settings.s3_region_name,
+            access_key=settings.s3_access_key_id.get_secret_value(),
+            secret_key=settings.s3_secret_access_key.get_secret_value(),
+            secure=settings.s3_secure,
+            cert_check=settings.s3_cert_verify,
+        )
 
-    @asynccontextmanager
-    async def get_client(self) -> AsyncGenerator[S3Client, None]:
-        async with self._session.create_client('s3', **self._config) as client:
-            yield client
+    def get_client(self) -> Minio:
+        return self._client
 
 
 def get_absolute_path(path: str) -> str:
@@ -49,135 +42,86 @@ def get_absolute_path(path: str) -> str:
     return path
 
 
-async def put_object(bucket_name: str, path: str, data: io.BytesIO, length: int, part_size: int = 0) -> str | None:
-    storage = ObjectStorage(s3_settings)
-
-    async with storage.get_client() as s3_client:
-        try:
-            data.seek(0)
-
-            await s3_client.put_object(Bucket=bucket_name, Key=path, Body=data, ContentLength=length)
-
-            logger.debug(f'Stored object into object storage: {bucket_name}:{path}')
-
-            return f's3://{bucket_name}/{path}'
+def put_object(bucket_name: str, path: str, data: io.BytesIO, length: int, part_size: int = 0) -> str | None:
+    if s3_client := ObjectStorage(s3_settings).get_client():
+        result = s3_client.put_object(bucket_name, path, data, length, part_size=part_size)
 
-        except Exception as e:
-            logger.error(f'Failed to put object to object storage: {bucket_name}:{path}', exc_info=e)
+        logger.debug(f'Stored object into object storage: {bucket_name}:{path}')
 
-            return None
+        return result.location
+    else:
+        logger.warning('No S3 client available, skipping object put')
 
 
-@asynccontextmanager
-async def get_object_stream(bucket_name: str, path: str) -> AsyncGenerator[StreamingBody]:
-    storage = ObjectStorage(s3_settings)
-
-    async with storage.get_client() as s3_client:
+@contextmanager
+def get_object_stream(bucket_name: str, path: str):
+    if s3_client := ObjectStorage(s3_settings).get_client():
         logger.debug(f'Getting object from object storage: {bucket_name}:{path}')
 
         try:
-            response = await s3_client.get_object(Bucket=bucket_name, Key=path)
-
-            async with response['Body'] as stream:
-                yield stream
+            response = s3_client.get_object(bucket_name, path)
         except Exception as e:
             logger.debug(f'Failed getting object from object storage: {bucket_name}:{path}', exc_info=e)
 
             raise
 
+        yield response
 
-async def get_object(bucket_name: str, path: str) -> bytes:
-    async with get_object_stream(bucket_name, path) as stream:
-        body = await stream.read()
+        response.close()
+        response.release_conn()
+    else:
+        logger.warning('No S3 client available, skipping object put')
 
-    logger.debug(f'Loaded object from object storage: {bucket_name}:{path}')
-
-    return body
-
-
-async def list_objects(bucket_name: str, prefix: str, recursive: bool = True) -> AsyncGenerator[Mapping, None]:
-    storage = ObjectStorage(s3_settings)
 
-    async with storage.get_client() as s3_client:
-        paginator = s3_client.get_paginator('list_objects_v2')
+def get_object(bucket_name: str, path: str) -> bytes:
+    with get_object_stream(bucket_name, path) as stream:
+        body = stream.read()
 
-        page_iterator = paginator.paginate(Bucket=bucket_name, Prefix=prefix, Delimiter='' if recursive else '/')
+    logger.debug(f'Loaded object from object storage: {bucket_name}:{path}')
 
-        async for page in page_iterator:
-            if 'Contents' in page:
-                for obj in page['Contents']:
-                    yield dict(obj)
+    return body
 
 
-async def get_object_streams(
-    bucket_name: str, path: str, recursive: bool = True
-) -> AsyncGenerator[tuple[str, datetime, StreamingBody], None]:
-    async for obj in list_objects(bucket_name, path, recursive):
-        object_name = obj['Key']
-        last_modified = obj['LastModified']
+def list_objects(bucket_name: str, prefix: str, recursive: bool = True) -> Generator[Object, None, None]:
+    s3_client = ObjectStorage(s3_settings).get_client()
 
-        async with get_object_stream(bucket_name, path) as stream:
-            yield object_name, last_modified, stream
+    yield from s3_client.list_objects(bucket_name, prefix=prefix, recursive=recursive)
 
 
-async def get_objects(
+def get_objects(
     bucket_name: str, path: str, recursive: bool = True
-) -> AsyncGenerator[tuple[str, datetime, bytes], None]:
-    async for object_name, last_modified, stream in get_object_streams(bucket_name, path, recursive):
-        data = await stream.read()
-
-        yield object_name, last_modified, data
+) -> Generator[tuple[str, datetime, bytes], None, None]:
+    for obj in list_objects(bucket_name, path, recursive):
+        object_name = obj.object_name
 
-
-async def remove_object(bucket_name: str, object_name: str):
-    storage = ObjectStorage(s3_settings)
-
-    async with storage.get_client() as s3_client:
-        try:
-            await s3_client.delete_object(Bucket=bucket_name, Key=object_name)
-            logger.debug(f'Removed object from object storage: {bucket_name}:{object_name}')
-        except Exception as e:
-            logger.error(f'Failed to remove object from object storage: {bucket_name}:{object_name}', exc_info=e)
-
-            raise
-
-
-async def remove_objects(
-    bucket_name: str, prefix: str = None, object_names: Iterable[str] = None
-) -> Sequence[Mapping] | None:
-    storage = ObjectStorage(s3_settings)
-
-    async with storage.get_client() as s3_client:
-        objects_to_delete = []
-
-        if prefix:
-            async for obj in list_objects(bucket_name, prefix, recursive=True):
-                objects_to_delete.append({'Key': obj['Key']})
-        elif object_names:
-            objects_to_delete = [{'Key': name} for name in object_names]
+        if obj.size:
+            data = get_object(bucket_name, object_name)
         else:
-            return None
-
-        if not objects_to_delete:
-            return None
-
-        try:
-            errors = []
-            # S3 delete_objects can handle up to 1000 objects at once
-            chunk_size = 1000
+            data = b''
 
-            for i in range(0, len(objects_to_delete), chunk_size):
-                chunk = objects_to_delete[i : i + chunk_size]
+        yield object_name, obj.last_modified, data
 
-                response = await s3_client.delete_objects(Bucket=bucket_name, Delete={'Objects': chunk})
 
-                if 'Errors' in response:
-                    errors.extend(response['Errors'])
+def remove_object(bucket_name: str, object_name: str):
+    s3_client = ObjectStorage(s3_settings).get_client()
+    s3_client.remove_object(bucket_name, object_name)
 
-            logger.debug(f'Removed {len(objects_to_delete)} objects from object storage: {bucket_name}')
 
-            return errors if errors else None
-        except Exception as e:
-            logger.error(f'Failed to remove objects from object storage: {bucket_name}', exc_info=e)
-
-            raise
+def remove_objects(
+    bucket_name: str, prefix: str = None, object_names: Iterable[str] = None
+) -> Iterable[DeleteError] | None:
+    s3_client = ObjectStorage(s3_settings).get_client()
+
+    if prefix:
+        delete_object_list = map(
+            lambda obj: DeleteObject(obj.object_name),
+            s3_client.list_objects(bucket_name, prefix=prefix, recursive=True),
+        )
+    elif object_names:
+        delete_object_list = map(DeleteObject, object_names)
+    else:
+        return None
+
+    errors = s3_client.remove_objects(bucket_name, delete_object_list)
+
+    return errors

{python3_commons-0.8.36.dist-info → python3_commons-0.8.38.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python3-commons
-Version: 0.8.36
+Version: 0.8.38
 Summary: Re-usable Python3 code
 Author-email: Oleg Korsak <kamikaze.is.waiting.you@gmail.com>
 License-Expression: GPL-3.0
@@ -8,20 +8,20 @@ Project-URL: Homepage, https://github.com/kamikaze/python3-commons
 Project-URL: Documentation, https://github.com/kamikaze/python3-commons/wiki
 Classifier: Development Status :: 4 - Beta
 Classifier: Programming Language :: Python
-Requires-Python: ==3.13.*
+Requires-Python: ==3.14.*
 Description-Content-Type: text/x-rst
 License-File: LICENSE
 License-File: AUTHORS.rst
-Requires-Dist: aiobotocore~=2.23.0
-Requires-Dist: aiohttp[speedups]~=3.12.14
+Requires-Dist: aiohttp[speedups]~=3.13.0
 Requires-Dist: asyncpg~=0.30.0
 Requires-Dist: fastapi-users-db-sqlalchemy~=7.0.0
 Requires-Dist: fastapi-users[sqlalchemy]~=14.0.1
 Requires-Dist: lxml~=6.0.0
+Requires-Dist: minio~=7.2.15
 Requires-Dist: msgpack~=1.1.1
 Requires-Dist: msgspec~=0.19.0
-Requires-Dist: pydantic[email]~=2.11.7
-Requires-Dist: pydantic-settings~=2.10.1
+Requires-Dist: pydantic[email]~=2.12.0
+Requires-Dist: pydantic-settings~=2.11.0
 Requires-Dist: python-jose==3.5.0
 Requires-Dist: SQLAlchemy[asyncio]~=2.0.40
 Requires-Dist: valkey[libvalkey]~=6.1.0

{python3_commons-0.8.36.dist-info → python3_commons-0.8.38.dist-info}/RECORD

@@ -5,12 +5,12 @@ python3_commons/auth.py,sha256=vVaiJ5MHUMSbiLF6TIxe4dqVPhBlLttf940jjODL3a4,2934
 python3_commons/cache.py,sha256=lf27LTD4Z9Iqi5GaK8jH8UC0cL9sHH8wicZ88YDp6Mg,7725
 python3_commons/conf.py,sha256=JenspXyTqTlYeEb8X9Njfje1AiLCee23nm0k8zhYLfs,2283
 python3_commons/fs.py,sha256=wfLjybXndwLqNlOxTpm_HRJnuTcC4wbrHEOaEeCo9Wc,337
-python3_commons/helpers.py,sha256=qSBQ1A02BEmIY8UPJkCoO28aZq0BPsMg6AGW9k-CThM,3166
-python3_commons/object_storage.py,sha256=eF0EOB29yFn3oVr8y0SegEVJLo1W5gJT3IYcINEtLOU,6301
+python3_commons/helpers.py,sha256=ygnTv3KYoiibOFIi99-g8EXaETKHLt5i3jvykGrv6aE,3079
+python3_commons/object_storage.py,sha256=nQsXca0zzzeSY35qhnjE6pLfkLuxn7jDul0-hw0jizE,3985
 python3_commons/permissions.py,sha256=bhjTp-tq-oaTGFMHNnSBlcVX5XQCTL0nWcu6SdPEAB4,1555
 python3_commons/db/__init__.py,sha256=5nArsGm17e-pelpOwAeBKy2n_Py20XqklZsNgkcJ-DQ,2947
 python3_commons/db/helpers.py,sha256=PY0h08aLiGx-J54wmP3GHPCgGCcLd60rayAUnR3aWdI,1742
-python3_commons/db/models/__init__.py,sha256=zjZCf0DNDkqmPZ49quJ6KZohtKH87viI_ijDG3E0PVE,554
+python3_commons/db/models/__init__.py,sha256=Utr5AJf1FwcrxNtdesgjq92WMK4zpK4VL_8z1JEkJw0,185
 python3_commons/db/models/auth.py,sha256=dmyD3BX7LVBgKiepPN-bxlY6J3PhcmUfVdQwhNR45fU,1187
 python3_commons/db/models/common.py,sha256=IwrVfMQhAkPqrPmPRkG9CAB0KRa6YG_0Mogs1aafAoA,1537
 python3_commons/db/models/rbac.py,sha256=7NNTUbS8whuPUHpm4oba_UWDdNiJlHrm8HBO7oGtk64,3185
@@ -21,9 +21,9 @@ python3_commons/serializers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMp
 python3_commons/serializers/json.py,sha256=91UaXLGKGj0yPyrnuMeNrkG2GuPUgcgAsmIokUgEwpU,808
 python3_commons/serializers/msgpack.py,sha256=WrvaPE187shSK8zkH4UHHMimEZNMv9RaDSwsBE2HlCw,1269
 python3_commons/serializers/msgspec.py,sha256=0AliXlEl5sewi0UENjI8St5ZScXE5DNRERKzqWKy2Ps,2674
-python3_commons-0.8.36.dist-info/licenses/AUTHORS.rst,sha256=3R9JnfjfjH5RoPWOeqKFJgxVShSSfzQPIrEr1nxIo9Q,90
-python3_commons-0.8.36.dist-info/licenses/LICENSE,sha256=xxILuojHm4fKQOrMHPSslbyy6WuKAN2RiG74HbrYfzM,34575
-python3_commons-0.8.36.dist-info/METADATA,sha256=SRAoOrNJjXa2lxTv7ncHdEq6yMyPCVhsoggAHQp_rwE,1134
-python3_commons-0.8.36.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-python3_commons-0.8.36.dist-info/top_level.txt,sha256=lJI6sCBf68eUHzupCnn2dzG10lH3jJKTWM_hrN1cQ7M,16
-python3_commons-0.8.36.dist-info/RECORD,,
+python3_commons-0.8.38.dist-info/licenses/AUTHORS.rst,sha256=3R9JnfjfjH5RoPWOeqKFJgxVShSSfzQPIrEr1nxIo9Q,90
+python3_commons-0.8.38.dist-info/licenses/LICENSE,sha256=xxILuojHm4fKQOrMHPSslbyy6WuKAN2RiG74HbrYfzM,34575
+python3_commons-0.8.38.dist-info/METADATA,sha256=L6V4wFohHTZLYLVs3HL61LUN4mrI6Khx6Hb4DHseSzc,1127
+python3_commons-0.8.38.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+python3_commons-0.8.38.dist-info/top_level.txt,sha256=lJI6sCBf68eUHzupCnn2dzG10lH3jJKTWM_hrN1cQ7M,16
+python3_commons-0.8.38.dist-info/RECORD,,