python3-commons 0.8.35__tar.gz → 0.8.36__tar.gz

{python3_commons-0.8.35 → python3_commons-0.8.36}/.pre-commit-config.yaml

@@ -1,12 +1,12 @@
 repos:
   - repo: https://github.com/astral-sh/uv-pre-commit
-    rev: 0.7.13
+    rev: 0.7.20
     hooks:
       - id: uv-lock
 #      - id: uv-export
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.12.0
+    rev: v0.12.3
     hooks:
       # Run the linter.
       - id: ruff-check

{python3_commons-0.8.35/src/python3_commons.egg-info → python3_commons-0.8.36}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python3-commons
-Version: 0.8.35
+Version: 0.8.36
 Summary: Re-usable Python3 code
 Author-email: Oleg Korsak <kamikaze.is.waiting.you@gmail.com>
 License-Expression: GPL-3.0
@@ -12,12 +12,12 @@ Requires-Python: ==3.13.*
 Description-Content-Type: text/x-rst
 License-File: LICENSE
 License-File: AUTHORS.rst
-Requires-Dist: aiohttp[speedups]~=3.12.13
+Requires-Dist: aiobotocore~=2.23.0
+Requires-Dist: aiohttp[speedups]~=3.12.14
 Requires-Dist: asyncpg~=0.30.0
 Requires-Dist: fastapi-users-db-sqlalchemy~=7.0.0
 Requires-Dist: fastapi-users[sqlalchemy]~=14.0.1
 Requires-Dist: lxml~=6.0.0
-Requires-Dist: minio~=7.2.15
 Requires-Dist: msgpack~=1.1.1
 Requires-Dist: msgspec~=0.19.0
 Requires-Dist: pydantic[email]~=2.11.7

{python3_commons-0.8.35 → python3_commons-0.8.36}/pyproject.toml

@@ -19,12 +19,12 @@ keywords = []
 requires-python = "==3.13.*"
 
 dependencies = [
-    "aiohttp[speedups]~=3.12.13",
+    "aiobotocore~=2.23.0",
+    "aiohttp[speedups]~=3.12.14",
     "asyncpg~=0.30.0",
     "fastapi-users-db-sqlalchemy~=7.0.0",
     "fastapi-users[sqlalchemy]~=14.0.1",
     "lxml~=6.0.0",
-    "minio~=7.2.15",
     "msgpack~=1.1.1",
     "msgspec~=0.19.0",
     "pydantic[email]~=2.11.7",
@@ -40,10 +40,11 @@ dev = [
     "build",
     "pip==25.1.1",
     "pre-commit==4.2.0",
-    "pyright==1.1.402",
-    "ruff==0.12.1",
+    "pyright==1.1.403",
+    "ruff==0.12.3",
     "setuptools==80.9.0",
     "setuptools_scm==8.3.1",
+    "types-aiobotocore-s3",
     "wheel==0.45.1",
 ]
 testing = [

python3_commons-0.8.36/src/python3_commons/db/models/__init__.py

@@ -0,0 +1,8 @@
+from python3_commons.db.models.auth import ApiKey as ApiKey
+from python3_commons.db.models.auth import User as User
+from python3_commons.db.models.auth import UserGroup as UserGroup
+from python3_commons.db.models.rbac import RBACApiKeyRole as RBACApiKeyRole
+from python3_commons.db.models.rbac import RBACPermission as RBACPermission
+from python3_commons.db.models.rbac import RBACRole as RBACRole
+from python3_commons.db.models.rbac import RBACRolePermission as RBACRolePermission
+from python3_commons.db.models.rbac import RBACUserRole as RBACUserRole

{python3_commons-0.8.35 → python3_commons-0.8.36}/src/python3_commons/helpers.py

@@ -1,6 +1,8 @@
 import logging
 import shlex
 import threading
+from abc import ABCMeta
+from collections import defaultdict
 from datetime import date, datetime, timedelta
 from decimal import ROUND_HALF_UP, Decimal
 from json import dumps
@@ -12,24 +14,24 @@ from python3_commons.serializers.json import CustomJSONEncoder
 logger = logging.getLogger(__name__)
 
 
-class SingletonMeta(type):
+class SingletonMeta(ABCMeta):
     """
     A metaclass that creates a Singleton base class when called.
     """
 
-    _instances = {}
-    _lock = threading.Lock()
+    __instances = {}
+    __locks = defaultdict(threading.Lock)
 
     def __call__(cls, *args, **kwargs):
         try:
-            return cls._instances[cls]
+            return cls.__instances[cls]
         except KeyError:
-            with cls._lock:
+            with cls.__locks[cls]:
                 try:
-                    return cls._instances[cls]
+                    return cls.__instances[cls]
                 except KeyError:
                     instance = super(SingletonMeta, cls).__call__(*args, **kwargs)
-                    cls._instances[cls] = instance
+                    cls.__instances[cls] = instance
 
                     return instance
 

python3_commons-0.8.36/src/python3_commons/object_storage.py

@@ -0,0 +1,183 @@
+from __future__ import annotations
+
+import io
+import logging
+from contextlib import asynccontextmanager
+from datetime import datetime
+from typing import TYPE_CHECKING, AsyncGenerator, Iterable, Mapping, Sequence
+
+import aiobotocore.session
+from aiobotocore.response import StreamingBody
+
+if TYPE_CHECKING:
+    from types_aiobotocore_s3.client import S3Client
+
+from python3_commons.conf import S3Settings, s3_settings
+from python3_commons.helpers import SingletonMeta
+
+logger = logging.getLogger(__name__)
+
+
+class ObjectStorage(metaclass=SingletonMeta):
+    def __init__(self, settings: S3Settings):
+        if not settings.s3_endpoint_url:
+            raise ValueError('s3_endpoint_url must be set')
+
+        self._session = aiobotocore.session.get_session()
+        self._config = {
+            'endpoint_url': settings.s3_endpoint_url,
+            'region_name': settings.s3_region_name,
+            'aws_access_key_id': settings.s3_access_key_id.get_secret_value(),
+            'aws_secret_access_key': settings.s3_secret_access_key.get_secret_value(),
+            'use_ssl': settings.s3_secure,
+            'verify': settings.s3_cert_verify,
+        }
+
+    @asynccontextmanager
+    async def get_client(self) -> AsyncGenerator[S3Client, None]:
+        async with self._session.create_client('s3', **self._config) as client:
+            yield client
+
+
+def get_absolute_path(path: str) -> str:
+    if path.startswith('/'):
+        path = path[1:]
+
+    if bucket_root := s3_settings.s3_bucket_root:
+        path = f'{bucket_root[:1] if bucket_root.startswith("/") else bucket_root}/{path}'
+
+    return path
+
+
+async def put_object(bucket_name: str, path: str, data: io.BytesIO, length: int, part_size: int = 0) -> str | None:
+    storage = ObjectStorage(s3_settings)
+
+    async with storage.get_client() as s3_client:
+        try:
+            data.seek(0)
+
+            await s3_client.put_object(Bucket=bucket_name, Key=path, Body=data, ContentLength=length)
+
+            logger.debug(f'Stored object into object storage: {bucket_name}:{path}')
+
+            return f's3://{bucket_name}/{path}'
+
+        except Exception as e:
+            logger.error(f'Failed to put object to object storage: {bucket_name}:{path}', exc_info=e)
+
+            return None
+
+
+@asynccontextmanager
+async def get_object_stream(bucket_name: str, path: str) -> AsyncGenerator[StreamingBody]:
+    storage = ObjectStorage(s3_settings)
+
+    async with storage.get_client() as s3_client:
+        logger.debug(f'Getting object from object storage: {bucket_name}:{path}')
+
+        try:
+            response = await s3_client.get_object(Bucket=bucket_name, Key=path)
+
+            async with response['Body'] as stream:
+                yield stream
+        except Exception as e:
+            logger.debug(f'Failed getting object from object storage: {bucket_name}:{path}', exc_info=e)
+
+            raise
+
+
+async def get_object(bucket_name: str, path: str) -> bytes:
+    async with get_object_stream(bucket_name, path) as stream:
+        body = await stream.read()
+
+    logger.debug(f'Loaded object from object storage: {bucket_name}:{path}')
+
+    return body
+
+
+async def list_objects(bucket_name: str, prefix: str, recursive: bool = True) -> AsyncGenerator[Mapping, None]:
+    storage = ObjectStorage(s3_settings)
+
+    async with storage.get_client() as s3_client:
+        paginator = s3_client.get_paginator('list_objects_v2')
+
+        page_iterator = paginator.paginate(Bucket=bucket_name, Prefix=prefix, Delimiter='' if recursive else '/')
+
+        async for page in page_iterator:
+            if 'Contents' in page:
+                for obj in page['Contents']:
+                    yield dict(obj)
+
+
+async def get_object_streams(
+    bucket_name: str, path: str, recursive: bool = True
+) -> AsyncGenerator[tuple[str, datetime, StreamingBody], None]:
+    async for obj in list_objects(bucket_name, path, recursive):
+        object_name = obj['Key']
+        last_modified = obj['LastModified']
+
+        async with get_object_stream(bucket_name, path) as stream:
+            yield object_name, last_modified, stream
+
+
+async def get_objects(
+    bucket_name: str, path: str, recursive: bool = True
+) -> AsyncGenerator[tuple[str, datetime, bytes], None]:
+    async for object_name, last_modified, stream in get_object_streams(bucket_name, path, recursive):
+        data = await stream.read()
+
+        yield object_name, last_modified, data
+
+
+async def remove_object(bucket_name: str, object_name: str):
+    storage = ObjectStorage(s3_settings)
+
+    async with storage.get_client() as s3_client:
+        try:
+            await s3_client.delete_object(Bucket=bucket_name, Key=object_name)
+            logger.debug(f'Removed object from object storage: {bucket_name}:{object_name}')
+        except Exception as e:
+            logger.error(f'Failed to remove object from object storage: {bucket_name}:{object_name}', exc_info=e)
+
+            raise
+
+
+async def remove_objects(
+    bucket_name: str, prefix: str = None, object_names: Iterable[str] = None
+) -> Sequence[Mapping] | None:
+    storage = ObjectStorage(s3_settings)
+
+    async with storage.get_client() as s3_client:
+        objects_to_delete = []
+
+        if prefix:
+            async for obj in list_objects(bucket_name, prefix, recursive=True):
+                objects_to_delete.append({'Key': obj['Key']})
+        elif object_names:
+            objects_to_delete = [{'Key': name} for name in object_names]
+        else:
+            return None
+
+        if not objects_to_delete:
+            return None
+
+        try:
+            errors = []
+            # S3 delete_objects can handle up to 1000 objects at once
+            chunk_size = 1000
+
+            for i in range(0, len(objects_to_delete), chunk_size):
+                chunk = objects_to_delete[i : i + chunk_size]
+
+                response = await s3_client.delete_objects(Bucket=bucket_name, Delete={'Objects': chunk})
+
+                if 'Errors' in response:
+                    errors.extend(response['Errors'])
+
+            logger.debug(f'Removed {len(objects_to_delete)} objects from object storage: {bucket_name}')
+
+            return errors if errors else None
+        except Exception as e:
+            logger.error(f'Failed to remove objects from object storage: {bucket_name}', exc_info=e)
+
+            raise

{python3_commons-0.8.35 → python3_commons-0.8.36/src/python3_commons.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python3-commons
-Version: 0.8.35
+Version: 0.8.36
 Summary: Re-usable Python3 code
 Author-email: Oleg Korsak <kamikaze.is.waiting.you@gmail.com>
 License-Expression: GPL-3.0
@@ -12,12 +12,12 @@ Requires-Python: ==3.13.*
 Description-Content-Type: text/x-rst
 License-File: LICENSE
 License-File: AUTHORS.rst
-Requires-Dist: aiohttp[speedups]~=3.12.13
+Requires-Dist: aiobotocore~=2.23.0
+Requires-Dist: aiohttp[speedups]~=3.12.14
 Requires-Dist: asyncpg~=0.30.0
 Requires-Dist: fastapi-users-db-sqlalchemy~=7.0.0
 Requires-Dist: fastapi-users[sqlalchemy]~=14.0.1
 Requires-Dist: lxml~=6.0.0
-Requires-Dist: minio~=7.2.15
 Requires-Dist: msgpack~=1.1.1
 Requires-Dist: msgspec~=0.19.0
 Requires-Dist: pydantic[email]~=2.11.7

{python3_commons-0.8.35 → python3_commons-0.8.36}/src/python3_commons.egg-info/requires.txt

@@ -1,9 +1,9 @@
-aiohttp[speedups]~=3.12.13
+aiobotocore~=2.23.0
+aiohttp[speedups]~=3.12.14
 asyncpg~=0.30.0
 fastapi-users-db-sqlalchemy~=7.0.0
 fastapi-users[sqlalchemy]~=14.0.1
 lxml~=6.0.0
-minio~=7.2.15
 msgpack~=1.1.1
 msgspec~=0.19.0
 pydantic[email]~=2.11.7