python3-commons 0.0.0__py3-none-any.whl

python3_commons/conf.py

@@ -0,0 +1,52 @@
+from pydantic import Field, HttpUrl, PostgresDsn, RedisDsn, SecretStr
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class CommonSettings(BaseSettings):
+    logging_level: str = 'INFO'
+    logging_format: str = '%(asctime)s [%(levelname)s] %(name)s: %(message)s'
+    logging_formatter: str = 'default'
+
+
+class OIDCSettings(BaseSettings):
+    model_config = SettingsConfigDict(env_prefix='OIDC_')
+
+    enabled: bool = True
+    authority_url: HttpUrl | None = None
+    client_id: str | None = None
+
+
+class ValkeySettings(BaseSettings):
+    model_config = SettingsConfigDict(env_prefix='VALKEY_')
+
+    dsn: RedisDsn | None = None
+    sentinel_dsn: RedisDsn | None = None
+
+
+class DBSettings(BaseSettings):
+    model_config = SettingsConfigDict(env_prefix='DB_')
+
+    dsn: PostgresDsn | None = Field(default=None, serialization_alias='url')
+    echo: bool = False
+    pool_size: int = 20
+    max_overflow: int = 0
+    pool_timeout: int = 30
+    pool_recycle: int = 1800  # 30 minutes
+
+
+class S3Settings(BaseSettings):
+    s3_endpoint_url: str | None = None
+    s3_region_name: str | None = None
+    s3_access_key_id: SecretStr = ''
+    s3_secret_access_key: SecretStr = ''
+    s3_secure: bool = True
+    s3_bucket: str | None = None
+    s3_bucket_root: str | None = None
+    s3_cert_verify: bool = True
+
+
+settings = CommonSettings()
+oidc_settings = OIDCSettings()
+valkey_settings = ValkeySettings()
+db_settings = DBSettings()
+s3_settings = S3Settings()

python3_commons/db/__init__.py

@@ -0,0 +1,83 @@
+import contextlib
+import logging
+from typing import AsyncGenerator, Callable, Mapping
+
+from sqlalchemy import MetaData
+from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_engine_from_config
+from sqlalchemy.ext.asyncio.session import async_sessionmaker
+from sqlalchemy.orm import declarative_base
+
+from python3_commons.conf import DBSettings
+
+logger = logging.getLogger(__name__)
+metadata = MetaData()
+Base = declarative_base(metadata=metadata)
+
+
+class AsyncSessionManager:
+    def __init__(self, db_settings: Mapping[str, DBSettings]):
+        self.db_settings: Mapping[str, DBSettings] = db_settings
+        self.engines: dict[str, AsyncEngine] = {}
+        self.session_makers: dict = {}
+
+    def get_db_settings(self, name: str) -> DBSettings:
+        try:
+            return self.db_settings[name]
+        except KeyError:
+            logger.error(f'Missing database settings: {name}')
+
+            raise
+
+    def async_engine_from_db_settings(self, name):
+        db_settings = self.get_db_settings(name)
+        configuration = db_settings.model_dump(by_alias=True)
+        configuration['url'] = str(configuration['url'])
+        engine = async_engine_from_config(configuration, prefix='')
+
+        return engine
+
+    def get_engine(self, name: str) -> AsyncEngine:
+        try:
+            engine = self.engines[name]
+        except KeyError:
+            logger.debug(f'Creating engine: {name}')
+            engine = self.async_engine_from_db_settings(name)
+            self.engines[name] = engine
+
+        return engine
+
+    def get_session_maker(self, name: str):
+        try:
+            session_maker = self.session_makers[name]
+        except KeyError:
+            logger.debug(f'Creating session maker: {name}')
+            engine = self.get_engine(name)
+            session_maker = async_sessionmaker(engine, expire_on_commit=False)
+            self.session_makers[name] = session_maker
+
+        return session_maker
+
+    def get_async_session(self, name: str) -> Callable[[], AsyncGenerator[AsyncSession, None]]:
+        async def get_session() -> AsyncGenerator[AsyncSession, None]:
+            session_maker = self.get_session_maker(name)
+
+            async with session_maker() as session:
+                yield session
+
+        return get_session
+
+    def get_session_context(self, name: str):
+        # TODO: cache
+        return contextlib.asynccontextmanager(self.get_async_session(name))
+
+
+async def is_healthy(engine: AsyncEngine) -> bool:
+    try:
+        async with engine.begin() as conn:
+            result = await conn.execute('SELECT 1;')
+
+            return result.scalar() == 1
+    except Exception as e:
+        logger.error(f'Database connection is not healthy: {e}')
+
+        return False

python3_commons/db/helpers.py

@@ -0,0 +1,62 @@
+import logging
+from typing import Mapping
+
+import sqlalchemy as sa
+from sqlalchemy import asc, desc, func
+from sqlalchemy.sql.elements import BooleanClauseList, UnaryExpression
+
+logger = logging.getLogger(__name__)
+
+
+def get_query(
+    search: Mapping[str, str] | None = None, order_by: str | None = None, columns: Mapping | None = None
+) -> tuple[BooleanClauseList, UnaryExpression]:
+    """
+    :columns:
+        Param name ->
+        0: Model column
+        1: case-insensitive if True
+        2: cast value to type
+        3: exact match if True, LIKE %value% if False
+    """
+
+    order_by_cols = {}
+
+    if order_by:
+        for order_by_col in order_by.split(','):
+            if order_by_col.startswith('-'):
+                direction = desc
+                order_by_col = order_by_col[1:]
+            else:
+                direction = asc
+
+            order_by_cols[order_by_col] = direction
+
+        order_by_clauses = tuple(
+            direction(columns[order_by_col][0]) for order_by_col, direction in order_by_cols.items()
+        )
+    else:
+        order_by_clauses = None
+
+    if search:
+        where_parts = [
+            *(
+                (func.upper(columns[k][0]) if columns[k][1] else columns[k][0]) == columns[k][2](v)
+                for k, v in search.items()
+                if columns[k][3]
+            ),
+            *(
+                (func.upper(columns[k][0]) if columns[k][1] else columns[k][0]).like(f'%{v.upper()}%')
+                for k, v in search.items()
+                if not columns[k][3]
+            ),
+        ]
+    else:
+        where_parts = None
+
+    if where_parts:
+        where_clause = sa.and_(*where_parts)
+    else:
+        where_clause = None
+
+    return where_clause, order_by_clauses

python3_commons/db/models/__init__.py

@@ -0,0 +1,2 @@
+from python3_commons.db.models.auth import ApiKey, User, UserGroup
+from python3_commons.db.models.rbac import RBACApiKeyRole, RBACPermission, RBACRole, RBACRolePermission, RBACUserRole

python3_commons/db/models/auth.py

@@ -0,0 +1,35 @@
+import uuid
+
+from fastapi_users_db_sqlalchemy import GUID, SQLAlchemyBaseUserTableUUID
+from pydantic import AwareDatetime
+from sqlalchemy import BIGINT, DateTime, ForeignKey, String
+from sqlalchemy.orm import Mapped, mapped_column
+
+from python3_commons.db import Base
+from python3_commons.db.models.common import BaseDBModel, BaseDBUUIDModel
+
+
+class UserGroup(BaseDBModel, Base):
+    __tablename__ = 'user_groups'
+
+    name: Mapped[str] = mapped_column(String, nullable=False)
+
+
+class User(SQLAlchemyBaseUserTableUUID, Base):
+    __tablename__ = 'users'
+
+    username: Mapped[str] = mapped_column(String, unique=True, index=True, nullable=False)
+    group_id: Mapped[int | None] = mapped_column(BIGINT, ForeignKey('user_groups.id'))
+
+
+class ApiKey(BaseDBUUIDModel, Base):
+    __tablename__ = 'api_keys'
+
+    user_id: Mapped[uuid.UUID | None] = mapped_column(
+        GUID,
+        ForeignKey('users.id', name='fk_api_key_user', ondelete='RESTRICT'),
+        index=True,
+    )
+    partner_name: Mapped[str] = mapped_column(String, unique=True)
+    key: Mapped[str] = mapped_column(String, unique=True)
+    expires_at: Mapped[AwareDatetime | None] = mapped_column(DateTime(timezone=True))

python3_commons/db/models/common.py

@@ -0,0 +1,39 @@
+from pydantic import AwareDatetime
+from sqlalchemy import BIGINT, DateTime
+from sqlalchemy.dialects.postgresql import UUID
+from sqlalchemy.ext.compiler import compiles
+from sqlalchemy.orm import Mapped, mapped_column
+from sqlalchemy.sql import expression
+from sqlalchemy.sql.ddl import CreateColumn
+
+
+class UTCNow(expression.FunctionElement):
+    type = DateTime(timezone=True)
+
+
+@compiles(UTCNow, 'postgresql')
+def pg_utcnow(element, compiler, **kw):
+    return "TIMEZONE('utc', CURRENT_TIMESTAMP)"
+
+
+@compiles(CreateColumn, 'postgresql')
+def use_identity(element, compiler, **kw):
+    result = compiler.visit_create_column(element, **kw).replace('SERIAL', 'INT GENERATED BY DEFAULT AS IDENTITY')
+
+    return result.replace('BIGSERIAL', 'BIGINT GENERATED BY DEFAULT AS IDENTITY')
+
+
+class BaseDBModel:
+    id: Mapped[int] = mapped_column(BIGINT, primary_key=True, sort_order=-3)
+    created_at: Mapped[AwareDatetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=UTCNow(), sort_order=-2
+    )
+    updated_at: Mapped[AwareDatetime] = mapped_column(DateTime(timezone=True), onupdate=UTCNow(), sort_order=-1)
+
+
+class BaseDBUUIDModel:
+    uid: Mapped[UUID] = mapped_column(UUID, primary_key=True, sort_order=-3)
+    created_at: Mapped[AwareDatetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=UTCNow(), sort_order=-2
+    )
+    updated_at: Mapped[AwareDatetime | None] = mapped_column(DateTime(timezone=True), onupdate=UTCNow(), sort_order=-1)

python3_commons/db/models/rbac.py

@@ -0,0 +1,91 @@
+import uuid
+
+from fastapi_users_db_sqlalchemy import GUID
+from pydantic import AwareDatetime
+from sqlalchemy import CheckConstraint, DateTime, ForeignKey, PrimaryKeyConstraint, String
+from sqlalchemy.dialects.postgresql import UUID
+from sqlalchemy.orm import Mapped, mapped_column
+
+from python3_commons.db import Base
+
+
+class RBACRole(Base):
+    __tablename__ = 'rbac_roles'
+
+    uid: Mapped[uuid.UUID] = mapped_column(UUID, primary_key=True)
+    name: Mapped[str] = mapped_column(String, unique=True, nullable=False)
+
+
+class RBACPermission(Base):
+    __tablename__ = 'rbac_permissions'
+
+    uid: Mapped[uuid.UUID] = mapped_column(UUID, primary_key=True)
+    name: Mapped[str] = mapped_column(String, unique=True, nullable=False)
+
+    __table_args__ = (CheckConstraint("name ~ '^[a-z0-9_.]+$'", name='check_rbac_permissions_name'),)
+
+
+class RBACRolePermission(Base):
+    __tablename__ = 'rbac_role_permissions'
+
+    role_uid: Mapped[uuid.UUID | None] = mapped_column(
+        UUID,
+        ForeignKey('rbac_roles.uid', name='fk_rbac_role_permissions_role', ondelete='CASCADE'),
+        index=True,
+    )
+    permission_uid: Mapped[uuid.UUID | None] = mapped_column(
+        UUID,
+        ForeignKey('rbac_permissions.uid', name='fk_rbac_role_permissions_permission', ondelete='CASCADE'),
+        index=True,
+    )
+
+    __table_args__ = (PrimaryKeyConstraint('role_uid', 'permission_uid', name='pk_rbac_role_permissions'),)
+
+
+class RBACUserRole(Base):
+    __tablename__ = 'rbac_user_roles'
+
+    user_id: Mapped[uuid.UUID | None] = mapped_column(
+        GUID,
+        ForeignKey('users.id', name='fk_rbac_user_roles_user', ondelete='CASCADE'),
+        index=True,
+    )
+    role_uid: Mapped[uuid.UUID | None] = mapped_column(
+        UUID,
+        ForeignKey('rbac_roles.uid', name='fk_rbac_user_roles_role', ondelete='CASCADE'),
+        index=True,
+    )
+    starts_at: Mapped[AwareDatetime] = mapped_column(DateTime(timezone=True), nullable=False)
+    expires_at: Mapped[AwareDatetime | None] = mapped_column(DateTime(timezone=True))
+
+    __table_args__ = (PrimaryKeyConstraint('user_id', 'role_uid', name='pk_rbac_user_roles'),)
+
+
+class RBACApiKeyRole(Base):
+    __tablename__ = 'rbac_api_key_roles'
+
+    api_key_uid: Mapped[uuid.UUID | None] = mapped_column(
+        UUID,
+        ForeignKey('api_keys.uid', name='fk_rbac_api_key_roles_user', ondelete='CASCADE'),
+        index=True,
+    )
+    role_uid: Mapped[uuid.UUID | None] = mapped_column(
+        UUID,
+        ForeignKey('rbac_roles.uid', name='fk_rbac_api_key_roles_role', ondelete='CASCADE'),
+        index=True,
+    )
+    starts_at: Mapped[AwareDatetime] = mapped_column(DateTime(timezone=True), nullable=False)
+    expires_at: Mapped[AwareDatetime | None] = mapped_column(DateTime(timezone=True))
+
+    __table_args__ = (PrimaryKeyConstraint('api_key_uid', 'role_uid', name='pk_rbac_api_key_roles'),)
+
+
+# class RBACRoleRelation(Base):
+#     __tablename__ = 'rbac_role_relations'
+#
+#     parent_uid: Mapped[uuid.UUID] = mapped_column(UUID)
+#     child_uid: Mapped[uuid.UUID] = mapped_column(UUID)
+#
+#     __table_args__ = (
+#         PrimaryKeyConstraint('parent_uid', 'child_uid', name='pk_rbac_role_relations'),
+#     )

python3_commons/fs.py

@@ -0,0 +1,10 @@
+from pathlib import Path
+from typing import Generator
+
+
+def iter_files(root: Path, recursive: bool = True) -> Generator[Path, None, None]:
+    for item in root.iterdir():
+        if item.is_file():
+            yield item
+        elif item.is_dir() and recursive and not item.name.startswith('.'):
+            yield from iter_files(item)

python3_commons/helpers.py

@@ -0,0 +1,108 @@
+import logging
+import shlex
+import threading
+from datetime import date, datetime, timedelta
+from decimal import ROUND_HALF_UP, Decimal
+from json import dumps
+from typing import Literal, Mapping, Sequence
+from urllib.parse import urlencode
+
+from python3_commons.serializers.json import CustomJSONEncoder
+
+logger = logging.getLogger(__name__)
+
+
+class SingletonMeta(type):
+    """
+    A metaclass that creates a Singleton base class when called.
+    """
+
+    _instances = {}
+    _lock = threading.Lock()
+
+    def __call__(cls, *args, **kwargs):
+        try:
+            return cls._instances[cls]
+        except KeyError:
+            with cls._lock:
+                try:
+                    return cls._instances[cls]
+                except KeyError:
+                    instance = super(SingletonMeta, cls).__call__(*args, **kwargs)
+                    cls._instances[cls] = instance
+
+                    return instance
+
+
+def date_from_string(string: str, fmt: str = '%d.%m.%Y') -> date:
+    try:
+        return datetime.strptime(string, fmt).date()
+    except ValueError:
+        return date.fromisoformat(string)
+
+
+def datetime_from_string(string: str) -> datetime:
+    try:
+        return datetime.strptime(string, '%d.%m.%Y %H:%M:%S')
+    except ValueError:
+        return datetime.fromisoformat(string)
+
+
+def date_range(start_date, end_date):
+    for n in range(int((end_date - start_date).days + 1)):
+        yield start_date + timedelta(days=n)
+
+
+def tries(times):
+    def func_wrapper(f):
+        async def wrapper(*args, **kwargs):
+            for time in range(times if times > 0 else 1):
+                # noinspection PyBroadException
+                try:
+                    return await f(*args, **kwargs)
+                except Exception as exc:
+                    if time >= times:
+                        raise exc
+
+        return wrapper
+
+    return func_wrapper
+
+
+def round_decimal(value: Decimal, decimal_places=2, rounding_mode=ROUND_HALF_UP) -> Decimal:
+    try:
+        return value.quantize(Decimal(10) ** -decimal_places, rounding=rounding_mode)
+    except AttributeError:
+        return value
+
+
+def request_to_curl(
+    url: str,
+    query: Mapping | None = None,
+    method: Literal['get', 'post', 'put', 'patch', 'options', 'head', 'delete'] = 'get',
+    headers: Mapping | None = None,
+    json: Mapping | Sequence | str | None = None,
+    data: bytes | None = None,
+) -> str:
+    if query:
+        url = f'{url}?{urlencode(query)}'
+
+    curl_cmd = ['curl', '-i', '-X', method.upper(), shlex.quote(url)]
+
+    if headers:
+        for key, value in headers.items():
+            header_line = f'{key}: {value}'
+            curl_cmd.append('-H')
+            curl_cmd.append(shlex.quote(header_line))
+
+    if json:
+        curl_cmd.append('-H')
+        curl_cmd.append(shlex.quote('Content-Type: application/json'))
+
+        curl_cmd.append('-d')
+        curl_cmd.append(shlex.quote(dumps(json, cls=CustomJSONEncoder)))
+    elif data:
+        curl_cmd.append('-d')
+        curl_cmd.append(shlex.quote(data.decode('utf-8')))
+
+    return ' '.join(curl_cmd)

python3_commons/logging/filters.py

@@ -0,0 +1,10 @@
+import logging
+
+
+def filter_maker(level):
+    level = getattr(logging, level)
+
+    def record_filter(record):
+        return record.levelno <= level
+
+    return record_filter

python3_commons/logging/formatters.py

@@ -0,0 +1,25 @@
+import json
+import logging
+import traceback
+from contextvars import ContextVar
+
+from python3_commons.serializers.json import CustomJSONEncoder
+
+correlation_id: ContextVar[str | None] = ContextVar('correlation_id', default=None)
+
+
+class JSONFormatter(logging.Formatter):
+    @staticmethod
+    def format_exception(exc_info):
+        return ''.join(traceback.format_exception(*exc_info))
+
+    def format(self, record):
+        if corr_id := correlation_id.get():
+            record.correlation_id = corr_id
+
+        if record.exc_info:
+            record.exc_text = self.format_exception(record.exc_info)
+        else:
+            record.exc_text = None
+
+        return json.dumps(record.__dict__, cls=CustomJSONEncoder)

python3_commons/object_storage.py

@@ -0,0 +1,127 @@
+import io
+import logging
+from contextlib import contextmanager
+from datetime import datetime
+from typing import Generator, Iterable
+
+from minio import Minio
+from minio.datatypes import Object
+from minio.deleteobjects import DeleteError, DeleteObject
+
+from python3_commons.conf import S3Settings, s3_settings
+from python3_commons.helpers import SingletonMeta
+
+logger = logging.getLogger(__name__)
+
+
+class ObjectStorage(metaclass=SingletonMeta):
+    def __init__(self, settings: S3Settings):
+        if not s3_settings.s3_endpoint_url:
+            raise ValueError('s3_settings.s3_endpoint_url must be set')
+
+        self._client = Minio(
+            settings.s3_endpoint_url,
+            region=settings.s3_region_name,
+            access_key=settings.s3_access_key_id.get_secret_value(),
+            secret_key=settings.s3_secret_access_key.get_secret_value(),
+            secure=settings.s3_secure,
+            cert_check=settings.s3_cert_verify,
+        )
+
+    def get_client(self) -> Minio:
+        return self._client
+
+
+def get_absolute_path(path: str) -> str:
+    if path.startswith('/'):
+        path = path[1:]
+
+    if bucket_root := s3_settings.s3_bucket_root:
+        path = f'{bucket_root[:1] if bucket_root.startswith("/") else bucket_root}/{path}'
+
+    return path
+
+
+def put_object(bucket_name: str, path: str, data: io.BytesIO, length: int, part_size: int = 0) -> str | None:
+    if s3_client := ObjectStorage(s3_settings).get_client():
+        result = s3_client.put_object(bucket_name, path, data, length, part_size=part_size)
+
+        logger.debug(f'Stored object into object storage: {bucket_name}:{path}')
+
+        return result.location
+    else:
+        logger.warning('No S3 client available, skipping object put')
+
+
+@contextmanager
+def get_object_stream(bucket_name: str, path: str):
+    if s3_client := ObjectStorage(s3_settings).get_client():
+        logger.debug(f'Getting object from object storage: {bucket_name}:{path}')
+
+        try:
+            response = s3_client.get_object(bucket_name, path)
+        except Exception as e:
+            logger.debug(f'Failed getting object from object storage: {bucket_name}:{path}', exc_info=e)
+
+            raise
+
+        yield response
+
+        response.close()
+        response.release_conn()
+    else:
+        logger.warning('No S3 client available, skipping object put')
+
+
+def get_object(bucket_name: str, path: str) -> bytes:
+    with get_object_stream(bucket_name, path) as stream:
+        body = stream.read()
+
+    logger.debug(f'Loaded object from object storage: {bucket_name}:{path}')
+
+    return body
+
+
+def list_objects(bucket_name: str, prefix: str, recursive: bool = True) -> Generator[Object, None, None]:
+    s3_client = ObjectStorage(s3_settings).get_client()
+
+    yield from s3_client.list_objects(bucket_name, prefix=prefix, recursive=recursive)
+
+
+def get_objects(
+    bucket_name: str, path: str, recursive: bool = True
+) -> Generator[tuple[str, datetime, bytes], None, None]:
+    for obj in list_objects(bucket_name, path, recursive):
+        object_name = obj.object_name
+
+        if obj.size:
+            data = get_object(bucket_name, object_name)
+        else:
+            data = b''
+
+        yield object_name, obj.last_modified, data
+
+
+def remove_object(bucket_name: str, object_name: str):
+    s3_client = ObjectStorage(s3_settings).get_client()
+    s3_client.remove_object(bucket_name, object_name)
+
+
+def remove_objects(
+    bucket_name: str, prefix: str = None, object_names: Iterable[str] = None
+) -> Iterable[DeleteError] | None:
+    s3_client = ObjectStorage(s3_settings).get_client()
+
+    if prefix:
+        delete_object_list = map(
+            lambda obj: DeleteObject(obj.object_name),
+            s3_client.list_objects(bucket_name, prefix=prefix, recursive=True),
+        )
+    elif object_names:
+        delete_object_list = map(DeleteObject, object_names)
+    else:
+        return None
+
+    errors = s3_client.remove_objects(bucket_name, delete_object_list)
+
+    return errors