arpakitlib 1.7.256__py3-none-any.whl → 1.8.0__py3-none-any.whl

arpakitlib/ar_operation_execution_util.py

@@ -1,504 +0,0 @@
-# arpakit
-
-from __future__ import annotations
-
-import asyncio
-import logging
-import traceback
-from datetime import timedelta, time
-from typing import Any, Callable
-
-from pydantic import ConfigDict
-from pydantic.v1 import BaseModel
-from sqlalchemy import asc
-from sqlalchemy.exc import NoResultFound
-from sqlalchemy.orm import Session
-
-from arpakitlib.ar_base_worker_util import BaseWorker
-from arpakitlib.ar_datetime_util import now_utc_dt
-from arpakitlib.ar_dict_util import combine_dicts
-from arpakitlib.ar_sleep_util import sync_safe_sleep, async_safe_sleep
-from arpakitlib.ar_sqlalchemy_model_util import OperationDBM, StoryLogDBM, BaseOperationTypes
-from arpakitlib.ar_sqlalchemy_util import SQLAlchemyDb
-from arpakitlib.ar_type_util import raise_for_type
-
-_ARPAKIT_LIB_MODULE_VERSION = "3.0"
-
-_logger = logging.getLogger(__name__)
-
-
-def get_operation_for_execution(
-        *,
-        session: Session | None = None,
-        sqlalchemy_db: SQLAlchemyDb | None = None,
-        filter_operation_types: list[str] | str | None = None,
-        lock: bool = False
-) -> OperationDBM | None:
-    if isinstance(filter_operation_types, str):
-        filter_operation_types = [filter_operation_types]
-
-    def func(session_: Session):
-        query = (
-            session_
-            .query(OperationDBM)
-            .filter(OperationDBM.status == OperationDBM.Statuses.waiting_for_execution)
-        )
-        if filter_operation_types:
-            query = query.filter(OperationDBM.type.in_(filter_operation_types))
-
-        if lock:
-            query = query.with_for_update()
-
-        query = query.order_by(asc(OperationDBM.creation_dt))
-        operation_dbm: OperationDBM | None = query.first()
-        return operation_dbm
-
-    if session is not None:
-        return func(session_=session)
-    elif sqlalchemy_db is not None:
-        with sqlalchemy_db.new_session() as session:
-            return func(session_=session)
-    else:
-        raise ValueError("session is None and sqlalchemy_db is None")
-
-
-def get_operation_by_id(
-        *,
-        session: Session | None = None,
-        sqlalchemy_db: SQLAlchemyDb | None = None,
-        filter_operation_id: int,
-        raise_if_not_found: bool = False,
-        lock: bool = False
-) -> OperationDBM | None:
-    def func(session_: Session):
-        query = (
-            session_
-            .query(OperationDBM)
-            .filter(OperationDBM.id == filter_operation_id)
-        )
-
-        if lock:
-            query = query.with_for_update()
-
-        if raise_if_not_found:
-            try:
-                return query.one()
-            except NoResultFound:
-                if raise_if_not_found:
-                    raise ValueError("Operation not found")
-        else:
-            return query.one_or_none()
-
-    if session is not None:
-        return func(session_=session)
-    elif sqlalchemy_db is not None:
-        with sqlalchemy_db.new_session() as session:
-            return func(session_=session)
-    else:
-        raise ValueError("session is None and sqlalchemy_db is None")
-
-
-def remove_operations(
-        *,
-        session: Session | None = None,
-        sqlalchemy_db: SQLAlchemyDb | None = None,
-        filter_operation_ids: list[int] | int | None = None,
-        filter_operation_types: list[str] | str | None = None,
-        filter_operation_statuses: list[str] | str | None = None
-):
-    if isinstance(filter_operation_ids, int):
-        filter_operation_ids = [filter_operation_ids]
-    if isinstance(filter_operation_types, str):
-        filter_operation_types = [filter_operation_types]
-    if isinstance(filter_operation_statuses, str):
-        filter_operation_statuses = [filter_operation_statuses]
-
-    if filter_operation_ids is not None:
-        raise_for_type(filter_operation_ids, list)
-    if filter_operation_types is not None:
-        raise_for_type(filter_operation_types, list)
-    if filter_operation_statuses is not None:
-        raise_for_type(filter_operation_statuses, list)
-
-    def func(session_: Session):
-        query = session_.query(OperationDBM)
-        if filter_operation_ids is not None:
-            query = query.filter(OperationDBM.id.in_(filter_operation_ids))
-        if filter_operation_types is not None:
-            query = query.filter(OperationDBM.type.in_(filter_operation_types))
-        if filter_operation_statuses is not None:
-            query = query.filter(OperationDBM.status.in_(filter_operation_statuses))
-        query.delete()
-        session_.commit()
-
-    if session is not None:
-        return func(session_=session)
-    elif sqlalchemy_db is not None:
-        with sqlalchemy_db.new_session() as session:
-            return func(session_=session)
-    else:
-        raise ValueError("session is None and sqlalchemy_db is None")
-
-
-class BaseOperationExecutor:
-    def __init__(self, *, sqlalchemy_db: SQLAlchemyDb, **kwargs):
-        self._logger = logging.getLogger(self.__class__.__name__)
-        self.sql_alchemy_db = sqlalchemy_db
-
-    def sync_execute_operation(self, operation_dbm: OperationDBM, session: Session) -> OperationDBM:
-        if operation_dbm.type == BaseOperationTypes.healthcheck_:
-            self._logger.info("healthcheck")
-        elif operation_dbm.type == BaseOperationTypes.raise_fake_exception_:
-            self._logger.info("raise_fake_exception")
-            raise Exception("raise_fake_exception")
-        return operation_dbm
-
-    def sync_safe_execute_operation(
-            self, operation_dbm: OperationDBM, worker: OperationExecutorWorker, session: Session
-    ) -> OperationDBM:
-        self._logger.info(
-            f"start "
-            f"operation_dbm.id={operation_dbm.id}, "
-            f"operation_dbm.type={operation_dbm.type}, "
-            f"operation_dbm.status={operation_dbm.status}"
-        )
-
-        operation_dbm.execution_start_dt = now_utc_dt()
-        operation_dbm.status = OperationDBM.Statuses.executing
-        operation_dbm.output_data = combine_dicts(
-            operation_dbm.output_data,
-            {
-                worker.worker_fullname: True
-            }
-        )
-        session.commit()
-
-        exception: Exception | None = None
-        traceback_str: str | None = None
-
-        try:
-            self.sync_execute_operation(operation_dbm=operation_dbm, session=session)
-        except Exception as exception_:
-            self._logger.error(
-                f"error in sync_execute_operation (id={operation_dbm.id}, type={operation_dbm.type})",
-                exc_info=exception_
-            )
-            exception = exception_
-            traceback_str = traceback.format_exc()
-
-        operation_dbm.execution_finish_dt = now_utc_dt()
-        if exception:
-            operation_dbm.status = OperationDBM.Statuses.executed_with_error
-            operation_dbm.error_data = combine_dicts(
-                {
-                    "exception_str": str(exception),
-                    "traceback_str": traceback_str
-                },
-                operation_dbm.error_data
-            )
-        else:
-            operation_dbm.status = OperationDBM.Statuses.executed_without_error
-        session.commit()
-
-        if exception:
-            story_log_dbm = StoryLogDBM(
-                level=StoryLogDBM.Levels.error,
-                title=f"error in sync_execute_operation (id={operation_dbm.id}, type={operation_dbm.type})",
-                data={
-                    "operation_id": operation_dbm.id,
-                    "exception_str": str(exception),
-                    "traceback_str": traceback_str
-                }
-            )
-            session.add(story_log_dbm)
-            session.commit()
-
-        session.refresh(operation_dbm)
-
-        self._logger.info(
-            f"finish sync_safe_execute_operation, "
-            f"operation_dbm.id={operation_dbm.id}, "
-            f"operation_dbm.type={operation_dbm.type}, "
-            f"operation_dbm.status={operation_dbm.status}, "
-            f"operation_dbm.duration={operation_dbm.duration}"
-        )
-
-        return operation_dbm
-
-    async def async_execute_operation(self, operation_dbm: OperationDBM, session: Session) -> OperationDBM:
-        if operation_dbm.type == BaseOperationTypes.healthcheck_:
-            self._logger.info("healthcheck")
-        elif operation_dbm.type == BaseOperationTypes.raise_fake_exception_:
-            self._logger.info("raise_fake_exception")
-            raise Exception("raise_fake_exception")
-        return operation_dbm
-
-    async def async_safe_execute_operation(
-            self, operation_dbm: OperationDBM, worker: OperationExecutorWorker, session: Session
-    ) -> OperationDBM:
-        self._logger.info(
-            f"start "
-            f"operation_dbm.id={operation_dbm.id}, "
-            f"operation_dbm.type={operation_dbm.type}, "
-            f"operation_dbm.status={operation_dbm.status}"
-        )
-
-        operation_dbm.execution_start_dt = now_utc_dt()
-        operation_dbm.status = OperationDBM.Statuses.executing
-        operation_dbm.output_data = combine_dicts(
-            operation_dbm.output_data,
-            {
-                worker.worker_fullname: True
-            }
-        )
-        session.commit()
-
-        exception: Exception | None = None
-        traceback_str: str | None = None
-
-        try:
-            await self.async_execute_operation(operation_dbm=operation_dbm, session=session)
-        except Exception as exception_:
-            self._logger.error(
-                f"error in async_execute_operation (id={operation_dbm.id}, type={operation_dbm.type})",
-                exc_info=exception_
-            )
-            exception = exception_
-            traceback_str = traceback.format_exc()
-
-        operation_dbm.execution_finish_dt = now_utc_dt()
-        if exception:
-            operation_dbm.status = OperationDBM.Statuses.executed_with_error
-            operation_dbm.error_data = combine_dicts(
-                {
-                    "exception_str": str(exception),
-                    "traceback_str": traceback_str
-                },
-                operation_dbm.error_data
-            )
-        else:
-            operation_dbm.status = OperationDBM.Statuses.executed_without_error
-        session.commit()
-
-        if exception:
-            story_log_dbm = StoryLogDBM(
-                level=StoryLogDBM.Levels.error,
-                title=f"error in async_execute_operation (id={operation_dbm.id}, type={operation_dbm.type})",
-                data={
-                    "operation_id": operation_dbm.id,
-                    "exception_str": str(exception),
-                    "traceback_str": traceback_str
-                }
-            )
-            session.add(story_log_dbm)
-            session.commit()
-
-        session.refresh(operation_dbm)
-
-        self._logger.info(
-            f"finish async_safe_execute_operation, "
-            f"operation_dbm.id={operation_dbm.id}, "
-            f"operation_dbm.type={operation_dbm.type}, "
-            f"operation_dbm.status={operation_dbm.status}, "
-            f"operation_dbm.duration={operation_dbm.duration}"
-        )
-
-        return operation_dbm
-
-
-class OperationExecutorWorker(BaseWorker):
-
-    def __init__(
-            self,
-            *,
-            sqlalchemy_db: SQLAlchemyDb,
-            operation_executor: BaseOperationExecutor | None = None,
-            filter_operation_types: str | list[str] | None = None,
-            startup_funcs: list[Any] | None = None,
-            **kwargs
-    ):
-        super().__init__(
-            timeout_after_run=timedelta(seconds=0.3),
-            timeout_after_err_in_run=timedelta(seconds=0.3),
-            startup_funcs=startup_funcs,
-        )
-        raise_for_type(sqlalchemy_db, SQLAlchemyDb)
-        self.sqlalchemy_db = sqlalchemy_db
-        if operation_executor is None:
-            operation_executor = BaseOperationExecutor(sqlalchemy_db=sqlalchemy_db)
-        self.operation_executor = operation_executor
-        if isinstance(filter_operation_types, str):
-            filter_operation_types = [filter_operation_types]
-        self.filter_operation_types = filter_operation_types
-
-    def sync_on_startup(self):
-        self.sqlalchemy_db.init()
-        self.sync_run_startup_funcs()
-
-    def sync_execute_operation(self, operation_dbm: OperationDBM, session: Session) -> OperationDBM:
-        return self.operation_executor.sync_safe_execute_operation(
-            operation_dbm=operation_dbm, worker=self, session=session
-        )
-
-    def sync_run(self):
-        with self.sqlalchemy_db.new_session() as session:
-            operation_dbm: OperationDBM | None = get_operation_for_execution(
-                session=session,
-                filter_operation_types=self.filter_operation_types,
-                lock=True
-            )
-            if not operation_dbm:
-                return
-            self.sync_execute_operation(operation_dbm=operation_dbm, session=session)
-
-    async def async_on_startup(self):
-        self.sqlalchemy_db.init()
-        await self.async_run_startup_funcs()
-
-    async def async_execute_operation(self, operation_dbm: OperationDBM, session: Session) -> OperationDBM:
-        return await self.operation_executor.async_safe_execute_operation(
-            operation_dbm=operation_dbm, worker=self, session=session
-        )
-
-    async def async_run(self):
-        with self.sqlalchemy_db.new_session() as session:
-            operation_dbm: OperationDBM | None = get_operation_for_execution(
-                sqlalchemy_db=self.sqlalchemy_db,
-                filter_operation_types=self.filter_operation_types,
-                lock=True
-            )
-            if not operation_dbm:
-                return
-            await self.async_execute_operation(operation_dbm=operation_dbm, session=session)
-
-
-class ScheduledOperation(BaseModel):
-    model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True, from_attributes=True)
-
-    type: str
-    input_data: dict[str, Any] | None = None
-    is_time_func: Callable[[], bool]
-    timeout_after_creation: timedelta | None = None
-
-
-class ScheduledOperationCreatorWorker(BaseWorker):
-    def __init__(
-            self,
-            *,
-            sqlalchemy_db: SQLAlchemyDb,
-            scheduled_operations: ScheduledOperation | list[ScheduledOperation] | None = None,
-            startup_funcs: list[Any] | None = None
-    ):
-        super().__init__(
-            timeout_after_run=timedelta(seconds=0.3),
-            timeout_after_err_in_run=timedelta(seconds=0.3),
-            startup_funcs=startup_funcs
-        )
-        raise_for_type(sqlalchemy_db, SQLAlchemyDb)
-        self.sqlalchemy_db = sqlalchemy_db
-        if scheduled_operations is None:
-            scheduled_operations = []
-        if isinstance(scheduled_operations, ScheduledOperation):
-            scheduled_operations = [scheduled_operations]
-        self.scheduled_operations = scheduled_operations
-
-    def sync_on_startup(self):
-        self.sqlalchemy_db.init()
-        self.sync_run_startup_funcs()
-
-    def sync_run(self):
-        timeout = None
-
-        for scheduled_operation in self.scheduled_operations:
-
-            if not scheduled_operation.is_time_func():
-                continue
-
-            with self.sqlalchemy_db.new_session() as session:
-                operation_dbm = OperationDBM(
-                    type=scheduled_operation.type,
-                    input_data=scheduled_operation.input_data
-                )
-                session.add(operation_dbm)
-                session.commit()
-                session.refresh(operation_dbm)
-                self._logger.info(f"scheduled operation (id={operation_dbm.id}) was created")
-
-            if scheduled_operation.timeout_after_creation is not None:
-                if timeout is not None:
-                    if scheduled_operation.timeout_after_creation > timeout:
-                        timeout = scheduled_operation.timeout_after_creation
-                else:
-                    timeout = scheduled_operation.timeout_after_creation
-
-        if timeout is not None:
-            sync_safe_sleep(n=timeout)
-
-    async def async_on_startup(self):
-        self.sqlalchemy_db.init()
-        await self.async_run_startup_funcs()
-
-    async def async_run(self):
-        timeout: timedelta | None = None
-
-        for scheduled_operation in self.scheduled_operations:
-
-            if not scheduled_operation.is_time_func():
-                continue
-
-            with self.sqlalchemy_db.new_session() as session:
-                operation_dbm = OperationDBM(
-                    type=scheduled_operation.type,
-                    input_data=scheduled_operation.input_data
-                )
-                session.add(operation_dbm)
-                session.commit()
-                session.refresh(operation_dbm)
-
-            if scheduled_operation.timeout_after_creation is not None:
-                if timeout is not None:
-                    if scheduled_operation.timeout_after_creation > timeout:
-                        timeout = scheduled_operation.timeout_after_creation
-                else:
-                    timeout = scheduled_operation.timeout_after_creation
-
-        if timeout is not None:
-            await async_safe_sleep(n=timeout)
-
-
-def every_timedelta_is_time_func(*, td: timedelta, now_dt_func: Callable = now_utc_dt) -> Callable:
-    last_now_utc_dt = now_utc_dt()
-
-    def func() -> bool:
-        nonlocal last_now_utc_dt
-        now_dt_func_ = now_dt_func()
-        if (now_dt_func_ - last_now_utc_dt) >= td:
-            last_now_utc_dt = now_dt_func_
-            return True
-        return False
-
-    return func
-
-
-def between_different_times_is_time_func(
-        *, from_time: time, to_time: time, now_dt_func: Callable = now_utc_dt
-) -> Callable:
-    def func() -> bool:
-        if from_time <= now_dt_func().time() <= to_time:
-            return True
-        return False
-
-    return func
-
-
-def __example():
-    pass
-
-
-async def __async_example():
-    pass
-
-
-if __name__ == '__main__':
-    __example()
-    asyncio.run(__async_example())

arpakitlib/ar_sqlalchemy_model_util.py

@@ -1,183 +0,0 @@
-# arpakit
-
-from datetime import datetime, timedelta
-from typing import Any
-from uuid import uuid4
-
-from sqlalchemy import inspect, INTEGER, TEXT, TIMESTAMP, func
-from sqlalchemy.dialects.postgresql import JSON
-from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
-
-from arpakitlib.ar_datetime_util import now_utc_dt
-from arpakitlib.ar_enumeration_util import Enumeration
-from arpakitlib.ar_json_util import safely_transfer_obj_to_json_str
-
-_ARPAKIT_LIB_MODULE_VERSION = "3.0"
-
-
-def generate_default_long_id():
-    return (
-        f"longid"
-        f"{str(uuid4()).replace('-', '')}"
-        f"{str(now_utc_dt().timestamp()).replace('.', '')}"
-    )
-
-
-class BaseDBM(DeclarativeBase):
-    __abstract__ = True
-    _bus_data: dict[str, Any] | None = None
-
-    @property
-    def bus_data(self) -> dict[str, Any]:
-        if self._bus_data is None:
-            self._bus_data = {}
-        return self._bus_data
-
-    def simple_dict(self, *, include_sd_properties: bool = True) -> dict[str, Any]:
-        res = {}
-
-        for c in inspect(self).mapper.column_attrs:
-            value = getattr(self, c.key)
-            if isinstance(value, BaseDBM):
-                res[c.key] = value.simple_dict(include_sd_properties=include_sd_properties)
-            elif isinstance(value, list):
-                res[c.key] = [
-                    item.simple_dict(include_sd_properties=include_sd_properties)
-                    if isinstance(item, BaseDBM) else item
-                    for item in value
-                ]
-            else:
-                res[c.key] = value
-
-        if include_sd_properties:
-            for attr_name in dir(self):
-                if attr_name.startswith("sdp_") and isinstance(getattr(type(self), attr_name, None), property):
-                    prop_name = attr_name.removeprefix("sdp_")
-                    value = getattr(self, attr_name)
-                    if isinstance(value, BaseDBM):
-                        res[prop_name] = value.simple_dict(include_sd_properties=include_sd_properties)
-                    elif isinstance(value, list):
-                        res[prop_name] = [
-                            item.simple_dict(include_sd_properties=include_sd_properties)
-                            if isinstance(item, BaseDBM) else item
-                            for item in value
-                        ]
-                    else:
-                        res[prop_name] = value
-
-        return res
-
-    def simple_dict_with_sd_properties(self) -> dict[str, Any]:
-        return self.simple_dict(include_sd_properties=True)
-
-    def simple_json(self, *, include_sd_properties: bool = True) -> str:
-        return safely_transfer_obj_to_json_str(self.simple_dict(include_sd_properties=include_sd_properties))
-
-
-class SimpleDBM(BaseDBM):
-    __abstract__ = True
-
-    id: Mapped[int] = mapped_column(
-        INTEGER, primary_key=True, autoincrement=True, sort_order=-3, nullable=False
-    )
-    long_id: Mapped[str] = mapped_column(
-        TEXT, insert_default=generate_default_long_id, server_default=func.gen_random_uuid(),
-        unique=True, sort_order=-2, nullable=False
-    )
-    creation_dt: Mapped[datetime] = mapped_column(
-        TIMESTAMP(timezone=True), insert_default=now_utc_dt, server_default=func.now(),
-        index=True, sort_order=-1, nullable=False
-    )
-
-    def __repr__(self):
-        return f"{self.__class__.__name__.removesuffix('DBM')} (id={self.id})"
-
-
-class StoryLogDBM(SimpleDBM):
-    __tablename__ = "story_log"
-
-    class Levels(Enumeration):
-        info = "info"
-        warning = "warning"
-        error = "error"
-
-    level: Mapped[str] = mapped_column(
-        TEXT, insert_default=Levels.info, server_default=Levels.info, index=True, nullable=False
-    )
-    title: Mapped[str | None] = mapped_column(TEXT, index=True, default=None, nullable=True)
-    data: Mapped[dict[str, Any]] = mapped_column(
-        JSON, insert_default={}, server_default="{}", nullable=False
-    )
-
-
-class BaseOperationTypes(Enumeration):
-    healthcheck_ = "healthcheck"
-    raise_fake_exception_ = "raise_fake_exception"
-
-
-class OperationDBM(SimpleDBM):
-    __tablename__ = "operation"
-
-    class Statuses(Enumeration):
-        waiting_for_execution = "waiting_for_execution"
-        executing = "executing"
-        executed_without_error = "executed_without_error"
-        executed_with_error = "executed_with_error"
-
-    status: Mapped[str] = mapped_column(
-        TEXT, index=True, insert_default=Statuses.waiting_for_execution,
-        server_default=Statuses.waiting_for_execution, nullable=False
-    )
-    type: Mapped[str] = mapped_column(
-        TEXT, index=True, insert_default=BaseOperationTypes.healthcheck_, nullable=False
-    )
-    execution_start_dt: Mapped[datetime | None] = mapped_column(TIMESTAMP(timezone=True), nullable=True)
-    execution_finish_dt: Mapped[datetime | None] = mapped_column(TIMESTAMP(timezone=True), nullable=True)
-    input_data: Mapped[dict[str, Any]] = mapped_column(
-        JSON,
-        insert_default={},
-        server_default="{}",
-        nullable=False
-    )
-    output_data: Mapped[dict[str, Any]] = mapped_column(JSON, insert_default={}, server_default="{}", nullable=False)
-    error_data: Mapped[dict[str, Any]] = mapped_column(JSON, insert_default={}, server_default="{}", nullable=False)
-
-    def raise_if_executed_with_error(self):
-        if self.status == self.Statuses.executed_with_error:
-            raise Exception(
-                f"Operation (id={self.id}, type={self.type}) executed with error, error_data={self.error_data}"
-            )
-
-    def raise_if_error_data(self):
-        if self.error_data:
-            raise Exception(
-                f"Operation (id={self.id}, type={self.type}) has error_data, error_data={self.error_data}"
-            )
-
-    @property
-    def duration(self) -> timedelta | None:
-        if self.execution_start_dt is None or self.execution_finish_dt is None:
-            return None
-        return self.execution_finish_dt - self.execution_start_dt
-
-    @property
-    def duration_total_seconds(self) -> float | None:
-        if self.duration is None:
-            return None
-        return self.duration.total_seconds()
-
-    @property
-    def sdp_duration_total_seconds(self) -> float | None:
-        return self.duration_total_seconds
-
-
-def import_ar_sqlalchemy_models():
-    pass
-
-
-def __example():
-    pass
-
-
-if __name__ == '__main__':
-    __example()