fennflow 0.1.0__py3-none-any.whl

fennflow/__init__.py

@@ -0,0 +1,7 @@
+__all__ = [
+    "ConfigDict",
+    "UnitOfWork",
+]
+
+from .core import ConfigDict
+from .uow import UnitOfWork

fennflow/_base_exception.py

@@ -0,0 +1,2 @@
+class FennFlowException(Exception):
+    """Base exception for all fennflow errors."""

fennflow/_datetime.py

@@ -0,0 +1,10 @@
+import datetime
+
+from pydantic import AwareDatetime as PydanticAwareDatetime
+
+AwareDatetime = PydanticAwareDatetime
+
+
+def now() -> AwareDatetime:
+    """Returns a timezone-aware datetime object for current time."""
+    return datetime.datetime.now(tz=datetime.timezone.utc)

fennflow/_decorators.py

@@ -0,0 +1,33 @@
+import asyncio
+import functools
+from collections.abc import Callable
+from typing import Any, TypeAlias
+
+CatchType: TypeAlias = type[Exception] | tuple[type[Exception], ...]
+
+
+def reraise_with(
+    exception: Exception,
+    catch: CatchType = Exception,
+) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
+    def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
+
+        @functools.wraps(func)
+        async def async_wrapper(*args: Any, **kwargs: Any) -> Any:
+            try:
+                return await func(*args, **kwargs)
+            except catch as e:
+                raise exception from e
+
+        @functools.wraps(func)
+        def sync_wrapper(*args: Any, **kwargs: Any) -> Any:
+            try:
+                return func(*args, **kwargs)
+            except catch as e:
+                raise exception from e
+
+        if asyncio.iscoroutinefunction(func):
+            return async_wrapper
+        return sync_wrapper
+
+    return decorator

fennflow/_new_types.py

@@ -0,0 +1,7 @@
+from typing_extensions import TypeAliasType
+
+UowQualName = TypeAliasType("UowQualName", str)
+StoragePath = TypeAliasType("StoragePath", str)
+Namespace = TypeAliasType("Namespace", str)
+BackendScope = TypeAliasType("BackendScope", str)
+BucketName = Namespace

fennflow/_operations/context/abstract.py

@@ -0,0 +1,6 @@
+from dataclasses import dataclass
+
+
+@dataclass(slots=True)
+class BaseContext:
+    pass

fennflow/_operations/context/delete.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from fennflow._operations.context.abstract import BaseContext
+
+if TYPE_CHECKING:
+    from fennflow._new_types import Namespace, StoragePath
+
+
+@dataclass(slots=True)
+class DeleteContext(BaseContext):
+    to_storage_path: StoragePath
+    to_namespace: Namespace

fennflow/_operations/context/put.py

@@ -0,0 +1,14 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from fennflow._operations.context.abstract import BaseContext
+
+if TYPE_CHECKING:
+    from fennflow.files.types import BinaryMedia
+
+
+@dataclass(slots=True)
+class PutContext(BaseContext):
+    file: BinaryMedia

fennflow/_operations/context/types.py

@@ -0,0 +1,7 @@
+from typing import TypeAlias
+
+from fennflow._operations.context.abstract import BaseContext
+from fennflow._operations.context.delete import DeleteContext
+from fennflow._operations.context.put import PutContext
+
+Context: TypeAlias = PutContext | DeleteContext | BaseContext

fennflow/_operations/dto.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+import datetime
+import uuid
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+from uuid import UUID
+
+from fennflow._datetime import now
+from fennflow._operations.context.abstract import BaseContext
+from fennflow._operations.enums import OperationStatusEnum, OperationTypeEnum
+from fennflow._sentinel import NOT_GIVEN
+
+if TYPE_CHECKING:
+    from fennflow._new_types import Namespace, StoragePath
+    from fennflow._operations.context.types import Context
+    from fennflow._sentinel import NotGiven
+    from fennflow.repositories.fields.base import RepoExtra
+
+
+@dataclass(slots=True)
+class OperationRecord:
+    session_id: uuid.UUID
+    storage_path: StoragePath
+    repo_extra: RepoExtra
+    operation_type: OperationTypeEnum
+    status: OperationStatusEnum
+    context: Context = field(default_factory=BaseContext)
+    operation_id: uuid.UUID = field(default_factory=uuid.uuid4)
+
+    created_at: datetime.datetime = field(
+        default_factory=now,
+    )
+    expired_at: datetime.datetime = field(
+        default_factory=lambda: now() + datetime.timedelta(seconds=30)
+    )
+    error: str | None | NotGiven = NOT_GIVEN
+
+    @property
+    def is_pending(self) -> bool:
+        return self.status == OperationStatusEnum.PENDING
+
+    @property
+    def is_uploaded(self) -> bool:
+        return self.status == OperationStatusEnum.UPLOADED
+
+    @property
+    def is_failed(self) -> bool:
+        return self.status == OperationStatusEnum.FAILED
+
+    @property
+    def is_deleted(self) -> bool:
+        return self.status == OperationStatusEnum.DELETED
+
+    @property
+    def namespace(self) -> Namespace:
+        return self.repo_extra["namespace"]
+
+    @property
+    def is_expired(self):
+        return self.expired_at < now()
+
+    @property
+    def is_dangling(self) -> bool:
+        return self.is_expired and not self.is_uploaded
+
+    def is_locked(self, current_session_id: UUID) -> bool:
+        return (
+            self.session_id != current_session_id
+            and self.is_pending
+            and not self.is_expired
+        )
+
+    def is_visible(self, requested_from_session_id: UUID) -> bool:
+        return self.is_uploaded or (
+            self.is_pending
+            and self.session_id == requested_from_session_id
+            and self.operation_type == OperationTypeEnum.PUT
+        )

fennflow/_operations/enums.py

@@ -0,0 +1,19 @@
+from enum import IntEnum, auto
+
+
+class OperationTypeEnum(IntEnum):
+    """Enum for operation types."""
+
+    PUT = auto()
+    GET = auto()
+    DELETE = auto()
+
+
+class OperationStatusEnum(IntEnum):
+    """Enum for operation status."""
+
+    PENDING = auto()
+    UPLOADED = auto()
+    FAILED = auto()
+    COMPENSATION_FAILED = auto()
+    DELETED = auto()

fennflow/_operations/executor.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from .registry import flow_registry
+
+if TYPE_CHECKING:
+    from fennflow.connectors.abstract import AbstractConnector
+
+    from .dto import OperationRecord
+
+
+class OperationExecutor:
+    def __init__(self, connector: AbstractConnector) -> None:
+        self.connector = connector
+
+    async def execute(
+        self,
+        operation: OperationRecord,
+        **provider_extra,
+    ):
+        operation_flow = flow_registry[operation.operation_type]
+        return await operation_flow().execute(
+            operation=operation,
+            connector=self.connector,
+            **provider_extra,
+        )
+
+    async def compensate(
+        self,
+        operation: OperationRecord,
+    ):
+        operation_flow = flow_registry[operation.operation_type]
+        return await operation_flow().compensate(
+            operation=operation,
+            connector=self.connector,
+        )
+
+    async def finalize(
+        self,
+        operation: OperationRecord,
+    ):
+        operation_flow = flow_registry[operation.operation_type]
+        return await operation_flow().finalize(
+            operation=operation,
+            connector=self.connector,
+        )

fennflow/_operations/flows/abstract.py

@@ -0,0 +1,33 @@
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from fennflow._operations.dto import OperationRecord
+    from fennflow.connectors.abstract import AbstractConnector
+
+
+class AbstractFlow(ABC):
+    @staticmethod
+    @abstractmethod
+    async def execute(
+        *,
+        operation: "OperationRecord",
+        connector: "AbstractConnector",
+        **provider_extra,
+    ): ...
+
+    @staticmethod
+    @abstractmethod
+    async def compensate(
+        *,
+        operation: "OperationRecord",
+        connector: "AbstractConnector",
+    ): ...
+
+    @staticmethod
+    @abstractmethod
+    async def finalize(
+        *,
+        operation: "OperationRecord",
+        connector: "AbstractConnector",
+    ): ...

fennflow/_operations/flows/delete.py

@@ -0,0 +1,69 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from fennflow._operations.enums import OperationStatusEnum
+from fennflow._operations.flows.abstract import AbstractFlow
+
+if TYPE_CHECKING:
+    from fennflow._operations.context.delete import DeleteContext
+    from fennflow._operations.dto import OperationRecord
+    from fennflow.connectors.abstract import AbstractConnector
+
+
+class DeleteFlow(AbstractFlow):
+    @staticmethod
+    async def execute(
+        *,
+        operation: OperationRecord,
+        connector: AbstractConnector,
+        **provider_extra,
+    ):
+        ctx: DeleteContext = operation.context
+        await connector.copy_object(
+            from_storage_path=operation.storage_path,
+            to_storage_path=ctx.to_storage_path,
+            to_namespace=ctx.to_namespace,
+            repo_extra=operation.repo_extra,
+            **provider_extra,
+        )
+        return await connector.delete(
+            storage_path=operation.storage_path,
+            repo_extra=operation.repo_extra,
+            **provider_extra,
+        )
+
+    @staticmethod
+    async def compensate(
+        *,
+        operation: OperationRecord,
+        connector: AbstractConnector,
+        **provider_extra,
+    ):
+        ctx: DeleteContext = operation.context
+        await connector.copy_object(
+            from_storage_path=ctx.to_storage_path,
+            to_storage_path=operation.storage_path,
+            to_namespace=operation.namespace,
+            repo_extra=operation.repo_extra,
+            **provider_extra,
+        )
+        await connector.delete(
+            storage_path=ctx.to_storage_path,
+            repo_extra=operation.repo_extra,
+        )
+        operation.status = OperationStatusEnum.UPLOADED
+
+    @staticmethod
+    async def finalize(
+        *,
+        operation: OperationRecord,
+        connector: AbstractConnector,
+        **provider_extra,
+    ):
+        ctx: DeleteContext = operation.context
+        await connector.delete(
+            storage_path=ctx.to_storage_path,
+            repo_extra=operation.repo_extra,
+            **provider_extra,
+        )

fennflow/_operations/flows/put.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from fennflow._operations.enums import OperationStatusEnum
+from fennflow._operations.flows.abstract import AbstractFlow
+
+if TYPE_CHECKING:
+    from fennflow._operations.context.put import PutContext
+    from fennflow._operations.dto import OperationRecord
+    from fennflow.connectors.abstract import AbstractConnector
+
+
+class PutFlow(AbstractFlow):
+    @staticmethod
+    async def execute(
+        *,
+        operation: OperationRecord,
+        connector: AbstractConnector,
+        **provider_extra,
+    ):
+        ctx: PutContext = operation.context
+        return await connector.put(
+            file=ctx.file,
+            repo_extra=operation.repo_extra,
+            **provider_extra,
+        )
+
+    @staticmethod
+    async def compensate(
+        *,
+        operation: OperationRecord,
+        connector: AbstractConnector,
+        **provider_extra,
+    ):
+
+        result = await connector.delete(
+            storage_path=operation.storage_path,
+            repo_extra=operation.repo_extra,
+            **provider_extra,
+        )
+        operation.status = OperationStatusEnum.FAILED
+        return result
+
+    @staticmethod
+    async def finalize(
+        *,
+        operation: OperationRecord,
+        connector: AbstractConnector,
+        **provider_extra,
+    ):
+        pass

fennflow/_operations/registry.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from .enums import OperationTypeEnum
+from .flows.delete import DeleteFlow
+from .flows.put import PutFlow
+
+if TYPE_CHECKING:
+    from .flows.abstract import AbstractFlow
+
+flow_registry: dict[OperationTypeEnum | str, type[AbstractFlow]] = {
+    OperationTypeEnum.PUT: PutFlow,
+    OperationTypeEnum.DELETE: DeleteFlow,
+}

fennflow/_path.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+import re
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from fennflow._new_types import StoragePath
+
+
+class Path:
+    @staticmethod
+    def normalize_path(path: str) -> StoragePath:
+        path = re.sub(r"/+", "/", path)
+        return path.strip("/")
+
+    @classmethod
+    def join_path(cls, *paths: str) -> StoragePath:
+        path = "/".join(path for path in paths)
+
+        return cls.normalize_path(path)
+
+    @classmethod
+    def normalize_folder(cls, path: str) -> str:
+        """Used for display of folder path.
+
+        Adds trailing slash to distinguish folders from file keys.
+        _path itself has no trailing slash as it's used directly in join operations.
+
+        """
+        normalized = cls.normalize_path(path)
+        return normalized + "/" if normalized else ""

fennflow/_reconciler/__init__.py

@@ -0,0 +1,10 @@
+__all__ = [
+    "ReconcileConfig",
+    "ReconcileFrequencyEnum",
+    "ReconcileStrategyEnum",
+    "Reconciler",
+]
+
+from .config import ReconcileConfig
+from .core import Reconciler
+from .enums import ReconcileFrequencyEnum, ReconcileStrategyEnum

fennflow/_reconciler/_orchestrator.py

@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+from typing import TYPE_CHECKING, Any, ClassVar
+
+from fennflow._reconciler.core import Reconciler
+from fennflow._reconciler.enums import ReconcileFrequencyEnum
+from fennflow.uow.inspector import UowInspector
+
+if TYPE_CHECKING:
+    from fennflow import UnitOfWork
+    from fennflow._new_types import UowQualName
+
+logger = logging.getLogger(__name__)
+
+
+class ReconcileOrchestrator:
+    _reconciled_on_startup: ClassVar[set[UowQualName]] = set()
+    _lock: asyncio.Lock = asyncio.Lock()
+
+    async def reconcile_if_needed(self, uow: UnitOfWork) -> None:
+        frequency = uow._resolved_config.reconcile.frequency
+
+        async with self._lock:
+            match frequency:
+                case ReconcileFrequencyEnum.NEVER:
+                    await self._handle_never(uow=uow)
+                case ReconcileFrequencyEnum.ON_START_APP:
+                    await self._handle_on_start_app(uow=uow)
+                case ReconcileFrequencyEnum.ON_SESSION_START:
+                    await self._handle_on_session_start(uow=uow)
+                case _:
+                    raise AssertionError("Unhandled frequency")
+
+    async def _handle_never(self, uow: UnitOfWork) -> None:
+        logger.debug(
+            "Skipping reconciliation.",
+            extra=self._log_extra(uow),
+        )
+
+    async def _handle_on_start_app(self, uow: UnitOfWork) -> None:
+        if uow.__class__.__qualname__ in self._reconciled_on_startup:
+            logger.debug(
+                "Skipping reconciliation.",
+                extra=self._log_extra(uow=uow),
+            )
+            return
+
+        logger.debug(
+            "Starting reconciliation...",
+            extra=self._log_extra(uow=uow),
+        )
+        await self._reconcile(uow=uow)
+        self._reconciled_on_startup.add(uow.__class__.__qualname__)
+
+    async def _handle_on_session_start(self, uow: UnitOfWork) -> None:
+        logger.debug("Starting reconciliation...", extra=self._log_extra(uow=uow))
+        await self._reconcile(uow=uow)
+
+    async def _reconcile(self, uow: UnitOfWork) -> None:
+        extractor = UowInspector(uow=uow)
+        reconcile = Reconciler(
+            uow_fields=extractor.get_repo_fields(),
+            connector=uow.connector,
+            backend=uow.backend,
+        )
+
+        await reconcile.reconcile(
+            session_id=uow._session_id,
+            batch_size=uow._resolved_config.reconcile.batch_size,
+            strategy=uow._resolved_config.reconcile.strategy,
+        )
+
+        logger.debug(
+            "Finished reconciliation.",
+            extra=self._log_extra(uow=uow),
+        )
+
+    @staticmethod
+    def _log_extra(uow: UnitOfWork) -> dict[str, Any]:
+        return {
+            "session_id": uow._session_id,
+            "uow_class_name": uow.__class__.__qualname__,
+            "frequency": uow._resolved_config.reconcile.frequency,
+        }

fennflow/_reconciler/config.py

@@ -0,0 +1,10 @@
+from dataclasses import dataclass
+
+from fennflow._reconciler.enums import ReconcileFrequencyEnum, ReconcileStrategyEnum
+
+
+@dataclass(slots=True, frozen=True)
+class ReconcileConfig:
+    frequency: ReconcileFrequencyEnum = ReconcileFrequencyEnum.ON_START_APP
+    strategy: ReconcileStrategyEnum = ReconcileStrategyEnum.FILL_IF_EMPTY
+    batch_size: int = 1000