refresh-engine 0.1.0__py3-none-any.whl

refresh_engine/events/publisher.py

@@ -0,0 +1,77 @@
+"""Ordered, subscriber-isolated lifecycle event publication."""
+
+from __future__ import annotations
+
+import asyncio
+import inspect
+import threading
+
+from refresh_engine.api.abc import EventPublisherABC
+from refresh_engine.api.protocols import EventHandler
+from refresh_engine.events.models import RefreshEvent
+
+
+class EventPublisher(EventPublisherABC):
+    """Queue ordered delivery without placing subscribers on the refresh path."""
+
+    def __init__(self) -> None:
+        self._handlers: list[EventHandler] = []
+        self._lock = threading.Lock()
+        self._errors: list[Exception] = []
+        self._tail: asyncio.Task[None] | None = None
+        self._tasks: set[asyncio.Task[None]] = set()
+
+    @property
+    def errors(self) -> tuple[Exception, ...]:
+        with self._lock:
+            return tuple(self._errors)
+
+    def subscribe(self, handler: EventHandler) -> None:
+        with self._lock:
+            if handler not in self._handlers:
+                self._handlers.append(handler)
+
+    def unsubscribe(self, handler: EventHandler) -> None:
+        with self._lock:
+            if handler in self._handlers:
+                self._handlers.remove(handler)
+
+    async def publish(self, event: RefreshEvent) -> None:
+        previous = self._tail
+        task = asyncio.create_task(
+            self._dispatch_after(previous, event),
+            name=f"refresh-event:{event.kind.value}",
+        )
+        self._tail = task
+        self._tasks.add(task)
+        task.add_done_callback(self._tasks.discard)
+        await asyncio.sleep(0)
+
+    async def drain(self) -> None:
+        """Wait until all events published so far have been delivered."""
+        tail = self._tail
+        if tail is not None:
+            await asyncio.shield(tail)
+
+    async def _dispatch_after(
+        self,
+        previous: asyncio.Task[None] | None,
+        event: RefreshEvent,
+    ) -> None:
+        if previous is not None:
+            await previous
+        with self._lock:
+            handlers = tuple(self._handlers)
+        for handler in handlers:
+            try:
+                if inspect.iscoroutinefunction(handler):
+                    await handler(event)
+                else:
+                    result = await asyncio.to_thread(handler, event)
+                    if inspect.isawaitable(result):
+                        await result
+            except asyncio.CancelledError:
+                raise
+            except Exception as error:
+                with self._lock:
+                    self._errors.append(error)

refresh_engine/execution/__init__.py

@@ -0,0 +1,5 @@
+"""Refresh execution."""
+
+from refresh_engine.execution.executor import RefreshExecutor
+
+__all__ = ["RefreshExecutor"]

refresh_engine/execution/coordinator.py

@@ -0,0 +1,128 @@
+"""Refresh request overlap coordination."""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass, field
+
+from refresh_engine.core.models import (
+    OverlapPolicy,
+    RefreshRequest,
+    RefreshResult,
+)
+
+Runner = Callable[[RefreshRequest], Awaitable[RefreshResult]]
+
+
+@dataclass(slots=True)
+class _Pending:
+    request: RefreshRequest
+    futures: list[asyncio.Future[RefreshResult]] = field(default_factory=list)
+
+
+class RefreshCoordinator:
+    """Serialize refreshes and apply an explicit overlap policy."""
+
+    def __init__(self, runner: Runner, policy: OverlapPolicy) -> None:
+        self._runner = runner
+        self._policy = policy
+        self._lock = asyncio.Lock()
+        self._pending: list[_Pending] = []
+        self._worker: asyncio.Task[None] | None = None
+        self._active: asyncio.Task[RefreshResult] | None = None
+        self._active_pending: _Pending | None = None
+        self._closed = False
+
+    async def submit(self, request: RefreshRequest) -> RefreshResult:
+        loop = asyncio.get_running_loop()
+        future: asyncio.Future[RefreshResult] = loop.create_future()
+        async with self._lock:
+            if self._closed:
+                raise RuntimeError("refresh coordinator is closed")
+            running = self._active is not None or bool(self._pending)
+            if running and self._policy is OverlapPolicy.SKIP_IF_RUNNING:
+                return RefreshResult.skipped("refresh already running")
+            if running and self._policy is OverlapPolicy.CANCEL_PREVIOUS:
+                if self._active is not None:
+                    self._active.cancel()
+                self._cancel_pending()
+            if running and self._policy is OverlapPolicy.COALESCE and self._pending:
+                pending = self._pending[-1]
+                pending.request = pending.request.merge(request)
+                pending.futures.append(future)
+            else:
+                self._pending.append(_Pending(request, [future]))
+            if self._worker is None or self._worker.done():
+                self._worker = asyncio.create_task(
+                    self._work(), name="refresh-engine-coordinator"
+                )
+        try:
+            return await future
+        except asyncio.CancelledError:
+            async with self._lock:
+                for pending in self._pending:
+                    if future in pending.futures:
+                        pending.futures.remove(future)
+                if (
+                    self._active_pending is not None
+                    and future in self._active_pending.futures
+                    and all(item.done() for item in self._active_pending.futures)
+                    and self._active is not None
+                ):
+                    self._active.cancel()
+            raise
+
+    async def close(self, *, cancel_active: bool) -> None:
+        async with self._lock:
+            self._closed = True
+            if cancel_active and self._active is not None:
+                self._active.cancel()
+            if cancel_active:
+                self._cancel_pending()
+            worker = self._worker
+        if worker is not None:
+            await asyncio.gather(worker, return_exceptions=True)
+
+    async def _work(self) -> None:
+        while True:
+            async with self._lock:
+                if not self._pending:
+                    self._active = None
+                    return
+                pending = self._pending.pop(0)
+                self._active = asyncio.create_task(
+                    self._run_one(pending.request),
+                    name="refresh-engine-active-refresh",
+                )
+                self._active_pending = pending
+                active = self._active
+            try:
+                result = await active
+            except asyncio.CancelledError:
+                for future in pending.futures:
+                    if not future.done():
+                        future.cancel()
+            except Exception as error:
+                for future in pending.futures:
+                    if not future.done():
+                        future.set_exception(error)
+            else:
+                for future in pending.futures:
+                    if not future.done():
+                        future.set_result(result)
+            finally:
+                async with self._lock:
+                    if self._active is active:
+                        self._active = None
+                        self._active_pending = None
+
+    async def _run_one(self, request: RefreshRequest) -> RefreshResult:
+        return await self._runner(request)
+
+    def _cancel_pending(self) -> None:
+        for pending in self._pending:
+            for future in pending.futures:
+                if not future.done():
+                    future.cancel()
+        self._pending.clear()

refresh_engine/execution/executor.py

@@ -0,0 +1,180 @@
+"""Bounded, retrying, dependency-ordered refresh execution."""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Mapping
+from time import perf_counter
+
+from tenacity import (
+    AsyncRetrying,
+    retry_if_exception_type,
+    stop_after_attempt,
+    wait_exponential,
+    wait_random,
+)
+
+from refresh_engine.api.protocols import RefreshOperation
+from refresh_engine.core.models import (
+    FailurePolicy,
+    PlanAction,
+    PlanItem,
+    RefreshConfig,
+    RefreshIssue,
+    RefreshPlan,
+    Resource,
+    ResourceExecutionResult,
+    ResourceSnapshot,
+)
+from refresh_engine.events import EventKind, EventPublisher, RefreshEvent
+
+
+class RefreshExecutor:
+    def __init__(
+        self,
+        operation: RefreshOperation,
+        config: RefreshConfig,
+        events: EventPublisher,
+    ) -> None:
+        self._operation = operation
+        self._config = config
+        self._events = events
+
+    async def execute(
+        self,
+        plan: RefreshPlan,
+        resources: Mapping[str, Resource],
+        snapshots: Mapping[str, ResourceSnapshot],
+    ) -> tuple[ResourceExecutionResult, ...]:
+        semaphore = asyncio.Semaphore(self._config.max_concurrency)
+        all_results: list[ResourceExecutionResult] = []
+        await self._events.publish(
+            RefreshEvent(EventKind.EXECUTION_STARTED, plan.refresh_id)
+        )
+        for level in plan.levels:
+            tasks = [
+                asyncio.create_task(
+                    self._execute_item(plan, item, resources, snapshots, semaphore),
+                    name=f"refresh:{plan.refresh_id}:{item.resource_id}",
+                )
+                for item in level
+            ]
+            try:
+                level_results = await asyncio.gather(*tasks)
+            except asyncio.CancelledError:
+                for task in tasks:
+                    task.cancel()
+                await asyncio.gather(*tasks, return_exceptions=True)
+                raise
+            all_results.extend(level_results)
+            if self._config.failure_policy is FailurePolicy.FAIL_FAST and any(
+                not result.succeeded for result in level_results
+            ):
+                break
+        await self._events.publish(
+            RefreshEvent(EventKind.EXECUTION_COMPLETED, plan.refresh_id)
+        )
+        return tuple(all_results)
+
+    async def _execute_item(
+        self,
+        plan: RefreshPlan,
+        item: PlanItem,
+        resources: Mapping[str, Resource],
+        snapshots: Mapping[str, ResourceSnapshot],
+        semaphore: asyncio.Semaphore,
+    ) -> ResourceExecutionResult:
+        async with semaphore:
+            started = perf_counter()
+            await self._events.publish(
+                RefreshEvent(
+                    EventKind.RESOURCE_REFRESH_STARTED,
+                    plan.refresh_id,
+                    item.resource_id,
+                )
+            )
+            attempts = 0
+            error: Exception | None = None
+            retry = self._config.retry
+            try:
+                if retry.max_attempts == 1:
+                    attempts = 1
+                    await self._run_operation(plan, item, resources, snapshots)
+                else:
+                    async for attempt in AsyncRetrying(
+                        stop=stop_after_attempt(retry.max_attempts),
+                        wait=wait_exponential(
+                            multiplier=retry.initial_delay,
+                            max=retry.max_delay,
+                            exp_base=retry.exponential_base,
+                        )
+                        + wait_random(0, retry.jitter),
+                        retry=retry_if_exception_type(retry.retryable_exceptions),
+                        reraise=True,
+                    ):
+                        attempts = attempt.retry_state.attempt_number
+                        with attempt:
+                            await self._run_operation(plan, item, resources, snapshots)
+            except asyncio.CancelledError:
+                raise
+            except Exception as caught:
+                error = caught
+            else:
+                await self._events.publish(
+                    RefreshEvent(
+                        EventKind.RESOURCE_REFRESH_COMPLETED,
+                        plan.refresh_id,
+                        item.resource_id,
+                    )
+                )
+                return ResourceExecutionResult(
+                    item.resource_id,
+                    item.action,
+                    True,
+                    attempts,
+                    perf_counter() - started,
+                )
+            assert error is not None
+            issue = RefreshIssue(
+                "execution",
+                str(error),
+                item.resource_id,
+                type(error).__name__,
+            )
+            await self._events.publish(
+                RefreshEvent(
+                    EventKind.RESOURCE_REFRESH_FAILED,
+                    plan.refresh_id,
+                    item.resource_id,
+                    attributes={"error_type": type(error).__name__},
+                )
+            )
+            return ResourceExecutionResult(
+                item.resource_id,
+                item.action,
+                False,
+                attempts,
+                perf_counter() - started,
+                issue,
+            )
+
+    async def _run_operation(
+        self,
+        plan: RefreshPlan,
+        item: PlanItem,
+        resources: Mapping[str, Resource],
+        snapshots: Mapping[str, ResourceSnapshot],
+    ) -> None:
+        resource = resources.get(item.resource_id)
+        if resource is None and item.action is PlanAction.DELETE:
+            resource = Resource(item.resource_id)
+        operation = self._operation(
+            resource,
+            snapshots.get(item.resource_id),
+            item.action,
+            plan.request,
+        )
+        if self._config.timeout.resource_refresh is None:
+            await operation
+        else:
+            await asyncio.wait_for(operation, self._config.timeout.resource_refresh)

refresh_engine/observability/__init__.py

@@ -0,0 +1,9 @@
+"""Logging and metrics hooks."""
+
+from refresh_engine.observability.logging import (
+    StructlogEventHandler,
+    default_redactor,
+)
+from refresh_engine.observability.metrics import InMemoryMetrics
+
+__all__ = ["InMemoryMetrics", "StructlogEventHandler", "default_redactor"]

refresh_engine/observability/logging.py

@@ -0,0 +1,49 @@
+"""Safe lifecycle event logging through structlog."""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Mapping
+from typing import Any
+
+import structlog
+from structlog.typing import FilteringBoundLogger
+
+from refresh_engine.events.models import RefreshEvent
+
+Redactor = Callable[[Mapping[str, Any]], Mapping[str, Any]]
+
+
+def default_redactor(values: Mapping[str, Any]) -> Mapping[str, Any]:
+    sensitive = {"api_key", "authorization", "password", "secret", "token"}
+    return {
+        key: "[REDACTED]" if key.lower() in sensitive else value
+        for key, value in values.items()
+    }
+
+
+class StructlogEventHandler:
+    """Translate refresh events into native structlog events.
+
+    The application retains ownership of global structlog processor and renderer
+    configuration. This handler only binds safe lifecycle fields.
+    """
+
+    def __init__(
+        self,
+        logger: FilteringBoundLogger | None = None,
+        redactor: Redactor = default_redactor,
+    ) -> None:
+        self._logger = logger or structlog.get_logger("refresh_engine")
+        self._redactor = redactor
+
+    def __call__(self, event: object) -> None:
+        if not isinstance(event, RefreshEvent):
+            return
+        fields = {
+            "refresh_id": event.refresh_id,
+            "resource_id": event.resource_id,
+            "event_kind": event.kind.value,
+            "timestamp": event.timestamp.isoformat(),
+            **event.attributes,
+        }
+        self._logger.info("refresh_event", **dict(self._redactor(fields)))

refresh_engine/observability/metrics.py

@@ -0,0 +1,44 @@
+"""Minimal provider-neutral metrics implementation."""
+
+from __future__ import annotations
+
+import threading
+from collections import defaultdict
+from collections.abc import Mapping
+
+from refresh_engine.api.abc import MetricsSinkABC
+
+
+class InMemoryMetrics(MetricsSinkABC):
+    def __init__(self) -> None:
+        self._counters: dict[str, int] = defaultdict(int)
+        self._observations: dict[str, list[float]] = defaultdict(list)
+        self._lock = threading.Lock()
+
+    def increment(
+        self, name: str, value: int = 1, attributes: Mapping[str, str] | None = None
+    ) -> None:
+        key = self._key(name, attributes)
+        with self._lock:
+            self._counters[key] += value
+
+    def observe(
+        self, name: str, value: float, attributes: Mapping[str, str] | None = None
+    ) -> None:
+        key = self._key(name, attributes)
+        with self._lock:
+            self._observations[key].append(value)
+
+    def snapshot(self) -> tuple[dict[str, int], dict[str, tuple[float, ...]]]:
+        with self._lock:
+            return (
+                dict(self._counters),
+                {key: tuple(values) for key, values in self._observations.items()},
+            )
+
+    @staticmethod
+    def _key(name: str, attributes: Mapping[str, str] | None) -> str:
+        if not attributes:
+            return name
+        suffix = ",".join(f"{key}={value}" for key, value in sorted(attributes.items()))
+        return f"{name}{{{suffix}}}"

refresh_engine/planning/__init__.py

@@ -0,0 +1,6 @@
+"""Dependency analysis and refresh planning."""
+
+from refresh_engine.planning.dependency import DependencyGraph, ImpactAnalyzer
+from refresh_engine.planning.planner import RefreshPlanner
+
+__all__ = ["DependencyGraph", "ImpactAnalyzer", "RefreshPlanner"]

refresh_engine/planning/dependency.py

@@ -0,0 +1,79 @@
+"""Dependency graph and impact analysis."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from typing import cast
+
+import networkx as nx
+
+from refresh_engine.core.models import ChangeSet, ChangeType, ImpactSet
+from refresh_engine.errors import DependencyCycleError
+
+
+class DependencyGraph:
+    """Directed graph where a node maps to the resources it depends on."""
+
+    def __init__(self) -> None:
+        self._graph: nx.DiGraph[str] = nx.DiGraph()
+
+    @property
+    def nodes(self) -> frozenset[str]:
+        return frozenset(self._graph.nodes)
+
+    def add_resource(self, resource_id: str) -> None:
+        self._graph.add_node(resource_id)
+
+    def add_dependency(self, resource_id: str, dependency_id: str) -> None:
+        self._graph.add_edge(dependency_id, resource_id)
+
+    def remove_dependency(self, resource_id: str, dependency_id: str) -> None:
+        if self._graph.has_edge(dependency_id, resource_id):
+            self._graph.remove_edge(dependency_id, resource_id)
+
+    def dependencies_of(self, resource_id: str) -> frozenset[str]:
+        if resource_id not in self._graph:
+            return frozenset()
+        return frozenset(self._graph.predecessors(resource_id))
+
+    def dependents_of(self, resource_id: str) -> frozenset[str]:
+        if resource_id not in self._graph:
+            return frozenset()
+        return frozenset(self._graph.successors(resource_id))
+
+    def transitive_dependents(self, resource_ids: Iterable[str]) -> frozenset[str]:
+        selected = set(resource_ids)
+        impacted = set(selected)
+        for resource_id in selected & self.nodes:
+            impacted.update(cast(set[str], nx.descendants(self._graph, resource_id)))
+        return frozenset(impacted)
+
+    def topological_levels(
+        self, resource_ids: Iterable[str] | None = None
+    ) -> tuple[tuple[str, ...], ...]:
+        selected = self.nodes if resource_ids is None else frozenset(resource_ids)
+        graph = self._graph.subgraph(selected)
+        try:
+            generations = cast(
+                Iterable[Iterable[str]], nx.topological_generations(graph)
+            )
+            return tuple(tuple(sorted(generation)) for generation in generations)
+        except nx.NetworkXUnfeasible as error:
+            cycle_edges = cast(list[tuple[str, str]], nx.find_cycle(graph))
+            cycle = (*tuple(edge[0] for edge in cycle_edges), cycle_edges[0][0])
+            raise DependencyCycleError(cycle) from error
+
+
+class ImpactAnalyzer:
+    def analyze(self, changes: ChangeSet, graph: DependencyGraph) -> ImpactSet:
+        direct = frozenset(
+            change.resource_id
+            for change in changes.changes
+            if change.change_type is not ChangeType.UNCHANGED
+        )
+        impacted = graph.transitive_dependents(direct)
+        return ImpactSet(
+            directly_changed=direct,
+            indirectly_impacted=impacted - direct,
+            unaffected=graph.nodes - impacted,
+        )

refresh_engine/planning/planner.py

@@ -0,0 +1,86 @@
+"""Refresh plan construction."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+
+from refresh_engine.core.models import (
+    ChangeSet,
+    ChangeType,
+    ImpactSet,
+    PlanAction,
+    PlanItem,
+    RefreshMode,
+    RefreshPlan,
+    RefreshRequest,
+    ResourceState,
+)
+from refresh_engine.planning.dependency import DependencyGraph
+
+
+class RefreshPlanner:
+    def plan(
+        self,
+        refresh_id: str,
+        request: RefreshRequest,
+        changes: ChangeSet,
+        impact: ImpactSet,
+        graph: DependencyGraph,
+        current: Mapping[str, ResourceState],
+    ) -> RefreshPlan:
+        changed = {change.resource_id: change for change in changes.changes}
+        if request.mode is RefreshMode.FULL:
+            selected = set(current) | {
+                resource_id
+                for resource_id, change in changed.items()
+                if change.change_type is ChangeType.DELETED
+            }
+        elif request.mode is RefreshMode.TARGETED:
+            selected = set(request.resource_ids) & set(changed)
+        elif request.mode is RefreshMode.DEPENDENCY_AWARE:
+            selected = set(impact.all_impacted)
+        else:
+            selected = {
+                resource_id
+                for resource_id, change in changed.items()
+                if change.change_type is not ChangeType.UNCHANGED
+            }
+
+        deleted = {
+            resource_id
+            for resource_id in selected
+            if changed.get(resource_id)
+            and changed[resource_id].change_type is ChangeType.DELETED
+        }
+        refreshable = selected - deleted
+        levels = []
+        for level in graph.topological_levels(refreshable):
+            levels.append(
+                tuple(
+                    PlanItem(
+                        resource_id,
+                        PlanAction.REFRESH,
+                        current.get(resource_id),
+                        graph.dependencies_of(resource_id),
+                    )
+                    for resource_id in level
+                )
+            )
+        if deleted:
+            levels.append(
+                tuple(
+                    PlanItem(
+                        resource_id,
+                        PlanAction.DELETE,
+                        None,
+                        frozenset(),
+                    )
+                    for resource_id in sorted(deleted)
+                )
+            )
+        return RefreshPlan(
+            refresh_id,
+            request,
+            tuple(levels),
+            skipped_count=max(0, len(changes.changes) - len(selected)),
+        )

refresh_engine/scheduling/__init__.py

@@ -0,0 +1,13 @@
+"""Background scheduler implementations."""
+
+from refresh_engine.scheduling.async_scheduler import (
+    AsyncScheduler,
+    SchedulerState,
+)
+from refresh_engine.scheduling.external import ExternalSchedulerAdapter
+
+__all__ = [
+    "AsyncScheduler",
+    "ExternalSchedulerAdapter",
+    "SchedulerState",
+]