refresh-engine 0.1.0__py3-none-any.whl

refresh_engine/core/models.py

@@ -0,0 +1,362 @@
+"""Immutable public models used by refresh-engine."""
+
+from __future__ import annotations
+
+from collections.abc import AsyncIterator, Mapping
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from enum import StrEnum, auto
+from types import MappingProxyType
+from typing import Any
+from uuid import uuid4
+
+ResourceId = str
+Metadata = Mapping[str, Any]
+
+
+def utc_now() -> datetime:
+    """Return a timezone-aware UTC timestamp."""
+    return datetime.now(UTC)
+
+
+def freeze_mapping(value: Mapping[str, Any] | None = None) -> Metadata:
+    """Return a shallow immutable copy of metadata."""
+    return MappingProxyType(dict(value or {}))
+
+
+@dataclass(frozen=True, slots=True)
+class Resource:
+    """Stable resource identity and lightweight discovery metadata."""
+
+    resource_id: ResourceId
+    metadata: Metadata = field(default_factory=freeze_mapping)
+    version: str | None = None
+    dependencies: frozenset[ResourceId] = frozenset()
+    source: str | None = None
+    cheap_indicator: str | None = None
+    cheap_indicator_reliable: bool = False
+
+    def __post_init__(self) -> None:
+        if not self.resource_id:
+            raise ValueError("resource_id must not be empty")
+        object.__setattr__(self, "metadata", freeze_mapping(self.metadata))
+        object.__setattr__(self, "dependencies", frozenset(self.dependencies))
+
+
+@dataclass(frozen=True, slots=True)
+class ResourceSnapshot:
+    """Observable resource state used to create a strong fingerprint."""
+
+    resource_id: ResourceId
+    content: Any = None
+    metadata: Metadata = field(default_factory=freeze_mapping)
+    version: str | None = None
+    version_token: str | None = None
+    etag: str | None = None
+    dependencies: frozenset[ResourceId] = frozenset()
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "metadata", freeze_mapping(self.metadata))
+        object.__setattr__(self, "dependencies", frozenset(self.dependencies))
+
+
+@dataclass(frozen=True, slots=True)
+class DiscoveryResult:
+    """A streaming discovery plus an explicit deletion-safety contract."""
+
+    resources: AsyncIterator[Resource]
+    complete: bool = True
+    diagnostics: tuple[str, ...] = ()
+
+
+@dataclass(frozen=True, slots=True)
+class ResourceFingerprint:
+    """A namespaced deterministic fingerprint."""
+
+    algorithm: str
+    value: str
+
+
+@dataclass(frozen=True, slots=True)
+class ResourceState:
+    """Last successfully processed resource state."""
+
+    resource_id: ResourceId
+    fingerprint: ResourceFingerprint
+    version: str | None = None
+    cheap_indicator: str | None = None
+    snapshot_metadata: Metadata = field(default_factory=freeze_mapping)
+    dependencies: frozenset[ResourceId] = frozenset()
+    refreshed_at: datetime = field(default_factory=utc_now)
+
+    def __post_init__(self) -> None:
+        object.__setattr__(
+            self, "snapshot_metadata", freeze_mapping(self.snapshot_metadata)
+        )
+        object.__setattr__(self, "dependencies", frozenset(self.dependencies))
+
+
+class ChangeType(StrEnum):
+    ADDED = auto()
+    MODIFIED = auto()
+    DELETED = auto()
+    UNCHANGED = auto()
+    DEPENDENCY_CHANGED = auto()
+
+
+@dataclass(frozen=True, slots=True)
+class Change:
+    resource_id: ResourceId
+    change_type: ChangeType
+    previous: ResourceState | None = None
+    current: ResourceState | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class ChangeSet:
+    changes: tuple[Change, ...]
+    diagnostics: tuple[str, ...] = ()
+
+    def of_type(self, change_type: ChangeType) -> tuple[Change, ...]:
+        return tuple(c for c in self.changes if c.change_type is change_type)
+
+    @property
+    def added(self) -> tuple[Change, ...]:
+        return self.of_type(ChangeType.ADDED)
+
+    @property
+    def modified(self) -> tuple[Change, ...]:
+        return self.of_type(ChangeType.MODIFIED)
+
+    @property
+    def deleted(self) -> tuple[Change, ...]:
+        return self.of_type(ChangeType.DELETED)
+
+    @property
+    def unchanged(self) -> tuple[Change, ...]:
+        return self.of_type(ChangeType.UNCHANGED)
+
+    @property
+    def dependency_changed(self) -> tuple[Change, ...]:
+        return self.of_type(ChangeType.DEPENDENCY_CHANGED)
+
+    @property
+    def counts(self) -> Mapping[ChangeType, int]:
+        return MappingProxyType({kind: len(self.of_type(kind)) for kind in ChangeType})
+
+    @property
+    def resource_ids(self) -> frozenset[ResourceId]:
+        return frozenset(change.resource_id for change in self.changes)
+
+
+@dataclass(frozen=True, slots=True)
+class ImpactSet:
+    directly_changed: frozenset[ResourceId]
+    indirectly_impacted: frozenset[ResourceId]
+    unaffected: frozenset[ResourceId]
+
+    @property
+    def all_impacted(self) -> frozenset[ResourceId]:
+        return self.directly_changed | self.indirectly_impacted
+
+
+class RefreshMode(StrEnum):
+    FULL = auto()
+    INCREMENTAL = auto()
+    TARGETED = auto()
+    DEPENDENCY_AWARE = auto()
+
+
+class TriggerSource(StrEnum):
+    MANUAL = auto()
+    SCHEDULED = auto()
+    EVENT = auto()
+    QUERY = auto()
+    EXTERNAL = auto()
+
+
+class OverlapPolicy(StrEnum):
+    SKIP_IF_RUNNING = auto()
+    QUEUE = auto()
+    COALESCE = auto()
+    CANCEL_PREVIOUS = auto()
+
+
+class FailurePolicy(StrEnum):
+    FAIL_FAST = auto()
+    BEST_EFFORT = auto()
+    RETRY_THEN_CONTINUE = auto()
+
+
+class RefreshStatus(StrEnum):
+    SUCCESS = auto()
+    PARTIAL = auto()
+    FAILED = auto()
+    CANCELLED = auto()
+    SKIPPED = auto()
+
+
+@dataclass(frozen=True, slots=True)
+class RefreshRequest:
+    mode: RefreshMode = RefreshMode.INCREMENTAL
+    resource_ids: frozenset[ResourceId] = frozenset()
+    trigger: TriggerSource = TriggerSource.MANUAL
+    priority: int = 0
+    metadata: Metadata = field(default_factory=freeze_mapping)
+    requested_at: datetime = field(default_factory=utc_now)
+    correlation_id: str | None = None
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "resource_ids", frozenset(self.resource_ids))
+        object.__setattr__(self, "metadata", freeze_mapping(self.metadata))
+        if self.mode is RefreshMode.TARGETED and not self.resource_ids:
+            raise ValueError("targeted refresh requires resource_ids")
+
+    @property
+    def deduplication_key(self) -> tuple[RefreshMode, frozenset[str], str | None]:
+        return self.mode, self.resource_ids, self.correlation_id
+
+    def merge(self, other: RefreshRequest) -> RefreshRequest:
+        mode = (
+            RefreshMode.FULL
+            if RefreshMode.FULL in (self.mode, other.mode)
+            else (
+                RefreshMode.DEPENDENCY_AWARE
+                if RefreshMode.DEPENDENCY_AWARE in (self.mode, other.mode)
+                else self.mode
+            )
+        )
+        return RefreshRequest(
+            mode=mode,
+            resource_ids=self.resource_ids | other.resource_ids,
+            trigger=other.trigger,
+            priority=max(self.priority, other.priority),
+            metadata={**self.metadata, **other.metadata},
+            requested_at=min(self.requested_at, other.requested_at),
+            correlation_id=other.correlation_id or self.correlation_id,
+        )
+
+
+class PlanAction(StrEnum):
+    REFRESH = auto()
+    DELETE = auto()
+
+
+@dataclass(frozen=True, slots=True)
+class PlanItem:
+    resource_id: ResourceId
+    action: PlanAction
+    state: ResourceState | None
+    dependencies: frozenset[ResourceId] = frozenset()
+
+
+@dataclass(frozen=True, slots=True)
+class RefreshPlan:
+    refresh_id: str
+    request: RefreshRequest
+    levels: tuple[tuple[PlanItem, ...], ...]
+    skipped_count: int
+
+    @property
+    def items(self) -> tuple[PlanItem, ...]:
+        return tuple(item for level in self.levels for item in level)
+
+
+@dataclass(frozen=True, slots=True)
+class RefreshIssue:
+    phase: str
+    message: str
+    resource_id: ResourceId | None = None
+    exception_type: str | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class ResourceExecutionResult:
+    resource_id: ResourceId
+    action: PlanAction
+    succeeded: bool
+    attempts: int
+    duration: float
+    error: RefreshIssue | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class RefreshResult:
+    refresh_id: str
+    status: RefreshStatus
+    started_at: datetime
+    completed_at: datetime
+    duration: float
+    discovered_count: int = 0
+    added_count: int = 0
+    modified_count: int = 0
+    deleted_count: int = 0
+    unchanged_count: int = 0
+    impacted_count: int = 0
+    refreshed_count: int = 0
+    skipped_count: int = 0
+    failed_count: int = 0
+    errors: tuple[RefreshIssue, ...] = ()
+    warnings: tuple[str, ...] = ()
+    executions: tuple[ResourceExecutionResult, ...] = ()
+
+    @classmethod
+    def skipped(cls, reason: str) -> RefreshResult:
+        now = utc_now()
+        return cls(
+            refresh_id=str(uuid4()),
+            status=RefreshStatus.SKIPPED,
+            started_at=now,
+            completed_at=now,
+            duration=0.0,
+            skipped_count=1,
+            warnings=(reason,),
+        )
+
+
+@dataclass(frozen=True, slots=True)
+class RetryConfig:
+    max_attempts: int = 1
+    initial_delay: float = 0.1
+    max_delay: float = 5.0
+    exponential_base: float = 2.0
+    jitter: float = 0.0
+    retryable_exceptions: tuple[type[Exception], ...] = (Exception,)
+
+    def __post_init__(self) -> None:
+        if self.max_attempts < 1:
+            raise ValueError("max_attempts must be at least 1")
+        if min(self.initial_delay, self.max_delay, self.jitter) < 0:
+            raise ValueError("retry delays must not be negative")
+        if self.exponential_base < 1:
+            raise ValueError("exponential_base must be at least 1")
+
+
+@dataclass(frozen=True, slots=True)
+class TimeoutConfig:
+    discovery: float | None = None
+    snapshot: float | None = None
+    resource_refresh: float | None = None
+    overall: float | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class RefreshConfig:
+    max_concurrency: int = 10
+    overlap_policy: OverlapPolicy = OverlapPolicy.COALESCE
+    failure_policy: FailurePolicy = FailurePolicy.BEST_EFFORT
+    retry: RetryConfig = field(default_factory=RetryConfig)
+    timeout: TimeoutConfig = field(default_factory=TimeoutConfig)
+    cancel_active_on_shutdown: bool = False
+
+    def __post_init__(self) -> None:
+        if self.max_concurrency < 1:
+            raise ValueError("max_concurrency must be at least 1")
+        for value in (
+            self.timeout.discovery,
+            self.timeout.snapshot,
+            self.timeout.resource_refresh,
+            self.timeout.overall,
+        ):
+            if value is not None and value <= 0:
+                raise ValueError("timeouts must be positive")

refresh_engine/detection/__init__.py

@@ -0,0 +1,21 @@
+"""Change detection and fingerprinting."""
+
+from refresh_engine.detection.detector import ChangeDetector
+from refresh_engine.detection.fingerprints import (
+    CompositeHash,
+    ContentHash,
+    ETagFingerprint,
+    MetadataHash,
+    VersionTokenFingerprint,
+    canonical_bytes,
+)
+
+__all__ = [
+    "ChangeDetector",
+    "CompositeHash",
+    "ContentHash",
+    "ETagFingerprint",
+    "MetadataHash",
+    "VersionTokenFingerprint",
+    "canonical_bytes",
+]

refresh_engine/detection/detector.py

@@ -0,0 +1,54 @@
+"""Change classification."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+
+from refresh_engine.core.models import (
+    Change,
+    ChangeSet,
+    ChangeType,
+    ResourceState,
+)
+
+
+class ChangeDetector:
+    """Compare current candidates with the last successful state."""
+
+    def detect(
+        self,
+        previous: Mapping[str, ResourceState],
+        current: Mapping[str, ResourceState],
+        *,
+        discovery_complete: bool,
+        diagnostics: tuple[str, ...] = (),
+    ) -> ChangeSet:
+        changes: list[Change] = []
+        for resource_id, state in current.items():
+            old = previous.get(resource_id)
+            if old is None:
+                kind = ChangeType.ADDED
+            elif old.fingerprint != state.fingerprint:
+                kind = ChangeType.MODIFIED
+            elif old.dependencies != state.dependencies:
+                kind = ChangeType.DEPENDENCY_CHANGED
+            else:
+                kind = ChangeType.UNCHANGED
+            changes.append(Change(resource_id, kind, old, state))
+
+        if discovery_complete:
+            for resource_id in previous.keys() - current.keys():
+                changes.append(
+                    Change(
+                        resource_id,
+                        ChangeType.DELETED,
+                        previous[resource_id],
+                        None,
+                    )
+                )
+        elif previous.keys() - current.keys():
+            diagnostics += (
+                "deletion inference suppressed because discovery was incomplete",
+            )
+        changes.sort(key=lambda change: change.resource_id)
+        return ChangeSet(tuple(changes), diagnostics)

refresh_engine/detection/fingerprints.py

@@ -0,0 +1,115 @@
+"""Deterministic fingerprint strategies."""
+
+from __future__ import annotations
+
+import base64
+import hashlib
+import json
+import math
+from collections.abc import Mapping, Sequence, Set
+from dataclasses import asdict, is_dataclass
+from datetime import date, datetime
+from enum import Enum
+from pathlib import PurePath
+from typing import Any
+
+from refresh_engine.api.abc import FingerprintStrategyABC
+from refresh_engine.core.models import ResourceFingerprint, ResourceSnapshot
+from refresh_engine.errors import FingerprintError
+
+
+def _canonical(value: Any) -> Any:
+    if value is None:
+        return ["null", None]
+    if isinstance(value, bool):
+        return ["bool", value]
+    if isinstance(value, int):
+        return ["int", str(value)]
+    if isinstance(value, float):
+        if not math.isfinite(value):
+            raise FingerprintError("non-finite floats cannot be fingerprinted")
+        return ["float", value.hex()]
+    if isinstance(value, str):
+        return ["str", value]
+    if isinstance(value, bytes):
+        return ["bytes", base64.b64encode(value).decode("ascii")]
+    if isinstance(value, (datetime, date)):
+        return [type(value).__name__, value.isoformat()]
+    if isinstance(value, PurePath):
+        return ["path", value.as_posix()]
+    if isinstance(value, Enum):
+        return [
+            "enum",
+            f"{type(value).__module__}.{type(value).__qualname__}",
+            value.value,
+        ]
+    if is_dataclass(value) and not isinstance(value, type):
+        return ["dataclass", _canonical(asdict(value))]
+    if isinstance(value, Mapping):
+        pairs = [(_canonical(key), _canonical(item)) for key, item in value.items()]
+        pairs.sort(key=lambda pair: _encode(pair[0]))
+        return ["mapping", pairs]
+    if isinstance(value, Set):
+        items = [_canonical(item) for item in value]
+        items.sort(key=_encode)
+        return ["set", items]
+    if isinstance(value, Sequence):
+        return ["sequence", [_canonical(item) for item in value]]
+    raise FingerprintError(f"unsupported fingerprint value: {type(value).__qualname__}")
+
+
+def _encode(value: Any) -> bytes:
+    return json.dumps(
+        value, ensure_ascii=False, separators=(",", ":"), sort_keys=True
+    ).encode("utf-8")
+
+
+def canonical_bytes(value: Any) -> bytes:
+    """Encode a supported value deterministically and with type information."""
+    return _encode(_canonical(value))
+
+
+def _sha256(namespace: str, value: Any) -> ResourceFingerprint:
+    digest = hashlib.sha256(canonical_bytes(value)).hexdigest()
+    return ResourceFingerprint(f"sha256:{namespace}", digest)
+
+
+class ContentHash(FingerprintStrategyABC):
+    async def fingerprint(self, snapshot: ResourceSnapshot) -> ResourceFingerprint:
+        return _sha256("content", snapshot.content)
+
+
+class MetadataHash(FingerprintStrategyABC):
+    async def fingerprint(self, snapshot: ResourceSnapshot) -> ResourceFingerprint:
+        return _sha256("metadata", snapshot.metadata)
+
+
+class CompositeHash(FingerprintStrategyABC):
+    def __init__(self, *, include_dependencies: bool = True) -> None:
+        self._include_dependencies = include_dependencies
+
+    async def fingerprint(self, snapshot: ResourceSnapshot) -> ResourceFingerprint:
+        value = {
+            "content": snapshot.content,
+            "metadata": snapshot.metadata,
+            "version": snapshot.version,
+            "version_token": snapshot.version_token,
+            "etag": snapshot.etag,
+        }
+        if self._include_dependencies:
+            value["dependencies"] = sorted(snapshot.dependencies)
+        return _sha256("composite", value)
+
+
+class VersionTokenFingerprint(FingerprintStrategyABC):
+    async def fingerprint(self, snapshot: ResourceSnapshot) -> ResourceFingerprint:
+        if snapshot.version_token is None:
+            raise FingerprintError("snapshot has no version_token")
+        return _sha256("version-token", snapshot.version_token)
+
+
+class ETagFingerprint(FingerprintStrategyABC):
+    async def fingerprint(self, snapshot: ResourceSnapshot) -> ResourceFingerprint:
+        if snapshot.etag is None:
+            raise FingerprintError("snapshot has no etag")
+        return _sha256("etag", snapshot.etag)

refresh_engine/errors.py

@@ -0,0 +1,57 @@
+"""Structured refresh-engine exceptions."""
+
+
+class RefreshError(Exception):
+    """Base exception for refresh-engine failures."""
+
+
+class ConfigurationError(RefreshError):
+    """Invalid configuration."""
+
+
+class DiscoveryError(RefreshError):
+    """Resource discovery failed."""
+
+
+class SnapshotError(RefreshError):
+    """Resource snapshotting failed."""
+
+
+class FingerprintError(RefreshError):
+    """Fingerprint creation failed."""
+
+
+class ChangeDetectionError(RefreshError):
+    """Change detection failed."""
+
+
+class DependencyError(RefreshError):
+    """Dependency graph operation failed."""
+
+
+class DependencyCycleError(DependencyError):
+    """A dependency cycle prevents deterministic execution."""
+
+    def __init__(self, cycle: tuple[str, ...]) -> None:
+        self.cycle = cycle
+        super().__init__(f"dependency cycle detected: {' -> '.join(cycle)}")
+
+
+class PlanningError(RefreshError):
+    """Refresh planning failed."""
+
+
+class ExecutionError(RefreshError):
+    """Refresh execution failed."""
+
+
+class StateStoreError(RefreshError):
+    """State persistence failed."""
+
+
+class SchedulerError(RefreshError):
+    """Scheduler lifecycle operation failed."""
+
+
+class CancellationError(RefreshError):
+    """A refresh was cancelled by policy."""

refresh_engine/events/__init__.py

@@ -0,0 +1,6 @@
+"""Provider-neutral lifecycle events."""
+
+from refresh_engine.events.models import EventKind, RefreshEvent
+from refresh_engine.events.publisher import EventPublisher
+
+__all__ = ["EventKind", "EventPublisher", "RefreshEvent"]

refresh_engine/events/models.py

@@ -0,0 +1,42 @@
+"""Structured lifecycle events."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import StrEnum, auto
+
+from refresh_engine.core.models import Metadata, freeze_mapping, utc_now
+
+
+class EventKind(StrEnum):
+    REFRESH_STARTED = auto()
+    DISCOVERY_STARTED = auto()
+    DISCOVERY_COMPLETED = auto()
+    DETECTION_STARTED = auto()
+    DETECTION_COMPLETED = auto()
+    PLANNING_STARTED = auto()
+    PLANNING_COMPLETED = auto()
+    EXECUTION_STARTED = auto()
+    RESOURCE_REFRESH_STARTED = auto()
+    RESOURCE_REFRESH_COMPLETED = auto()
+    RESOURCE_REFRESH_FAILED = auto()
+    EXECUTION_COMPLETED = auto()
+    STATE_PERSISTED = auto()
+    REFRESH_COMPLETED = auto()
+    REFRESH_FAILED = auto()
+    REFRESH_CANCELLED = auto()
+    SCHEDULER_STARTED = auto()
+    SCHEDULER_STOPPED = auto()
+
+
+@dataclass(frozen=True, slots=True)
+class RefreshEvent:
+    kind: EventKind
+    refresh_id: str | None = None
+    resource_id: str | None = None
+    timestamp: datetime = field(default_factory=utc_now)
+    attributes: Metadata = field(default_factory=freeze_mapping)
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "attributes", freeze_mapping(self.attributes))