agentevolve-optimizer 0.5.0__py3-none-any.whl

agent_evolve/infrastructure/events/jsonl.py

@@ -0,0 +1,193 @@
+"""Durable append-only JSONL EventStore.
+
+Correctness is favored over indexing in this first milestone: streams are
+revalidated before reads and appends. Event payloads remain small because large
+blobs will live in the artifact store added by the second half of M1.
+"""
+
+from __future__ import annotations
+
+import os
+import threading
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Dict, Iterator, List, Tuple
+
+from agent_evolve.domain.event import (
+    EventCodecError,
+    EventEnvelope,
+    event_from_json,
+    event_to_json,
+)
+from agent_evolve.domain.ids import EventId, RunId
+from agent_evolve.infrastructure.events._validation import (
+    prepare_event_for_append,
+    validate_append,
+    validate_loaded_stream,
+)
+from agent_evolve.ports.event_store import (
+    CorruptEventLogError,
+    DuplicateEventIdError,
+    EventRunMismatchError,
+    EventSequenceError,
+)
+
+try:  # pragma: no cover - exercised on Unix CI; fallback is thread-only.
+    import fcntl
+except ImportError:  # pragma: no cover
+    fcntl = None
+
+
+class JsonlEventStore:
+    """One canonical ``<run_id>.jsonl`` stream per run under *root*."""
+
+    def __init__(self, root: str | os.PathLike[str]) -> None:
+        self.root = Path(root)
+        self.root.mkdir(parents=True, exist_ok=True)
+        self._thread_lock = threading.RLock()
+        self._lock_path = self.root / ".event-store.lock"
+        # Fail early on malformed/tampered existing streams.
+        with self._locked():
+            self._load_all_unlocked()
+
+    def path_for_run(self, run_id: RunId) -> Path:
+        if type(run_id) is not RunId:
+            raise TypeError("run_id must be an exact RunId")
+        canonical_run_id = RunId(run_id.value)
+        return self.root / f"{canonical_run_id.value}.jsonl"
+
+    @contextmanager
+    def _locked(self) -> Iterator[None]:
+        with self._thread_lock:
+            lock_fd = os.open(self._lock_path, os.O_RDWR | os.O_CREAT, 0o600)
+            try:
+                if fcntl is not None:
+                    fcntl.flock(lock_fd, fcntl.LOCK_EX)
+                yield
+            finally:
+                if fcntl is not None:
+                    fcntl.flock(lock_fd, fcntl.LOCK_UN)
+                os.close(lock_fd)
+
+    def _read_file_unlocked(self, path: Path) -> Tuple[EventEnvelope, ...]:
+        data = path.read_bytes()
+        if not data:
+            return ()
+        if not data.endswith(b"\n"):
+            raise CorruptEventLogError(f"Event log has a truncated final line: {path}")
+        events: List[EventEnvelope] = []
+        for line_number, raw_line in enumerate(data.splitlines(), 1):
+            if not raw_line.strip():
+                raise CorruptEventLogError(f"Blank event line at {path}:{line_number}")
+            try:
+                line = raw_line.decode("utf-8")
+            except UnicodeDecodeError:
+                line = None
+            if line is None:
+                raise CorruptEventLogError(
+                    f"Malformed event at {path}:{line_number}"
+                )
+            try:
+                event = event_from_json(line)
+            except EventCodecError:
+                event = None
+            if event is None:
+                raise CorruptEventLogError(
+                    f"Malformed event at {path}:{line_number}"
+                )
+            events.append(event)
+        return tuple(events)
+
+    def _load_all_unlocked(
+        self,
+    ) -> tuple[Dict[RunId, Tuple[EventEnvelope, ...]], set[EventId]]:
+        streams: Dict[RunId, Tuple[EventEnvelope, ...]] = {}
+        event_ids: set[EventId] = set()
+        for path in sorted(self.root.glob("*.jsonl")):
+            events = self._read_file_unlocked(path)
+            if not events:
+                # Empty files have no authoritative run identity yet.
+                continue
+            run_id = events[0].run_id
+            expected_name = f"{run_id.value}.jsonl"
+            if path.name != expected_name:
+                raise EventRunMismatchError(
+                    f"Stream file {path.name!r} contains run {run_id}; "
+                    f"expected file {expected_name!r}"
+                )
+            try:
+                validate_loaded_stream(run_id, events)
+            except (EventSequenceError, EventRunMismatchError, DuplicateEventIdError) as exc:
+                raise CorruptEventLogError(f"Invalid event stream {path}: {exc}") from exc
+            duplicate = event_ids.intersection(event.event_id for event in events)
+            if duplicate:
+                repeated = sorted(str(event_id) for event_id in duplicate)
+                raise CorruptEventLogError(f"Duplicate event ID across streams: {repeated}")
+            event_ids.update(event.event_id for event in events)
+            streams[run_id] = events
+        return streams, event_ids
+
+    @staticmethod
+    def _append_bytes(path: Path, data: bytes) -> None:
+        fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600)
+        try:
+            view = memoryview(data)
+            while view:
+                written = os.write(fd, view)
+                if written <= 0:  # pragma: no cover - defensive OS failure guard.
+                    raise OSError("zero-byte write while appending event")
+                view = view[written:]
+            os.fsync(fd)
+        finally:
+            os.close(fd)
+
+    def _fsync_root_directory(self) -> None:
+        """Make creation of a new stream name durable on POSIX filesystems."""
+
+        if os.name != "posix":  # pragma: no cover - directory fsync is POSIX-specific.
+            return
+        flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
+        fd = os.open(self.root, flags)
+        try:
+            os.fsync(fd)
+        finally:
+            os.close(fd)
+
+    def append(
+        self,
+        event: EventEnvelope,
+        *,
+        expected_previous_sequence: int,
+    ) -> None:
+        snapshot = prepare_event_for_append(event)
+        encoded = (event_to_json(snapshot) + "\n").encode("utf-8")
+        with self._locked():
+            streams, event_ids = self._load_all_unlocked()
+            existing = streams.get(snapshot.run_id, ())
+            validate_append(
+                existing,
+                snapshot,
+                expected_previous_sequence=expected_previous_sequence,
+                known_event_ids=event_ids,
+            )
+            self._append_bytes(self.path_for_run(snapshot.run_id), encoded)
+            self._fsync_root_directory()
+
+    def read(
+        self,
+        run_id: RunId,
+        *,
+        after_sequence: int = 0,
+    ) -> Tuple[EventEnvelope, ...]:
+        if type(after_sequence) is not int or after_sequence < 0:
+            raise ValueError("after_sequence must be a non-negative integer")
+        if type(run_id) is not RunId:
+            raise TypeError("run_id must be an exact RunId")
+        canonical_run_id = RunId(run_id.value)
+        with self._locked():
+            streams, _ = self._load_all_unlocked()
+            return tuple(
+                event
+                for event in streams.get(canonical_run_id, ())
+                if event.sequence_number > after_sequence
+            )

agent_evolve/infrastructure/exception_provenance.py

@@ -0,0 +1,215 @@
+"""Privacy-safe, bounded exception-origin projection.
+
+This module is framework-neutral.  It converts Python exception topology into
+closed domain values without reading exception messages, reprs, arguments,
+tracebacks, response objects, URLs, payloads, or arbitrary metadata.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from collections import deque
+
+from agent_evolve.domain.llm_task_queue import (
+    MAX_EXCEPTION_PROVENANCE_NODES,
+    ExceptionOriginFamily,
+    ExceptionProvenanceLink,
+    SanitizedExceptionProvenance,
+    SanitizedExceptionProvenanceNode,
+)
+
+
+EXCEPTION_TYPE_IDENTITY_FINGERPRINT_ALGORITHM = (
+    "sha256_domain_and_bounded_module_qualname_projection_v1"
+)
+_EXCEPTION_TYPE_IDENTITY_DOMAIN = (
+    b"agent-evolve:sanitized-exception-type-identity:v1\x00"
+)
+EXCEPTION_TYPE_IDENTITY_DOMAIN_SHA256 = hashlib.sha256(
+    _EXCEPTION_TYPE_IDENTITY_DOMAIN
+).hexdigest()
+_MAX_TYPE_COMPONENT_CHARS = 512
+_TYPE_COMPONENT_EDGE_CHARS = 128
+
+
+def _runtime_type_text(exception_type: type[BaseException], name: str) -> str | None:
+    """Read type metadata without invoking a custom metaclass override."""
+
+    try:
+        value = type.__getattribute__(exception_type, name)
+    except BaseException:
+        return None
+    return value if type(value) is str else None
+
+
+def _bounded_type_component(value: str | None) -> dict[str, object]:
+    """Project type metadata with fixed memory independent of input length."""
+
+    if value is None:
+        return {"state": "absent", "length": 0, "prefix": "", "suffix": ""}
+    length = len(value)
+    if length <= _MAX_TYPE_COMPONENT_CHARS:
+        return {
+            "state": "complete",
+            "length": length,
+            "prefix": value,
+            "suffix": "",
+        }
+    return {
+        "state": "truncated",
+        "length": length,
+        "prefix": value[:_TYPE_COMPONENT_EDGE_CHARS],
+        "suffix": value[-_TYPE_COMPONENT_EDGE_CHARS:],
+    }
+
+
+def exception_type_identity_sha256(exception_type: type[BaseException]) -> str:
+    """Return a stable, redacted fingerprint for one runtime exception type."""
+
+    if not isinstance(exception_type, type) or not issubclass(
+        exception_type, BaseException
+    ):
+        raise TypeError("exception_type must be a BaseException type")
+    record = {
+        "schema_version": 1,
+        "module": _bounded_type_component(
+            _runtime_type_text(exception_type, "__module__")
+        ),
+        "qualname": _bounded_type_component(
+            _runtime_type_text(exception_type, "__qualname__")
+        ),
+    }
+    canonical = json.dumps(
+        record,
+        ensure_ascii=True,
+        allow_nan=False,
+        separators=(",", ":"),
+        sort_keys=True,
+    ).encode("ascii")
+    return hashlib.sha256(_EXCEPTION_TYPE_IDENTITY_DOMAIN + canonical).hexdigest()
+
+
+def _origin_family(exception_type: type[BaseException]) -> ExceptionOriginFamily:
+    module = _runtime_type_text(exception_type, "__module__")
+    if module is None:
+        return ExceptionOriginFamily.OTHER
+    roots = (
+        ("builtins", ExceptionOriginFamily.BUILTINS),
+        ("asyncio", ExceptionOriginFamily.ASYNCIO),
+        ("anyio", ExceptionOriginFamily.ANYIO),
+        ("httpx", ExceptionOriginFamily.HTTPX),
+        ("httpcore", ExceptionOriginFamily.HTTPCORE),
+        ("openai", ExceptionOriginFamily.OPENAI),
+        ("pydantic_ai", ExceptionOriginFamily.PYDANTIC_AI),
+        ("pydantic", ExceptionOriginFamily.PYDANTIC),
+        ("agent_evolve", ExceptionOriginFamily.AGENT_EVOLVE),
+    )
+    for root, family in roots:
+        if module == root or module.startswith(root + "."):
+            return family
+    return ExceptionOriginFamily.OTHER
+
+
+def _base_exception_link(
+    value: BaseException,
+    attribute: str,
+) -> BaseException | None:
+    try:
+        linked = BaseException.__getattribute__(value, attribute)
+    except BaseException:
+        return None
+    return linked if isinstance(linked, BaseException) else None
+
+
+def _group_members(value: BaseException) -> tuple[BaseException, ...]:
+    if not isinstance(value, BaseExceptionGroup):
+        return ()
+    try:
+        members = BaseExceptionGroup.exceptions.__get__(value, type(value))
+    except BaseException:
+        return ()
+    if type(members) is not tuple:
+        return ()
+    # One extra member is enough to authenticate that the bounded graph was
+    # truncated; never enqueue an attacker-sized group in full.
+    return tuple(
+        member
+        for member in members[: MAX_EXCEPTION_PROVENANCE_NODES + 1]
+        if isinstance(member, BaseException)
+    )
+
+
+def sanitized_exception_provenance(
+    exc: BaseException,
+) -> SanitizedExceptionProvenance:
+    """Project root/cause/context/group topology into at most sixteen nodes."""
+
+    if not isinstance(exc, BaseException):
+        raise TypeError("exc must be a BaseException")
+    pending: deque[tuple[BaseException, int | None, ExceptionProvenanceLink]] = deque(
+        [(exc, None, ExceptionProvenanceLink.ROOT)]
+    )
+    nodes: list[SanitizedExceptionProvenanceNode] = []
+    seen: set[int] = set()
+    overflowed = False
+
+    while pending and len(nodes) < MAX_EXCEPTION_PROVENANCE_NODES:
+        current, parent_index, link = pending.popleft()
+        identity = id(current)
+        if identity in seen:
+            continue
+        seen.add(identity)
+        node_index = len(nodes)
+        exception_type = type(current)
+        nodes.append(
+            SanitizedExceptionProvenanceNode(
+                parent_index=parent_index,
+                link=link,
+                family=_origin_family(exception_type),
+                type_identity_sha256=exception_type_identity_sha256(exception_type),
+            )
+        )
+
+        linked_values = (
+            (
+                _base_exception_link(current, "__cause__"),
+                ExceptionProvenanceLink.CAUSE,
+            ),
+            (
+                _base_exception_link(current, "__context__"),
+                ExceptionProvenanceLink.CONTEXT,
+            ),
+        )
+        children = [
+            (linked, node_index, child_link)
+            for linked, child_link in linked_values
+            if linked is not None and id(linked) not in seen
+        ]
+        children.extend(
+            (member, node_index, ExceptionProvenanceLink.GROUP_MEMBER)
+            for member in _group_members(current)
+            if id(member) not in seen
+        )
+        for child in children:
+            # Bound pending storage as well as the emitted graph. The extra
+            # slot only records that more topology existed.
+            if len(nodes) + len(pending) < MAX_EXCEPTION_PROVENANCE_NODES:
+                pending.append(child)
+            else:
+                overflowed = True
+
+    if pending:
+        overflowed = True
+    return SanitizedExceptionProvenance(
+        nodes=tuple(nodes),
+        truncated=overflowed,
+    )
+
+
+__all__ = [
+    "EXCEPTION_TYPE_IDENTITY_DOMAIN_SHA256",
+    "EXCEPTION_TYPE_IDENTITY_FINGERPRINT_ALGORITHM",
+    "exception_type_identity_sha256",
+    "sanitized_exception_provenance",
+]

agent_evolve/infrastructure/ids.py

@@ -0,0 +1,118 @@
+"""Production and deterministic ID factories."""
+
+from __future__ import annotations
+
+import threading
+import uuid
+from collections import defaultdict
+from typing import DefaultDict, Type, TypeVar
+
+from agent_evolve.domain.ids import (
+    CandidateId,
+    CorrelationId,
+    EvaluationAttemptId,
+    EvaluationId,
+    EventId,
+    GenerationId,
+    InsightId,
+    LLMCallId,
+    OperatorInvocationId,
+    ProviderAttemptId,
+    RunId,
+    StableId,
+    validate_id_namespace,
+)
+
+IdT = TypeVar("IdT", bound=StableId)
+class UuidIdFactory:
+    """Allocate opaque UUID4-backed IDs.
+
+    Ordering comes from the event sequence, so IDs do not encode wall time and do
+    not risk clock-regression ambiguity.
+    """
+
+    @staticmethod
+    def _new(id_type: Type[IdT]) -> IdT:
+        return id_type(f"{id_type.PREFIX}_{uuid.uuid4().hex}")
+
+    def new_run_id(self) -> RunId:
+        return self._new(RunId)
+
+    def new_event_id(self) -> EventId:
+        return self._new(EventId)
+
+    def new_generation_id(self) -> GenerationId:
+        return self._new(GenerationId)
+
+    def new_candidate_id(self) -> CandidateId:
+        return self._new(CandidateId)
+
+    def new_insight_id(self) -> InsightId:
+        return self._new(InsightId)
+
+    def new_operator_invocation_id(self) -> OperatorInvocationId:
+        return self._new(OperatorInvocationId)
+
+    def new_llm_call_id(self) -> LLMCallId:
+        return self._new(LLMCallId)
+
+    def new_provider_attempt_id(self) -> ProviderAttemptId:
+        return self._new(ProviderAttemptId)
+
+    def new_evaluation_id(self) -> EvaluationId:
+        return self._new(EvaluationId)
+
+    def new_evaluation_attempt_id(self) -> EvaluationAttemptId:
+        return self._new(EvaluationAttemptId)
+
+    def new_correlation_id(self) -> CorrelationId:
+        return self._new(CorrelationId)
+
+
+class DeterministicIdFactory:
+    """Per-ID-type deterministic allocator for tests and recorded replay."""
+
+    def __init__(self, namespace: str = "test") -> None:
+        validate_id_namespace(namespace)
+        self._namespace = namespace
+        self._counters: DefaultDict[Type[StableId], int] = defaultdict(int)
+        self._lock = threading.Lock()
+
+    def _new(self, id_type: Type[IdT]) -> IdT:
+        with self._lock:
+            self._counters[id_type] += 1
+            ordinal = self._counters[id_type]
+        return id_type(f"{id_type.PREFIX}_{self._namespace}_{ordinal:06d}")
+
+    def new_run_id(self) -> RunId:
+        return self._new(RunId)
+
+    def new_event_id(self) -> EventId:
+        return self._new(EventId)
+
+    def new_generation_id(self) -> GenerationId:
+        return self._new(GenerationId)
+
+    def new_candidate_id(self) -> CandidateId:
+        return self._new(CandidateId)
+
+    def new_insight_id(self) -> InsightId:
+        return self._new(InsightId)
+
+    def new_operator_invocation_id(self) -> OperatorInvocationId:
+        return self._new(OperatorInvocationId)
+
+    def new_llm_call_id(self) -> LLMCallId:
+        return self._new(LLMCallId)
+
+    def new_provider_attempt_id(self) -> ProviderAttemptId:
+        return self._new(ProviderAttemptId)
+
+    def new_evaluation_id(self) -> EvaluationId:
+        return self._new(EvaluationId)
+
+    def new_evaluation_attempt_id(self) -> EvaluationAttemptId:
+        return self._new(EvaluationAttemptId)
+
+    def new_correlation_id(self) -> CorrelationId:
+        return self._new(CorrelationId)