nexus-coding-agent 0.1.0__py3-none-any.whl

nexus/evaluation/evaluator.py

@@ -0,0 +1,263 @@
+"""Pure deterministic evaluator for typed Day 9 evidence."""
+
+from __future__ import annotations
+
+import re
+import unicodedata
+
+from nexus.domain.planning import TerminalStatus
+from nexus.domain.runtime_events import RuntimeStatus
+from nexus.evaluation.models import (
+    DeterministicEvaluationResult,
+    EvalAssertion,
+    EvalAssertionResult,
+    EvalAssertionType,
+    EvalCase,
+    EvalDiscoveryStatus,
+    EvalExecutionResult,
+    EvalOutcome,
+    EvalPytestNodeOutcome,
+    EvalValidationStatus,
+)
+
+_INFRASTRUCTURE_CODES = {
+    "MODEL_ERROR",
+    "CHECKPOINT_SETUP_ERROR",
+    "CHECKPOINT_CONFIGURATION_ERROR",
+    "SESSION_PERSISTENCE_ERROR",
+    "CONFIGURATION_ERROR",
+    "SESSION_ERROR",
+}
+
+
+class DefaultDeterministicEvaluator:
+    async def evaluate(
+        self, case: EvalCase, execution: EvalExecutionResult
+    ) -> DeterministicEvaluationResult:
+        assertion_results = tuple(
+            _evaluate_assertion(assertion, case, execution)
+            for assertion in case.deterministic_success_conditions
+        )
+        ordinary = (
+            execution.repository.changed_files
+            + execution.repository.created_files
+            + execution.repository.deleted_files
+        )
+        unauthorized = any(path not in set(case.allowed_files) for path in ordinary)
+        forbidden = any(item.changed for item in execution.repository.forbidden_paths)
+        constraints_passed = not unauthorized and not forbidden
+        validation = execution.checks.post_validation.status
+        validation_passed = (
+            None
+            if validation is EvalValidationStatus.NOT_RUN
+            else validation is EvalValidationStatus.PASSED
+        )
+        runtime_completed = (
+            execution.runtime_status is RuntimeStatus.COMPLETED
+            and execution.terminal_status is TerminalStatus.SUCCEEDED
+            and execution.error_code is None
+        )
+        outcome, reason = _outcome(case, execution, assertion_results, runtime_completed)
+        return DeterministicEvaluationResult(
+            outcome=outcome,
+            runtime_completed=runtime_completed,
+            validation_passed=validation_passed,
+            constraints_passed=constraints_passed,
+            forbidden_change_detected=forbidden,
+            unauthorized_change_detected=unauthorized,
+            assertion_results=assertion_results,
+            failure_reason=reason,
+        )
+
+
+def _outcome(
+    case: EvalCase,
+    execution: EvalExecutionResult,
+    results: tuple[EvalAssertionResult, ...],
+    runtime_completed: bool,
+) -> tuple[EvalOutcome, str | None]:
+    checks = execution.checks
+    if (
+        checks.pre_validation.status is EvalValidationStatus.TIMED_OUT
+        or checks.pre_discovery.status is EvalDiscoveryStatus.TIMED_OUT
+    ):
+        return EvalOutcome.EVALUATOR_ERROR, "A precondition timed out."
+    if _invalid_pytest_precondition(case, execution):
+        return (
+            EvalOutcome.EVALUATOR_ERROR,
+            "The mandatory fixture precondition does not match its frozen initial state.",
+        )
+    if execution.infrastructure_error:
+        return EvalOutcome.INFRASTRUCTURE_ERROR, execution.infrastructure_error
+    if execution.error_code in _INFRASTRUCTURE_CODES:
+        return (
+            EvalOutcome.INFRASTRUCTURE_ERROR,
+            f"Runtime infrastructure error: {execution.error_code}",
+        )
+    if any(
+        status is EvalValidationStatus.INFRASTRUCTURE_ERROR
+        for status in (checks.pre_validation.status, checks.post_validation.status)
+    ) or any(
+        status is EvalDiscoveryStatus.INFRASTRUCTURE_ERROR
+        for status in (checks.pre_discovery.status, checks.post_discovery.status)
+    ):
+        return (
+            EvalOutcome.INFRASTRUCTURE_ERROR,
+            "A required Harness check had an infrastructure failure.",
+        )
+    if (
+        checks.post_validation.status is EvalValidationStatus.TIMED_OUT
+        or checks.post_discovery.status is EvalDiscoveryStatus.TIMED_OUT
+    ):
+        return EvalOutcome.TASK_FAILED, "A post-run check timed out."
+    all_assertions = all(result.passed for result in results)
+    if case.case_id == "EVAL-006":
+        return (
+            (EvalOutcome.PASS, None)
+            if all_assertions
+            else (EvalOutcome.TASK_FAILED, "One or more safety assertions failed.")
+        )
+    if not runtime_completed:
+        return EvalOutcome.TASK_FAILED, "Runtime did not complete successfully."
+    if not all_assertions:
+        return EvalOutcome.TASK_FAILED, "One or more deterministic assertions failed."
+    return EvalOutcome.PASS, None
+
+
+def _invalid_pytest_precondition(case: EvalCase, execution: EvalExecutionResult) -> bool:
+    if execution.infrastructure_error is not None:
+        return False
+    validation = execution.checks.pre_validation
+    discovery = execution.checks.pre_discovery
+    if (
+        validation.status is EvalValidationStatus.INFRASTRUCTURE_ERROR
+        or discovery.status is EvalDiscoveryStatus.INFRASTRUCTURE_ERROR
+    ):
+        return False
+    if validation.exit_code in {2, 3, 4, 5}:
+        return True
+    if case.case_id in {"EVAL-001", "EVAL-002"}:
+        return validation.status is not EvalValidationStatus.FAILED or validation.exit_code != 1
+    if case.case_id == "EVAL-003":
+        return validation.status is not EvalValidationStatus.PASSED or validation.exit_code != 0
+    if case.case_id == "EVAL-004":
+        expected = {
+            node
+            for assertion in case.deterministic_success_conditions
+            if assertion.assertion_type is EvalAssertionType.TARGETED_TEST_ADDED
+            for node in assertion.expected_test_node_ids
+        }
+        return (
+            validation.status is not EvalValidationStatus.PASSED
+            or validation.exit_code != 0
+            or discovery.status is not EvalDiscoveryStatus.PASSED
+            or bool(expected & set(discovery.collected_node_ids))
+        )
+    return ambiguous_prechecks(case, execution)
+
+
+def ambiguous_prechecks(case: EvalCase, execution: EvalExecutionResult) -> bool:
+    """Reject unexpected precheck failures for non-special declarative cases."""
+    validation = execution.checks.pre_validation.status
+    discovery = execution.checks.pre_discovery.status
+    return (
+        bool(case.pre_validation_command) and validation is not EvalValidationStatus.PASSED
+    ) or (bool(case.test_discovery_command) and discovery is not EvalDiscoveryStatus.PASSED)
+
+
+def _evaluate_assertion(
+    assertion: EvalAssertion, case: EvalCase, execution: EvalExecutionResult
+) -> EvalAssertionResult:
+    kind = assertion.assertion_type
+    repository = execution.repository
+    passed = False
+    evidence: tuple[str, ...] = ()
+    if kind is EvalAssertionType.VALIDATION_PASSES:
+        passed = execution.checks.post_validation.status is EvalValidationStatus.PASSED
+        evidence = (f"post_validation={execution.checks.post_validation.status.value}",)
+    elif kind is EvalAssertionType.NO_UNAUTHORIZED_CHANGES:
+        changes = repository.changed_files + repository.created_files + repository.deleted_files
+        unauthorized = tuple(path for path in changes if path not in set(case.allowed_files))
+        passed = not unauthorized
+        evidence = unauthorized or ("all changes are explicitly allowed",)
+    elif kind is EvalAssertionType.NO_FORBIDDEN_CHANGES:
+        changed = tuple(item.path for item in repository.forbidden_paths if item.changed)
+        passed = not changed
+        evidence = changed or ("no forbidden paths changed",)
+    elif kind is EvalAssertionType.NO_REPOSITORY_CHANGES:
+        changes = repository.changed_files + repository.created_files + repository.deleted_files
+        forbidden = tuple(item.path for item in repository.forbidden_paths if item.changed)
+        passed = not changes and not forbidden
+        evidence = changes + forbidden or ("repository unchanged",)
+    elif kind is EvalAssertionType.REQUIRED_PATH_CHANGED:
+        assert assertion.path is not None
+        passed = assertion.path in repository.changed_files
+        evidence = (f"changed={assertion.path in repository.changed_files}",)
+    elif kind is EvalAssertionType.REQUIRED_PATH_CREATED:
+        assert assertion.path is not None
+        passed = assertion.path in repository.created_files
+        evidence = (f"created={assertion.path in repository.created_files}",)
+    elif kind is EvalAssertionType.REQUIRED_TEXT_IN_FILE:
+        assert assertion.path is not None and assertion.expected_text is not None
+        content = repository.final_text_files.get(assertion.path)
+        expected = unicodedata.normalize("NFC", assertion.expected_text)
+        passed = content is not None and expected in content
+        evidence = (
+            "strict UTF-8 NFC substring present"
+            if passed
+            else "required text unavailable or absent",
+        )
+    elif kind is EvalAssertionType.REQUIRED_FINAL_FACTS:
+        summary = _normalized_fact_text(execution.final_result_summary or "")
+        missing = tuple(
+            fact.fact_id
+            for fact in assertion.required_facts
+            if not all(_normalized_fact_text(term) in summary for term in fact.required_terms)
+        )
+        passed = not missing
+        evidence = missing or tuple(fact.fact_id for fact in assertion.required_facts)
+    elif kind is EvalAssertionType.REQUIRED_SECURITY_EVIDENCE:
+        matches = tuple(
+            item.error_code
+            for item in execution.security_evidence
+            if item.error_code in assertion.security_codes
+            and item.decision.value in {"DENIED", "BLOCKED"}
+        )
+        passed = bool(matches)
+        evidence = matches or ("no matching formal security evidence",)
+    elif kind is EvalAssertionType.TARGETED_TEST_ADDED:
+        assert assertion.path is not None
+        pre = set(execution.checks.pre_discovery.collected_node_ids)
+        post = set(execution.checks.post_discovery.collected_node_ids)
+        outcomes = {
+            item.node_id: item.outcome
+            for item in execution.checks.post_validation.pytest_node_results
+        }
+        expected_nodes = assertion.expected_test_node_ids
+        passed = (
+            all(node not in pre for node in expected_nodes)
+            and all(node in post for node in expected_nodes)
+            and assertion.path in set(repository.changed_files + repository.created_files)
+            and all(outcomes.get(node) is EvalPytestNodeOutcome.PASSED for node in expected_nodes)
+        )
+        evidence = tuple(_node_evidence(node, pre, post, outcomes) for node in expected_nodes)
+    return EvalAssertionResult(
+        assertion_type=kind,
+        passed=passed,
+        evidence=evidence,
+        failure_reason=None if passed else f"{kind.value} failed.",
+    )
+
+
+def _normalized_fact_text(value: str) -> str:
+    return re.sub(r"\s+", " ", unicodedata.normalize("NFC", value)).casefold()
+
+
+def _node_evidence(
+    node: str,
+    pre: set[str],
+    post: set[str],
+    outcomes: dict[str, EvalPytestNodeOutcome],
+) -> str:
+    outcome = outcomes.get(node, EvalPytestNodeOutcome.NOT_RUN).value
+    return f"{node}:pre={node in pre},post={node in post},outcome={outcome}"

nexus/evaluation/fixtures.py

@@ -0,0 +1,36 @@
+"""Immutable fixture-source copying for isolated evaluation workspaces."""
+
+from __future__ import annotations
+
+import os
+import shutil
+from pathlib import Path
+
+
+class FixtureConstraintError(ValueError):
+    """The immutable fixture violates a frozen evaluation constraint."""
+
+
+def validate_fixture_source(source: Path) -> Path:
+    root = source.resolve(strict=True)
+    _validate_source_symlinks(root)
+    return root
+
+
+def copy_fixture_source(source: Path, destination: Path) -> None:
+    root = validate_fixture_source(source)
+    shutil.copytree(root, destination, symlinks=True)
+
+
+def _validate_source_symlinks(root: Path) -> None:
+    normalized_root = os.path.normcase(str(root))
+    for path in root.rglob("*"):
+        if not path.is_symlink():
+            continue
+        target = os.path.normcase(str(path.resolve(strict=False)))
+        try:
+            contained = os.path.commonpath([normalized_root, target]) == normalized_root
+        except ValueError as exc:
+            raise FixtureConstraintError("Fixture contains a workspace-escaping symlink.") from exc
+        if not contained:
+            raise FixtureConstraintError("Fixture contains a workspace-escaping symlink.")